feat(status): suppress false negative provider readiness

This commit is contained in:
phamnazage-jpg
2026-05-30 14:42:51 +08:00
parent 7dbd6769db
commit 15b7437edd
6 changed files with 225 additions and 11 deletions

View File

@@ -281,7 +281,10 @@ func (s *ImportService) Import(ctx context.Context, req ImportRequest) (report I
report.BatchStatus = BatchStatusSucceeded report.BatchStatus = BatchStatusSucceeded
report.ProviderStatus = ProviderStatusActive report.ProviderStatus = ProviderStatusActive
if failedAccounts > 0 || !GatewayAccessReady(gateway) { if failedAccounts > 0 {
report.BatchStatus = BatchStatusPartial
}
if !GatewayAccessReady(gateway) {
report.BatchStatus = BatchStatusPartial report.BatchStatus = BatchStatusPartial
report.ProviderStatus = ProviderStatusDegraded report.ProviderStatus = ProviderStatusDegraded
} }

View File

@@ -291,6 +291,57 @@ func TestImportServiceTreatsForbiddenProbeRaceAsAdvisoryWhenGatewaySucceeds(t *t
} }
} }
func TestImportServiceKeepsProviderActiveWhenGatewayReadyDespiteSingleAccountProbeFailure(t *testing.T) {
host := &fakeHostAdapter{
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "minimax-01"}},
testResults: map[string]sub2api.ProbeResult{
"account_1": {
OK: false,
Status: "failed",
Message: `API returned 401: {"code":"INVALID_API_KEY","message":"Invalid API key"}`,
},
},
models: map[string][]sub2api.AccountModel{
"account_1": {{ID: "MiniMax-M2.7-highspeed"}},
},
gatewayResult: sub2api.GatewayAccessResult{
OK: true,
StatusCode: 200,
HasExpectedModel: true,
Models: []string{"MiniMax-M2.5-highspeed", "MiniMax-M2.7-highspeed"},
CompletionOK: true,
CompletionStatus: 200,
CompletionType: "text/event-stream",
},
}
report, err := NewImportService(host).Import(context.Background(), ImportRequest{
Provider: sampleProviderManifestWithSmokeModel("MiniMax-M2.7-highspeed"),
Mode: ImportModePartial,
Access: AccessRequest{
Mode: AccessModeSubscription,
ProbeAPIKey: "user-key",
Subscriptions: []SubscriptionTarget{{UserID: "user_1", DurationDays: 30}},
},
Keys: []string{"key-1"},
})
if err != nil {
t.Fatalf("Import() error = %v", err)
}
if report.BatchStatus != BatchStatusPartial {
t.Fatalf("BatchStatus = %q, want %q", report.BatchStatus, BatchStatusPartial)
}
if report.ProviderStatus != ProviderStatusActive {
t.Fatalf("ProviderStatus = %q, want %q", report.ProviderStatus, ProviderStatusActive)
}
if report.AccessStatus != AccessStatusSubscriptionReady {
t.Fatalf("AccessStatus = %q, want %q", report.AccessStatus, AccessStatusSubscriptionReady)
}
if got := report.Accounts[0].ValidationStatus(); got != AccountStatusFailed {
t.Fatalf("ValidationStatus = %q, want %q", got, AccountStatusFailed)
}
}
func TestImportServiceTreatsBare404ProbeAsAdvisoryWhenGatewaySucceeds(t *testing.T) { func TestImportServiceTreatsBare404ProbeAsAdvisoryWhenGatewaySucceeds(t *testing.T) {
host := &fakeHostAdapter{ host := &fakeHostAdapter{
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "deepseek-01"}}, batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "deepseek-01"}},
@@ -629,6 +680,25 @@ func sampleProviderManifest() pack.ProviderManifest {
} }
} }
func sampleProviderManifestWithSmokeModel(smokeModel string) pack.ProviderManifest {
provider := sampleProviderManifest()
provider.ProviderID = "minimax-53hk"
provider.DisplayName = "MiniMax 53hk"
provider.BaseURL = "https://api.minimax.chat/v1"
provider.DefaultModels = []string{"MiniMax-M2.5-highspeed", strings.TrimSpace(smokeModel)}
provider.SmokeTestModel = strings.TrimSpace(smokeModel)
provider.ChannelTemplate = pack.ChannelTemplate{
Name: "MiniMax 53hk 默认渠道",
ModelMapping: map[string]string{
"MiniMax-M2.5-highspeed": "MiniMax-M2.5-highspeed",
strings.TrimSpace(smokeModel): strings.TrimSpace(smokeModel),
},
}
provider.GroupTemplate = pack.GroupTemplate{Name: "MiniMax 53hk 默认分组", RateMultiplier: 1}
provider.PlanTemplate = pack.PlanTemplate{Name: "MiniMax 53hk 默认套餐", Price: 19.9, ValidityDays: 30, ValidityUnit: "day"}
return provider
}
func sampleChatOnlyProviderManifest() pack.ProviderManifest { func sampleChatOnlyProviderManifest() pack.ProviderManifest {
provider := sampleProviderManifest() provider := sampleProviderManifest()
provider.ProviderID = "kimi-a7m" provider.ProviderID = "kimi-a7m"

View File

@@ -175,10 +175,7 @@ func deriveProviderStatus(batchStatus, accessStatus, reconcileStatus string) str
case BatchStatusFailed, BatchStatusRolledBack: case BatchStatusFailed, BatchStatusRolledBack:
return ProviderStatusFailed return ProviderStatusFailed
} }
if (batchStatus == BatchStatusSucceeded || batchStatus == BatchStatusPartial) && accessStatus != "" && accessStatus != AccessStatusBroken && reconcileStatus == ProviderStatusActive { if (batchStatus == BatchStatusSucceeded || batchStatus == BatchStatusPartial) && providerAccessReady(accessStatus) {
return ProviderStatusActive
}
if batchStatus == BatchStatusSucceeded && accessStatus != "" && accessStatus != AccessStatusBroken {
return ProviderStatusActive return ProviderStatusActive
} }
if reconcileStatus != "" && reconcileStatus != "not_run" { if reconcileStatus != "" && reconcileStatus != "not_run" {
@@ -193,3 +190,8 @@ func deriveProviderStatus(batchStatus, accessStatus, reconcileStatus string) str
return firstNonEmpty(batchStatus, "unknown") return firstNonEmpty(batchStatus, "unknown")
} }
} }
func providerAccessReady(accessStatus string) bool {
accessStatus = strings.TrimSpace(accessStatus)
return accessStatus != "" && accessStatus != AccessStatusBroken
}

View File

@@ -332,6 +332,43 @@ func TestProviderStatusServicePromotesRecoveredPartialBatchAfterActiveReconcile(
} }
} }
func TestProviderStatusServicePromotesReadyPartialBatchWithoutReconcile(t *testing.T) {
store := openProvisionTestStore(t)
defer closeProvisionTestStore(t, store)
ctx := context.Background()
hostID := seedProvisionHost(t, store, "host-1", "https://sub2api.example.com")
packID, err := store.Packs().Create(ctx, sqlite.Pack{PackID: "openai-cn-pack", Version: "1.0.0", Checksum: "checksum-1"})
if err != nil {
t.Fatalf("Packs().Create() error = %v", err)
}
providerID, err := store.Providers().Create(ctx, sqlite.Provider{
PackID: packID, ProviderID: "deepseek", DisplayName: "DeepSeek", BaseURL: "https://api.deepseek.com", Platform: "openai",
})
if err != nil {
t.Fatalf("Providers().Create() error = %v", err)
}
batchID, err := store.ImportBatches().Create(ctx, sqlite.ImportBatch{
HostID: hostID, PackID: packID, ProviderID: providerID, Mode: ImportModePartial, BatchStatus: BatchStatusPartial, AccessStatus: AccessStatusSubscriptionReady,
})
if err != nil {
t.Fatalf("ImportBatches().Create() error = %v", err)
}
if _, err := store.AccessClosures().Create(ctx, sqlite.AccessClosureRecord{
BatchID: batchID, ClosureType: AccessModeSubscription, Status: AccessStatusSubscriptionReady, DetailsJSON: `{"completion_ok":true}`,
}); err != nil {
t.Fatalf("AccessClosures().Create() error = %v", err)
}
snapshot, err := NewProviderStatusService(store).GetStatus(ctx, ProviderQuery{ProviderID: "deepseek", HostID: "host-1"})
if err != nil {
t.Fatalf("GetStatus() error = %v", err)
}
if snapshot.ProviderStatus != ProviderStatusActive {
t.Fatalf("ProviderStatus = %q, want %q", snapshot.ProviderStatus, ProviderStatusActive)
}
}
func TestProviderStatusServiceKeepsRolledBackBatchFailedEvenWithActiveReconcile(t *testing.T) { func TestProviderStatusServiceKeepsRolledBackBatchFailedEvenWithActiveReconcile(t *testing.T) {
store := openProvisionTestStore(t) store := openProvisionTestStore(t)
defer closeProvisionTestStore(t, store) defer closeProvisionTestStore(t, store)

View File

@@ -499,3 +499,65 @@ func TestSyncProviderAccountsFromImportBatchLeavesRouteEmptyOnAmbiguousShadowBin
t.Fatalf("provider account route binding = %+v, want empty route on ambiguous shadow binding", account) t.Fatalf("provider account route binding = %+v, want empty route on ambiguous shadow binding", account)
} }
} }
func TestSyncProviderAccountsFromImportBatchPromotesSingleReadyGatewayAccount(t *testing.T) {
t.Parallel()
store := openTestDBWithFK(t)
ctx := context.Background()
hostID := createTestHost(t, store)
packID := createTestPack(t, store)
providerID, err := store.Providers().Create(ctx, Provider{
PackID: packID,
ProviderID: "minimax-provider",
DisplayName: "MiniMax Provider",
BaseURL: "https://api.minimax.chat/v1",
Platform: "openai",
})
if err != nil {
t.Fatalf("Providers().Create() error = %v", err)
}
batchID, err := store.ImportBatches().Create(ctx, ImportBatch{
HostID: hostID,
PackID: packID,
ProviderID: providerID,
Mode: "partial",
BatchStatus: "partially_succeeded",
AccessStatus: "subscription_ready",
})
if err != nil {
t.Fatalf("ImportBatches().Create() error = %v", err)
}
if _, err := store.ImportBatchItems().Create(ctx, ImportBatchItem{
BatchID: batchID,
KeyFingerprint: "sha256:key1",
AccountStatus: "failed",
ProbeSummaryJSON: `{"account_id":"account-1","probe_status":"failed","probe_advisory":false,` +
`"validation_status":"failed","smoke_model_seen":true}`,
}); err != nil {
t.Fatalf("ImportBatchItems().Create() error = %v", err)
}
for _, resource := range []ManagedResource{
{BatchID: batchID, HostID: hostID, ResourceType: "group", HostResourceID: "group-1", ResourceName: "MiniMax Group"},
{BatchID: batchID, HostID: hostID, ResourceType: "account", HostResourceID: "account-1", ResourceName: "minimax-01"},
} {
if _, err := store.ManagedResources().Create(ctx, resource); err != nil {
t.Fatalf("ManagedResources().Create(%s) error = %v", resource.ResourceType, err)
}
}
if err := SyncProviderAccountsFromImportBatch(ctx, store, batchID); err != nil {
t.Fatalf("SyncProviderAccountsFromImportBatch() error = %v", err)
}
account, err := store.ProviderAccounts().GetByHostIDAndAccountID(ctx, hostID, "account-1")
if err != nil {
t.Fatalf("ProviderAccounts().GetByHostIDAndAccountID() error = %v", err)
}
if account.AccountStatus != ProviderAccountStatusActive {
t.Fatalf("AccountStatus = %q, want %q", account.AccountStatus, ProviderAccountStatusActive)
}
if account.LastProbeStatus != "gateway_ready" {
t.Fatalf("LastProbeStatus = %q, want gateway_ready", account.LastProbeStatus)
}
}

View File

@@ -89,6 +89,7 @@ func SyncProviderAccountsFromImportBatch(ctx context.Context, store *DB, batchID
if !ok && index < len(unmatchedItems) { if !ok && index < len(unmatchedItems) {
match = unmatchedItems[index] match = unmatchedItems[index]
} }
accountStatus, probeStatus := deriveProviderAccountState(batch, len(accountResources), match)
row := ProviderAccount{ row := ProviderAccount{
HostID: batch.HostID, HostID: batch.HostID,
ProviderID: batch.ProviderID, ProviderID: batch.ProviderID,
@@ -97,8 +98,8 @@ func SyncProviderAccountsFromImportBatch(ctx context.Context, store *DB, batchID
HostAccountID: hostAccountID, HostAccountID: hostAccountID,
KeyFingerprint: fallbackString(match.KeyFingerprint, "legacy:"+hostAccountID), KeyFingerprint: fallbackString(match.KeyFingerprint, "legacy:"+hostAccountID),
AccountName: fallbackString(resource.ResourceName, hostAccountID), AccountName: fallbackString(resource.ResourceName, hostAccountID),
AccountStatus: providerAccountStatusFromLegacy(match.AccountStatus), AccountStatus: accountStatus,
LastProbeStatus: strings.TrimSpace(match.ProbeStatus), LastProbeStatus: probeStatus,
LastProbeAt: nowText, LastProbeAt: nowText,
} }
if existing, err := store.ProviderAccounts().GetByHostIDAndAccountID(ctx, batch.HostID, hostAccountID); err == nil { if existing, err := store.ProviderAccounts().GetByHostIDAndAccountID(ctx, batch.HostID, hostAccountID); err == nil {
@@ -145,8 +146,11 @@ func resolveProviderAccountRouteBinding(ctx context.Context, store *DB, shadowHo
type legacyBatchAccountProjection struct { type legacyBatchAccountProjection struct {
KeyFingerprint string KeyFingerprint string
AccountStatus string AccountStatus string
ValidationStatus string
ProbeStatus string ProbeStatus string
AccountID string AccountID string
ProbeAdvisory bool
SmokeModelSeen bool
} }
func indexBatchItemsByAccountID(items []ImportBatchItem) (map[string]legacyBatchAccountProjection, []legacyBatchAccountProjection) { func indexBatchItemsByAccountID(items []ImportBatchItem) (map[string]legacyBatchAccountProjection, []legacyBatchAccountProjection) {
@@ -165,6 +169,15 @@ func indexBatchItemsByAccountID(items []ImportBatchItem) (map[string]legacyBatch
if value, ok := payload["account_id"].(string); ok { if value, ok := payload["account_id"].(string); ok {
projection.AccountID = strings.TrimSpace(value) projection.AccountID = strings.TrimSpace(value)
} }
if value, ok := payload["validation_status"].(string); ok {
projection.ValidationStatus = strings.TrimSpace(value)
}
if value, ok := payload["probe_advisory"].(bool); ok {
projection.ProbeAdvisory = value
}
if value, ok := payload["smoke_model_seen"].(bool); ok {
projection.SmokeModelSeen = value
}
} }
if projection.AccountID != "" { if projection.AccountID != "" {
indexed[projection.AccountID] = projection indexed[projection.AccountID] = projection
@@ -188,6 +201,33 @@ func providerAccountStatusFromLegacy(accountStatus string) string {
} }
} }
func deriveProviderAccountState(batch ImportBatch, accountResourceCount int, projection legacyBatchAccountProjection) (string, string) {
legacyStatus := fallbackString(projection.ValidationStatus, projection.AccountStatus)
probeStatus := strings.TrimSpace(projection.ProbeStatus)
if projection.ProbeAdvisory || strings.EqualFold(legacyStatus, "warning") {
legacyStatus = "warning"
if probeStatus == "" || strings.EqualFold(probeStatus, "failed") {
probeStatus = "warning"
}
}
if importBatchAccessReady(batch.AccessStatus) &&
accountResourceCount == 1 &&
projection.SmokeModelSeen &&
strings.EqualFold(legacyStatus, "failed") {
return ProviderAccountStatusActive, "gateway_ready"
}
return providerAccountStatusFromLegacy(legacyStatus), probeStatus
}
func importBatchAccessReady(accessStatus string) bool {
switch strings.TrimSpace(accessStatus) {
case "subscription_ready", "self_service_ready", "fully_ready":
return true
default:
return false
}
}
func fallbackString(values ...string) string { func fallbackString(values ...string) string {
for _, value := range values { for _, value := range values {
if trimmed := strings.TrimSpace(value); trimmed != "" { if trimmed := strings.TrimSpace(value); trimmed != "" {