feat(status): suppress false negative provider readiness
This commit is contained in:
@@ -281,7 +281,10 @@ func (s *ImportService) Import(ctx context.Context, req ImportRequest) (report I
|
|||||||
|
|
||||||
report.BatchStatus = BatchStatusSucceeded
|
report.BatchStatus = BatchStatusSucceeded
|
||||||
report.ProviderStatus = ProviderStatusActive
|
report.ProviderStatus = ProviderStatusActive
|
||||||
if failedAccounts > 0 || !GatewayAccessReady(gateway) {
|
if failedAccounts > 0 {
|
||||||
|
report.BatchStatus = BatchStatusPartial
|
||||||
|
}
|
||||||
|
if !GatewayAccessReady(gateway) {
|
||||||
report.BatchStatus = BatchStatusPartial
|
report.BatchStatus = BatchStatusPartial
|
||||||
report.ProviderStatus = ProviderStatusDegraded
|
report.ProviderStatus = ProviderStatusDegraded
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -291,6 +291,57 @@ func TestImportServiceTreatsForbiddenProbeRaceAsAdvisoryWhenGatewaySucceeds(t *t
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestImportServiceKeepsProviderActiveWhenGatewayReadyDespiteSingleAccountProbeFailure(t *testing.T) {
|
||||||
|
host := &fakeHostAdapter{
|
||||||
|
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "minimax-01"}},
|
||||||
|
testResults: map[string]sub2api.ProbeResult{
|
||||||
|
"account_1": {
|
||||||
|
OK: false,
|
||||||
|
Status: "failed",
|
||||||
|
Message: `API returned 401: {"code":"INVALID_API_KEY","message":"Invalid API key"}`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
models: map[string][]sub2api.AccountModel{
|
||||||
|
"account_1": {{ID: "MiniMax-M2.7-highspeed"}},
|
||||||
|
},
|
||||||
|
gatewayResult: sub2api.GatewayAccessResult{
|
||||||
|
OK: true,
|
||||||
|
StatusCode: 200,
|
||||||
|
HasExpectedModel: true,
|
||||||
|
Models: []string{"MiniMax-M2.5-highspeed", "MiniMax-M2.7-highspeed"},
|
||||||
|
CompletionOK: true,
|
||||||
|
CompletionStatus: 200,
|
||||||
|
CompletionType: "text/event-stream",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
report, err := NewImportService(host).Import(context.Background(), ImportRequest{
|
||||||
|
Provider: sampleProviderManifestWithSmokeModel("MiniMax-M2.7-highspeed"),
|
||||||
|
Mode: ImportModePartial,
|
||||||
|
Access: AccessRequest{
|
||||||
|
Mode: AccessModeSubscription,
|
||||||
|
ProbeAPIKey: "user-key",
|
||||||
|
Subscriptions: []SubscriptionTarget{{UserID: "user_1", DurationDays: 30}},
|
||||||
|
},
|
||||||
|
Keys: []string{"key-1"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Import() error = %v", err)
|
||||||
|
}
|
||||||
|
if report.BatchStatus != BatchStatusPartial {
|
||||||
|
t.Fatalf("BatchStatus = %q, want %q", report.BatchStatus, BatchStatusPartial)
|
||||||
|
}
|
||||||
|
if report.ProviderStatus != ProviderStatusActive {
|
||||||
|
t.Fatalf("ProviderStatus = %q, want %q", report.ProviderStatus, ProviderStatusActive)
|
||||||
|
}
|
||||||
|
if report.AccessStatus != AccessStatusSubscriptionReady {
|
||||||
|
t.Fatalf("AccessStatus = %q, want %q", report.AccessStatus, AccessStatusSubscriptionReady)
|
||||||
|
}
|
||||||
|
if got := report.Accounts[0].ValidationStatus(); got != AccountStatusFailed {
|
||||||
|
t.Fatalf("ValidationStatus = %q, want %q", got, AccountStatusFailed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestImportServiceTreatsBare404ProbeAsAdvisoryWhenGatewaySucceeds(t *testing.T) {
|
func TestImportServiceTreatsBare404ProbeAsAdvisoryWhenGatewaySucceeds(t *testing.T) {
|
||||||
host := &fakeHostAdapter{
|
host := &fakeHostAdapter{
|
||||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "deepseek-01"}},
|
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "deepseek-01"}},
|
||||||
@@ -629,6 +680,25 @@ func sampleProviderManifest() pack.ProviderManifest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func sampleProviderManifestWithSmokeModel(smokeModel string) pack.ProviderManifest {
|
||||||
|
provider := sampleProviderManifest()
|
||||||
|
provider.ProviderID = "minimax-53hk"
|
||||||
|
provider.DisplayName = "MiniMax 53hk"
|
||||||
|
provider.BaseURL = "https://api.minimax.chat/v1"
|
||||||
|
provider.DefaultModels = []string{"MiniMax-M2.5-highspeed", strings.TrimSpace(smokeModel)}
|
||||||
|
provider.SmokeTestModel = strings.TrimSpace(smokeModel)
|
||||||
|
provider.ChannelTemplate = pack.ChannelTemplate{
|
||||||
|
Name: "MiniMax 53hk 默认渠道",
|
||||||
|
ModelMapping: map[string]string{
|
||||||
|
"MiniMax-M2.5-highspeed": "MiniMax-M2.5-highspeed",
|
||||||
|
strings.TrimSpace(smokeModel): strings.TrimSpace(smokeModel),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
provider.GroupTemplate = pack.GroupTemplate{Name: "MiniMax 53hk 默认分组", RateMultiplier: 1}
|
||||||
|
provider.PlanTemplate = pack.PlanTemplate{Name: "MiniMax 53hk 默认套餐", Price: 19.9, ValidityDays: 30, ValidityUnit: "day"}
|
||||||
|
return provider
|
||||||
|
}
|
||||||
|
|
||||||
func sampleChatOnlyProviderManifest() pack.ProviderManifest {
|
func sampleChatOnlyProviderManifest() pack.ProviderManifest {
|
||||||
provider := sampleProviderManifest()
|
provider := sampleProviderManifest()
|
||||||
provider.ProviderID = "kimi-a7m"
|
provider.ProviderID = "kimi-a7m"
|
||||||
|
|||||||
@@ -175,10 +175,7 @@ func deriveProviderStatus(batchStatus, accessStatus, reconcileStatus string) str
|
|||||||
case BatchStatusFailed, BatchStatusRolledBack:
|
case BatchStatusFailed, BatchStatusRolledBack:
|
||||||
return ProviderStatusFailed
|
return ProviderStatusFailed
|
||||||
}
|
}
|
||||||
if (batchStatus == BatchStatusSucceeded || batchStatus == BatchStatusPartial) && accessStatus != "" && accessStatus != AccessStatusBroken && reconcileStatus == ProviderStatusActive {
|
if (batchStatus == BatchStatusSucceeded || batchStatus == BatchStatusPartial) && providerAccessReady(accessStatus) {
|
||||||
return ProviderStatusActive
|
|
||||||
}
|
|
||||||
if batchStatus == BatchStatusSucceeded && accessStatus != "" && accessStatus != AccessStatusBroken {
|
|
||||||
return ProviderStatusActive
|
return ProviderStatusActive
|
||||||
}
|
}
|
||||||
if reconcileStatus != "" && reconcileStatus != "not_run" {
|
if reconcileStatus != "" && reconcileStatus != "not_run" {
|
||||||
@@ -193,3 +190,8 @@ func deriveProviderStatus(batchStatus, accessStatus, reconcileStatus string) str
|
|||||||
return firstNonEmpty(batchStatus, "unknown")
|
return firstNonEmpty(batchStatus, "unknown")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func providerAccessReady(accessStatus string) bool {
|
||||||
|
accessStatus = strings.TrimSpace(accessStatus)
|
||||||
|
return accessStatus != "" && accessStatus != AccessStatusBroken
|
||||||
|
}
|
||||||
|
|||||||
@@ -332,6 +332,43 @@ func TestProviderStatusServicePromotesRecoveredPartialBatchAfterActiveReconcile(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProviderStatusServicePromotesReadyPartialBatchWithoutReconcile(t *testing.T) {
|
||||||
|
store := openProvisionTestStore(t)
|
||||||
|
defer closeProvisionTestStore(t, store)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
hostID := seedProvisionHost(t, store, "host-1", "https://sub2api.example.com")
|
||||||
|
packID, err := store.Packs().Create(ctx, sqlite.Pack{PackID: "openai-cn-pack", Version: "1.0.0", Checksum: "checksum-1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Packs().Create() error = %v", err)
|
||||||
|
}
|
||||||
|
providerID, err := store.Providers().Create(ctx, sqlite.Provider{
|
||||||
|
PackID: packID, ProviderID: "deepseek", DisplayName: "DeepSeek", BaseURL: "https://api.deepseek.com", Platform: "openai",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Providers().Create() error = %v", err)
|
||||||
|
}
|
||||||
|
batchID, err := store.ImportBatches().Create(ctx, sqlite.ImportBatch{
|
||||||
|
HostID: hostID, PackID: packID, ProviderID: providerID, Mode: ImportModePartial, BatchStatus: BatchStatusPartial, AccessStatus: AccessStatusSubscriptionReady,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ImportBatches().Create() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.AccessClosures().Create(ctx, sqlite.AccessClosureRecord{
|
||||||
|
BatchID: batchID, ClosureType: AccessModeSubscription, Status: AccessStatusSubscriptionReady, DetailsJSON: `{"completion_ok":true}`,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("AccessClosures().Create() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot, err := NewProviderStatusService(store).GetStatus(ctx, ProviderQuery{ProviderID: "deepseek", HostID: "host-1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStatus() error = %v", err)
|
||||||
|
}
|
||||||
|
if snapshot.ProviderStatus != ProviderStatusActive {
|
||||||
|
t.Fatalf("ProviderStatus = %q, want %q", snapshot.ProviderStatus, ProviderStatusActive)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestProviderStatusServiceKeepsRolledBackBatchFailedEvenWithActiveReconcile(t *testing.T) {
|
func TestProviderStatusServiceKeepsRolledBackBatchFailedEvenWithActiveReconcile(t *testing.T) {
|
||||||
store := openProvisionTestStore(t)
|
store := openProvisionTestStore(t)
|
||||||
defer closeProvisionTestStore(t, store)
|
defer closeProvisionTestStore(t, store)
|
||||||
|
|||||||
@@ -499,3 +499,65 @@ func TestSyncProviderAccountsFromImportBatchLeavesRouteEmptyOnAmbiguousShadowBin
|
|||||||
t.Fatalf("provider account route binding = %+v, want empty route on ambiguous shadow binding", account)
|
t.Fatalf("provider account route binding = %+v, want empty route on ambiguous shadow binding", account)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSyncProviderAccountsFromImportBatchPromotesSingleReadyGatewayAccount(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := openTestDBWithFK(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
hostID := createTestHost(t, store)
|
||||||
|
packID := createTestPack(t, store)
|
||||||
|
providerID, err := store.Providers().Create(ctx, Provider{
|
||||||
|
PackID: packID,
|
||||||
|
ProviderID: "minimax-provider",
|
||||||
|
DisplayName: "MiniMax Provider",
|
||||||
|
BaseURL: "https://api.minimax.chat/v1",
|
||||||
|
Platform: "openai",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Providers().Create() error = %v", err)
|
||||||
|
}
|
||||||
|
batchID, err := store.ImportBatches().Create(ctx, ImportBatch{
|
||||||
|
HostID: hostID,
|
||||||
|
PackID: packID,
|
||||||
|
ProviderID: providerID,
|
||||||
|
Mode: "partial",
|
||||||
|
BatchStatus: "partially_succeeded",
|
||||||
|
AccessStatus: "subscription_ready",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ImportBatches().Create() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := store.ImportBatchItems().Create(ctx, ImportBatchItem{
|
||||||
|
BatchID: batchID,
|
||||||
|
KeyFingerprint: "sha256:key1",
|
||||||
|
AccountStatus: "failed",
|
||||||
|
ProbeSummaryJSON: `{"account_id":"account-1","probe_status":"failed","probe_advisory":false,` +
|
||||||
|
`"validation_status":"failed","smoke_model_seen":true}`,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("ImportBatchItems().Create() error = %v", err)
|
||||||
|
}
|
||||||
|
for _, resource := range []ManagedResource{
|
||||||
|
{BatchID: batchID, HostID: hostID, ResourceType: "group", HostResourceID: "group-1", ResourceName: "MiniMax Group"},
|
||||||
|
{BatchID: batchID, HostID: hostID, ResourceType: "account", HostResourceID: "account-1", ResourceName: "minimax-01"},
|
||||||
|
} {
|
||||||
|
if _, err := store.ManagedResources().Create(ctx, resource); err != nil {
|
||||||
|
t.Fatalf("ManagedResources().Create(%s) error = %v", resource.ResourceType, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := SyncProviderAccountsFromImportBatch(ctx, store, batchID); err != nil {
|
||||||
|
t.Fatalf("SyncProviderAccountsFromImportBatch() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
account, err := store.ProviderAccounts().GetByHostIDAndAccountID(ctx, hostID, "account-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ProviderAccounts().GetByHostIDAndAccountID() error = %v", err)
|
||||||
|
}
|
||||||
|
if account.AccountStatus != ProviderAccountStatusActive {
|
||||||
|
t.Fatalf("AccountStatus = %q, want %q", account.AccountStatus, ProviderAccountStatusActive)
|
||||||
|
}
|
||||||
|
if account.LastProbeStatus != "gateway_ready" {
|
||||||
|
t.Fatalf("LastProbeStatus = %q, want gateway_ready", account.LastProbeStatus)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ func SyncProviderAccountsFromImportBatch(ctx context.Context, store *DB, batchID
|
|||||||
if !ok && index < len(unmatchedItems) {
|
if !ok && index < len(unmatchedItems) {
|
||||||
match = unmatchedItems[index]
|
match = unmatchedItems[index]
|
||||||
}
|
}
|
||||||
|
accountStatus, probeStatus := deriveProviderAccountState(batch, len(accountResources), match)
|
||||||
row := ProviderAccount{
|
row := ProviderAccount{
|
||||||
HostID: batch.HostID,
|
HostID: batch.HostID,
|
||||||
ProviderID: batch.ProviderID,
|
ProviderID: batch.ProviderID,
|
||||||
@@ -97,8 +98,8 @@ func SyncProviderAccountsFromImportBatch(ctx context.Context, store *DB, batchID
|
|||||||
HostAccountID: hostAccountID,
|
HostAccountID: hostAccountID,
|
||||||
KeyFingerprint: fallbackString(match.KeyFingerprint, "legacy:"+hostAccountID),
|
KeyFingerprint: fallbackString(match.KeyFingerprint, "legacy:"+hostAccountID),
|
||||||
AccountName: fallbackString(resource.ResourceName, hostAccountID),
|
AccountName: fallbackString(resource.ResourceName, hostAccountID),
|
||||||
AccountStatus: providerAccountStatusFromLegacy(match.AccountStatus),
|
AccountStatus: accountStatus,
|
||||||
LastProbeStatus: strings.TrimSpace(match.ProbeStatus),
|
LastProbeStatus: probeStatus,
|
||||||
LastProbeAt: nowText,
|
LastProbeAt: nowText,
|
||||||
}
|
}
|
||||||
if existing, err := store.ProviderAccounts().GetByHostIDAndAccountID(ctx, batch.HostID, hostAccountID); err == nil {
|
if existing, err := store.ProviderAccounts().GetByHostIDAndAccountID(ctx, batch.HostID, hostAccountID); err == nil {
|
||||||
@@ -145,8 +146,11 @@ func resolveProviderAccountRouteBinding(ctx context.Context, store *DB, shadowHo
|
|||||||
type legacyBatchAccountProjection struct {
|
type legacyBatchAccountProjection struct {
|
||||||
KeyFingerprint string
|
KeyFingerprint string
|
||||||
AccountStatus string
|
AccountStatus string
|
||||||
|
ValidationStatus string
|
||||||
ProbeStatus string
|
ProbeStatus string
|
||||||
AccountID string
|
AccountID string
|
||||||
|
ProbeAdvisory bool
|
||||||
|
SmokeModelSeen bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func indexBatchItemsByAccountID(items []ImportBatchItem) (map[string]legacyBatchAccountProjection, []legacyBatchAccountProjection) {
|
func indexBatchItemsByAccountID(items []ImportBatchItem) (map[string]legacyBatchAccountProjection, []legacyBatchAccountProjection) {
|
||||||
@@ -165,6 +169,15 @@ func indexBatchItemsByAccountID(items []ImportBatchItem) (map[string]legacyBatch
|
|||||||
if value, ok := payload["account_id"].(string); ok {
|
if value, ok := payload["account_id"].(string); ok {
|
||||||
projection.AccountID = strings.TrimSpace(value)
|
projection.AccountID = strings.TrimSpace(value)
|
||||||
}
|
}
|
||||||
|
if value, ok := payload["validation_status"].(string); ok {
|
||||||
|
projection.ValidationStatus = strings.TrimSpace(value)
|
||||||
|
}
|
||||||
|
if value, ok := payload["probe_advisory"].(bool); ok {
|
||||||
|
projection.ProbeAdvisory = value
|
||||||
|
}
|
||||||
|
if value, ok := payload["smoke_model_seen"].(bool); ok {
|
||||||
|
projection.SmokeModelSeen = value
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if projection.AccountID != "" {
|
if projection.AccountID != "" {
|
||||||
indexed[projection.AccountID] = projection
|
indexed[projection.AccountID] = projection
|
||||||
@@ -188,6 +201,33 @@ func providerAccountStatusFromLegacy(accountStatus string) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func deriveProviderAccountState(batch ImportBatch, accountResourceCount int, projection legacyBatchAccountProjection) (string, string) {
|
||||||
|
legacyStatus := fallbackString(projection.ValidationStatus, projection.AccountStatus)
|
||||||
|
probeStatus := strings.TrimSpace(projection.ProbeStatus)
|
||||||
|
if projection.ProbeAdvisory || strings.EqualFold(legacyStatus, "warning") {
|
||||||
|
legacyStatus = "warning"
|
||||||
|
if probeStatus == "" || strings.EqualFold(probeStatus, "failed") {
|
||||||
|
probeStatus = "warning"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if importBatchAccessReady(batch.AccessStatus) &&
|
||||||
|
accountResourceCount == 1 &&
|
||||||
|
projection.SmokeModelSeen &&
|
||||||
|
strings.EqualFold(legacyStatus, "failed") {
|
||||||
|
return ProviderAccountStatusActive, "gateway_ready"
|
||||||
|
}
|
||||||
|
return providerAccountStatusFromLegacy(legacyStatus), probeStatus
|
||||||
|
}
|
||||||
|
|
||||||
|
func importBatchAccessReady(accessStatus string) bool {
|
||||||
|
switch strings.TrimSpace(accessStatus) {
|
||||||
|
case "subscription_ready", "self_service_ready", "fully_ready":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func fallbackString(values ...string) string {
|
func fallbackString(values ...string) string {
|
||||||
for _, value := range values {
|
for _, value := range values {
|
||||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||||
|
|||||||
Reference in New Issue
Block a user