fix(provision): stabilize kimi a7m import closure
Downgrade the first third-party account test 403 to an advisory warning when models are already present, and retry transient gateway completion 503 responses during access closure. Add regression coverage for the probe race and completion retry paths, update the execution board, and store the final v0.1.129 Kimi A7M fresh-host acceptance artifact that now reaches succeeded/active/subscription_ready.
This commit is contained in:
phamnazage-jpg
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"source": "manual:a7m-kimi:freshhost",
|
||||
"provider_id": "kimi-a7m",
|
||||
"upstream_key_prefix": "sk-FKg61gc51",
|
||||
"upstream_key_suffix": "w6rqf2",
|
||||
"host_base": "http://127.0.0.1:18109",
|
||||
"db_name": "sub2api_kimi_fresh_20260522_120829",
|
||||
"redis_db": "9"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"crm_base": "http://127.0.0.1:18100",
|
||||
"host_base": "http://127.0.0.1:18109",
|
||||
"provider_id": "kimi-a7m",
|
||||
"subscription_user_id": "2",
|
||||
"subscription_user_key_prefix": "sk-177942307",
|
||||
"db_name": "sub2api_kimi_fresh_20260522_120829",
|
||||
"redis_db": "9"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"host_id": "local-v0129-kimi-fresh-1779423075", "base_url": "http://127.0.0.1:18109", "host_version": "0.1.129", "auth_type": "bearer", "status": "unsupported", "capabilities": {"groups": true, "channels": true, "plans": true, "accounts": true, "account_test": false, "account_models": true, "subscriptions": true}}
|
||||
@@ -0,0 +1,5 @@
|
||||
HTTP/1.1 200 OK
|
||||
Content-Type: application/json
|
||||
Date: Fri, 22 May 2026 04:27:09 GMT
|
||||
Content-Length: 1001
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"accepted_keys_count":1,"access_status":"subscription_ready","accounts_count":1,"batch_id":38,"batch_status":"succeeded","channel":{"id":"1","name":"Kimi A7M 默认渠道-subscription"},"gateway":{"ok":true,"status_code":200,"models":["kimi-k2.6"],"has_expected_model":true,"completion_ok":true,"completion_status":200,"completion_content_type":"application/json","completion_body_preview":"{\"id\":\"msg_e428d081-aeac-4f7d-be2a-7c6caa19d737\",\"model\":\"kimi-k2.6\",\"object\":\"chat.completion\",\"created\":1779424029,\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Pong! 🏓\\n\\nI'm\"},\"finish_reason\":\"length\"}],\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":8,\"total_tokens\":18,\"prompt_tokens_details\":{\"cached_tokens\":0,\"text_tokens\":0,\"audio_tokens\":0,\"image_tokens\":0},\"completion"},"group":{"id":"2","name":"Kimi A7M 默认分组-subscription"},"plan":{"id":"1","name":"Kimi A7M 默认套餐-subscription"},"provider_status":"active"}
|
||||
@@ -0,0 +1 @@
|
||||
{"access_closures":[{"ID":43,"BatchID":38,"ClosureType":"subscription","Status":"subscription_ready","DetailsJSON":"{\"completion_ok\":true,\"completion_preview\":\"{\\\"id\\\":\\\"msg_e428d081-aeac-4f7d-be2a-7c6caa19d737\\\",\\\"model\\\":\\\"kimi-k2.6\\\",\\\"object\\\":\\\"chat.completion\\\",\\\"created\\\":1779424029,\\\"choices\\\":[{\\\"index\\\":0,\\\"message\\\":{\\\"role\\\":\\\"assistant\\\",\\\"content\\\":\\\"Pong! 🏓\\\\n\\\\nI'm\\\"},\\\"finish_reason\\\":\\\"length\\\"}],\\\"usage\\\":{\\\"prompt_tokens\\\":10,\\\"completion_tokens\\\":8,\\\"total_tokens\\\":18,\\\"prompt_tokens_details\\\":{\\\"cached_tokens\\\":0,\\\"text_tokens\\\":0,\\\"audio_tokens\\\":0,\\\"image_tokens\\\":0},\\\"completion\",\"completion_status\":200,\"completion_type\":\"application/json\",\"has_expected_model\":true,\"models\":[\"kimi-k2.6\"],\"ok\":true,\"status_code\":200}"}],"access_count":1,"batch":{"access_status":"subscription_ready","batch_status":"succeeded","host_id":3,"id":38,"mode":"partial","pack_id":38,"provider_id":48},"items":[{"account_status":"warning","batch_id":38,"id":38,"key_fingerprint":"sha256:100830605ca92c766278d12cea58d6fc5d5eb27902a491d6c8e4fe13900a3bbe","probe_summary_json":"{\"account_id\":\"4\",\"models\":[{\"id\":\"kimi-k2.6\",\"display_name\":\"kimi-k2.6\",\"type\":\"model\"}],\"probe_advisory\":true,\"probe_message\":\"API returned 403: Forbidden\",\"probe_ok\":false,\"probe_status\":\"failed\",\"smoke_model_seen\":true,\"validation_status\":\"warning\"}"}],"items_count":1,"managed_count":1,"managed_resources":[{"ID":65,"BatchID":38,"HostID":3,"ResourceType":"account","HostResourceID":"4","ResourceName":"kimi-a7m-01"}],"reconcile_count":0,"reconcile_runs":[]}
|
||||
@@ -0,0 +1,25 @@
|
||||
BEGIN;
|
||||
UPDATE users
|
||||
SET balance = CASE WHEN balance < 10 THEN 10 ELSE balance END,
|
||||
updated_at = now()
|
||||
WHERE id = 2;
|
||||
UPDATE api_keys
|
||||
SET group_id = 2,
|
||||
updated_at = now()
|
||||
WHERE key = 'sk-1779423075-ae5f2b09';
|
||||
INSERT INTO user_subscriptions (
|
||||
user_id, group_id, starts_at, expires_at, status, assigned_by, assigned_at, notes, created_at, updated_at, deleted_at
|
||||
) VALUES (
|
||||
2, 2, now(), now() + interval '30 days', 'active', 1, now(), 'local v0.1.129 freshhost kimi validation', now(), now(), NULL
|
||||
)
|
||||
ON CONFLICT (user_id, group_id) WHERE deleted_at IS NULL
|
||||
DO UPDATE SET
|
||||
starts_at = EXCLUDED.starts_at,
|
||||
expires_at = EXCLUDED.expires_at,
|
||||
status = 'active',
|
||||
assigned_by = EXCLUDED.assigned_by,
|
||||
assigned_at = EXCLUDED.assigned_at,
|
||||
notes = EXCLUDED.notes,
|
||||
updated_at = now(),
|
||||
deleted_at = NULL;
|
||||
COMMIT;
|
||||
@@ -0,0 +1,5 @@
|
||||
BEGIN
|
||||
UPDATE 1
|
||||
UPDATE 1
|
||||
INSERT 0 1
|
||||
COMMIT
|
||||
@@ -0,0 +1,4 @@
|
||||
auth_cache_key=apikey:auth:fc26062e46b5878e43d427af92c522baefef87a59512163b32da3189005b523a
|
||||
balance_cache_key=billing:balance:2
|
||||
subscription_cache_key=billing:sub:2:2
|
||||
0
|
||||
@@ -0,0 +1 @@
|
||||
{"group_id" : 2, "group" : {"id":2,"name":"Kimi A7M 默认分组-subscription","description":"","rate_multiplier":1.0000,"is_exclusive":false,"status":"active","created_at":"2026-05-22T04:11:15.989375+00:00","updated_at":"2026-05-22T04:11:15.989375+00:00","deleted_at":null,"platform":"openai","subscription_type":"subscription","daily_limit_usd":null,"weekly_limit_usd":null,"monthly_limit_usd":null,"default_validity_days":0,"image_price_1k":null,"image_price_2k":null,"image_price_4k":null,"claude_code_only":false,"fallback_group_id":null,"model_routing":{},"model_routing_enabled":false,"fallback_group_id_on_invalid_request":null,"mcp_xml_inject":true,"supported_model_scopes":null,"sort_order":0,"allow_messages_dispatch":false,"default_mapped_model":"","require_oauth_only":false,"require_privacy_set":false,"messages_dispatch_model_config":{},"rpm_limit":0,"allow_image_generation":false,"image_rate_independent":false,"image_rate_multiplier":1.0000}, "subscription" : {"id":2,"user_id":2,"group_id":2,"starts_at":"2026-05-22T04:27:09.353018+00:00","expires_at":"2026-06-21T04:27:09.353018+00:00","status":"active","daily_window_start":"2026-05-21T16:00:00+00:00","weekly_window_start":"2026-05-21T16:00:00+00:00","monthly_window_start":"2026-05-21T16:00:00+00:00","daily_usage_usd":0.0000000000,"weekly_usage_usd":0.0000000000,"monthly_usage_usd":0.0000000000,"assigned_by":1,"assigned_at":"2026-05-22T04:27:09.353018+00:00","notes":"local v0.1.129 freshhost kimi validation","created_at":"2026-05-22T04:11:17.172098+00:00","updated_at":"2026-05-22T04:27:09.353018+00:00","deleted_at":null}, "key" : {"id":1,"user_id":2,"key":"sk-1779423075-ae5f2b09","name":"relay-sub-1779423075-ae5f2b09-key","group_id":2,"status":"active","created_at":"2026-05-22T04:11:15.624575+00:00","updated_at":"2026-05-22T04:27:09.353018+00:00","deleted_at":null,"ip_whitelist":null,"ip_blacklist":null,"quota":0.00000000,"quota_used":0.00000000,"expires_at":null,"last_used_at":"2026-05-22T04:22:18.019347+00:00","rate_limit_5h":0.00000000,"rate_limit_1d":0.00000000,"rate_limit_7d":0.00000000,"usage_5h":0.00000000,"usage_1d":0.00000000,"usage_7d":0.00000000,"window_5h_start":null,"window_1d_start":null,"window_7d_start":null}}
|
||||
@@ -0,0 +1,9 @@
|
||||
HTTP/1.1 200 OK
|
||||
Content-Type: application/json; charset=utf-8
|
||||
Referrer-Policy: strict-origin-when-cross-origin
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: DENY
|
||||
X-Request-Id: 3f344999-9686-4357-8673-7b56a40efb74
|
||||
Date: Fri, 22 May 2026 04:27:09 GMT
|
||||
Content-Length: 123
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"data":[{"id":"kimi-k2.6","type":"model","display_name":"kimi-k2.6","created_at":"2024-01-01T00:00:00Z"}],"object":"list"}
|
||||
@@ -0,0 +1,10 @@
|
||||
HTTP/1.1 200 OK
|
||||
Content-Type: application/json
|
||||
Date: Fri, 22 May 2026 04:27:10 GMT
|
||||
Referrer-Policy: strict-origin-when-cross-origin
|
||||
Vary: Accept-Encoding
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: DENY
|
||||
X-Request-Id: b03fe0d7-0318-4d2e-9606-b0f9e1006a96
|
||||
Content-Length: 611
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"id":"msg_0aec7be5-3b0c-459c-a4a6-f5e95671ad48","model":"kimi-k2.6","object":"chat.completion","created":1779424030,"choices":[{"index":0,"message":{"role":"assistant","content":"Pong! 🏓\n\nI'm"},"finish_reason":"length"}],"usage":{"prompt_tokens":10,"completion_tokens":8,"total_tokens":18,"prompt_tokens_details":{"cached_tokens":0,"text_tokens":0,"audio_tokens":0,"image_tokens":0},"completion_tokens_details":{"text_tokens":0,"audio_tokens":0,"reasoning_tokens":0},"input_tokens":0,"output_tokens":0,"input_tokens_details":null,"claude_cache_creation_5_m_tokens":0,"claude_cache_creation_1_h_tokens":0}}
|
||||
@@ -0,0 +1 @@
|
||||
{"access_closures_count":1,"batch":{"access_status":"subscription_ready","batch_status":"succeeded","id":38,"mode":"partial"},"host":{"base_url":"http://127.0.0.1:18109","host_id":"local-v0129-kimi-fresh-1779423075","host_version":"0.1.129"},"latest_access_status":"subscription_ready","latest_reconcile_status":"not_run","latest_reconcile_summary":{},"managed_resources_count":1,"pack":{"pack_id":"openai-cn-pack-kimi-a7m","version":"1.0.0"},"provider":{"display_name":"Kimi A7M OpenAI Compatible","platform":"openai","provider_id":"kimi-a7m"},"provider_status":"active","reconcile_runs_count":0}
|
||||
@@ -0,0 +1 @@
|
||||
{"batch_access_status":"subscription_ready","batch_id":38,"closures_count":1,"latest_access_status":"subscription_ready","latest_closure":{"closure_type":"subscription","details_json":"{\"completion_ok\":true,\"completion_preview\":\"{\\\"id\\\":\\\"msg_e428d081-aeac-4f7d-be2a-7c6caa19d737\\\",\\\"model\\\":\\\"kimi-k2.6\\\",\\\"object\\\":\\\"chat.completion\\\",\\\"created\\\":1779424029,\\\"choices\\\":[{\\\"index\\\":0,\\\"message\\\":{\\\"role\\\":\\\"assistant\\\",\\\"content\\\":\\\"Pong! 🏓\\\\n\\\\nI'm\\\"},\\\"finish_reason\\\":\\\"length\\\"}],\\\"usage\\\":{\\\"prompt_tokens\\\":10,\\\"completion_tokens\\\":8,\\\"total_tokens\\\":18,\\\"prompt_tokens_details\\\":{\\\"cached_tokens\\\":0,\\\"text_tokens\\\":0,\\\"audio_tokens\\\":0,\\\"image_tokens\\\":0},\\\"completion\",\"completion_status\":200,\"completion_type\":\"application/json\",\"has_expected_model\":true,\"models\":[\"kimi-k2.6\"],\"ok\":true,\"status_code\":200}","id":43,"status":"subscription_ready"},"pack_id":"openai-cn-pack-kimi-a7m","provider_id":"kimi-a7m"}
|
||||
@@ -0,0 +1 @@
|
||||
{"provider_id":"kimi-a7m","mode":"subscription","available":true,"message":"latest access status: subscription_ready"}
|
||||
@@ -0,0 +1 @@
|
||||
{"access_closures":[{"ID":43,"BatchID":38,"ClosureType":"subscription","Status":"subscription_ready","DetailsJSON":"{\"completion_ok\":true,\"completion_preview\":\"{\\\"id\\\":\\\"msg_e428d081-aeac-4f7d-be2a-7c6caa19d737\\\",\\\"model\\\":\\\"kimi-k2.6\\\",\\\"object\\\":\\\"chat.completion\\\",\\\"created\\\":1779424029,\\\"choices\\\":[{\\\"index\\\":0,\\\"message\\\":{\\\"role\\\":\\\"assistant\\\",\\\"content\\\":\\\"Pong! 🏓\\\\n\\\\nI'm\\\"},\\\"finish_reason\\\":\\\"length\\\"}],\\\"usage\\\":{\\\"prompt_tokens\\\":10,\\\"completion_tokens\\\":8,\\\"total_tokens\\\":18,\\\"prompt_tokens_details\\\":{\\\"cached_tokens\\\":0,\\\"text_tokens\\\":0,\\\"audio_tokens\\\":0,\\\"image_tokens\\\":0},\\\"completion\",\"completion_status\":200,\"completion_type\":\"application/json\",\"has_expected_model\":true,\"models\":[\"kimi-k2.6\"],\"ok\":true,\"status_code\":200}"}],"access_count":1,"batch":{"access_status":"subscription_ready","batch_status":"succeeded","host_id":3,"id":38,"mode":"partial","pack_id":38,"provider_id":48},"items":[{"account_status":"warning","batch_id":38,"id":38,"key_fingerprint":"sha256:100830605ca92c766278d12cea58d6fc5d5eb27902a491d6c8e4fe13900a3bbe","probe_summary_json":"{\"account_id\":\"4\",\"models\":[{\"id\":\"kimi-k2.6\",\"display_name\":\"kimi-k2.6\",\"type\":\"model\"}],\"probe_advisory\":true,\"probe_message\":\"API returned 403: Forbidden\",\"probe_ok\":false,\"probe_status\":\"failed\",\"smoke_model_seen\":true,\"validation_status\":\"warning\"}"}],"items_count":1,"managed_count":1,"managed_resources":[{"ID":65,"BatchID":38,"HostID":3,"ResourceType":"account","HostResourceID":"4","ResourceName":"kimi-a7m-01"}],"reconcile_count":0,"reconcile_runs":[]}
|
||||
@@ -0,0 +1,11 @@
|
||||
HTTP/1.1 200 Connection established
|
||||
|
||||
HTTP/2 200
|
||||
alt-svc: h3=":443"; ma=2592000
|
||||
content-type: application/json; charset=utf-8
|
||||
date: Fri, 22 May 2026 04:27:10 GMT
|
||||
via: 1.1 Caddy
|
||||
x-new-api-version: v0.0.0
|
||||
x-oneapi-request-id: 20260522042710560170306YIKBSr5Q
|
||||
content-length: 168
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"data":[{"id":"kimi-k2.6","object":"model","created":1626777600,"owned_by":"custom","supported_endpoint_types":["anthropic","openai"]}],"object":"list","success":true}
|
||||
@@ -0,0 +1,17 @@
|
||||
HTTP/1.1 200 Connection established
|
||||
|
||||
HTTP/2 200
|
||||
alt-svc: h3=":443"; ma=2592000
|
||||
content-type: application/json
|
||||
date: Fri, 22 May 2026 04:27:11 GMT
|
||||
req-arrive-time: 1779424031233
|
||||
req-cost-time: 816
|
||||
resp-start-time: 1779424032049
|
||||
server: istio-envoy
|
||||
set-cookie: acw_tc=180dd8dd17794240312333635e2900ebe766d23471b5c1c302fd300fee23fb;path=/;HttpOnly;Max-Age=1800
|
||||
via: 1.1 Caddy
|
||||
x-envoy-upstream-service-time: 812
|
||||
x-new-api-version: v0.0.0
|
||||
x-oneapi-request-id: 20260522042710706076631DSLRiX1b
|
||||
content-length: 611
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"id":"msg_b392a616-1575-4d7b-a4d6-7be85de3408c","model":"kimi-k2.6","object":"chat.completion","created":1779424032,"choices":[{"index":0,"message":{"role":"assistant","content":"Pong! 🏓\n\nI'm"},"finish_reason":"length"}],"usage":{"prompt_tokens":10,"completion_tokens":8,"total_tokens":18,"prompt_tokens_details":{"cached_tokens":0,"text_tokens":0,"audio_tokens":0,"image_tokens":0},"completion_tokens_details":{"text_tokens":0,"audio_tokens":0,"reasoning_tokens":0},"input_tokens":0,"output_tokens":0,"input_tokens_details":null,"claude_cache_creation_5_m_tokens":0,"claude_cache_creation_1_h_tokens":0}}
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"artifact_dir": "/home/long/project/sub2api-cn-relay-manager/artifacts/real-host-acceptance/20260522_122706_local_v0129_kimi_a7m_subscription_freshhost",
|
||||
"provider_id": "kimi-a7m",
|
||||
"model_name": "kimi-k2.6",
|
||||
"host_id": "local-v0129-kimi-fresh-1779423075",
|
||||
"host_version_from_create_host": "0.1.129",
|
||||
"batch_id": 38,
|
||||
"batch_status": "succeeded",
|
||||
"access_status_from_import": "subscription_ready",
|
||||
"provider_status_from_import": "active",
|
||||
"provider_status_latest": "active",
|
||||
"latest_access_status": "subscription_ready",
|
||||
"preview_available": true,
|
||||
"direct_models_http200": true,
|
||||
"direct_models_has_expected_model": true,
|
||||
"direct_models": [
|
||||
"kimi-k2.6"
|
||||
],
|
||||
"direct_chat_http200": true,
|
||||
"direct_chat_status": 200,
|
||||
"upstream_models_http200": true,
|
||||
"upstream_models_has_expected_model": true,
|
||||
"upstream_models": [
|
||||
"kimi-k2.6"
|
||||
],
|
||||
"upstream_chat_status": 200,
|
||||
"account_probe_summary": "{\"account_id\":\"4\",\"models\":[{\"id\":\"kimi-k2.6\",\"display_name\":\"kimi-k2.6\",\"type\":\"model\"}],\"probe_advisory\":true,\"probe_message\":\"API returned 403: Forbidden\",\"probe_ok\":false,\"probe_status\":\"failed\",\"smoke_model_seen\":true,\"validation_status\":\"warning\"}"
|
||||
}
|
||||
@@ -11,6 +11,7 @@
|
||||
- `subscription` 主链路已通过 latest fresh-host 复验：
|
||||
- MiniMax 53hk：`artifacts/real-host-acceptance/20260521_191418_remote43_minimax_key_import/21-summary.json`
|
||||
- DeepSeek 2166：`artifacts/real-host-acceptance/20260521_201509_remote43_deepseek_key_import/21-summary.json`
|
||||
- Kimi A7M（local host `v0.1.129`）：`artifacts/real-host-acceptance/20260522_122706_local_v0129_kimi_a7m_subscription_freshhost/21-summary.json`
|
||||
- `self_service` 主链路已通过 latest-head 标准 fresh-host 复验：
|
||||
- `artifacts/real-host-acceptance/20260521_210403/05-import.json`
|
||||
- `artifacts/real-host-acceptance/20260521_210403/07-access-status.json`
|
||||
@@ -64,6 +65,10 @@
|
||||
- 主目录 `artifacts/real-host-acceptance/` 当前只保留最终证据
|
||||
- 历史失败/半成功/试错样本已迁到 `artifacts/real-host-acceptance-archive/`
|
||||
- 分类规则见：`docs/REAL_HOST_ARTIFACT_RETENTION.md`
|
||||
13. relay-manager latest-head 已收口 Kimi A7M 两段竞态
|
||||
- account test 首次 `403 Forbidden` 已降级为 advisory warning；只要 `/models` 已命中 `smoke_test_model`，不会再把 batch 误判为 blocking failure
|
||||
- access closure 对导入后瞬时 `503 / no available accounts` 增加短暂 completion retry，避免宿主异步 probe / account warm-up 窗口把真实可用链路误记成 `broken`
|
||||
- `20260522_122706_local_v0129_kimi_a7m_subscription_freshhost` 已证明：在修复后的 relay-manager + patched host 组合下，`kimi-a7m / kimi-k2.6` 可落到 `batch_status=succeeded`、`provider_status=active`、`latest_access_status=subscription_ready`
|
||||
|
||||
## 已验证门禁
|
||||
|
||||
@@ -106,6 +111,11 @@
|
||||
- official MiniMax 模板 live 样本
|
||||
- 模板链路打通，但当前验证 key 命中 upstream `429`
|
||||
|
||||
7. `artifacts/real-host-acceptance/20260522_122706_local_v0129_kimi_a7m_subscription_freshhost`
|
||||
- latest-head relay-manager 对 patched host `v0.1.129` 的 Kimi A7M `subscription` 最终成功样本
|
||||
- `21-summary.json` 已到 `batch_status=succeeded`、`provider_status=active`
|
||||
- `account_probe_summary` 明确记录 `probe_advisory=true`、`validation_status=warning`，证明 403 probe race 已被 relay-manager 正确降级
|
||||
|
||||
## 剩余项（P2 / 运营前置，不阻塞按 PRD 首版范围上线）
|
||||
|
||||
1. 运营前置
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sub2api-cn-relay-manager/internal/host/sub2api"
|
||||
)
|
||||
@@ -11,6 +12,9 @@ import (
|
||||
const (
|
||||
ModeSubscription = "subscription"
|
||||
ModeSelfService = "self_service"
|
||||
|
||||
gatewayCompletionRetryAttempts = 3
|
||||
gatewayCompletionRetryDelay = 300 * time.Millisecond
|
||||
)
|
||||
|
||||
type SubscriptionTarget struct {
|
||||
@@ -96,7 +100,7 @@ func (s *Service) Close(ctx context.Context, req ClosureRequest) (sub2api.Gatewa
|
||||
return sub2api.GatewayAccessResult{}, fmt.Errorf("check gateway access: %w", err)
|
||||
}
|
||||
if result.OK && result.HasExpectedModel && strings.TrimSpace(req.ExpectedModel) != "" {
|
||||
completion, err := s.host.CheckGatewayCompletion(ctx, sub2api.GatewayCompletionCheckRequest{
|
||||
completion, err := s.checkGatewayCompletionWithRetry(ctx, sub2api.GatewayCompletionCheckRequest{
|
||||
APIKey: probeAPIKey,
|
||||
Model: req.ExpectedModel,
|
||||
Prompt: req.Prompt,
|
||||
@@ -112,3 +116,39 @@ func (s *Service) Close(ctx context.Context, req ClosureRequest) (sub2api.Gatewa
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) checkGatewayCompletionWithRetry(ctx context.Context, req sub2api.GatewayCompletionCheckRequest) (sub2api.GatewayCompletionResult, error) {
|
||||
var last sub2api.GatewayCompletionResult
|
||||
for attempt := 1; attempt <= gatewayCompletionRetryAttempts; attempt++ {
|
||||
completion, err := s.host.CheckGatewayCompletion(ctx, req)
|
||||
if err != nil {
|
||||
return sub2api.GatewayCompletionResult{}, err
|
||||
}
|
||||
last = completion
|
||||
if completion.OK || !isTransientGatewayCompletionFailure(completion) || attempt == gatewayCompletionRetryAttempts {
|
||||
return completion, nil
|
||||
}
|
||||
timer := time.NewTimer(gatewayCompletionRetryDelay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return last, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return last, nil
|
||||
}
|
||||
|
||||
func isTransientGatewayCompletionFailure(result sub2api.GatewayCompletionResult) bool {
|
||||
if result.OK {
|
||||
return false
|
||||
}
|
||||
if result.StatusCode != 0 && result.StatusCode != 429 && result.StatusCode != 502 && result.StatusCode != 503 && result.StatusCode != 504 {
|
||||
return false
|
||||
}
|
||||
body := strings.ToLower(strings.TrimSpace(result.BodyPreview))
|
||||
return strings.Contains(body, "service temporarily unavailable") ||
|
||||
strings.Contains(body, "no available accounts") ||
|
||||
strings.Contains(body, "temporar") ||
|
||||
strings.Contains(body, "try again")
|
||||
}
|
||||
|
||||
@@ -114,16 +114,47 @@ func TestServiceCloseReturnsSubscriptionErrorBeforeGatewayProbe(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceCloseRetriesTransientGatewayCompletionFailure(t *testing.T) {
|
||||
host := &fakeClosureHost{
|
||||
gatewayResult: sub2api.GatewayAccessResult{OK: true, StatusCode: 200, HasExpectedModel: true, Models: []string{"kimi-k2.6"}},
|
||||
completionResults: []sub2api.GatewayCompletionResult{
|
||||
{OK: false, StatusCode: 503, ContentType: "application/json", BodyPreview: `{"error":{"message":"Service temporarily unavailable","type":"api_error"}}`},
|
||||
{OK: true, StatusCode: 200, ContentType: "application/json"},
|
||||
},
|
||||
managedAccess: map[string]sub2api.SubscriptionAccessRef{
|
||||
"user-1": {UserID: "host-user-1", APIKey: "managed-user-key"},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := NewService(host).Close(context.Background(), ClosureRequest{
|
||||
Mode: "subscription",
|
||||
GroupID: "group-1",
|
||||
ExpectedModel: "kimi-k2.6",
|
||||
Subscriptions: []SubscriptionTarget{{UserID: "user-1", DurationDays: 30}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
if !result.CompletionOK || result.CompletionStatus != 200 {
|
||||
t.Fatalf("completion result = %+v, want retried success", result)
|
||||
}
|
||||
if host.completionCalls != 2 {
|
||||
t.Fatalf("completion calls = %d, want 2", host.completionCalls)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeClosureHost struct {
|
||||
assigned []sub2api.AssignSubscriptionRequest
|
||||
managedAccess map[string]sub2api.SubscriptionAccessRef
|
||||
assignErr error
|
||||
gatewayProbe sub2api.GatewayAccessCheckRequest
|
||||
gatewayResult sub2api.GatewayAccessResult
|
||||
gatewayErr error
|
||||
completionProbe sub2api.GatewayCompletionCheckRequest
|
||||
completionResult sub2api.GatewayCompletionResult
|
||||
completionErr error
|
||||
assigned []sub2api.AssignSubscriptionRequest
|
||||
managedAccess map[string]sub2api.SubscriptionAccessRef
|
||||
assignErr error
|
||||
gatewayProbe sub2api.GatewayAccessCheckRequest
|
||||
gatewayResult sub2api.GatewayAccessResult
|
||||
gatewayErr error
|
||||
completionProbe sub2api.GatewayCompletionCheckRequest
|
||||
completionCalls int
|
||||
completionResults []sub2api.GatewayCompletionResult
|
||||
completionResult sub2api.GatewayCompletionResult
|
||||
completionErr error
|
||||
}
|
||||
|
||||
func (f *fakeClosureHost) EnsureSubscriptionAccess(_ context.Context, req sub2api.EnsureSubscriptionAccessRequest) (sub2api.SubscriptionAccessRef, error) {
|
||||
@@ -151,8 +182,16 @@ func (f *fakeClosureHost) CheckGatewayAccess(_ context.Context, req sub2api.Gate
|
||||
|
||||
func (f *fakeClosureHost) CheckGatewayCompletion(_ context.Context, req sub2api.GatewayCompletionCheckRequest) (sub2api.GatewayCompletionResult, error) {
|
||||
f.completionProbe = req
|
||||
f.completionCalls++
|
||||
if f.completionErr != nil {
|
||||
return sub2api.GatewayCompletionResult{}, f.completionErr
|
||||
}
|
||||
if len(f.completionResults) > 0 {
|
||||
idx := f.completionCalls - 1
|
||||
if idx >= len(f.completionResults) {
|
||||
idx = len(f.completionResults) - 1
|
||||
}
|
||||
return f.completionResults[idx], nil
|
||||
}
|
||||
return f.completionResult, nil
|
||||
}
|
||||
|
||||
@@ -118,6 +118,16 @@ func isAdvisoryAccountProbeFailure(probe sub2api.ProbeResult) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// OpenAI-compatible third-party upstreams such as Kimi/DeepSeek may
|
||||
// create accounts and expose /models immediately, but the host's
|
||||
// asynchronous /responses capability probe can complete slightly later.
|
||||
// During that race window, the first /accounts/:id/test still takes the
|
||||
// default /responses path and returns a plain 403 Forbidden even though
|
||||
// the actual chat/completions route already works.
|
||||
if strings.Contains(message, "api returned 403: forbidden") {
|
||||
return true
|
||||
}
|
||||
|
||||
if !strings.Contains(message, "responses api") {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -244,6 +244,104 @@ func TestImportServiceTreatsTransientProbeFailureAsAdvisoryWhenGatewaySucceeds(t
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportServiceTreatsForbiddenProbeRaceAsAdvisoryWhenGatewaySucceeds(t *testing.T) {
|
||||
host := &fakeHostAdapter{
|
||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "kimi-a7m-01"}},
|
||||
testResults: map[string]sub2api.ProbeResult{
|
||||
"account_1": {
|
||||
OK: false,
|
||||
Status: "failed",
|
||||
Message: "API returned 403: Forbidden",
|
||||
},
|
||||
},
|
||||
models: map[string][]sub2api.AccountModel{
|
||||
"account_1": {{ID: "deepseek-chat"}},
|
||||
},
|
||||
gatewayResult: sub2api.GatewayAccessResult{
|
||||
OK: true,
|
||||
StatusCode: 200,
|
||||
HasExpectedModel: true,
|
||||
Models: []string{"deepseek-chat"},
|
||||
CompletionOK: true,
|
||||
CompletionStatus: 200,
|
||||
CompletionType: "application/json",
|
||||
},
|
||||
}
|
||||
|
||||
report, err := NewImportService(host).Import(context.Background(), ImportRequest{
|
||||
Provider: sampleProviderManifest(),
|
||||
Mode: ImportModePartial,
|
||||
Access: AccessRequest{Mode: AccessModeSelfService, ProbeAPIKey: "user-key"},
|
||||
Keys: []string{"key-1"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Import() error = %v", err)
|
||||
}
|
||||
if report.BatchStatus != BatchStatusSucceeded {
|
||||
t.Fatalf("BatchStatus = %q, want %q", report.BatchStatus, BatchStatusSucceeded)
|
||||
}
|
||||
if report.ProviderStatus != ProviderStatusActive {
|
||||
t.Fatalf("ProviderStatus = %q, want %q", report.ProviderStatus, ProviderStatusActive)
|
||||
}
|
||||
if report.AccessStatus != AccessStatusSelfServiceReady {
|
||||
t.Fatalf("AccessStatus = %q, want %q", report.AccessStatus, AccessStatusSelfServiceReady)
|
||||
}
|
||||
if got := report.Accounts[0].ValidationStatus(); got != AccountStatusWarning {
|
||||
t.Fatalf("ValidationStatus = %q, want %q", got, AccountStatusWarning)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportServiceRetriesTransientGatewayCompletionFailure(t *testing.T) {
|
||||
host := &fakeHostAdapter{
|
||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1", Name: "kimi-a7m-01"}},
|
||||
testResults: map[string]sub2api.ProbeResult{
|
||||
"account_1": {
|
||||
OK: false,
|
||||
Status: "failed",
|
||||
Message: "API returned 403: Forbidden",
|
||||
},
|
||||
},
|
||||
models: map[string][]sub2api.AccountModel{
|
||||
"account_1": {{ID: "deepseek-chat"}},
|
||||
},
|
||||
gatewayResult: sub2api.GatewayAccessResult{
|
||||
OK: true,
|
||||
StatusCode: 200,
|
||||
HasExpectedModel: true,
|
||||
Models: []string{"deepseek-chat"},
|
||||
},
|
||||
completionResults: []sub2api.GatewayCompletionResult{
|
||||
{OK: false, StatusCode: 503, ContentType: "application/json", BodyPreview: `{"error":{"message":"Service temporarily unavailable","type":"api_error"}}`},
|
||||
{OK: true, StatusCode: 200, ContentType: "application/json"},
|
||||
},
|
||||
}
|
||||
|
||||
report, err := NewImportService(host).Import(context.Background(), ImportRequest{
|
||||
Provider: sampleProviderManifest(),
|
||||
Mode: ImportModePartial,
|
||||
Access: AccessRequest{Mode: AccessModeSelfService, ProbeAPIKey: "user-key"},
|
||||
Keys: []string{"key-1"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Import() error = %v", err)
|
||||
}
|
||||
if report.BatchStatus != BatchStatusSucceeded {
|
||||
t.Fatalf("BatchStatus = %q, want %q", report.BatchStatus, BatchStatusSucceeded)
|
||||
}
|
||||
if report.ProviderStatus != ProviderStatusActive {
|
||||
t.Fatalf("ProviderStatus = %q, want %q", report.ProviderStatus, ProviderStatusActive)
|
||||
}
|
||||
if report.AccessStatus != AccessStatusSelfServiceReady {
|
||||
t.Fatalf("AccessStatus = %q, want %q", report.AccessStatus, AccessStatusSelfServiceReady)
|
||||
}
|
||||
if !report.Gateway.CompletionOK || report.Gateway.CompletionStatus != 200 {
|
||||
t.Fatalf("Gateway completion = %+v, want retried success", report.Gateway)
|
||||
}
|
||||
if host.completionCalls != 2 {
|
||||
t.Fatalf("completion calls = %d, want 2", host.completionCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportServiceStrictModeRollsBackCreatedResources(t *testing.T) {
|
||||
host := &fakeHostAdapter{
|
||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1"}, {ID: "account_2"}},
|
||||
@@ -520,6 +618,8 @@ type fakeHostAdapter struct {
|
||||
updateChannelID string
|
||||
updateChannelReq sub2api.CreateChannelRequest
|
||||
callSequence []string
|
||||
completionCalls int
|
||||
completionResults []sub2api.GatewayCompletionResult
|
||||
completionResult sub2api.GatewayCompletionResult
|
||||
completionErr error
|
||||
testedModels map[string]string
|
||||
@@ -620,9 +720,17 @@ func (f *fakeHostAdapter) CheckGatewayAccess(_ context.Context, req sub2api.Gate
|
||||
func (f *fakeHostAdapter) CheckGatewayCompletion(_ context.Context, req sub2api.GatewayCompletionCheckRequest) (sub2api.GatewayCompletionResult, error) {
|
||||
f.callSequence = append(f.callSequence, "completion")
|
||||
f.completionProbe = req
|
||||
f.completionCalls++
|
||||
if f.completionErr != nil {
|
||||
return sub2api.GatewayCompletionResult{}, f.completionErr
|
||||
}
|
||||
if len(f.completionResults) > 0 {
|
||||
idx := f.completionCalls - 1
|
||||
if idx >= len(f.completionResults) {
|
||||
idx = len(f.completionResults) - 1
|
||||
}
|
||||
return f.completionResults[idx], nil
|
||||
}
|
||||
if f.completionResult.StatusCode == 0 && !f.completionResult.OK {
|
||||
return sub2api.GatewayCompletionResult{OK: true, StatusCode: 200, ContentType: "application/json"}, nil
|
||||
}
|
||||
|
||||
@@ -236,6 +236,70 @@ func TestRuntimeImportServicePersistsWarningAccountStatusForAdvisoryProbeFailure
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeImportServicePersistsWarningAccountStatusForForbiddenProbeRace(t *testing.T) {
|
||||
store := openProvisionTestStore(t)
|
||||
defer closeProvisionTestStore(t, store)
|
||||
|
||||
seedProvisionHost(t, store, "host-1", "https://sub2api.example.com")
|
||||
|
||||
host := &fakeHostAdapter{
|
||||
batchAccounts: []sub2api.AccountRef{{ID: "account_1"}},
|
||||
testResults: map[string]sub2api.ProbeResult{
|
||||
"account_1": {
|
||||
OK: false,
|
||||
Status: "failed",
|
||||
Message: "API returned 403: Forbidden",
|
||||
},
|
||||
},
|
||||
models: map[string][]sub2api.AccountModel{
|
||||
"account_1": {{ID: "deepseek-chat"}},
|
||||
},
|
||||
gatewayResult: sub2api.GatewayAccessResult{
|
||||
OK: true,
|
||||
StatusCode: 200,
|
||||
HasExpectedModel: true,
|
||||
Models: []string{"deepseek-chat"},
|
||||
CompletionOK: true,
|
||||
CompletionStatus: 200,
|
||||
},
|
||||
}
|
||||
|
||||
svc := NewRuntimeImportService(store, host)
|
||||
result, err := svc.Import(context.Background(), RuntimeImportRequest{
|
||||
HostID: "host-1",
|
||||
HostBaseURL: "https://sub2api.example.com",
|
||||
Pack: pack.LoadedPack{
|
||||
Manifest: pack.Manifest{PackID: "openai-cn-pack", Version: "1.0.0", TargetHost: "sub2api", MinHostVersion: "0.1.126", MaxHostVersion: "0.2.x"},
|
||||
Checksum: "checksum-1",
|
||||
},
|
||||
Provider: sampleProviderManifest(),
|
||||
Mode: ImportModePartial,
|
||||
Keys: []string{"key-1"},
|
||||
Access: AccessRequest{
|
||||
Mode: AccessModeSelfService,
|
||||
ProbeAPIKey: "user-key",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RuntimeImportService.Import() error = %v", err)
|
||||
}
|
||||
if result.Report.BatchStatus != BatchStatusSucceeded {
|
||||
t.Fatalf("BatchStatus = %q, want %q", result.Report.BatchStatus, BatchStatusSucceeded)
|
||||
}
|
||||
|
||||
var accountStatus string
|
||||
var summary string
|
||||
if err := store.SQLDB().QueryRowContext(context.Background(), "SELECT account_status, probe_summary_json FROM import_batch_items WHERE batch_id = ? ORDER BY id LIMIT 1", result.BatchID).Scan(&accountStatus, &summary); err != nil {
|
||||
t.Fatalf("query import batch item: %v", err)
|
||||
}
|
||||
if accountStatus != AccountStatusWarning {
|
||||
t.Fatalf("account_status = %q, want %q", accountStatus, AccountStatusWarning)
|
||||
}
|
||||
if !strings.Contains(summary, "\"probe_advisory\":true") {
|
||||
t.Fatalf("probe_summary_json = %s, want probe_advisory=true", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeImportServicePersistsPartialManagedResourcesOnAccessFailure(t *testing.T) {
|
||||
store := openProvisionTestStore(t)
|
||||
defer closeProvisionTestStore(t, store)
|
||||
|
||||
Reference in New Issue
Block a user