492f33a129
- DEFAULT_CHAIN_ADMISSION.md: reviewed and approved, real artifact refs added - DEFAULT_DATA_IDEMPOTENT_RELEASE_GATE.md: reviewed and approved - scripts/setup_default_data.sh: idempotent init with --dry-run/--apply/artifact - scripts/test/test_default_data.sh: 4 test cases all pass - scripts/acceptance/verify_user_key_self_service.sh: Phase 0 skeleton - .gitignore: add generated artifact directories
174 lines
8.7 KiB
Bash
174 lines
8.7 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
# shellcheck disable=SC1091
|
|
source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"
|
|
|
|
CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
|
|
TS="${TS:-$(timestamp_token)}"
|
|
ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_host_pool_routing}"
|
|
|
|
GROUP_ID="${GROUP_ID:-p2t4-pool-${TS}}"
|
|
PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}"
|
|
PRIMARY_ROUTE_ID="${PRIMARY_ROUTE_ID:-primary-${TS}}"
|
|
SECONDARY_ROUTE_ID="${SECONDARY_ROUTE_ID:-secondary-${TS}}"
|
|
PRIMARY_ROUTE_PRIORITY="${PRIMARY_ROUTE_PRIORITY:-10}"
|
|
SECONDARY_ROUTE_PRIORITY="${SECONDARY_ROUTE_PRIORITY:-20}"
|
|
PRIMARY_SHADOW_MODEL="${PRIMARY_SHADOW_MODEL:-$PUBLIC_MODEL}"
|
|
SECONDARY_SHADOW_MODEL="${SECONDARY_SHADOW_MODEL:-$PUBLIC_MODEL}"
|
|
PRIMARY_SHADOW_HOST_ID="${PRIMARY_SHADOW_HOST_ID:?PRIMARY_SHADOW_HOST_ID required}"
|
|
PRIMARY_SHADOW_GROUP_ID="${PRIMARY_SHADOW_GROUP_ID:?PRIMARY_SHADOW_GROUP_ID required}"
|
|
SECONDARY_SHADOW_HOST_ID="${SECONDARY_SHADOW_HOST_ID:?SECONDARY_SHADOW_HOST_ID required}"
|
|
SECONDARY_SHADOW_GROUP_ID="${SECONDARY_SHADOW_GROUP_ID:?SECONDARY_SHADOW_GROUP_ID required}"
|
|
REQUEST_ID_PRIMARY="${REQUEST_ID_PRIMARY:-req-p2t4-pool-primary-${TS}}"
|
|
REQUEST_ID_FAILOVER="${REQUEST_ID_FAILOVER:-req-p2t4-pool-failover-${TS}}"
|
|
SUBJECT_ID_PRIMARY="${SUBJECT_ID_PRIMARY:-conv-p2t4-pool-primary-${TS}}"
|
|
SUBJECT_ID_FAILOVER="${SUBJECT_ID_FAILOVER:-conv-p2t4-pool-failover-${TS}}"
|
|
COOLDOWN_REASON="${COOLDOWN_REASON:-degraded}"
|
|
COOLDOWN_TTL_SECONDS="${COOLDOWN_TTL_SECONDS:-600}"
|
|
|
|
if [[ -z "${SUBSCRIPTION_USER_ID:-}" && -z "${GATEWAY_API_KEY:-}" ]]; then
|
|
echo "missing pool-routing auth: set SUBSCRIPTION_USER_ID or GATEWAY_API_KEY" >&2
|
|
exit 1
|
|
fi
|
|
|
|
crm_auth_init
|
|
ensure_artifact_dir
|
|
|
|
create_group_payload="$(python3 - "$GROUP_ID" <<'PY2'
|
|
import json, sys
|
|
group_id = sys.argv[1]
|
|
print(json.dumps({
|
|
"logical_group_id": group_id,
|
|
"display_name": f"P2T4 Pool Routing {group_id}",
|
|
"status": "active",
|
|
"description": "P2-T4 dual vendor same-model routing verification group",
|
|
"route_policy": "priority",
|
|
"sticky_mode": "conversation_preferred",
|
|
"conversation_ttl_seconds": 1200,
|
|
"user_model_ttl_seconds": 600,
|
|
"failover_threshold": 1,
|
|
"cooldown_seconds": 300,
|
|
}, ensure_ascii=False))
|
|
PY2
|
|
)"
|
|
save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")"
|
|
save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{"public_model":"$PUBLIC_MODEL","status":"active"}")"
|
|
|
|
create_route_payload() {
|
|
python3 - "$1" "$2" "$3" "$4" "$5" <<'PY2'
|
|
import json, sys
|
|
route_id, name, priority, shadow_group_id, shadow_host_id = sys.argv[1:6]
|
|
print(json.dumps({
|
|
"route_id": route_id,
|
|
"name": name,
|
|
"status": "active",
|
|
"priority": int(priority),
|
|
"weight": 100,
|
|
"shadow_group_id": shadow_group_id,
|
|
"shadow_host_id": shadow_host_id,
|
|
"upstream_base_url_hint": "https://real-shadow.example/v1",
|
|
}, ensure_ascii=False))
|
|
PY2
|
|
}
|
|
|
|
save_json 03-create-primary-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "$(create_route_payload "$PRIMARY_ROUTE_ID" "Primary $PRIMARY_ROUTE_ID" "$PRIMARY_ROUTE_PRIORITY" "$PRIMARY_SHADOW_GROUP_ID" "$PRIMARY_SHADOW_HOST_ID")")"
|
|
save_json 04-add-primary-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$PRIMARY_ROUTE_ID/models" "{"public_model":"$PUBLIC_MODEL","shadow_model":"$PRIMARY_SHADOW_MODEL","status":"active"}")"
|
|
save_json 05-create-secondary-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "$(create_route_payload "$SECONDARY_ROUTE_ID" "Secondary $SECONDARY_ROUTE_ID" "$SECONDARY_ROUTE_PRIORITY" "$SECONDARY_SHADOW_GROUP_ID" "$SECONDARY_SHADOW_HOST_ID")")"
|
|
save_json 06-add-secondary-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$SECONDARY_ROUTE_ID/models" "{"public_model":"$PUBLIC_MODEL","shadow_model":"$SECONDARY_SHADOW_MODEL","status":"active"}")"
|
|
|
|
build_route_chat_payload() {
|
|
python3 - "$1" "$2" "$3" "$4" "$5" <<'PY2'
|
|
import json, os, sys
|
|
logical_group_id, public_model, request_id, subject_id, gateway_api_key = sys.argv[1:6]
|
|
payload = {
|
|
"logical_group_id": logical_group_id,
|
|
"model": public_model,
|
|
"scope": "conversation",
|
|
"subject_id": subject_id,
|
|
"request_id": request_id,
|
|
"sync": True,
|
|
}
|
|
subscription_user_id = os.environ.get("SUBSCRIPTION_USER_ID", "").strip()
|
|
if subscription_user_id:
|
|
payload["subscription_user_id"] = subscription_user_id
|
|
if gateway_api_key.strip():
|
|
payload["gateway_api_key"] = gateway_api_key
|
|
print(json.dumps(payload, ensure_ascii=False))
|
|
PY2
|
|
}
|
|
|
|
save_json 07-route-chat-primary "$(crm_curl_json POST "/api/routing/chat/completions" "$(build_route_chat_payload "$GROUP_ID" "$PUBLIC_MODEL" "$REQUEST_ID_PRIMARY" "$SUBJECT_ID_PRIMARY" "${GATEWAY_API_KEY:-}")")"
|
|
save_json 08-set-primary-cooldown "$(crm_curl_json POST "/api/routing/sticky/cooldowns" "{"route_id":"$PRIMARY_ROUTE_ID","reason":"$COOLDOWN_REASON","ttl_seconds":$COOLDOWN_TTL_SECONDS}")"
|
|
save_json 09-get-primary-cooldown "$(crm_curl_json GET "/api/routing/sticky/cooldowns?route_id=$PRIMARY_ROUTE_ID")"
|
|
save_json 10-route-chat-failover "$(crm_curl_json POST "/api/routing/chat/completions" "$(build_route_chat_payload "$GROUP_ID" "$PUBLIC_MODEL" "$REQUEST_ID_FAILOVER" "$SUBJECT_ID_FAILOVER" "${GATEWAY_API_KEY:-}")")"
|
|
save_json 11-failover-logs "$(crm_curl_json GET "/api/routing/logs/failovers?request_id=$REQUEST_ID_FAILOVER&limit=5")"
|
|
save_json 12-route-health "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")"
|
|
|
|
python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$PUBLIC_MODEL" "$PRIMARY_ROUTE_ID" "$SECONDARY_ROUTE_ID" "$PRIMARY_SHADOW_HOST_ID" "$SECONDARY_SHADOW_HOST_ID" "$PRIMARY_SHADOW_GROUP_ID" "$SECONDARY_SHADOW_GROUP_ID" "$COOLDOWN_REASON" "$REQUEST_ID_PRIMARY" "$REQUEST_ID_FAILOVER" >"$ARTIFACT_DIR/13-summary.json" <<'PY2'
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
(
|
|
art_dir,
|
|
group_id,
|
|
public_model,
|
|
primary_route_id,
|
|
secondary_route_id,
|
|
primary_shadow_host_id,
|
|
secondary_shadow_host_id,
|
|
primary_shadow_group_id,
|
|
secondary_shadow_group_id,
|
|
cooldown_reason,
|
|
request_id_primary,
|
|
request_id_failover,
|
|
) = sys.argv[1:13]
|
|
art = Path(art_dir)
|
|
primary = json.loads((art / "07-route-chat-primary.json").read_text())
|
|
cooldown_set = json.loads((art / "08-set-primary-cooldown.json").read_text())
|
|
cooldown_get = json.loads((art / "09-get-primary-cooldown.json").read_text())
|
|
failover = json.loads((art / "10-route-chat-failover.json").read_text())
|
|
failover_logs = json.loads((art / "11-failover-logs.json").read_text()).get("failover_events", [])
|
|
route_health = json.loads((art / "12-route-health.json").read_text()).get("route_health", [])
|
|
assert primary["selected_route"]["route_id"] == primary_route_id
|
|
assert primary["selected_route"]["shadow_host_id"] == primary_shadow_host_id
|
|
assert primary["selected_route"]["shadow_group_id"] == primary_shadow_group_id
|
|
assert primary["model"] == public_model
|
|
assert cooldown_set["route_cooldown"]["route_id"] == primary_route_id
|
|
assert cooldown_get["route_cooldown"]["route_id"] == primary_route_id
|
|
assert cooldown_get["route_cooldown"]["reason"] == cooldown_reason
|
|
assert failover["selected_route"]["route_id"] == secondary_route_id
|
|
assert failover["selected_route"]["shadow_host_id"] == secondary_shadow_host_id
|
|
assert failover["selected_route"]["shadow_group_id"] == secondary_shadow_group_id
|
|
assert failover["model"] == public_model
|
|
assert any(item.get("from_route_id") == primary_route_id and item.get("to_route_id") == secondary_route_id and cooldown_reason in item.get("reason", "") for item in failover_logs), failover_logs
|
|
health_by_route = {item["route_id"]: item for item in route_health}
|
|
assert primary_route_id in health_by_route, route_health
|
|
assert secondary_route_id in health_by_route, route_health
|
|
assert health_by_route[primary_route_id]["runtime_status"] == "cooldown"
|
|
assert health_by_route[secondary_route_id]["runtime_status"] in {"healthy", "failing"}
|
|
summary = {
|
|
"artifact_dir": str(art),
|
|
"logical_group_id": group_id,
|
|
"public_model": public_model,
|
|
"primary_request_id": request_id_primary,
|
|
"failover_request_id": request_id_failover,
|
|
"primary_selected_route": primary["selected_route"]["route_id"],
|
|
"failover_selected_route": failover["selected_route"]["route_id"],
|
|
"primary_runtime_status": health_by_route[primary_route_id]["runtime_status"],
|
|
"secondary_runtime_status": health_by_route[secondary_route_id]["runtime_status"],
|
|
"failover_event_count": len(failover_logs),
|
|
"checks": {
|
|
"primary_route_serves_model": True,
|
|
"cooldown_recorded": True,
|
|
"secondary_route_takes_over": True,
|
|
"failover_event_recorded": True,
|
|
"route_health_reflects_cooldown": True
|
|
}
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
PY2
|
|
|
|
cat "$ARTIFACT_DIR/13-summary.json"
|