#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" # shellcheck disable=SC1091 source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh" CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}" TS="${TS:-$(timestamp_token)}" ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_host_pool_routing}" GROUP_ID="${GROUP_ID:-p2t4-pool-${TS}}" PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}" PRIMARY_ROUTE_ID="${PRIMARY_ROUTE_ID:-primary-${TS}}" SECONDARY_ROUTE_ID="${SECONDARY_ROUTE_ID:-secondary-${TS}}" PRIMARY_ROUTE_PRIORITY="${PRIMARY_ROUTE_PRIORITY:-10}" SECONDARY_ROUTE_PRIORITY="${SECONDARY_ROUTE_PRIORITY:-20}" PRIMARY_SHADOW_MODEL="${PRIMARY_SHADOW_MODEL:-$PUBLIC_MODEL}" SECONDARY_SHADOW_MODEL="${SECONDARY_SHADOW_MODEL:-$PUBLIC_MODEL}" PRIMARY_SHADOW_HOST_ID="${PRIMARY_SHADOW_HOST_ID:?PRIMARY_SHADOW_HOST_ID required}" PRIMARY_SHADOW_GROUP_ID="${PRIMARY_SHADOW_GROUP_ID:?PRIMARY_SHADOW_GROUP_ID required}" SECONDARY_SHADOW_HOST_ID="${SECONDARY_SHADOW_HOST_ID:?SECONDARY_SHADOW_HOST_ID required}" SECONDARY_SHADOW_GROUP_ID="${SECONDARY_SHADOW_GROUP_ID:?SECONDARY_SHADOW_GROUP_ID required}" REQUEST_ID_PRIMARY="${REQUEST_ID_PRIMARY:-req-p2t4-pool-primary-${TS}}" REQUEST_ID_FAILOVER="${REQUEST_ID_FAILOVER:-req-p2t4-pool-failover-${TS}}" SUBJECT_ID_PRIMARY="${SUBJECT_ID_PRIMARY:-conv-p2t4-pool-primary-${TS}}" SUBJECT_ID_FAILOVER="${SUBJECT_ID_FAILOVER:-conv-p2t4-pool-failover-${TS}}" COOLDOWN_REASON="${COOLDOWN_REASON:-degraded}" COOLDOWN_TTL_SECONDS="${COOLDOWN_TTL_SECONDS:-600}" if [[ -z "${SUBSCRIPTION_USER_ID:-}" && -z "${GATEWAY_API_KEY:-}" ]]; then echo "missing pool-routing auth: set SUBSCRIPTION_USER_ID or GATEWAY_API_KEY" >&2 exit 1 fi crm_auth_init ensure_artifact_dir create_group_payload="$(python3 - "$GROUP_ID" <<'PY2' import json, sys group_id = sys.argv[1] print(json.dumps({ "logical_group_id": group_id, "display_name": f"P2T4 Pool Routing {group_id}", "status": "active", "description": "P2-T4 dual vendor same-model routing verification group", "route_policy": "priority", "sticky_mode": "conversation_preferred", "conversation_ttl_seconds": 1200, "user_model_ttl_seconds": 600, "failover_threshold": 1, "cooldown_seconds": 300, }, ensure_ascii=False)) PY2 )" save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")" save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{"public_model":"$PUBLIC_MODEL","status":"active"}")" create_route_payload() { python3 - "$1" "$2" "$3" "$4" "$5" <<'PY2' import json, sys route_id, name, priority, shadow_group_id, shadow_host_id = sys.argv[1:6] print(json.dumps({ "route_id": route_id, "name": name, "status": "active", "priority": int(priority), "weight": 100, "shadow_group_id": shadow_group_id, "shadow_host_id": shadow_host_id, "upstream_base_url_hint": "https://real-shadow.example/v1", }, ensure_ascii=False)) PY2 } save_json 03-create-primary-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "$(create_route_payload "$PRIMARY_ROUTE_ID" "Primary $PRIMARY_ROUTE_ID" "$PRIMARY_ROUTE_PRIORITY" "$PRIMARY_SHADOW_GROUP_ID" "$PRIMARY_SHADOW_HOST_ID")")" save_json 04-add-primary-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$PRIMARY_ROUTE_ID/models" "{"public_model":"$PUBLIC_MODEL","shadow_model":"$PRIMARY_SHADOW_MODEL","status":"active"}")" save_json 05-create-secondary-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "$(create_route_payload "$SECONDARY_ROUTE_ID" "Secondary $SECONDARY_ROUTE_ID" "$SECONDARY_ROUTE_PRIORITY" "$SECONDARY_SHADOW_GROUP_ID" "$SECONDARY_SHADOW_HOST_ID")")" save_json 06-add-secondary-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$SECONDARY_ROUTE_ID/models" "{"public_model":"$PUBLIC_MODEL","shadow_model":"$SECONDARY_SHADOW_MODEL","status":"active"}")" build_route_chat_payload() { python3 - "$1" "$2" "$3" "$4" "$5" <<'PY2' import json, os, sys logical_group_id, public_model, request_id, subject_id, gateway_api_key = sys.argv[1:6] payload = { "logical_group_id": logical_group_id, "model": public_model, "scope": "conversation", "subject_id": subject_id, "request_id": request_id, "sync": True, } subscription_user_id = os.environ.get("SUBSCRIPTION_USER_ID", "").strip() if subscription_user_id: payload["subscription_user_id"] = subscription_user_id if gateway_api_key.strip(): payload["gateway_api_key"] = gateway_api_key print(json.dumps(payload, ensure_ascii=False)) PY2 } save_json 07-route-chat-primary "$(crm_curl_json POST "/api/routing/chat/completions" "$(build_route_chat_payload "$GROUP_ID" "$PUBLIC_MODEL" "$REQUEST_ID_PRIMARY" "$SUBJECT_ID_PRIMARY" "${GATEWAY_API_KEY:-}")")" save_json 08-set-primary-cooldown "$(crm_curl_json POST "/api/routing/sticky/cooldowns" "{"route_id":"$PRIMARY_ROUTE_ID","reason":"$COOLDOWN_REASON","ttl_seconds":$COOLDOWN_TTL_SECONDS}")" save_json 09-get-primary-cooldown "$(crm_curl_json GET "/api/routing/sticky/cooldowns?route_id=$PRIMARY_ROUTE_ID")" save_json 10-route-chat-failover "$(crm_curl_json POST "/api/routing/chat/completions" "$(build_route_chat_payload "$GROUP_ID" "$PUBLIC_MODEL" "$REQUEST_ID_FAILOVER" "$SUBJECT_ID_FAILOVER" "${GATEWAY_API_KEY:-}")")" save_json 11-failover-logs "$(crm_curl_json GET "/api/routing/logs/failovers?request_id=$REQUEST_ID_FAILOVER&limit=5")" save_json 12-route-health "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")" python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$PUBLIC_MODEL" "$PRIMARY_ROUTE_ID" "$SECONDARY_ROUTE_ID" "$PRIMARY_SHADOW_HOST_ID" "$SECONDARY_SHADOW_HOST_ID" "$PRIMARY_SHADOW_GROUP_ID" "$SECONDARY_SHADOW_GROUP_ID" "$COOLDOWN_REASON" "$REQUEST_ID_PRIMARY" "$REQUEST_ID_FAILOVER" >"$ARTIFACT_DIR/13-summary.json" <<'PY2' import json import sys from pathlib import Path ( art_dir, group_id, public_model, primary_route_id, secondary_route_id, primary_shadow_host_id, secondary_shadow_host_id, primary_shadow_group_id, secondary_shadow_group_id, cooldown_reason, request_id_primary, request_id_failover, ) = sys.argv[1:13] art = Path(art_dir) primary = json.loads((art / "07-route-chat-primary.json").read_text()) cooldown_set = json.loads((art / "08-set-primary-cooldown.json").read_text()) cooldown_get = json.loads((art / "09-get-primary-cooldown.json").read_text()) failover = json.loads((art / "10-route-chat-failover.json").read_text()) failover_logs = json.loads((art / "11-failover-logs.json").read_text()).get("failover_events", []) route_health = json.loads((art / "12-route-health.json").read_text()).get("route_health", []) assert primary["selected_route"]["route_id"] == primary_route_id assert primary["selected_route"]["shadow_host_id"] == primary_shadow_host_id assert primary["selected_route"]["shadow_group_id"] == primary_shadow_group_id assert primary["model"] == public_model assert cooldown_set["route_cooldown"]["route_id"] == primary_route_id assert cooldown_get["route_cooldown"]["route_id"] == primary_route_id assert cooldown_get["route_cooldown"]["reason"] == cooldown_reason assert failover["selected_route"]["route_id"] == secondary_route_id assert failover["selected_route"]["shadow_host_id"] == secondary_shadow_host_id assert failover["selected_route"]["shadow_group_id"] == secondary_shadow_group_id assert failover["model"] == public_model assert any(item.get("from_route_id") == primary_route_id and item.get("to_route_id") == secondary_route_id and cooldown_reason in item.get("reason", "") for item in failover_logs), failover_logs health_by_route = {item["route_id"]: item for item in route_health} assert primary_route_id in health_by_route, route_health assert secondary_route_id in health_by_route, route_health assert health_by_route[primary_route_id]["runtime_status"] == "cooldown" assert health_by_route[secondary_route_id]["runtime_status"] in {"healthy", "failing"} summary = { "artifact_dir": str(art), "logical_group_id": group_id, "public_model": public_model, "primary_request_id": request_id_primary, "failover_request_id": request_id_failover, "primary_selected_route": primary["selected_route"]["route_id"], "failover_selected_route": failover["selected_route"]["route_id"], "primary_runtime_status": health_by_route[primary_route_id]["runtime_status"], "secondary_runtime_status": health_by_route[secondary_route_id]["runtime_status"], "failover_event_count": len(failover_logs), "checks": { "primary_route_serves_model": True, "cooldown_recorded": True, "secondary_route_takes_over": True, "failover_event_recorded": True, "route_health_reflects_cooldown": True } } print(json.dumps(summary, ensure_ascii=False, indent=2)) PY2 cat "$ARTIFACT_DIR/13-summary.json"