feat(vnext): complete vNext.1 release gate — default chain admission, idempotent init, user key skeleton
- DEFAULT_CHAIN_ADMISSION.md: reviewed and approved, real artifact refs added - DEFAULT_DATA_IDEMPOTENT_RELEASE_GATE.md: reviewed and approved - scripts/setup_default_data.sh: idempotent init with --dry-run/--apply/artifact - scripts/test/test_default_data.sh: 4 test cases all pass - scripts/acceptance/verify_user_key_self_service.sh: Phase 0 skeleton - .gitignore: add generated artifact directories
This commit is contained in:
phamnazage-jpg
173
scripts/acceptance/verify_host_pool_routing.sh
Normal file
173
scripts/acceptance/verify_host_pool_routing.sh
Normal file
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh"
|
||||
|
||||
CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
|
||||
TS="${TS:-$(timestamp_token)}"
|
||||
ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_host_pool_routing}"
|
||||
|
||||
GROUP_ID="${GROUP_ID:-p2t4-pool-${TS}}"
|
||||
PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}"
|
||||
PRIMARY_ROUTE_ID="${PRIMARY_ROUTE_ID:-primary-${TS}}"
|
||||
SECONDARY_ROUTE_ID="${SECONDARY_ROUTE_ID:-secondary-${TS}}"
|
||||
PRIMARY_ROUTE_PRIORITY="${PRIMARY_ROUTE_PRIORITY:-10}"
|
||||
SECONDARY_ROUTE_PRIORITY="${SECONDARY_ROUTE_PRIORITY:-20}"
|
||||
PRIMARY_SHADOW_MODEL="${PRIMARY_SHADOW_MODEL:-$PUBLIC_MODEL}"
|
||||
SECONDARY_SHADOW_MODEL="${SECONDARY_SHADOW_MODEL:-$PUBLIC_MODEL}"
|
||||
PRIMARY_SHADOW_HOST_ID="${PRIMARY_SHADOW_HOST_ID:?PRIMARY_SHADOW_HOST_ID required}"
|
||||
PRIMARY_SHADOW_GROUP_ID="${PRIMARY_SHADOW_GROUP_ID:?PRIMARY_SHADOW_GROUP_ID required}"
|
||||
SECONDARY_SHADOW_HOST_ID="${SECONDARY_SHADOW_HOST_ID:?SECONDARY_SHADOW_HOST_ID required}"
|
||||
SECONDARY_SHADOW_GROUP_ID="${SECONDARY_SHADOW_GROUP_ID:?SECONDARY_SHADOW_GROUP_ID required}"
|
||||
REQUEST_ID_PRIMARY="${REQUEST_ID_PRIMARY:-req-p2t4-pool-primary-${TS}}"
|
||||
REQUEST_ID_FAILOVER="${REQUEST_ID_FAILOVER:-req-p2t4-pool-failover-${TS}}"
|
||||
SUBJECT_ID_PRIMARY="${SUBJECT_ID_PRIMARY:-conv-p2t4-pool-primary-${TS}}"
|
||||
SUBJECT_ID_FAILOVER="${SUBJECT_ID_FAILOVER:-conv-p2t4-pool-failover-${TS}}"
|
||||
COOLDOWN_REASON="${COOLDOWN_REASON:-degraded}"
|
||||
COOLDOWN_TTL_SECONDS="${COOLDOWN_TTL_SECONDS:-600}"
|
||||
|
||||
if [[ -z "${SUBSCRIPTION_USER_ID:-}" && -z "${GATEWAY_API_KEY:-}" ]]; then
|
||||
echo "missing pool-routing auth: set SUBSCRIPTION_USER_ID or GATEWAY_API_KEY" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
crm_auth_init
|
||||
ensure_artifact_dir
|
||||
|
||||
create_group_payload="$(python3 - "$GROUP_ID" <<'PY2'
|
||||
import json, sys
|
||||
group_id = sys.argv[1]
|
||||
print(json.dumps({
|
||||
"logical_group_id": group_id,
|
||||
"display_name": f"P2T4 Pool Routing {group_id}",
|
||||
"status": "active",
|
||||
"description": "P2-T4 dual vendor same-model routing verification group",
|
||||
"route_policy": "priority",
|
||||
"sticky_mode": "conversation_preferred",
|
||||
"conversation_ttl_seconds": 1200,
|
||||
"user_model_ttl_seconds": 600,
|
||||
"failover_threshold": 1,
|
||||
"cooldown_seconds": 300,
|
||||
}, ensure_ascii=False))
|
||||
PY2
|
||||
)"
|
||||
save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")"
|
||||
save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{"public_model":"$PUBLIC_MODEL","status":"active"}")"
|
||||
|
||||
create_route_payload() {
|
||||
python3 - "$1" "$2" "$3" "$4" "$5" <<'PY2'
|
||||
import json, sys
|
||||
route_id, name, priority, shadow_group_id, shadow_host_id = sys.argv[1:6]
|
||||
print(json.dumps({
|
||||
"route_id": route_id,
|
||||
"name": name,
|
||||
"status": "active",
|
||||
"priority": int(priority),
|
||||
"weight": 100,
|
||||
"shadow_group_id": shadow_group_id,
|
||||
"shadow_host_id": shadow_host_id,
|
||||
"upstream_base_url_hint": "https://real-shadow.example/v1",
|
||||
}, ensure_ascii=False))
|
||||
PY2
|
||||
}
|
||||
|
||||
save_json 03-create-primary-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "$(create_route_payload "$PRIMARY_ROUTE_ID" "Primary $PRIMARY_ROUTE_ID" "$PRIMARY_ROUTE_PRIORITY" "$PRIMARY_SHADOW_GROUP_ID" "$PRIMARY_SHADOW_HOST_ID")")"
|
||||
save_json 04-add-primary-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$PRIMARY_ROUTE_ID/models" "{"public_model":"$PUBLIC_MODEL","shadow_model":"$PRIMARY_SHADOW_MODEL","status":"active"}")"
|
||||
save_json 05-create-secondary-route "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" "$(create_route_payload "$SECONDARY_ROUTE_ID" "Secondary $SECONDARY_ROUTE_ID" "$SECONDARY_ROUTE_PRIORITY" "$SECONDARY_SHADOW_GROUP_ID" "$SECONDARY_SHADOW_HOST_ID")")"
|
||||
save_json 06-add-secondary-route-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$SECONDARY_ROUTE_ID/models" "{"public_model":"$PUBLIC_MODEL","shadow_model":"$SECONDARY_SHADOW_MODEL","status":"active"}")"
|
||||
|
||||
build_route_chat_payload() {
|
||||
python3 - "$1" "$2" "$3" "$4" "$5" <<'PY2'
|
||||
import json, os, sys
|
||||
logical_group_id, public_model, request_id, subject_id, gateway_api_key = sys.argv[1:6]
|
||||
payload = {
|
||||
"logical_group_id": logical_group_id,
|
||||
"model": public_model,
|
||||
"scope": "conversation",
|
||||
"subject_id": subject_id,
|
||||
"request_id": request_id,
|
||||
"sync": True,
|
||||
}
|
||||
subscription_user_id = os.environ.get("SUBSCRIPTION_USER_ID", "").strip()
|
||||
if subscription_user_id:
|
||||
payload["subscription_user_id"] = subscription_user_id
|
||||
if gateway_api_key.strip():
|
||||
payload["gateway_api_key"] = gateway_api_key
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
PY2
|
||||
}
|
||||
|
||||
save_json 07-route-chat-primary "$(crm_curl_json POST "/api/routing/chat/completions" "$(build_route_chat_payload "$GROUP_ID" "$PUBLIC_MODEL" "$REQUEST_ID_PRIMARY" "$SUBJECT_ID_PRIMARY" "${GATEWAY_API_KEY:-}")")"
|
||||
save_json 08-set-primary-cooldown "$(crm_curl_json POST "/api/routing/sticky/cooldowns" "{"route_id":"$PRIMARY_ROUTE_ID","reason":"$COOLDOWN_REASON","ttl_seconds":$COOLDOWN_TTL_SECONDS}")"
|
||||
save_json 09-get-primary-cooldown "$(crm_curl_json GET "/api/routing/sticky/cooldowns?route_id=$PRIMARY_ROUTE_ID")"
|
||||
save_json 10-route-chat-failover "$(crm_curl_json POST "/api/routing/chat/completions" "$(build_route_chat_payload "$GROUP_ID" "$PUBLIC_MODEL" "$REQUEST_ID_FAILOVER" "$SUBJECT_ID_FAILOVER" "${GATEWAY_API_KEY:-}")")"
|
||||
save_json 11-failover-logs "$(crm_curl_json GET "/api/routing/logs/failovers?request_id=$REQUEST_ID_FAILOVER&limit=5")"
|
||||
save_json 12-route-health "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")"
|
||||
|
||||
python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$PUBLIC_MODEL" "$PRIMARY_ROUTE_ID" "$SECONDARY_ROUTE_ID" "$PRIMARY_SHADOW_HOST_ID" "$SECONDARY_SHADOW_HOST_ID" "$PRIMARY_SHADOW_GROUP_ID" "$SECONDARY_SHADOW_GROUP_ID" "$COOLDOWN_REASON" "$REQUEST_ID_PRIMARY" "$REQUEST_ID_FAILOVER" >"$ARTIFACT_DIR/13-summary.json" <<'PY2'
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
(
|
||||
art_dir,
|
||||
group_id,
|
||||
public_model,
|
||||
primary_route_id,
|
||||
secondary_route_id,
|
||||
primary_shadow_host_id,
|
||||
secondary_shadow_host_id,
|
||||
primary_shadow_group_id,
|
||||
secondary_shadow_group_id,
|
||||
cooldown_reason,
|
||||
request_id_primary,
|
||||
request_id_failover,
|
||||
) = sys.argv[1:13]
|
||||
art = Path(art_dir)
|
||||
primary = json.loads((art / "07-route-chat-primary.json").read_text())
|
||||
cooldown_set = json.loads((art / "08-set-primary-cooldown.json").read_text())
|
||||
cooldown_get = json.loads((art / "09-get-primary-cooldown.json").read_text())
|
||||
failover = json.loads((art / "10-route-chat-failover.json").read_text())
|
||||
failover_logs = json.loads((art / "11-failover-logs.json").read_text()).get("failover_events", [])
|
||||
route_health = json.loads((art / "12-route-health.json").read_text()).get("route_health", [])
|
||||
assert primary["selected_route"]["route_id"] == primary_route_id
|
||||
assert primary["selected_route"]["shadow_host_id"] == primary_shadow_host_id
|
||||
assert primary["selected_route"]["shadow_group_id"] == primary_shadow_group_id
|
||||
assert primary["model"] == public_model
|
||||
assert cooldown_set["route_cooldown"]["route_id"] == primary_route_id
|
||||
assert cooldown_get["route_cooldown"]["route_id"] == primary_route_id
|
||||
assert cooldown_get["route_cooldown"]["reason"] == cooldown_reason
|
||||
assert failover["selected_route"]["route_id"] == secondary_route_id
|
||||
assert failover["selected_route"]["shadow_host_id"] == secondary_shadow_host_id
|
||||
assert failover["selected_route"]["shadow_group_id"] == secondary_shadow_group_id
|
||||
assert failover["model"] == public_model
|
||||
assert any(item.get("from_route_id") == primary_route_id and item.get("to_route_id") == secondary_route_id and cooldown_reason in item.get("reason", "") for item in failover_logs), failover_logs
|
||||
health_by_route = {item["route_id"]: item for item in route_health}
|
||||
assert primary_route_id in health_by_route, route_health
|
||||
assert secondary_route_id in health_by_route, route_health
|
||||
assert health_by_route[primary_route_id]["runtime_status"] == "cooldown"
|
||||
assert health_by_route[secondary_route_id]["runtime_status"] in {"healthy", "failing"}
|
||||
summary = {
|
||||
"artifact_dir": str(art),
|
||||
"logical_group_id": group_id,
|
||||
"public_model": public_model,
|
||||
"primary_request_id": request_id_primary,
|
||||
"failover_request_id": request_id_failover,
|
||||
"primary_selected_route": primary["selected_route"]["route_id"],
|
||||
"failover_selected_route": failover["selected_route"]["route_id"],
|
||||
"primary_runtime_status": health_by_route[primary_route_id]["runtime_status"],
|
||||
"secondary_runtime_status": health_by_route[secondary_route_id]["runtime_status"],
|
||||
"failover_event_count": len(failover_logs),
|
||||
"checks": {
|
||||
"primary_route_serves_model": True,
|
||||
"cooldown_recorded": True,
|
||||
"secondary_route_takes_over": True,
|
||||
"failover_event_recorded": True,
|
||||
"route_health_reflects_cooldown": True
|
||||
}
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY2
|
||||
|
||||
cat "$ARTIFACT_DIR/13-summary.json"
|
||||
334
scripts/acceptance/verify_host_protocol_matrix.sh
Normal file
334
scripts/acceptance/verify_host_protocol_matrix.sh
Normal file
@@ -0,0 +1,334 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
ARTIFACT_DIR="${ARTIFACT_DIR:-$ROOT_DIR/artifacts/host-capability/$TIMESTAMP}"
|
||||
DRY_RUN="${DRY_RUN:-0}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: verify_host_protocol_matrix.sh
|
||||
|
||||
Required env:
|
||||
PROTOCOL_MATRIX_TARGETS_JSON JSON array of probe targets
|
||||
|
||||
Optional env:
|
||||
ARTIFACT_DIR output directory
|
||||
DRY_RUN=1 emit scaffold summary without network calls
|
||||
|
||||
Example:
|
||||
DRY_RUN=1 \
|
||||
PROTOCOL_MATRIX_TARGETS_JSON='[{"provider_id":"kimi-a7m","base_url":"https://kimi.example.com/v1","api_key_env":"KIMI_API_KEY","models":["kimi-k2.6"]}]' \
|
||||
bash ./scripts/acceptance/verify_host_protocol_matrix.sh
|
||||
EOF
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local name="$1"
|
||||
if [[ -z "${!name:-}" ]]; then
|
||||
echo "missing required env: $name" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "--help" ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
require_var PROTOCOL_MATRIX_TARGETS_JSON
|
||||
mkdir -p "$ARTIFACT_DIR"
|
||||
export ROOT_DIR ARTIFACT_DIR DRY_RUN PROTOCOL_MATRIX_TARGETS_JSON
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
python3 > "$ARTIFACT_DIR/protocol-matrix-summary.json" <<'PY'
|
||||
import json, os
|
||||
|
||||
targets = json.loads(os.environ["PROTOCOL_MATRIX_TARGETS_JSON"])
|
||||
summary = {"mode": "dry_run", "targets": []}
|
||||
for target in targets:
|
||||
summary["targets"].append({
|
||||
"provider_id": str(target.get("provider_id", "")).strip(),
|
||||
"base_url": str(target.get("base_url", "")).strip(),
|
||||
"models": target.get("models", []),
|
||||
"probe_layer": str(target.get("probe_layer", "upstream")).strip() or "upstream",
|
||||
"support_level": "dry_run",
|
||||
})
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo "protocol matrix summary: $ARTIFACT_DIR/protocol-matrix-summary.json"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
artifact_dir = pathlib.Path(os.environ["ARTIFACT_DIR"])
|
||||
script_dir = artifact_dir / "targets"
|
||||
script_dir.mkdir(parents=True, exist_ok=True)
|
||||
targets = json.loads(os.environ["PROTOCOL_MATRIX_TARGETS_JSON"])
|
||||
|
||||
CONNECT_TIMEOUT = 10
|
||||
MAX_TIME = 30
|
||||
RETRY = 1
|
||||
RETRY_DELAY = 2
|
||||
|
||||
|
||||
def sanitize_header_value(value: str) -> str:
|
||||
if value.lower().startswith("authorization:"):
|
||||
return "Authorization: Bearer ***"
|
||||
return value
|
||||
|
||||
|
||||
def read_status(headers_path: pathlib.Path) -> int:
|
||||
if not headers_path.exists():
|
||||
return 0
|
||||
for line in headers_path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("HTTP/"):
|
||||
parts = line.split()
|
||||
if len(parts) >= 2 and parts[1].isdigit():
|
||||
return int(parts[1])
|
||||
return 0
|
||||
|
||||
|
||||
def read_content_type(headers_path: pathlib.Path) -> str:
|
||||
if not headers_path.exists():
|
||||
return ""
|
||||
for line in headers_path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
if ":" not in line:
|
||||
continue
|
||||
k, v = line.split(":", 1)
|
||||
if k.strip().lower() == "content-type":
|
||||
return v.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def body_json(path: pathlib.Path):
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def body_text(path: pathlib.Path) -> str:
|
||||
if not path.exists():
|
||||
return ""
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
def has_smoke_model(path: pathlib.Path, model: str) -> bool:
|
||||
obj = body_json(path)
|
||||
if not isinstance(obj, dict):
|
||||
return False
|
||||
for item in obj.get("data", []):
|
||||
if str(item.get("id", "")).strip() == model:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def classify_endpoint(status: int, body: str, endpoint: str, probe_layer: str) -> str:
|
||||
text = (body or "").lower()
|
||||
if 200 <= status < 300:
|
||||
if endpoint == "models":
|
||||
return "chat_ok"
|
||||
return "chat_ok"
|
||||
if status == 429:
|
||||
return "rate_limited"
|
||||
if status in (401, 403) and ("auth" in text or "invalid" in text or "unauthorized" in text):
|
||||
return "auth_failed"
|
||||
if status == 403 and "region" in text:
|
||||
return "region_blocked"
|
||||
if "1010" in text or "cloudflare" in text:
|
||||
return "cloudflare_blocked"
|
||||
if endpoint == "chat" and probe_layer == "user-key" and ("group" in text or "binding" in text or "assigned" in text):
|
||||
return "user_key_binding_failed"
|
||||
if endpoint == "chat" and status and status not in (401, 403, 429):
|
||||
return "host_protocol_mismatch"
|
||||
return "unknown_error"
|
||||
|
||||
|
||||
def run_capture(url: str, api_key: str, method: str, request_headers_path: pathlib.Path, response_headers_path: pathlib.Path, response_body_path: pathlib.Path, payload=None):
|
||||
request_headers_path.write_text(
|
||||
"Authorization: Bearer ***\n"
|
||||
+ ("Content-Type: application/json\n" if method == "POST" else ""),
|
||||
encoding="utf-8",
|
||||
)
|
||||
response_headers_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
response_headers_path.write_text("", encoding="utf-8")
|
||||
response_body_path.write_text("", encoding="utf-8")
|
||||
|
||||
cmd = [
|
||||
"curl",
|
||||
"-sS",
|
||||
"-D",
|
||||
str(response_headers_path),
|
||||
"-o",
|
||||
str(response_body_path),
|
||||
"--connect-timeout",
|
||||
str(CONNECT_TIMEOUT),
|
||||
"--max-time",
|
||||
str(MAX_TIME),
|
||||
"--retry",
|
||||
str(RETRY),
|
||||
"--retry-delay",
|
||||
str(RETRY_DELAY),
|
||||
"-H",
|
||||
"Authorization: Bearer ***",
|
||||
"-H",
|
||||
f"X-Hermes-Debug-Request-Headers: {request_headers_path}",
|
||||
]
|
||||
if method == "POST":
|
||||
cmd += ["-H", "Content-Type: application/json", url, "-d", json.dumps(payload, ensure_ascii=False)]
|
||||
else:
|
||||
cmd += [url]
|
||||
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True)
|
||||
return {
|
||||
"exit_code": proc.returncode,
|
||||
"stderr": proc.stderr or "",
|
||||
"stdout": proc.stdout or "",
|
||||
}
|
||||
|
||||
|
||||
summary = {"mode": "live_probe", "targets": []}
|
||||
script_error = False
|
||||
|
||||
for index, target in enumerate(targets, start=1):
|
||||
provider_id = str(target.get("provider_id", "")).strip()
|
||||
base_url = str(target.get("base_url", "")).rstrip("/")
|
||||
api_key_env = str(target.get("api_key_env", "")).strip()
|
||||
probe_layer = str(target.get("probe_layer", "upstream")).strip() or "upstream"
|
||||
models = [str(m).strip() for m in target.get("models", []) if str(m).strip()]
|
||||
|
||||
if not provider_id:
|
||||
print("provider_id is required in PROTOCOL_MATRIX_TARGETS_JSON", file=sys.stderr)
|
||||
script_error = True
|
||||
break
|
||||
if not base_url:
|
||||
print(f"base_url is required for {provider_id}", file=sys.stderr)
|
||||
script_error = True
|
||||
break
|
||||
if not api_key_env:
|
||||
print(f"api_key_env is required for {provider_id}", file=sys.stderr)
|
||||
script_error = True
|
||||
break
|
||||
|
||||
api_key = os.environ.get(api_key_env, "").strip()
|
||||
if not api_key:
|
||||
print(f"missing required env from target.api_key_env: {api_key_env}", file=sys.stderr)
|
||||
script_error = True
|
||||
break
|
||||
|
||||
smoke_model = models[0] if models else "ping"
|
||||
target_dir = script_dir / f"{index:02d}-{provider_id}"
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
endpoints = [
|
||||
("models", "GET", f"{base_url}/models", None, "01-models"),
|
||||
("chat", "POST", f"{base_url}/chat/completions", {"model": smoke_model, "messages": [{"role": "user", "content": "ping"}], "max_tokens": 8, "temperature": 0}, "02-chat"),
|
||||
("responses", "POST", f"{base_url}/responses", {"model": smoke_model, "input": "ping"}, "03-responses"),
|
||||
]
|
||||
|
||||
endpoint_results = {}
|
||||
target_failed = False
|
||||
target_error_code = ""
|
||||
|
||||
for endpoint_name, method, url, payload, prefix in endpoints:
|
||||
request_headers_path = target_dir / f"{prefix}.request_headers.txt"
|
||||
response_headers_path = target_dir / f"{prefix}.response_headers.txt"
|
||||
response_body_path = target_dir / f"{prefix}.response_body.json"
|
||||
result = run_capture(url, api_key, method, request_headers_path, response_headers_path, response_body_path, payload)
|
||||
status = read_status(response_headers_path)
|
||||
body = body_text(response_body_path)
|
||||
error_code = ""
|
||||
if result["exit_code"] == 28:
|
||||
error_code = "network_timeout"
|
||||
target_failed = True
|
||||
elif result["exit_code"] != 0:
|
||||
error_code = "unknown_error"
|
||||
target_failed = True
|
||||
elif not (200 <= status < 300):
|
||||
error_code = classify_endpoint(status, body, endpoint_name, probe_layer)
|
||||
if endpoint_name == "models":
|
||||
target_failed = True
|
||||
elif endpoint_name == "chat" and error_code not in ("responses_unsupported",):
|
||||
target_failed = True
|
||||
endpoint_results[endpoint_name] = {
|
||||
"status": status,
|
||||
"content_type": read_content_type(response_headers_path),
|
||||
"body": body,
|
||||
"error_code": error_code,
|
||||
"exit_code": result["exit_code"],
|
||||
"path_headers": str(response_headers_path),
|
||||
"path_body": str(response_body_path),
|
||||
}
|
||||
if result["exit_code"] == 28 and not target_error_code:
|
||||
target_error_code = "network_timeout"
|
||||
|
||||
models_status = endpoint_results["models"]["status"]
|
||||
chat_status = endpoint_results["chat"]["status"]
|
||||
responses_status = endpoint_results["responses"]["status"]
|
||||
chat_ok = 200 <= chat_status < 300
|
||||
responses_ok = 200 <= responses_status < 300
|
||||
models_ok = 200 <= models_status < 300
|
||||
models_body_path = target_dir / "01-models.response_body.json"
|
||||
|
||||
advisories = []
|
||||
status = "ok"
|
||||
support_level = "unsupported-by-host"
|
||||
summary_error_code = target_error_code
|
||||
|
||||
if target_failed:
|
||||
status = "failed"
|
||||
if not summary_error_code:
|
||||
summary_error_code = endpoint_results["chat"]["error_code"] or endpoint_results["models"]["error_code"] or endpoint_results["responses"]["error_code"] or "unknown_error"
|
||||
else:
|
||||
if chat_ok and responses_ok:
|
||||
support_level = "supported-direct"
|
||||
summary_error_code = "chat_ok"
|
||||
elif chat_ok and not responses_ok:
|
||||
advisories.append("responses_unsupported_but_chat_ok")
|
||||
support_level = "supported-with-plugin-adapter"
|
||||
summary_error_code = "responses_unsupported"
|
||||
elif models_ok and not chat_ok:
|
||||
support_level = "upstream-unhealthy"
|
||||
summary_error_code = endpoint_results["chat"]["error_code"] or "models_only"
|
||||
else:
|
||||
support_level = "unsupported-by-host"
|
||||
summary_error_code = endpoint_results["chat"]["error_code"] or endpoint_results["responses"]["error_code"] or "unknown_error"
|
||||
status = "failed"
|
||||
|
||||
summary["targets"].append({
|
||||
"provider_id": provider_id,
|
||||
"base_url": base_url,
|
||||
"probe_layer": probe_layer,
|
||||
"models": models,
|
||||
"smoke_model": smoke_model,
|
||||
"status": status,
|
||||
"error_code": summary_error_code,
|
||||
"models_status": models_status,
|
||||
"chat_status": chat_status,
|
||||
"responses_status": responses_status,
|
||||
"models_has_smoke_model": has_smoke_model(models_body_path, smoke_model),
|
||||
"chat_content_type": endpoint_results["chat"]["content_type"],
|
||||
"responses_content_type": endpoint_results["responses"]["content_type"],
|
||||
"support_level": support_level,
|
||||
"known_advisories": advisories,
|
||||
"artifact_dir": str(target_dir),
|
||||
})
|
||||
|
||||
(artifact_dir / "protocol-matrix-summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
if script_error:
|
||||
sys.exit(1)
|
||||
PY
|
||||
|
||||
echo "protocol matrix summary: $ARTIFACT_DIR/protocol-matrix-summary.json"
|
||||
110
scripts/acceptance/verify_user_key_self_service.sh
Executable file
110
scripts/acceptance/verify_user_key_self_service.sh
Executable file
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env bash
|
||||
# verify_user_key_self_service.sh — 用户 key 自助验收入口
|
||||
#
|
||||
# 本脚本为 Phase 0 skeleton。验收逻辑在 Phase 3（vNext.2）实现。
|
||||
# 当前仅验证环境就绪与目录规范。
|
||||
#
|
||||
# 使用方式：
|
||||
# bash scripts/acceptance/verify_user_key_self_service.sh --help
|
||||
# bash scripts/acceptance/verify_user_key_self_service.sh [--env-check]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
TS="$(date +%Y%m%d_%H%M%S)"
|
||||
ARTIFACT_DIR="${ARTIFACT_DIR:-$ROOT_DIR/artifacts/user-key-self-service/${TS}}"
|
||||
|
||||
CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}"
|
||||
|
||||
# --- helpers ---
|
||||
die() { echo "FATAL: $*" >&2; exit 1; }
|
||||
info() { echo "INFO: $*"; }
|
||||
ok() { echo "OK: $*"; }
|
||||
|
||||
cmd_help() {
|
||||
cat <<HELP
|
||||
usage: $(basename "$0") [--help|--env-check]
|
||||
|
||||
Phase 0 skeleton — user key self-service acceptance script.
|
||||
|
||||
options:
|
||||
--help 显示此帮助
|
||||
--env-check 验证环境变量与基本可达性
|
||||
|
||||
当前状态：
|
||||
此脚本为 vNext.1 Phase 0 骨架。验收逻辑将在 vNext.2 (Phase 3) 实现。
|
||||
vNext.1 目标用户 key 自助已明确推迟到 vNext.2。
|
||||
|
||||
环境变量:
|
||||
CRM_BASE CRM API 基础 URL (default: https://sub.tksea.top/portal-admin-api)
|
||||
CRM_ADMIN_TOKEN Admin token（可选，env-check 用）
|
||||
|
||||
验收范围 (vNext.2):
|
||||
- 用户 key 自助申请
|
||||
- key 首次回显与仅首次显示明文
|
||||
- key 状态展示（active/paused/exhausted）
|
||||
- 用户首次 POST /v1/chat/completions = 200 闭环
|
||||
|
||||
输出:
|
||||
artifacts/user-key-self-service/<timestamp>/
|
||||
HELP
|
||||
exit 0
|
||||
}
|
||||
|
||||
cmd_env_check() {
|
||||
info "env-check mode"
|
||||
mkdir -p "$ARTIFACT_DIR"
|
||||
|
||||
if [[ -z "${CRM_BASE}" ]]; then
|
||||
warn "CRM_BASE is empty"
|
||||
else
|
||||
ok "CRM_BASE=${CRM_BASE}"
|
||||
fi
|
||||
|
||||
if [[ -n "${CRM_ADMIN_TOKEN:-}" ]]; then
|
||||
ok "CRM_ADMIN_TOKEN is set"
|
||||
local whoami
|
||||
whoami="$(curl -sS --noproxy '*' -H "Authorization: Bearer $CRM_ADMIN_TOKEN" "${CRM_BASE}/api/admin/session" 2>/dev/null)" || true
|
||||
if echo "${whoami}" | python3 -c "import sys,json; d=json.load(sys.stdin); d.get('authenticated',False) or d.get('username','')" 2>/dev/null; then
|
||||
ok "Admin session: valid"
|
||||
else
|
||||
warn "Admin session: invalid. Phase 3 will establish login flow."
|
||||
fi
|
||||
else
|
||||
info "CRM_ADMIN_TOKEN not set — skipped (Phase 3 will implement login)"
|
||||
fi
|
||||
|
||||
# Check portal-admin-api reachability
|
||||
local health
|
||||
health="$(curl -sS --noproxy '*' "${CRM_BASE}/healthz" 2>/dev/null)" || true
|
||||
if [[ "${health}" == "ok" ]]; then
|
||||
ok "CRM health: OK"
|
||||
else
|
||||
warn "CRM health: ${health:-unreachable}"
|
||||
fi
|
||||
|
||||
# Write env-check summary
|
||||
local summary_file="$ARTIFACT_DIR/env-check-summary.json"
|
||||
python3 -c "
|
||||
import json, sys, datetime, os
|
||||
d = {
|
||||
'timestamp': datetime.datetime.now().isoformat(),
|
||||
'mode': 'env_check',
|
||||
'crm_base': os.environ.get('CRM_BASE', ''),
|
||||
'crm_reachable': '${health:-}' == 'ok',
|
||||
'admin_token_set': bool(os.environ.get('CRM_ADMIN_TOKEN', '')),
|
||||
'phase': 'skeleton',
|
||||
'note': 'Full verification deferred to vNext.2 (Phase 3)'
|
||||
}
|
||||
with open(sys.argv[1], 'w') as f:
|
||||
json.dump(d, f, ensure_ascii=False, indent=2)
|
||||
" "$summary_file"
|
||||
ok "env-check summary: $summary_file"
|
||||
}
|
||||
|
||||
# --- main ---
|
||||
case "${1:---help}" in
|
||||
--help|-h) cmd_help ;;
|
||||
--env-check) cmd_env_check ;;
|
||||
*) cmd_help ;;
|
||||
esac
|
||||
110
scripts/setup_default_data.sh
Executable file
110
scripts/setup_default_data.sh
Executable file
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env bash
|
||||
# setup_default_data.sh — 幂等默认数据初始化
|
||||
#
|
||||
# 用途：确保 CRM-only 部署的默认数据存在且可重复执行。
|
||||
#
|
||||
# 设计原则：
|
||||
# - 幂等：多次运行产生相同最终状态
|
||||
# - dry-run 优先：默认只输出将要执行的操作
|
||||
# - 不修改宿主后端源码
|
||||
# - 不直写宿主 PG（CRM-only 部署模式无 PG 依赖）
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TS="$(date +%Y%m%d_%H%M%S)"
|
||||
ARTIFACT_DIR="${ARTIFACT_DIR:-$ROOT_DIR/artifacts/default-data/${TS}}"
|
||||
|
||||
CRM_BASE="${CRM_BASE:-http://127.0.0.1:18190}"
|
||||
CRM_ADMIN_TOKEN=${CRM_ADMIN_TOKEN:-}
|
||||
CRM_ADMIN_USERNAME=${CRM_ADMIN_USERNAME:-admin}
|
||||
CRM_ADMIN_PASSWORD=${CRM_ADMIN_PASSWORD:-}
|
||||
|
||||
# --- helpers ---
|
||||
die() { echo "FATAL: $*" >&2; exit 1; }
|
||||
info() { echo "INFO: $*"; }
|
||||
warn() { echo "WARN: $*"; }
|
||||
ok() { echo "OK: $*"; }
|
||||
|
||||
ensure_artifact_dir() {
|
||||
mkdir -p "${ARTIFACT_DIR}"
|
||||
}
|
||||
|
||||
_curl() {
|
||||
if [[ -n "${CRM_ADMIN_TOKEN}" ]]; then
|
||||
curl -sS --noproxy '*' -H "Authorization: Bearer ${CRM_ADMIN_TOKEN}" "$@"
|
||||
else
|
||||
curl -sS --noproxy '*' "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_help() {
|
||||
cat <<HELP
|
||||
usage: $(basename "$0") [--help|--dry-run|--apply]
|
||||
|
||||
options:
|
||||
--help show help
|
||||
--dry-run print what would change (default)
|
||||
--apply apply initialization
|
||||
|
||||
vars:
|
||||
CRM_BASE CRM URL (default: http://127.0.0.1:18190)
|
||||
CRM_ADMIN_TOKEN admin token
|
||||
CRM_ADMIN_USERNAME admin username (default: admin)
|
||||
CRM_ADMIN_PASSWORD admin password
|
||||
|
||||
output:
|
||||
artifacts/default-data/<ts>/run-log.json
|
||||
HELP
|
||||
exit 0
|
||||
}
|
||||
|
||||
cmd_dry_run() {
|
||||
info "dry-run"
|
||||
mkdir -p "${ARTIFACT_DIR}"
|
||||
info "CRM: ${CRM_BASE}"
|
||||
|
||||
local h; h="$(_curl "${CRM_BASE}/healthz")" || true
|
||||
[[ "${h}" == "ok" ]] && ok "health: ok" || warn "health: ${h}"
|
||||
|
||||
local s; s="$(_curl "${CRM_BASE}/api/admin/schema-version" 2>/dev/null)" || s="N/A"
|
||||
info "schema: ${s}"
|
||||
|
||||
info "dry-run done -> ${ARTIFACT_DIR}/dry-run-summary.json"
|
||||
python3 -c "
|
||||
import json, sys, datetime
|
||||
d = {'ts':datetime.datetime.now().isoformat(),'mode':'dry_run','crm':sys.argv[1],'health':sys.argv[2],'schema':sys.argv[3]}
|
||||
f=open(sys.argv[4],'w'); json.dump(d,f,ensure_ascii=False,indent=2); f.close()
|
||||
" "${CRM_BASE}" "${h}" "${s}" "${ARTIFACT_DIR}/dry-run-summary.json"
|
||||
}
|
||||
|
||||
cmd_apply() {
|
||||
info "apply"
|
||||
mkdir -p "${ARTIFACT_DIR}"
|
||||
local actions=()
|
||||
|
||||
local h; h="$(_curl "${CRM_BASE}/healthz")" || true
|
||||
[[ "${h}" != "ok" ]] && die "CRM dead: ${h}"
|
||||
ok "health ok"; actions+=("h:ok")
|
||||
|
||||
actions+=("schema:$(_curl ${CRM_BASE}/api/admin/schema-version 2>/dev/null || echo N/A)")
|
||||
|
||||
local a; a="$(_curl "${CRM_BASE}/api/provider-accounts?limit=100" 2>/dev/null)" || a='{}'
|
||||
local n; n=$(echo "${a}" | python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('accounts',d.get('data',[]))))" 2>/dev/null || echo 0)
|
||||
ok "accounts: ${n}"; actions+=("acct:${n}")
|
||||
|
||||
python3 -c "
|
||||
import json, sys, datetime
|
||||
acts = sys.argv[4].split(';') if sys.argv[4] else []
|
||||
d = {'ts':datetime.datetime.now().isoformat(),'mode':'applied','crm':sys.argv[1],'schema':sys.argv[2],'accts':int(sys.argv[3]),'actions':acts}
|
||||
f=open(sys.argv[5],'w'); json.dump(d,f,ensure_ascii=False,indent=2); f.close()
|
||||
" "${CRM_BASE}" "${s:-N/A}" "${n:-0}" "$(IFS=';'; echo "${actions[*]}")" "${ARTIFACT_DIR}/apply-summary.json"
|
||||
ok "applied -> ${ARTIFACT_DIR}/apply-summary.json"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
--help|-h) cmd_help ;;
|
||||
--dry-run) cmd_dry_run ;;
|
||||
--apply) cmd_apply ;;
|
||||
*) cmd_help ;;
|
||||
esac
|
||||
37
scripts/test/test_default_data.sh
Executable file
37
scripts/test/test_default_data.sh
Executable file
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# test_default_data.sh — 验证 setup_default_data.sh 的基本功能
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
info() { echo "INFO: $*"; }
|
||||
ok() { echo "OK: $*"; }
|
||||
die() { echo "FAIL: $*" >&2; exit 1; }
|
||||
|
||||
# Test 1: --help exits cleanly
|
||||
info "Test 1: --help"
|
||||
output="$("$ROOT_DIR/scripts/setup_default_data.sh" --help 2>&1)" || true
|
||||
echo "$output" | grep -q "CRM_ADMIN_TOKEN" || die "help missing expected content"
|
||||
ok "Test 1 passed"
|
||||
|
||||
# Test 2: --dry-run with no CRM (should still produce help-like output)
|
||||
info "Test 2: --dry-run"
|
||||
output="$("$ROOT_DIR/scripts/setup_default_data.sh" --dry-run 2>&1)" || true
|
||||
echo "$output" | grep -q "dry-run" && ok "Test 2 passed" || warn "dry-run on local machine: $output"
|
||||
|
||||
# Test 3: --apply without running CRM should fail gracefully
|
||||
info "Test 3: --apply without CRM"
|
||||
output="$("$ROOT_DIR/scripts/setup_default_data.sh" --apply 2>&1)" || true
|
||||
if echo "$output" | grep -qi "dead\|not healthy\|FATAL"; then
|
||||
ok "Test 3 passed (correctly rejected)"
|
||||
else
|
||||
warn "Test 3 unexpected output: $output"
|
||||
fi
|
||||
|
||||
# Test 4: Script has no syntax errors
|
||||
info "Test 4: bash syntax check"
|
||||
bash -n "$ROOT_DIR/scripts/setup_default_data.sh" || die "syntax error"
|
||||
ok "Test 4 passed"
|
||||
|
||||
echo ""
|
||||
ok "All tests passed"
|
||||
201
scripts/test/test_host_protocol_matrix_script.sh
Normal file
201
scripts/test/test_host_protocol_matrix_script.sh
Normal file
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
SCRIPT="$ROOT_DIR/scripts/acceptance/verify_host_protocol_matrix.sh"
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
assert_contains() {
|
||||
local haystack="$1"
|
||||
local needle="$2"
|
||||
if [[ "$haystack" != *"$needle"* ]]; then
|
||||
fail "expected to find [$needle] in [$haystack]"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_file_contains() {
|
||||
local file="$1"
|
||||
local needle="$2"
|
||||
[[ -f "$file" ]] || fail "missing file: $file"
|
||||
local text
|
||||
text="$(cat "$file")"
|
||||
assert_contains "$text" "$needle"
|
||||
}
|
||||
|
||||
[[ -f "$SCRIPT" ]] || fail "missing $SCRIPT"
|
||||
|
||||
help_output="$(bash "$SCRIPT" --help)"
|
||||
assert_contains "$help_output" "Usage: verify_host_protocol_matrix.sh"
|
||||
assert_contains "$help_output" "PROTOCOL_MATRIX_TARGETS_JSON"
|
||||
assert_contains "$help_output" "DRY_RUN=1"
|
||||
|
||||
tmpdir="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmpdir"' EXIT
|
||||
|
||||
set +e
|
||||
missing_env_output="$(ARTIFACT_DIR="$tmpdir" bash "$SCRIPT" 2>&1)"
|
||||
missing_env_status=$?
|
||||
set -e
|
||||
if [[ $missing_env_status -eq 0 ]]; then
|
||||
fail "expected missing env invocation to fail"
|
||||
fi
|
||||
assert_contains "$missing_env_output" "missing required env: PROTOCOL_MATRIX_TARGETS_JSON"
|
||||
|
||||
dry_run_output="$(ARTIFACT_DIR="$tmpdir" DRY_RUN=1 PROTOCOL_MATRIX_TARGETS_JSON='[{"provider_id":"kimi-a7m","base_url":"https://kimi.example.com/v1","api_key_env":"KIMI_API_KEY","models":["kimi-k2.6"]}]' bash "$SCRIPT")"
|
||||
assert_contains "$dry_run_output" "protocol matrix summary"
|
||||
assert_contains "$dry_run_output" "$tmpdir"
|
||||
|
||||
summary_file="$(find "$tmpdir" -name protocol-matrix-summary.json | head -n 1)"
|
||||
[[ -n "$summary_file" ]] || fail "missing protocol-matrix-summary.json"
|
||||
summary_text="$(cat "$summary_file")"
|
||||
assert_contains "$summary_text" '"provider_id": "kimi-a7m"'
|
||||
assert_contains "$summary_text" '"mode": "dry_run"'
|
||||
assert_contains "$summary_text" '"support_level": "dry_run"'
|
||||
assert_contains "$summary_text" '"probe_layer": "upstream"'
|
||||
|
||||
fakebin="$tmpdir/bin"
|
||||
mkdir -p "$fakebin"
|
||||
cat > "$fakebin/curl" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
headers_file=""
|
||||
body_file=""
|
||||
url=""
|
||||
request_headers_file=""
|
||||
request_body=""
|
||||
prev=""
|
||||
log_file="${FAKE_CURL_LOG:-}"
|
||||
for arg in "$@"; do
|
||||
case "$prev" in
|
||||
-D)
|
||||
headers_file="$arg"
|
||||
prev=""
|
||||
continue
|
||||
;;
|
||||
-o)
|
||||
body_file="$arg"
|
||||
prev=""
|
||||
continue
|
||||
;;
|
||||
-d)
|
||||
request_body="$arg"
|
||||
prev=""
|
||||
continue
|
||||
;;
|
||||
-H)
|
||||
if [[ "$arg" == X-Hermes-Debug-Request-Headers:* ]]; then
|
||||
request_headers_file="${arg#X-Hermes-Debug-Request-Headers: }"
|
||||
fi
|
||||
prev=""
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
case "$arg" in
|
||||
-D|-o|-d|-H)
|
||||
prev="$arg"
|
||||
continue
|
||||
;;
|
||||
http://*|https://*)
|
||||
url="$arg"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
[[ -n "$headers_file" && -n "$body_file" && -n "$url" ]] || {
|
||||
echo "missing curl capture args: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ -n "$log_file" ]]; then
|
||||
printf '%s\n' "$*" >> "$log_file"
|
||||
fi
|
||||
if [[ -n "$request_headers_file" ]]; then
|
||||
printf 'Authorization: Bearer ***\n' > "$request_headers_file"
|
||||
printf 'Content-Type: application/json\n' >> "$request_headers_file"
|
||||
fi
|
||||
case "$url" in
|
||||
https://kimi.example.com/v1/models)
|
||||
printf 'HTTP/1.1 200 OK\nContent-Type: application/json\n' > "$headers_file"
|
||||
printf '{"data":[{"id":"kimi-k2.6"}]}' > "$body_file"
|
||||
;;
|
||||
https://kimi.example.com/v1/chat/completions)
|
||||
printf 'HTTP/1.1 200 OK\nContent-Type: application/json\n' > "$headers_file"
|
||||
printf '{"choices":[{"message":{"content":"pong"}}]}' > "$body_file"
|
||||
;;
|
||||
https://kimi.example.com/v1/responses)
|
||||
printf 'HTTP/1.1 403 Forbidden\nContent-Type: application/json\n' > "$headers_file"
|
||||
printf '{"error":{"message":"unsupported"}}' > "$body_file"
|
||||
;;
|
||||
https://timeout.example.com/v1/models)
|
||||
printf 'HTTP/1.1 200 OK\nContent-Type: application/json\n' > "$headers_file"
|
||||
printf '{"data":[{"id":"timeout-model"}]}' > "$body_file"
|
||||
;;
|
||||
https://timeout.example.com/v1/chat/completions)
|
||||
: > "$headers_file"
|
||||
: > "$body_file"
|
||||
exit 28
|
||||
;;
|
||||
https://timeout.example.com/v1/responses)
|
||||
: > "$headers_file"
|
||||
: > "$body_file"
|
||||
exit 28
|
||||
;;
|
||||
*)
|
||||
echo "unexpected curl url: $url" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
EOF
|
||||
chmod +x "$fakebin/curl"
|
||||
|
||||
live_dir="$tmpdir/live"
|
||||
curl_log="$tmpdir/fake-curl.log"
|
||||
set +e
|
||||
live_output="$(PATH="$fakebin:$PATH" FAKE_CURL_LOG="$curl_log" ARTIFACT_DIR="$live_dir" KIMI_API_KEY='kimi-key' TIMEOUT_API_KEY='timeout-key' PROTOCOL_MATRIX_TARGETS_JSON='[
|
||||
{"provider_id":"kimi-a7m","base_url":"https://kimi.example.com/v1","api_key_env":"KIMI_API_KEY","models":["kimi-k2.6"]},
|
||||
{"provider_id":"timeout-provider","base_url":"https://timeout.example.com/v1","api_key_env":"TIMEOUT_API_KEY","models":["timeout-model"],"probe_layer":"host"}
|
||||
]' bash "$SCRIPT")"
|
||||
live_status=$?
|
||||
set -e
|
||||
if [[ $live_status -ne 0 ]]; then
|
||||
fail "expected partial failure run to exit 0, got $live_status"
|
||||
fi
|
||||
assert_contains "$live_output" "protocol matrix summary"
|
||||
live_summary="$live_dir/protocol-matrix-summary.json"
|
||||
[[ -f "$live_summary" ]] || fail "missing live protocol-matrix-summary.json"
|
||||
live_summary_text="$(cat "$live_summary")"
|
||||
assert_contains "$live_summary_text" '"provider_id": "kimi-a7m"'
|
||||
assert_contains "$live_summary_text" '"models_status": 200'
|
||||
assert_contains "$live_summary_text" '"chat_status": 200'
|
||||
assert_contains "$live_summary_text" '"responses_status": 403'
|
||||
assert_contains "$live_summary_text" '"support_level": "supported-with-plugin-adapter"'
|
||||
assert_contains "$live_summary_text" '"error_code": "responses_unsupported"'
|
||||
assert_contains "$live_summary_text" '"probe_layer": "upstream"'
|
||||
assert_contains "$live_summary_text" '"provider_id": "timeout-provider"'
|
||||
assert_contains "$live_summary_text" '"status": "failed"'
|
||||
assert_contains "$live_summary_text" '"error_code": "network_timeout"'
|
||||
assert_contains "$live_summary_text" '"probe_layer": "host"'
|
||||
|
||||
first_target_dir="$live_dir/targets/01-kimi-a7m"
|
||||
second_target_dir="$live_dir/targets/02-timeout-provider"
|
||||
[[ -d "$first_target_dir" ]] || fail "missing first target artifact dir"
|
||||
[[ -d "$second_target_dir" ]] || fail "missing second target artifact dir"
|
||||
assert_file_contains "$first_target_dir/01-models.request_headers.txt" 'Authorization: Bearer ***'
|
||||
assert_file_contains "$first_target_dir/01-models.request_headers.txt" 'Content-Type: application/json'
|
||||
assert_file_contains "$first_target_dir/01-models.response_headers.txt" 'HTTP/1.1 200 OK'
|
||||
assert_file_contains "$first_target_dir/03-responses.response_body.json" 'unsupported'
|
||||
assert_file_contains "$second_target_dir/02-chat.response_body.json" ''
|
||||
|
||||
curl_log_text="$(cat "$curl_log")"
|
||||
assert_contains "$curl_log_text" '--connect-timeout 10'
|
||||
assert_contains "$curl_log_text" '--max-time 30'
|
||||
assert_contains "$curl_log_text" '--retry 1'
|
||||
assert_contains "$curl_log_text" '--retry-delay 2'
|
||||
|
||||
if grep -R --line-number 'kimi-key\|timeout-key' "$live_dir"; then
|
||||
fail "artifact contains unsanitized secrets"
|
||||
fi
|
||||
|
||||
echo "PASS: host protocol matrix script regression checks"
|
||||
Reference in New Issue
Block a user