1065 lines
42 KiB
Bash
Executable File
1065 lines
42 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||
|
||
fail() {
|
||
echo "FAIL: $*" >&2
|
||
exit 1
|
||
}
|
||
|
||
assert_contains() {
|
||
local haystack="$1"
|
||
local needle="$2"
|
||
if [[ "$haystack" != *"$needle"* ]]; then
|
||
fail "expected to find [$needle] in [$haystack]"
|
||
fi
|
||
}
|
||
|
||
assert_not_contains() {
|
||
local haystack="$1"
|
||
local needle="$2"
|
||
if [[ "$haystack" == *"$needle"* ]]; then
|
||
fail "expected to avoid [$needle] in [$haystack]"
|
||
fi
|
||
}
|
||
|
||
run_test_build_subscription_access_prep_sql() {
|
||
# shellcheck disable=SC1091
|
||
source "$ROOT_DIR/scripts/acceptance/host_access_prep_lib.sh"
|
||
|
||
local sql
|
||
sql="$(build_subscription_access_prep_sql 42 'sk-test-123' 7 10 30 1 'hermes remote subscription validation')"
|
||
|
||
assert_contains "$sql" "UPDATE users"
|
||
assert_contains "$sql" "balance < 10"
|
||
assert_contains "$sql" "UPDATE api_keys"
|
||
assert_contains "$sql" "group_id = 7"
|
||
assert_contains "$sql" "key = 'sk-test-123'"
|
||
assert_contains "$sql" "INSERT INTO user_subscriptions"
|
||
assert_contains "$sql" "ON CONFLICT (user_id, group_id) WHERE deleted_at IS NULL"
|
||
assert_contains "$sql" "now() + interval '30 days'"
|
||
|
||
local quoted_sql
|
||
quoted_sql="$(build_bind_api_key_group_sql "sk-o'reilly" 7)"
|
||
assert_contains "$quoted_sql" "WHERE key = 'sk-o''reilly'"
|
||
|
||
local auth_cache_key balance_cache_key subscription_cache_key
|
||
auth_cache_key="$(build_api_key_auth_cache_key 'user-key')"
|
||
balance_cache_key="$(build_user_balance_cache_key 42)"
|
||
subscription_cache_key="$(build_subscription_billing_cache_key 42 7)"
|
||
assert_contains "$auth_cache_key" "apikey:auth:"
|
||
assert_contains "$balance_cache_key" "billing:balance:42"
|
||
assert_contains "$subscription_cache_key" "billing:sub:42:7"
|
||
}
|
||
|
||
run_test_real_host_acceptance_after_import_hook() {
|
||
local tmpdir fakebin artifact_dir hook_file guide_file stdout_file
|
||
tmpdir="$(mktemp -d)"
|
||
trap 'rm -rf "$tmpdir"' RETURN
|
||
fakebin="$tmpdir/bin"
|
||
artifact_dir="$tmpdir/artifacts"
|
||
hook_file="$artifact_dir/hook.txt"
|
||
guide_file="$artifact_dir/00-artifact-guide.txt"
|
||
stdout_file="$tmpdir/real_host_acceptance.stdout.txt"
|
||
mkdir -p "$fakebin"
|
||
|
||
cat > "$fakebin/curl" <<'EOF'
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
url=""
|
||
for arg in "$@"; do
|
||
if [[ "$arg" == *'***'* ]]; then
|
||
echo "unexpected redacted auth placeholder in curl args: $*" >&2
|
||
exit 1
|
||
fi
|
||
if [[ "$arg" == http://* || "$arg" == https://* ]]; then
|
||
url="$arg"
|
||
fi
|
||
done
|
||
[[ -n "$url" ]] || {
|
||
echo "missing url in curl args: $*" >&2
|
||
exit 1
|
||
}
|
||
case "$url" in
|
||
*/api/hosts)
|
||
printf '%s\n' '{"host_id":"test-host"}'
|
||
;;
|
||
*/api/hosts/test-host)
|
||
printf '%s\n' '{"host_id":"test-host"}'
|
||
;;
|
||
*/api/hosts/test-host/probe)
|
||
printf '%s\n' '{"ok":true}'
|
||
;;
|
||
*/api/packs/install)
|
||
printf '%s\n' '{"pack_id":1}'
|
||
;;
|
||
*/api/providers/deepseek/preview-import)
|
||
printf '%s\n' '{"available":true}'
|
||
;;
|
||
*/api/providers/deepseek/import)
|
||
printf '%s\n' '{"batch_id":123,"batch_status":"partially_succeeded","access_status":"broken"}'
|
||
;;
|
||
*/api/import-batches/123)
|
||
printf '%s\n' '{"managed_resources":[{"ResourceType":"group","HostResourceID":"7","ResourceName":"DeepSeek 默认分组"}]}'
|
||
;;
|
||
*/api/providers/deepseek/access/preview)
|
||
printf '%s\n' '{"available":true}'
|
||
;;
|
||
*/api/providers/deepseek/access/status)
|
||
printf '%s\n' '{"latest_access_status":"subscription_ready"}'
|
||
;;
|
||
*/api/providers/deepseek/status)
|
||
printf '%s\n' '{"status":"ready"}'
|
||
;;
|
||
*/api/providers/deepseek/reconcile)
|
||
printf '%s\n' '{"status":"in_sync"}'
|
||
;;
|
||
*/api/import-batches/123/rollback)
|
||
printf '%s\n' '{"status":"rolled_back"}'
|
||
;;
|
||
*)
|
||
echo "unexpected curl url: $url" >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
EOF
|
||
chmod +x "$fakebin/curl"
|
||
|
||
PATH="$fakebin:$PATH" \
|
||
ARTIFACT_DIR="$artifact_dir" \
|
||
CRM_BASE_URL="http://crm.example.com" \
|
||
CRM_ADMIN_TOKEN="token" \
|
||
HOST_NAME="test-host" \
|
||
HOST_BASE_URL="http://host.example.com" \
|
||
PACK_PATH="/tmp/openai-pack" \
|
||
PROVIDER_ID="deepseek" \
|
||
HOST_API_KEY="host-key" \
|
||
MODE="partial" \
|
||
ACCESS_MODE="subscription" \
|
||
ACCESS_API_KEY="user-key" \
|
||
SUBSCRIPTION_USERS="42" \
|
||
SKIP_ROLLBACK="1" \
|
||
AFTER_IMPORT_HOOK_COMMAND='printf "%s\n" "$BATCH_ID:$BATCH_DETAIL_FILE:$ACCESS_MODE" > "$ARTIFACT_DIR/hook.txt"' \
|
||
"$ROOT_DIR/scripts/acceptance/real_host_acceptance.sh" >"$stdout_file"
|
||
|
||
[[ -f "$hook_file" ]] || fail "after-import hook did not create $hook_file"
|
||
[[ -f "$guide_file" ]] || fail "artifact guide was not created"
|
||
local hook_contents
|
||
hook_contents="$(cat "$hook_file")"
|
||
assert_contains "$hook_contents" "123:"
|
||
assert_contains "$hook_contents" "05a-batch-detail-pre-access.json:subscription"
|
||
|
||
local guide_contents stdout_contents import_json
|
||
guide_contents="$(cat "$guide_file")"
|
||
stdout_contents="$(cat "$stdout_file")"
|
||
import_json="$(cat "$artifact_dir/05-import.json")"
|
||
assert_contains "$guide_contents" "清单 4（必须分层留证据，不可混用）"
|
||
assert_contains "$guide_contents" "artifact security mode: safe"
|
||
assert_contains "$guide_contents" "repository-safe: yes"
|
||
assert_contains "$stdout_contents" "artifact guide: $artifact_dir/00-artifact-guide.txt"
|
||
assert_contains "$stdout_contents" "checklist layered evidence: see 05b-after-import-hook.stdout.txt / 05b-after-import-hook.stderr.txt"
|
||
assert_not_contains "$import_json" "host-key"
|
||
assert_not_contains "$import_json" "user-key"
|
||
}
|
||
|
||
run_test_check_deepseek_completion_split() {
|
||
local tmpdir fakebin artifact_dir summary_file stdout_file
|
||
tmpdir="$(mktemp -d)"
|
||
trap 'rm -rf "$tmpdir"' RETURN
|
||
fakebin="$tmpdir/bin"
|
||
artifact_dir="$tmpdir/artifacts"
|
||
summary_file="$artifact_dir/summary.json"
|
||
stdout_file="$tmpdir/check_deepseek_completion_split.stdout.txt"
|
||
mkdir -p "$fakebin" "$artifact_dir"
|
||
|
||
cat > "$fakebin/curl" <<'EOF'
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
headers_file=""
|
||
body_file=""
|
||
url=""
|
||
prev=""
|
||
for arg in "$@"; do
|
||
case "$prev" in
|
||
-D)
|
||
headers_file="$arg"
|
||
prev=""
|
||
continue
|
||
;;
|
||
-o)
|
||
body_file="$arg"
|
||
prev=""
|
||
continue
|
||
;;
|
||
esac
|
||
case "$arg" in
|
||
-D|-o)
|
||
prev="$arg"
|
||
continue
|
||
;;
|
||
http://*|https://*)
|
||
url="$arg"
|
||
;;
|
||
esac
|
||
done
|
||
[[ -n "$headers_file" && -n "$body_file" && -n "$url" ]] || {
|
||
echo "missing curl capture args: $*" >&2
|
||
exit 1
|
||
}
|
||
case "$url" in
|
||
http://host.example.com/v1/models)
|
||
printf '%s
|
||
Content-Type: application/json
|
||
' 'HTTP/1.1 200 OK' > "$headers_file"
|
||
printf '%s
|
||
' '{"data":[{"id":"deepseek-v4-flash"},{"id":"deepseek-v4-pro"}]}' > "$body_file"
|
||
;;
|
||
http://host.example.com/v1/chat/completions)
|
||
printf '%s
|
||
Content-Type: application/json
|
||
' 'HTTP/1.1 502 Bad Gateway' > "$headers_file"
|
||
printf '%s
|
||
' '{"error":{"message":"Upstream service temporarily unavailable","type":"upstream_error"}}' > "$body_file"
|
||
;;
|
||
https://upstream.example.com/v1/chat/completions)
|
||
printf '%s
|
||
Content-Type: text/event-stream
|
||
' 'HTTP/1.1 200 OK' > "$headers_file"
|
||
printf '%s
|
||
' 'data: {"choices":[{"delta":{"content":"pong"}}]}' > "$body_file"
|
||
;;
|
||
*)
|
||
echo "unexpected curl url: $url" >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
EOF
|
||
chmod +x "$fakebin/curl"
|
||
|
||
PATH="$fakebin:$PATH" \
|
||
ARTIFACT_DIR="$artifact_dir" \
|
||
HOST_BASE="http://host.example.com" \
|
||
HOST_MANAGED_KEY="managed-key" \
|
||
UPSTREAM_BASE="https://upstream.example.com/v1" \
|
||
UPSTREAM_API_KEY="upstream-key" \
|
||
MODEL="deepseek-v4-flash" \
|
||
bash "$ROOT_DIR/scripts/acceptance/check_deepseek_completion_split.sh" >"$stdout_file"
|
||
|
||
[[ -f "$summary_file" ]] || fail "missing summary file: $summary_file"
|
||
local summary stdout_contents host_headers upstream_headers
|
||
summary="$(cat "$summary_file")"
|
||
stdout_contents="$(cat "$stdout_file")"
|
||
host_headers="$(cat "$artifact_dir/01-host-models.headers.txt")"
|
||
upstream_headers="$(cat "$artifact_dir/05-upstream-chat.headers.txt")"
|
||
assert_contains "$summary" '"classification": "host_compatibility_gap"'
|
||
assert_contains "$summary" '"host_models_status": 200'
|
||
assert_contains "$summary" '"host_chat_status": 502'
|
||
assert_contains "$summary" '"upstream_chat_status": 200'
|
||
assert_contains "$summary" '"upstream_chat_content_type": "text/event-stream"'
|
||
assert_contains "$stdout_contents" '"classification": "host_compatibility_gap"'
|
||
assert_not_contains "$host_headers" "Authorization:"
|
||
assert_not_contains "$upstream_headers" "Authorization:"
|
||
}
|
||
|
||
run_test_verify_quality_gates_script() {
|
||
local script threshold_file script_contents
|
||
script="$ROOT_DIR/scripts/test/verify_quality_gates.sh"
|
||
threshold_file="$ROOT_DIR/tests/quality/coverage_thresholds.tsv"
|
||
|
||
[[ -f "$script" ]] || fail "missing $script"
|
||
[[ -f "$threshold_file" ]] || fail "missing $threshold_file"
|
||
|
||
script_contents="$(cat "$script")"
|
||
assert_contains "$script_contents" "gofmt -l ."
|
||
assert_contains "$script_contents" "go vet ./..."
|
||
assert_contains "$script_contents" "go test -cover ./internal/..."
|
||
assert_contains "$script_contents" "go test ./tests/integration/... -count=1"
|
||
assert_contains "$script_contents" "Coverage Gate Report"
|
||
assert_contains "$script_contents" "tests/quality/coverage_thresholds.tsv"
|
||
assert_contains "$script_contents" "tier_by_package"
|
||
}
|
||
|
||
run_test_import_remote43_provider_subscription_prep() {
|
||
local tmpdir fakebin artifact_dir ssh_log summary_file pack_dir
|
||
tmpdir="$(mktemp -d)"
|
||
trap 'rm -rf "$tmpdir"' RETURN
|
||
fakebin="$tmpdir/bin"
|
||
artifact_dir="$tmpdir/artifacts"
|
||
ssh_log="$artifact_dir/ssh-log.txt"
|
||
summary_file="$artifact_dir/run/05-subscription-access-prep.summary.json"
|
||
pack_dir="$tmpdir/pack"
|
||
mkdir -p "$fakebin"
|
||
mkdir -p "$pack_dir/providers"
|
||
printf '%s\n' '{"pack_id":"openai-cn-pack","version":"1.1.3"}' > "$pack_dir/pack.json"
|
||
printf '%s\n' '{"provider_id":"deepseek","base_url":"https://upstream.example.com/v1"}' > "$pack_dir/providers/deepseek.json"
|
||
|
||
cat > "$fakebin/curl" <<'EOF'
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
headers_file=""
|
||
body_file=""
|
||
url=""
|
||
prev=""
|
||
for arg in "$@"; do
|
||
if [[ "$arg" == *'***'* ]]; then
|
||
echo "unexpected redacted auth placeholder in curl args: $*" >&2
|
||
exit 1
|
||
fi
|
||
case "$prev" in
|
||
-D)
|
||
headers_file="$arg"
|
||
prev=""
|
||
continue
|
||
;;
|
||
-o)
|
||
body_file="$arg"
|
||
prev=""
|
||
continue
|
||
;;
|
||
esac
|
||
case "$arg" in
|
||
-D|-o)
|
||
prev="$arg"
|
||
continue
|
||
;;
|
||
http://*|https://*)
|
||
url="$arg"
|
||
;;
|
||
esac
|
||
done
|
||
|
||
write_headers() {
|
||
[[ -n "$headers_file" ]] && printf '%s\n' 'HTTP/1.1 200 OK' > "$headers_file"
|
||
}
|
||
|
||
write_body() {
|
||
local body="$1"
|
||
if [[ -n "$body_file" ]]; then
|
||
printf '%s\n' "$body" > "$body_file"
|
||
else
|
||
printf '%s\n' "$body"
|
||
fi
|
||
}
|
||
|
||
case "$url" in
|
||
*/api/admin/session/login)
|
||
write_body '{"authenticated":true,"username":"portal-admin"}'
|
||
;;
|
||
*/api/admin/session)
|
||
write_body '{"authenticated":true,"login_enabled":true,"username":"portal-admin"}'
|
||
;;
|
||
*/api/hosts)
|
||
write_body '{"host_id":"remote43-current-host"}'
|
||
;;
|
||
*/api/providers/deepseek/import)
|
||
write_headers
|
||
write_body '{"batch_id":123,"batch_status":"partially_succeeded","access_status":"broken","provider_status":"ready","accepted_keys_count":1,"group":{"id":"7","name":"DeepSeek 默认分组"}}'
|
||
;;
|
||
*/api/import-batches/123)
|
||
write_body '{"managed_resources":[{"ResourceType":"group","HostResourceID":"7","ResourceName":"DeepSeek 默认分组"}]}'
|
||
;;
|
||
*/api/providers/deepseek/status\?pack_id=openai-cn-pack\&host_id=remote43-current-host)
|
||
write_body '{"status":"ready"}'
|
||
;;
|
||
*/api/providers/deepseek/access/status\?pack_id=openai-cn-pack\&host_id=remote43-current-host)
|
||
write_body '{"latest_access_status":"subscription_ready"}'
|
||
;;
|
||
*/api/providers/deepseek/access/preview\?pack_id=openai-cn-pack\&host_id=remote43-current-host)
|
||
write_body '{"available":true}'
|
||
;;
|
||
*)
|
||
echo "unexpected curl url: $url" >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
EOF
|
||
chmod +x "$fakebin/curl"
|
||
|
||
cat > "$fakebin/ssh" <<'EOF'
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
log_dir="${FAKE_REMOTE_LOG_DIR:?missing FAKE_REMOTE_LOG_DIR}"
|
||
cmd="${*: -1}"
|
||
printf '%s\n' "$cmd" >> "$log_dir/ssh-log.txt"
|
||
if [[ "$cmd" == *'***'* ]]; then
|
||
echo "unexpected redacted auth placeholder in ssh command: $cmd" >&2
|
||
exit 1
|
||
fi
|
||
case "$cmd" in
|
||
"sudo -n docker ps --format '{{.Names}}\t{{.Ports}}'"*)
|
||
printf '%s\n' 'sub2api-fresh-deepseek-20260519_115244-app-1 127.0.0.1:18093->8080/tcp'
|
||
;;
|
||
*"/api/v1/auth/login"*)
|
||
printf '%s\n' 'host-bearer-token'
|
||
;;
|
||
*"grep ^SUB2API_CRM_ADMIN_TOKEN="*)
|
||
printf '%s\n' 'crm-token'
|
||
;;
|
||
*"select value from settings where key='admin_api_key'"*)
|
||
printf '%s\n' 'admin-key'
|
||
;;
|
||
*"select id from users where role='admin'"*)
|
||
printf '%s\n' '1'
|
||
;;
|
||
*"select id from users where email like 'relay-sub-%@sub2api.local'"*)
|
||
printf '%s\n' '42'
|
||
;;
|
||
*"select k.key from users u join api_keys k on k.user_id=u.id"*)
|
||
printf '%s\n' 'user-key'
|
||
;;
|
||
*"/api/providers/deepseek/import"*)
|
||
printf '%s\n' '{"batch_id":123,"batch_status":"partially_succeeded","access_status":"broken","group":{"id":"7","name":"DeepSeek 默认分组"}}' > /tmp/import_body.json
|
||
printf '%s\n' 'HTTP/1.1 200 OK' > /tmp/import_headers.txt
|
||
;;
|
||
"cat /tmp/import_headers.txt")
|
||
cat /tmp/import_headers.txt
|
||
;;
|
||
"cat /tmp/import_body.json")
|
||
cat /tmp/import_body.json
|
||
;;
|
||
*"/api/import-batches/123"*)
|
||
printf '%s\n' '{"managed_resources":[{"ResourceType":"account","HostResourceID":"8","ResourceName":"deepseek-01"}]}'
|
||
;;
|
||
*"curl -sS -D /tmp/models_headers.txt"*)
|
||
printf '%s\n' 'HTTP/1.1 200 OK' > /tmp/models_headers.txt
|
||
printf '%s\n' '{"data":[{"id":"gpt-4"},{"id":"gpt-4.1"}]}' > /tmp/models_body.json
|
||
;;
|
||
"cat /tmp/models_headers.txt")
|
||
cat /tmp/models_headers.txt
|
||
;;
|
||
"cat /tmp/models_body.json")
|
||
cat /tmp/models_body.json
|
||
;;
|
||
*"curl -sS -D /tmp/chat_headers.txt"*)
|
||
printf '%s\n' 'HTTP/1.1 200 OK' > /tmp/chat_headers.txt
|
||
printf '%s\n' '{"choices":[{"message":{"content":"pong"}}]}' > /tmp/chat_body.json
|
||
;;
|
||
"cat /tmp/chat_headers.txt")
|
||
cat /tmp/chat_headers.txt
|
||
;;
|
||
"cat /tmp/chat_body.json")
|
||
cat /tmp/chat_body.json
|
||
;;
|
||
*"curl -sS -D /tmp/upstream_models_headers.txt"*)
|
||
printf '%s\n' 'HTTP/1.1 200 OK' > /tmp/upstream_models_headers.txt
|
||
printf '%s\n' '{"data":[{"id":"openai/gpt-4"},{"id":"openai/gpt-4.1"}]}' > /tmp/upstream_models_body.json
|
||
;;
|
||
"cat /tmp/upstream_models_headers.txt")
|
||
cat /tmp/upstream_models_headers.txt
|
||
;;
|
||
"cat /tmp/upstream_models_body.json")
|
||
cat /tmp/upstream_models_body.json
|
||
;;
|
||
*"curl -sS -D /tmp/upstream_chat_headers.txt"*)
|
||
printf '%s\n' 'HTTP/1.1 200 OK' > /tmp/upstream_chat_headers.txt
|
||
printf '%s\n' '{"choices":[{"message":{"content":"upstream-pong"}}]}' > /tmp/upstream_chat_body.txt
|
||
;;
|
||
"cat /tmp/upstream_chat_headers.txt")
|
||
cat /tmp/upstream_chat_headers.txt
|
||
;;
|
||
"cat /tmp/upstream_chat_body.txt")
|
||
cat /tmp/upstream_chat_body.txt
|
||
;;
|
||
*"/api/providers/deepseek/status"*)
|
||
printf '%s\n' '{"status":"ready"}'
|
||
;;
|
||
*"/api/providers/deepseek/access/status"*)
|
||
printf '%s\n' '{"latest_access_status":"subscription_ready"}'
|
||
;;
|
||
*"/api/providers/deepseek/access/preview"*)
|
||
printf '%s\n' '{"available":true}'
|
||
;;
|
||
*"/api/providers/deepseek/reconcile"*)
|
||
printf '%s\n' '{"status":"in_sync"}'
|
||
;;
|
||
*"sudo -n docker exec -i sub2api-fresh-deepseek-20260519_115244-postgres-1 psql -U sub2api -d sub2api -At -F ''"*)
|
||
printf '%s\n' '{"group_id":7,"subscription":{"status":"active"},"key":{"group_id":7}}'
|
||
;;
|
||
*"sudo -n docker exec -i sub2api-fresh-deepseek-20260519_115244-postgres-1 psql -U sub2api -d sub2api"*)
|
||
CMD="$cmd" LOG_DIR="$log_dir" python3 - <<'PY'
|
||
import base64, os, re, sys
|
||
cmd = os.environ['CMD']
|
||
match = re.search(r"printf '%s' '([^']+)' \| base64 -d", cmd)
|
||
if not match:
|
||
raise SystemExit(f'failed to extract base64 payload from: {cmd}')
|
||
sql = base64.b64decode(match.group(1)).decode()
|
||
if "select id from users where email like 'relay-sub-%@sub2api.local' and not exists" in sql:
|
||
print('')
|
||
elif "select k.key from users u join api_keys k on k.user_id=u.id" in sql and "not exists" in sql:
|
||
print('')
|
||
elif "UPDATE users" in sql and "INSERT INTO user_subscriptions" in sql:
|
||
print('')
|
||
elif "INSERT INTO users" in sql and "INSERT INTO api_keys" in sql:
|
||
print('84\tuser-key-fresh')
|
||
elif "SELECT json_build_object(" in sql:
|
||
print('{"group_id":7,"subscription":{"status":"active"},"key":{"group_id":7}}')
|
||
else:
|
||
print('')
|
||
PY
|
||
;;
|
||
*"sudo -n docker exec sub2api-fresh-deepseek-20260519_115244-redis-1 redis-cli DEL apikey:auth:"*" billing:balance:"*" billing:sub:"*":7"*)
|
||
printf '%s\n' '3'
|
||
;;
|
||
*)
|
||
echo "unexpected ssh command: $cmd" >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
EOF
|
||
chmod +x "$fakebin/ssh"
|
||
|
||
PATH="$fakebin:$PATH" \
|
||
FAKE_REMOTE_LOG_DIR="$artifact_dir" \
|
||
KEY="/does/not/matter" \
|
||
REMOTE="fake@host" \
|
||
CRM_BASE="http://127.0.0.1:18088" \
|
||
CRM_ADMIN_USERNAME="portal-admin" \
|
||
CRM_ADMIN_PASSWORD="portal-pass" \
|
||
HOST_BASE="http://127.0.0.1:18087" \
|
||
CRM_HOST_BASE="http://127.0.0.1:18093" \
|
||
REMOTE_HOST_BASE="http://127.0.0.1:18093" \
|
||
HOST_NAME="human-friendly-host-name" \
|
||
ROOT="$artifact_dir/root" \
|
||
ART="$artifact_dir/run" \
|
||
PACK_PATH="$pack_dir" \
|
||
UPSTREAM_KEY="upstream-test-key" \
|
||
SUBSCRIPTION_DAYS=30 \
|
||
MIN_BALANCE=10 \
|
||
SKIP_ROLLBACK=1 \
|
||
bash "$ROOT_DIR/scripts/acceptance/import_remote43_provider.sh" deepseek gpt-4 UPSTREAM_KEY >/dev/null
|
||
|
||
[[ -f "$summary_file" ]] || fail "prep summary was not captured"
|
||
local prep_summary
|
||
prep_summary="$(cat "$summary_file")"
|
||
assert_contains "$prep_summary" '"subscription_group_id": 7'
|
||
assert_contains "$prep_summary" '"min_balance": 10'
|
||
assert_contains "$prep_summary" '"subscription_days": 30'
|
||
assert_not_contains "$prep_summary" '"prefix": "user-key'
|
||
|
||
local runtime_context invalidation_log subscription_state models_body chat_body upstream_models upstream_chat summary_json local_key_source
|
||
runtime_context="$(cat "$artifact_dir/run/01-runtime-context.json")"
|
||
assert_contains "$runtime_context" '"crm_host_base": "http://127.0.0.1:18093"'
|
||
assert_contains "$runtime_context" '"remote_host_base": "http://127.0.0.1:18093"'
|
||
assert_contains "$runtime_context" '"subscription_user_id_hash"'
|
||
assert_not_contains "$runtime_context" '"subscription_user_id":'
|
||
assert_not_contains "$runtime_context" '"managed_user_email":'
|
||
|
||
local_key_source="$(cat "$artifact_dir/run/00-local-key-source.json")"
|
||
assert_contains "$local_key_source" '"fingerprint"'
|
||
assert_not_contains "$local_key_source" '"upstream_key":'
|
||
|
||
invalidation_log="$(cat "$artifact_dir/run/07-redis-targeted-invalidation.json")"
|
||
assert_contains "$invalidation_log" '"auth_cache_invalidated": true'
|
||
assert_contains "$invalidation_log" '"balance_cache_invalidated": true'
|
||
assert_contains "$invalidation_log" '"subscription_cache_invalidated": true'
|
||
assert_not_contains "$invalidation_log" 'apikey:auth:'
|
||
|
||
subscription_state="$(cat "$artifact_dir/run/08-subscription-group-state.json")"
|
||
assert_contains "$subscription_state" '"group_id": 7'
|
||
assert_contains "$subscription_state" '"status": "active"'
|
||
assert_contains "$subscription_state" '"redacted"'
|
||
assert_not_contains "$subscription_state" '"key": "'
|
||
|
||
models_body="$(cat "$artifact_dir/run/10-models.body.json")"
|
||
chat_body="$(cat "$artifact_dir/run/12-chat.body.json")"
|
||
upstream_models="$(cat "$artifact_dir/run/18-upstream-models.body.json")"
|
||
upstream_chat="$(cat "$artifact_dir/run/20-upstream-chat.body.txt")"
|
||
summary_json="$(cat "$artifact_dir/run/21-summary.json" 2>/dev/null || true)"
|
||
assert_contains "$models_body" '"id":"gpt-4"'
|
||
assert_contains "$chat_body" '"content":"pong"'
|
||
assert_contains "$upstream_models" '"id":"openai/gpt-4"'
|
||
assert_contains "$upstream_chat" '"content":"upstream-pong"'
|
||
assert_contains "$summary_json" '"upstream_models_has_expected_model": true'
|
||
assert_contains "$summary_json" '"completion_classification": "unknown"'
|
||
[[ -s "$ssh_log" ]] || fail "ssh log was empty"
|
||
local ssh_contents
|
||
ssh_contents="$(cat "$ssh_log")"
|
||
assert_contains "$ssh_contents" "sudo -n docker ps --format"
|
||
assert_contains "$ssh_contents" "http://127.0.0.1:18093/v1/models"
|
||
assert_contains "$ssh_contents" "http://127.0.0.1:18093/v1/chat/completions"
|
||
assert_not_contains "$ssh_contents" "http://127.0.0.1:18087/v1/models"
|
||
assert_not_contains "$ssh_contents" "http://127.0.0.1:18087/v1/chat/completions"
|
||
assert_not_contains "$ssh_contents" "user-key"
|
||
|
||
local provider_status
|
||
provider_status="$(cat "$artifact_dir/run/13-provider-status.json")"
|
||
assert_contains "$provider_status" '"status":"ready"'
|
||
|
||
local access_status
|
||
access_status="$(cat "$artifact_dir/run/14-access-status.json")"
|
||
assert_contains "$access_status" '"latest_access_status":"subscription_ready"'
|
||
}
|
||
|
||
run_test_migrate_historical_artifacts() {
|
||
local tmpdir src_root sensitive_root target_dir
|
||
tmpdir="$(mktemp -d)"
|
||
trap 'rm -rf "$tmpdir"' RETURN
|
||
src_root="$tmpdir/artifacts/real-host-acceptance"
|
||
sensitive_root="$tmpdir/artifacts/real-host-acceptance-sensitive"
|
||
target_dir="$src_root/20260522_foo"
|
||
mkdir -p "$target_dir"
|
||
|
||
cat > "$target_dir/00-local-key-source.json" <<'EOF'
|
||
{"source":"env:UPSTREAM_KEY","provider_id":"deepseek","upstream_key_prefix":"sk-live-secret","upstream_key_suffix":"cret42"}
|
||
EOF
|
||
cat > "$target_dir/01-runtime-context.json" <<'EOF'
|
||
{"subscription_user_id":"42","subscription_user_key_prefix":"user-key-secr","managed_user_email":"relay-sub-abc@sub2api.local","managed_probe_key_prefix":"sk-relay-secret-123456","crm_host_base":"http://127.0.0.1:18093","remote_host_base":"http://127.0.0.1:18093"}
|
||
EOF
|
||
cat > "$target_dir/05-subscription-access-prep.sql" <<'EOF'
|
||
BEGIN;
|
||
UPDATE api_keys SET group_id = 7 WHERE key = 'user-key-secret';
|
||
COMMIT;
|
||
EOF
|
||
cat > "$target_dir/07-redis-targeted-invalidation.txt" <<'EOF'
|
||
auth_cache_key=apikey:auth:abcd
|
||
balance_cache_key=billing:balance:42
|
||
subscription_cache_key=billing:sub:42:7
|
||
3
|
||
EOF
|
||
cat > "$target_dir/08-subscription-group-state.json" <<'EOF'
|
||
{"group_id":7,"subscription":{"user_id":42,"status":"active"},"key":{"id":9,"group_id":7,"status":"active","key":"user-key-secret"}}
|
||
EOF
|
||
cat > "$target_dir/09-models.headers.txt" <<'EOF'
|
||
HTTP/1.1 200 OK
|
||
Authorization: Bearer managed-secret
|
||
Content-Type: application/json
|
||
EOF
|
||
cat > "$target_dir/00-managed-key.txt" <<'EOF'
|
||
sk-managed-secret
|
||
EOF
|
||
cat > "$target_dir/00-managed-key-corrected.txt" <<'EOF'
|
||
sk-managed-secret-corrected
|
||
EOF
|
||
cat > "$target_dir/00-raw-user-key.txt" <<'EOF'
|
||
sk-user-secret
|
||
EOF
|
||
cat > "$target_dir/summary.json" <<'EOF'
|
||
{"provider_id":"deepseek","subscription_user_id":"24","gateway_key_prefix":"sk-deepseek-","host_account":{"data":{"credentials":{"api_key":"sk-live-123456"}}}}
|
||
EOF
|
||
cat > "$target_dir/99-semantic-summary.json" <<'EOF'
|
||
{"raw_user_id":"2","raw_key":"sk-raw-probe-20260523b","requested_probe_api_key":"sk-raw-probe-20260523b"}
|
||
EOF
|
||
cat > "$target_dir/05a-batch-detail-pre-access.json" <<'EOF'
|
||
{"access_closures":[{"DetailsJSON":"{\"requested_probe_api_key\":\"sk-raw-probe-20260523b\",\"subscription_users\":[\"crm-user\"]}"}]}
|
||
EOF
|
||
|
||
python3 "$ROOT_DIR/scripts/acceptance/migrate_historical_artifacts.py" "$src_root" >/dev/null
|
||
|
||
local migrated_runtime migrated_key_source migrated_invalidation migrated_group_state headers_text summary_json semantic_json details_json
|
||
migrated_runtime="$(cat "$target_dir/01-runtime-context.json")"
|
||
migrated_key_source="$(cat "$target_dir/00-local-key-source.json")"
|
||
migrated_invalidation="$(cat "$target_dir/07-redis-targeted-invalidation.json")"
|
||
migrated_group_state="$(cat "$target_dir/08-subscription-group-state.json")"
|
||
headers_text="$(cat "$target_dir/09-models.headers.txt")"
|
||
summary_json="$(cat "$target_dir/summary.json")"
|
||
semantic_json="$(cat "$target_dir/99-semantic-summary.json")"
|
||
details_json="$(cat "$target_dir/05a-batch-detail-pre-access.json")"
|
||
|
||
assert_contains "$migrated_runtime" '"subscription_user_id_hash"'
|
||
assert_not_contains "$migrated_runtime" '"subscription_user_id":'
|
||
assert_not_contains "$migrated_runtime" '"managed_user_email":'
|
||
assert_contains "$migrated_key_source" '"redacted"'
|
||
assert_not_contains "$migrated_key_source" 'upstream_key_prefix'
|
||
assert_contains "$migrated_invalidation" '"auth_cache_invalidated": true'
|
||
assert_not_contains "$migrated_invalidation" 'apikey:auth:'
|
||
assert_contains "$migrated_group_state" '"redacted"'
|
||
assert_not_contains "$migrated_group_state" 'user-key-secret'
|
||
assert_not_contains "$headers_text" 'Authorization:'
|
||
assert_contains "$summary_json" '"api_key": {'
|
||
assert_not_contains "$summary_json" 'sk-live-123456'
|
||
assert_contains "$semantic_json" '"raw_key": {'
|
||
assert_not_contains "$semantic_json" 'sk-raw-probe-20260523b'
|
||
assert_contains "$details_json" '\"requested_probe_api_key\": {'
|
||
assert_not_contains "$details_json" 'sk-raw-probe-20260523b'
|
||
[[ -f "$target_dir/05-subscription-access-prep.summary.json" ]] || fail "sql summary was not created"
|
||
[[ -f "$sensitive_root/20260522_foo/00-managed-key.txt" ]] || fail "managed key was not moved to sensitive mirror"
|
||
[[ -f "$sensitive_root/20260522_foo/00-managed-key-corrected.txt" ]] || fail "managed key corrected file was not moved to sensitive mirror"
|
||
[[ -f "$sensitive_root/20260522_foo/05-subscription-access-prep.sql" ]] || fail "sql file was not moved to sensitive mirror"
|
||
}
|
||
|
||
run_test_verify_route_control_plane_script() {
|
||
local tmpdir fakebin artifact_dir stdout_file
|
||
tmpdir="$(mktemp -d)"
|
||
trap 'rm -rf "$tmpdir"' RETURN
|
||
fakebin="$tmpdir/bin"
|
||
artifact_dir="$tmpdir/artifacts"
|
||
stdout_file="$tmpdir/verify_route_control_plane.stdout.txt"
|
||
mkdir -p "$fakebin" "$artifact_dir"
|
||
|
||
cat > "$fakebin/curl" <<'EOF'
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
method="GET"
|
||
url=""
|
||
payload=""
|
||
prev=""
|
||
for arg in "$@"; do
|
||
case "$prev" in
|
||
-X) method="$arg"; prev=""; continue ;;
|
||
-d|--data) payload="$arg"; prev=""; continue ;;
|
||
esac
|
||
case "$arg" in
|
||
-X|-d|--data) prev="$arg"; continue ;;
|
||
http://*|https://*) url="$arg" ;;
|
||
esac
|
||
done
|
||
case "$method $url" in
|
||
"POST http://crm.example.com/api/logical-groups")
|
||
printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane p2t4-cp-1700000000","status":"active"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/models")
|
||
printf '%s\n' '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes")
|
||
printf '%s\n' '{"route":{"route_id":"primary-1700000000","logical_group_id":"p2t4-cp-1700000000","name":"Primary primary-1700000000","status":"active","priority":10,"weight":100,"shadow_group_id":"shadow-group-1700000000","shadow_host_id":"shadow-host-1700000000"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000/models")
|
||
printf '%s\n' '{"route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4","status":"active"}}'
|
||
;;
|
||
"GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000")
|
||
printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane p2t4-cp-1700000000","status":"active","models":[{"public_model":"gpt-5.4"}],"routes":[{"route_id":"primary-1700000000"}]}}'
|
||
;;
|
||
"PUT http://crm.example.com/api/logical-groups/p2t4-cp-1700000000")
|
||
printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-cp-1700000000","display_name":"P2T4 Control Plane Updated p2t4-cp-1700000000","status":"active"}}'
|
||
;;
|
||
"PUT http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000")
|
||
printf '%s\n' '{"route":{"route_id":"primary-1700000000","logical_group_id":"p2t4-cp-1700000000","name":"Primary Route Updated","status":"active","priority":12,"weight":80,"shadow_group_id":"shadow-group-1700000000","shadow_host_id":"shadow-host-1700000000"}}'
|
||
;;
|
||
"GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes")
|
||
printf '%s\n' '{"routes":[{"route_id":"primary-1700000000","weight":80}]}'
|
||
;;
|
||
"GET http://crm.example.com/api/logical-groups/p2t4-cp-1700000000/routes/primary-1700000000/models")
|
||
printf '%s\n' '{"route_models":[{"public_model":"gpt-5.4","shadow_model":"gpt-5.4","status":"active"}]}'
|
||
;;
|
||
*)
|
||
echo "unexpected curl request: $method $url payload=$payload" >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
EOF
|
||
chmod +x "$fakebin/curl"
|
||
|
||
PATH="$fakebin:$PATH" \
|
||
CRM_BASE="http://crm.example.com" \
|
||
CRM_ADMIN_TOKEN="token" \
|
||
TS="1700000000" \
|
||
ARTIFACT_DIR="$artifact_dir" \
|
||
bash "$ROOT_DIR/scripts/acceptance/verify_route_control_plane.sh" >"$stdout_file"
|
||
|
||
local summary stdout_text
|
||
summary="$(cat "$artifact_dir/10-summary.json")"
|
||
stdout_text="$(cat "$stdout_file")"
|
||
assert_contains "$summary" '"group_id": "p2t4-cp-1700000000"'
|
||
assert_contains "$summary" '"route_id": "primary-1700000000"'
|
||
assert_contains "$summary" '"route_updated": true'
|
||
assert_contains "$stdout_text" '"route_model_listed": true'
|
||
}
|
||
|
||
run_test_verify_route_data_plane_script() {
|
||
local tmpdir fakebin artifact_dir stdout_file payload_log
|
||
tmpdir="$(mktemp -d)"
|
||
trap 'rm -rf "$tmpdir"' RETURN
|
||
fakebin="$tmpdir/bin"
|
||
artifact_dir="$tmpdir/artifacts"
|
||
stdout_file="$tmpdir/verify_route_data_plane.stdout.txt"
|
||
payload_log="$tmpdir/payload.log"
|
||
mkdir -p "$fakebin" "$artifact_dir"
|
||
|
||
cat > "$fakebin/curl" <<'EOF'
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
method="GET"
|
||
url=""
|
||
payload=""
|
||
prev=""
|
||
for arg in "$@"; do
|
||
case "$prev" in
|
||
-X) method="$arg"; prev=""; continue ;;
|
||
-d|--data) payload="$arg"; prev=""; continue ;;
|
||
esac
|
||
case "$arg" in
|
||
-X|-d|--data) prev="$arg"; continue ;;
|
||
http://*|https://*) url="$arg" ;;
|
||
esac
|
||
done
|
||
printf '%s\n' "$payload" >> "${PAYLOAD_LOG:?missing PAYLOAD_LOG}"
|
||
case "$method $url" in
|
||
"POST http://crm.example.com/api/logical-groups")
|
||
printf '%s\n' '{"logical_group":{"logical_group_id":"p2t4-dp-1700000001","display_name":"P2T4 Data Plane p2t4-dp-1700000001","status":"active"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/models")
|
||
printf '%s\n' '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/routes")
|
||
printf '%s\n' '{"route":{"route_id":"primary-1700000001","shadow_group_id":"shadow-group-9","shadow_host_id":"shadow-host-real"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/logical-groups/p2t4-dp-1700000001/routes/primary-1700000001/models")
|
||
printf '%s\n' '{"route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/routing/chat/completions")
|
||
printf '%s\n' '{"request_id":"req-p2t4-dp-1700000001","logical_group_id":"p2t4-dp-1700000001","model":"gpt-5.4","selected_route":{"route_id":"primary-1700000001","shadow_host_id":"shadow-host-real","shadow_group_id":"shadow-group-9","shadow_model":"gpt-5.4"},"forward":{"upstream_status":200,"effective_gateway_key_source":"managed_subscription"}}'
|
||
;;
|
||
"GET http://crm.example.com/api/routing/logs/decisions?request_id=req-p2t4-dp-1700000001&limit=5")
|
||
printf '%s\n' '{"decision_logs":[{"request_id":"req-p2t4-dp-1700000001","selected_route_id":"primary-1700000001"}]}'
|
||
;;
|
||
*)
|
||
echo "unexpected curl request: $method $url payload=$payload" >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
EOF
|
||
chmod +x "$fakebin/curl"
|
||
|
||
PATH="$fakebin:$PATH" \
|
||
PAYLOAD_LOG="$payload_log" \
|
||
CRM_BASE="http://crm.example.com" \
|
||
CRM_ADMIN_TOKEN="token" \
|
||
TS="1700000001" \
|
||
SHADOW_HOST_ID="shadow-host-real" \
|
||
SHADOW_GROUP_ID="shadow-group-9" \
|
||
SUBSCRIPTION_USER_ID="36" \
|
||
ARTIFACT_DIR="$artifact_dir" \
|
||
bash "$ROOT_DIR/scripts/acceptance/verify_route_data_plane.sh" >"$stdout_file"
|
||
|
||
local summary payloads
|
||
summary="$(cat "$artifact_dir/07-summary.json")"
|
||
payloads="$(cat "$payload_log")"
|
||
assert_contains "$summary" '"forward_upstream_status": 200'
|
||
assert_contains "$summary" '"effective_gateway_key_source": "managed_subscription"'
|
||
assert_contains "$payloads" '"subscription_user_id": "36"'
|
||
}
|
||
|
||
run_test_verify_route_health_ui_script() {
|
||
local tmpdir fakebin artifact_dir stdout_file
|
||
tmpdir="$(mktemp -d)"
|
||
trap 'rm -rf "$tmpdir"' RETURN
|
||
fakebin="$tmpdir/bin"
|
||
artifact_dir="$tmpdir/artifacts"
|
||
stdout_file="$tmpdir/verify_route_health_ui.stdout.txt"
|
||
mkdir -p "$fakebin" "$artifact_dir"
|
||
|
||
cat > "$fakebin/curl" <<'EOF'
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
method="GET"
|
||
url=""
|
||
payload=""
|
||
output_file=""
|
||
prev=""
|
||
for arg in "$@"; do
|
||
case "$prev" in
|
||
-X) method="$arg"; prev=""; continue ;;
|
||
-d|--data) payload="$arg"; prev=""; continue ;;
|
||
-o) output_file="$arg"; prev=""; continue ;;
|
||
esac
|
||
case "$arg" in
|
||
-X|-d|--data|-o) prev="$arg"; continue ;;
|
||
http://*|https://*) url="$arg" ;;
|
||
esac
|
||
done
|
||
write_body() {
|
||
local body="$1"
|
||
if [[ -n "$output_file" ]]; then
|
||
printf '%s\n' "$body" > "$output_file"
|
||
else
|
||
printf '%s\n' "$body"
|
||
fi
|
||
}
|
||
case "$method $url" in
|
||
"GET http://portal.example.com/route-health.html")
|
||
write_body '<html><title>Route Health Admin</title><body>Route Health Admin</body></html>'
|
||
;;
|
||
"POST http://crm.example.com/api/logical-groups")
|
||
write_body '{"logical_group":{"logical_group_id":"p2t4-health-1700000002","status":"active"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/models")
|
||
write_body '{"logical_group_model":{"public_model":"gpt-5.4","status":"active"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes")
|
||
if [[ "$payload" == *'"route_id":"primary-1700000002"'* ]]; then
|
||
write_body '{"route":{"route_id":"primary-1700000002"}}'
|
||
elif [[ "$payload" == *'"route_id":"fallback-1700000002"'* ]]; then
|
||
write_body '{"route":{"route_id":"fallback-1700000002"}}'
|
||
else
|
||
write_body '{"route":{"route_id":"failing-1700000002"}}'
|
||
fi
|
||
;;
|
||
"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/primary-1700000002/models"|"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/fallback-1700000002/models"|"POST http://crm.example.com/api/logical-groups/p2t4-health-1700000002/routes/failing-1700000002/models")
|
||
write_body '{"route_model":{"public_model":"gpt-5.4","shadow_model":"gpt-5.4"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/routing/sticky/cooldowns")
|
||
write_body '{"route_cooldown":{"route_id":"primary-1700000002","reason":"degraded"}}'
|
||
;;
|
||
"POST http://crm.example.com/api/routing/sticky/route-failures")
|
||
write_body '{"route_failure":{"route_id":"failing-1700000002","failure_count":2,"last_error_class":"timeout"}}'
|
||
;;
|
||
"GET http://crm.example.com/api/routing/routes/health?logical_group_id=p2t4-health-1700000002")
|
||
if [[ ! -f /tmp/p2t4-health-switch ]]; then
|
||
write_body '{"route_health":[{"route_id":"primary-1700000002","runtime_status":"cooldown"},{"route_id":"fallback-1700000002","runtime_status":"healthy","recent_failover_count":0},{"route_id":"failing-1700000002","runtime_status":"failing"}]}'
|
||
else
|
||
write_body '{"route_health":[{"route_id":"primary-1700000002","runtime_status":"cooldown"},{"route_id":"fallback-1700000002","runtime_status":"healthy","recent_failover_count":1,"last_selected_at":"2026-05-29T12:00:00Z"},{"route_id":"failing-1700000002","runtime_status":"failing"}]}'
|
||
fi
|
||
;;
|
||
"POST http://crm.example.com/api/routing/resolve")
|
||
: > /tmp/p2t4-health-switch
|
||
write_body '{"resolve":{"request_id":"req-p2t4-health-1700000002","route_id":"fallback-1700000002","fallback_used":true}}'
|
||
;;
|
||
"GET http://crm.example.com/api/routing/logs/failovers?request_id=req-p2t4-health-1700000002&limit=5")
|
||
write_body '{"failover_events":[{"from_route_id":"primary-1700000002","to_route_id":"fallback-1700000002","reason":"active_cooldown:degraded"}]}'
|
||
;;
|
||
*)
|
||
echo "unexpected curl request: $method $url payload=$payload" >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
EOF
|
||
chmod +x "$fakebin/curl"
|
||
|
||
PATH="$fakebin:$PATH" \
|
||
CRM_BASE="http://crm.example.com" \
|
||
CRM_ADMIN_TOKEN="token" \
|
||
ROUTE_HEALTH_PAGE_URL="http://portal.example.com/route-health.html" \
|
||
TS="1700000002" \
|
||
ARTIFACT_DIR="$artifact_dir" \
|
||
bash "$ROOT_DIR/scripts/acceptance/verify_route_health_ui.sh" >"$stdout_file"
|
||
|
||
local summary
|
||
summary="$(cat "$artifact_dir/12-summary.json")"
|
||
assert_contains "$summary" '"resolve_route_id": "fallback-1700000002"'
|
||
assert_contains "$summary" '"fallback_recent_failover_count": 1'
|
||
}
|
||
|
||
run_test_remote43_patched_stack_renderers() {
|
||
# shellcheck disable=SC1091
|
||
source "$ROOT_DIR/scripts/deploy/remote43_patched_stack_lib.sh"
|
||
|
||
local host_env crm_env bootstrap
|
||
host_env="$(render_remote43_host_env "stack-pg" "stack-redis" "db-pass" "sub2api" "admin@sub2api.local" "admin-pass" "jwt-secret" "totp-secret")"
|
||
crm_env="$(render_remote43_crm_env "18143" "file:/tmp/sub2api.db?_foreign_keys=on" "crm-token" "/home/ubuntu/sub2api-cn-relay-manager-git-current" "portal-admin" "portal-pass")"
|
||
bootstrap="$(render_remote43_bootstrap_script \
|
||
"/home/ubuntu/test-stack" \
|
||
"/home/ubuntu/test-stack/.env.host" \
|
||
"/home/ubuntu/test-stack/.env.crm" \
|
||
"sub2api-patched" \
|
||
"sub2api-cn-relay-manager-server" \
|
||
"/home/ubuntu/test-stack/data" \
|
||
"/home/ubuntu/test-stack/sub2api-cn-relay-manager.db" \
|
||
"/home/ubuntu/test-stack/crm.pid" \
|
||
"/home/ubuntu/test-stack/crm.log" \
|
||
"test-stack-app" \
|
||
"test-stack-pg" \
|
||
"test-stack-redis" \
|
||
"test-stack-net" \
|
||
"weishaw/sub2api:0.1.129" \
|
||
"postgres:16-alpine" \
|
||
"redis:7-alpine" \
|
||
"db-pass" \
|
||
"sub2api" \
|
||
"18139" \
|
||
"18143" \
|
||
"8080" \
|
||
"/home/ubuntu/sub2api-cn-relay-manager-git-current" \
|
||
"/home/ubuntu/test-stack/sub2api-cn-relay-manager.bundle")"
|
||
|
||
assert_contains "$host_env" "AUTO_SETUP=true"
|
||
assert_contains "$host_env" "DATABASE_HOST=stack-pg"
|
||
assert_contains "$host_env" "REDIS_HOST=stack-redis"
|
||
assert_contains "$crm_env" "SUB2API_CRM_LISTEN_ADDR=127.0.0.1:18143"
|
||
assert_contains "$crm_env" "SUB2API_CRM_SQLITE_DSN="
|
||
assert_contains "$crm_env" "SUB2API_CRM_ADMIN_TOKEN=crm-token"
|
||
assert_contains "$crm_env" "SUB2API_CRM_ADMIN_USERNAME=portal-admin"
|
||
assert_contains "$crm_env" "SUB2API_CRM_ADMIN_PASSWORD=portal-pass"
|
||
assert_contains "$crm_env" "SUB2API_CRM_ADMIN_SESSION_TTL=12h"
|
||
assert_contains "$crm_env" "SUB2API_CRM_REPO_ROOT=/home/ubuntu/sub2api-cn-relay-manager-git-current"
|
||
local sourced_dsn
|
||
sourced_dsn="$(bash -lc 'set -a; source /dev/stdin; set +a; printf "%s" "$SUB2API_CRM_SQLITE_DSN"' <<<"$crm_env")"
|
||
[[ "$sourced_dsn" == "file:/tmp/sub2api.db?_foreign_keys=on" ]] || fail "crm env dsn did not survive bash source"
|
||
assert_contains "$bootstrap" 'rm -f "$DATA_DIR/install.lock" "$DATA_DIR/config.yaml" "$DATA_DIR/.installed"'
|
||
assert_contains "$bootstrap" '-v "$HOST_BINARY:/app/sub2api:ro"'
|
||
assert_contains "$bootstrap" '-p "127.0.0.1:$HOST_PORT:$HOST_CONTAINER_PORT"'
|
||
assert_contains "$bootstrap" 'REMOTE_REPO_ROOT=/home/ubuntu/sub2api-cn-relay-manager-git-current'
|
||
assert_contains "$bootstrap" 'REMOTE_REPO_BUNDLE=/home/ubuntu/test-stack/sub2api-cn-relay-manager.bundle'
|
||
assert_contains "$bootstrap" 'git -C "$REMOTE_REPO_ROOT" fetch "$REMOTE_REPO_BUNDLE" main'
|
||
assert_contains "$bootstrap" 'git clone "$REMOTE_REPO_BUNDLE" "$REMOTE_REPO_ROOT"'
|
||
assert_contains "$bootstrap" '/api/v1/auth/login'
|
||
assert_contains "$bootstrap" '/healthz'
|
||
assert_contains "$bootstrap" 'source "$1"; set +a; exec "$2"'
|
||
}
|
||
|
||
run_test_setup_remote43_patched_stack_dry_run() {
|
||
local tmpdir pack_dir shared_pack_dir repo_bundle host_bin crm_bin operator_env tunnel_script stdout_file ssh_key
|
||
tmpdir="$(mktemp -d)"
|
||
trap 'rm -rf "$tmpdir"' RETURN
|
||
pack_dir="$tmpdir/pack"
|
||
shared_pack_dir="$tmpdir/shared-pack"
|
||
repo_bundle="$tmpdir/sub2api-cn-relay-manager-main.bundle"
|
||
host_bin="$tmpdir/sub2api-patched"
|
||
crm_bin="$tmpdir/server"
|
||
operator_env="$tmpdir/operator.env"
|
||
tunnel_script="$tmpdir/tunnel.sh"
|
||
stdout_file="$tmpdir/setup.stdout.txt"
|
||
ssh_key="$tmpdir/remote43.pem"
|
||
|
||
mkdir -p "$pack_dir/providers"
|
||
printf '%s\n' '{"pack_id":"openai-cn-pack","version":"1.1.3"}' > "$pack_dir/pack.json"
|
||
printf '%s\n' '{"provider_id":"kimi-a7m"}' > "$pack_dir/providers/kimi-a7m.json"
|
||
printf '%s\n' '#!/usr/bin/env bash' > "$host_bin"
|
||
printf '%s\n' '#!/usr/bin/env bash' > "$crm_bin"
|
||
printf '%s\n' 'dummy-key' > "$ssh_key"
|
||
chmod +x "$host_bin" "$crm_bin"
|
||
|
||
git -C "$ROOT_DIR" bundle create "$repo_bundle" main >/dev/null
|
||
|
||
KEY="$ssh_key" \
|
||
REMOTE="ubuntu@example.com" \
|
||
STACK_NAME="test-stack" \
|
||
HOST_PORT=18139 \
|
||
CRM_PORT=18143 \
|
||
HOST_BINARY="$host_bin" \
|
||
CRM_BINARY="$crm_bin" \
|
||
PACK_DIR="$pack_dir" \
|
||
LOCAL_SHARED_PACK_DIR="$shared_pack_dir" \
|
||
LOCAL_REPO_BUNDLE="$repo_bundle" \
|
||
LOCAL_OPERATOR_ENV_FILE="$operator_env" \
|
||
LOCAL_TUNNEL_SCRIPT="$tunnel_script" \
|
||
REMOTE_ROOT="/home/ubuntu/test-stack" \
|
||
DRY_RUN=1 \
|
||
bash "$ROOT_DIR/scripts/deploy/setup_remote43_patched_stack.sh" >"$stdout_file"
|
||
|
||
[[ -f "$operator_env" ]] || fail "operator env file was not created"
|
||
[[ -f "$tunnel_script" ]] || fail "tunnel script was not created"
|
||
[[ -f "$shared_pack_dir/pack.json" ]] || fail "shared pack mirror was not created"
|
||
|
||
local stdout_text operator_env_text tunnel_text
|
||
stdout_text="$(cat "$stdout_file")"
|
||
operator_env_text="$(cat "$operator_env")"
|
||
tunnel_text="$(cat "$tunnel_script")"
|
||
|
||
assert_contains "$stdout_text" "remote43 patched stack prepared"
|
||
assert_contains "$stdout_text" "local operator env file: $operator_env"
|
||
assert_contains "$stdout_text" "remote repo root: /home/ubuntu/sub2api-cn-relay-manager-git-current"
|
||
assert_contains "$stdout_text" "DRY_RUN: ssh -i $ssh_key"
|
||
assert_contains "$operator_env_text" "CRM_BASE=http://127.0.0.1:18143"
|
||
assert_contains "$operator_env_text" "HOST_BASE=http://127.0.0.1:18139"
|
||
assert_contains "$operator_env_text" "PACK_PATH=$shared_pack_dir"
|
||
assert_contains "$operator_env_text" "REMOTE_HOST_ENV_FILE=/home/ubuntu/test-stack/.env.host"
|
||
assert_contains "$operator_env_text" "REMOTE_REPO_ROOT=/home/ubuntu/sub2api-cn-relay-manager-git-current"
|
||
assert_contains "$tunnel_text" "-L 18143:127.0.0.1:18143"
|
||
assert_contains "$tunnel_text" "-L 18139:127.0.0.1:18139"
|
||
}
|
||
|
||
run_test_build_subscription_access_prep_sql
|
||
run_test_real_host_acceptance_after_import_hook
|
||
run_test_check_deepseek_completion_split
|
||
run_test_import_remote43_provider_subscription_prep
|
||
run_test_migrate_historical_artifacts
|
||
run_test_verify_route_control_plane_script
|
||
run_test_verify_route_data_plane_script
|
||
run_test_verify_route_health_ui_script
|
||
run_test_remote43_patched_stack_renderers
|
||
run_test_setup_remote43_patched_stack_dry_run
|
||
run_test_verify_quality_gates_script
|
||
|
||
echo "PASS: real host script regression checks"
|