fix(provision): harden batch scoping and remote43 import validation

2026-05-19 20:21:21 +08:00
parent 0ecce50a48
commit 18e1b085eb
16 changed files with 412 additions and 101 deletions
--- a/scripts/host_access_prep_lib.sh
+++ b/scripts/host_access_prep_lib.sh
@@ -34,6 +34,24 @@ WHERE key = $(sql_literal "$api_key");
 SQL
 }

+build_api_key_auth_cache_key() {
+  local api_key="$1"
+  local digest
+  digest="$(printf '%s' "$api_key" | sha256sum | awk '{print $1}')"
+  printf 'apikey:auth:%s' "$digest"
+}
+
+build_user_balance_cache_key() {
+  local user_id="$1"
+  printf 'billing:balance:%s' "$user_id"
+}
+
+build_subscription_billing_cache_key() {
+  local user_id="$1"
+  local group_id="$2"
+  printf 'billing:sub:%s:%s' "$user_id" "$group_id"
+}
+
 build_upsert_subscription_sql() {
  local user_id="$1"
  local group_id="$2"
--- a/scripts/import_remote43_provider.sh
+++ b/scripts/import_remote43_provider.sh
@@ -14,6 +14,7 @@ KEY="${KEY:-/home/long/下载/zjsea.pem}"
 REMOTE="${REMOTE:-ubuntu@43.155.133.187}"
 CRM_BASE="${CRM_BASE:-http://127.0.0.1:18088}"
 HOST_BASE="${HOST_BASE:-http://127.0.0.1:18087}"
+CRM_HOST_BASE="${CRM_HOST_BASE:-$HOST_BASE}"
 PACK_PATH="${PACK_PATH:-/home/ubuntu/sub2api-cn-relay-manager/packs/openai-cn-pack}"
 ROOT="${ROOT:-$ROOT_DIR/artifacts/real-host-acceptance}"
 ART="${ART:-$ROOT/$(date +%Y%m%d_%H%M%S)_remote43_${provider_id}_key_import}"
@@ -46,6 +47,13 @@ remote_pg_exec() {
  ssh_cmd "printf '%s' '$encoded' | base64 -d | sudo -n docker exec -i sub2api-relaymgr-pg psql -U sub2api -d sub2api"
 }

+remote_pg_query() {
+  local sql="$1"
+  local encoded
+  encoded="$(printf '%s' "$sql" | base64 -w0)"
+  ssh_cmd "printf '%s' '$encoded' | base64 -d | sudo -n docker exec -i sub2api-relaymgr-pg psql -U sub2api -d sub2api -At -F $'\t'"
+}
+
 remote_fetch_group_state() {
  local group_id="$1"
  local user_id="$2"
@@ -98,24 +106,109 @@ admin_key="$(ssh_cmd "sudo -n docker exec sub2api-relaymgr-pg psql -U sub2api -d
 admin_key="${admin_key##*$'\n'}"
 admin_uid="$(ssh_cmd "sudo -n docker exec sub2api-relaymgr-pg psql -U sub2api -d sub2api -Atc \"select id from users where role='admin' order by id asc limit 1;\"")"
 admin_uid="${admin_uid##*$'\n'}"
-sub_uid="$(ssh_cmd "sudo -n docker exec sub2api-relaymgr-pg psql -U sub2api -d sub2api -Atc \"select id from users where email like 'relay-sub-%@sub2api.local' order by id desc limit 1;\"")"
+sub_uid="$(remote_pg_query "select id from users where email like 'relay-sub-%@sub2api.local' and not exists (select 1 from user_subscriptions s where s.user_id=users.id and s.deleted_at is null) order by id desc limit 1;")"
 sub_uid="${sub_uid##*$'\n'}"
-sub_key="$(ssh_cmd "sudo -n docker exec sub2api-relaymgr-pg psql -U sub2api -d sub2api -Atc \"select k.key from users u join api_keys k on k.user_id=u.id where u.email like 'relay-sub-%@sub2api.local' order by u.id desc limit 1;\"")"
+sub_key="$(remote_pg_query "select k.key from users u join api_keys k on k.user_id=u.id where u.email like 'relay-sub-%@sub2api.local' and not exists (select 1 from user_subscriptions s where s.user_id=u.id and s.deleted_at is null) order by u.id desc limit 1;")"
 sub_key="${sub_key##*$'\n'}"
+if [[ -z "$sub_uid" || -z "$sub_key" ]]; then
+  fresh_seed="$(python3 - <<'PY'
+import secrets, time
+print(f"{int(time.time())}-{secrets.token_hex(4)}")
+PY
+)"
+  fresh_email="relay-sub-${fresh_seed}@sub2api.local"
+  fresh_username="relay-sub-${fresh_seed}"
+  fresh_key="sk-${fresh_seed}"
+  create_user_sql="$(python3 - "$fresh_email" "$fresh_username" "$fresh_key" <<'PY'
+import sys
+email, username, api_key = sys.argv[1:4]

-python3 - "$ART/01-runtime-context.json" "$CRM_BASE" "$HOST_BASE" "$provider_id" "$sub_uid" "$sub_key" <<'PY'
+def sql_quote(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+print(f'''
+WITH seed AS (
+  SELECT password_hash
+  FROM users
+  WHERE role = 'admin'
+  ORDER BY id ASC
+  LIMIT 1
+),
+ins_user AS (
+  INSERT INTO users (
+    email, password_hash, role, balance, concurrency, status, username, notes, wechat,
+    totp_secret_encrypted, totp_enabled, balance_notify_enabled, balance_notify_threshold,
+    balance_notify_extra_emails, balance_notify_threshold_type, total_recharged, signup_source,
+    rpm_limit
+  )
+  SELECT
+    {sql_quote(email)},
+    password_hash,
+    'user',
+    10,
+    5,
+    'active',
+    {sql_quote(username)},
+    'hermes remote subscription validation',
+    '',
+    '',
+    false,
+    true,
+    NULL,
+    '[]',
+    'fixed',
+    0,
+    'email',
+    0
+  FROM seed
+  RETURNING id
+),
+ins_key AS (
+  INSERT INTO api_keys (
+    user_id, key, name, group_id, status, quota, quota_used,
+    rate_limit_5h, rate_limit_1d, rate_limit_7d,
+    usage_5h, usage_1d, usage_7d
+  )
+  SELECT
+    id,
+    {sql_quote(api_key)},
+    {sql_quote(username + '-key')},
+    NULL,
+    'active',
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0
+  FROM ins_user
+  RETURNING user_id, key
+)
+SELECT user_id, key FROM ins_key;
+'''.strip())
+PY
+)"
+  read -r sub_uid sub_key <<EOF
+$(remote_pg_query "$create_user_sql")
+EOF
+fi
+
+python3 - "$ART/01-runtime-context.json" "$CRM_BASE" "$HOST_BASE" "$CRM_HOST_BASE" "$provider_id" "$sub_uid" "$sub_key" <<'PY'
 import json, sys, pathlib
-path, crm, host, provider_id, sub_uid, sub_key = sys.argv[1:7]
+path, crm, host, crm_host, provider_id, sub_uid, sub_key = sys.argv[1:8]
 pathlib.Path(path).write_text(json.dumps({
  'crm_base': crm,
  'host_base': host,
+  'crm_host_base': crm_host,
  'provider_id': provider_id,
  'subscription_user_id': sub_uid,
  'subscription_user_key_prefix': sub_key[:12],
 }, ensure_ascii=False, indent=2), encoding='utf-8')
 PY

-payload="$(python3 - "$HOST_BASE" "$admin_key" "$PACK_PATH" "$provider_id" "$upstream_key" "$sub_key" "$sub_uid" "$SUBSCRIPTION_DAYS" <<'PY'
+payload="$(python3 - "$CRM_HOST_BASE" "$admin_key" "$PACK_PATH" "$provider_id" "$upstream_key" "$sub_key" "$sub_uid" "$SUBSCRIPTION_DAYS" <<'PY'
 import json, sys
 host_base, admin_key, pack_path, provider_id, upstream_key, sub_key, sub_uid, subscription_days = sys.argv[1:9]
 print(json.dumps({
@@ -144,18 +237,28 @@ print(obj['batch_id'])
 PY
 )"

-ssh_cmd "curl -sS -H 'Authorization: Bearer *** $CRM_BASE/api/import-batches/$batch_id" > "$ART/04-batch-detail-initial.json"
-subscription_group_id="$(python3 - "$ART/04-batch-detail-initial.json" <<'PY'
+ssh_cmd "curl -sS -H 'Authorization: Bearer $crm_token' $CRM_BASE/api/import-batches/$batch_id" > "$ART/04-batch-detail-initial.json"
+subscription_group_id="$(python3 - "$ART/03-import.body.json" "$ART/04-batch-detail-initial.json" <<'PY'
 import json, pathlib, sys
-obj = json.loads(pathlib.Path(sys.argv[1]).read_text())
-for item in obj.get('managed_resources', []):
+import_obj = json.loads(pathlib.Path(sys.argv[1]).read_text())
+batch_obj = json.loads(pathlib.Path(sys.argv[2]).read_text())
+
+group = import_obj.get('group') or {}
+if group.get('id'):
+    print(group['id'])
+    raise SystemExit(0)
+
+for item in batch_obj.get('managed_resources', []):
    if item.get('ResourceType') == 'group':
        print(item.get('HostResourceID', ''))
-        break
-else:
-    raise SystemExit('missing managed group in batch detail')
+        raise SystemExit(0)
+
+raise SystemExit('missing managed group in import response and batch detail')
 PY
 )"
+auth_cache_key="$(build_api_key_auth_cache_key "$sub_key")"
+balance_cache_key="$(build_user_balance_cache_key "$sub_uid")"
+subscription_cache_key="$(build_subscription_billing_cache_key "$sub_uid" "$subscription_group_id")"

 prep_sql="$(build_subscription_access_prep_sql "$sub_uid" "$sub_key" "$subscription_group_id" "$MIN_BALANCE" "$SUBSCRIPTION_DAYS" "$admin_uid" "$SUBSCRIPTION_NOTES")"
 python3 - "$ART/05-subscription-access-prep.sql" "$prep_sql" <<'PY'
@@ -163,15 +266,21 @@ import pathlib, sys
 pathlib.Path(sys.argv[1]).write_text(sys.argv[2], encoding='utf-8')
 PY
 remote_pg_exec "$prep_sql" > "$ART/06-subscription-access-prep.psql.txt"
-ssh_cmd "sudo -n docker exec sub2api-relaymgr-redis redis-cli FLUSHDB" > "$ART/07-redis-flush.txt"
+{
+  printf 'auth_cache_key=%s\n' "$auth_cache_key"
+  printf 'balance_cache_key=%s\n' "$balance_cache_key"
+  printf 'subscription_cache_key=%s\n' "$subscription_cache_key"
+  ssh_cmd "sudo -n docker exec sub2api-relaymgr-redis redis-cli DEL $auth_cache_key $balance_cache_key $subscription_cache_key"
+} > "$ART/07-redis-targeted-invalidation.txt"
 remote_fetch_group_state "$subscription_group_id" "$sub_uid" "$sub_key" "$ART/08-subscription-group-state.json"

-python3 - "$ART/01-runtime-context.json" "$CRM_BASE" "$HOST_BASE" "$provider_id" "$sub_uid" "$sub_key" "$subscription_group_id" "$admin_uid" <<'PY'
+python3 - "$ART/01-runtime-context.json" "$CRM_BASE" "$HOST_BASE" "$CRM_HOST_BASE" "$provider_id" "$sub_uid" "$sub_key" "$subscription_group_id" "$admin_uid" <<'PY'
 import json, sys, pathlib
-path, crm, host, provider_id, sub_uid, sub_key, group_id, admin_uid = sys.argv[1:9]
+path, crm, host, crm_host, provider_id, sub_uid, sub_key, group_id, admin_uid = sys.argv[1:10]
 pathlib.Path(path).write_text(json.dumps({
  'crm_base': crm,
  'host_base': host,
+  'crm_host_base': crm_host,
  'provider_id': provider_id,
  'subscription_user_id': sub_uid,
  'subscription_user_key_prefix': sub_key[:12],
@@ -190,30 +299,42 @@ print(json.dumps({
 }, ensure_ascii=False))
 PY
 )"
-ssh_cmd "curl -sS -D /tmp/chat_headers.txt -o /tmp/chat_body.json -H 'Authorization: Bearer *** -H 'Content-Type: application/json' $HOST_BASE/v1/chat/completions -d $(printf %q "$probe_payload")"
-ssh_cmd "cat /tmp/chat_headers.txt" > "$ART/09-chat.headers.txt"
-ssh_cmd "cat /tmp/chat_body.json" > "$ART/10-chat.body.json"
+ssh_cmd "curl -sS -D /tmp/models_headers.txt -o /tmp/models_body.json -H 'Authorization: Bearer *** $HOST_BASE/v1/models"
+ssh_cmd "cat /tmp/models_headers.txt" > "$ART/09-models.headers.txt"
+ssh_cmd "cat /tmp/models_body.json" > "$ART/10-models.body.json"

-ssh_cmd "curl -sS -H 'Authorization: Bearer *** $CRM_BASE/api/providers/$provider_id/status" > "$ART/11-provider-status.json"
-ssh_cmd "curl -sS -H 'Authorization: Bearer *** $CRM_BASE/api/providers/$provider_id/access/status" > "$ART/12-access-status.json"
+ssh_cmd "curl -sS -D /tmp/chat_headers.txt -o /tmp/chat_body.json -H 'Authorization: Bearer *** -H 'Content-Type: application/json' $HOST_BASE/v1/chat/completions -d $(printf %q "$probe_payload")"
+ssh_cmd "cat /tmp/chat_headers.txt" > "$ART/11-chat.headers.txt"
+ssh_cmd "cat /tmp/chat_body.json" > "$ART/12-chat.body.json"
+
+ssh_cmd "curl -sS -H 'Authorization: Bearer *** $CRM_BASE/api/providers/$provider_id/status" > "$ART/13-provider-status.json"
+ssh_cmd "curl -sS -H 'Authorization: Bearer *** $CRM_BASE/api/providers/$provider_id/access/status" > "$ART/14-access-status.json"
 preview_payload="$(python3 - "$provider_id" <<'PY'
 import json, sys
 print(json.dumps({'provider_id': sys.argv[1], 'mode': 'subscription'}, ensure_ascii=False))
 PY
 )"
-ssh_cmd "curl -sS -X POST -H 'Authorization: Bearer *** -H 'Content-Type: application/json' $CRM_BASE/api/providers/$provider_id/access/preview -d $(printf %q "$preview_payload")" > "$ART/13-access-preview.json"
-ssh_cmd "curl -sS -H 'Authorization: Bearer *** $CRM_BASE/api/import-batches/$batch_id" > "$ART/14-batch-detail-final.json"
+ssh_cmd "curl -sS -X POST -H 'Authorization: Bearer *** -H 'Content-Type: application/json' $CRM_BASE/api/providers/$provider_id/access/preview -d $(printf %q "$preview_payload")" > "$ART/15-access-preview.json"
+ssh_cmd "curl -sS -H 'Authorization: Bearer *** $CRM_BASE/api/import-batches/$batch_id" > "$ART/16-batch-detail-final.json"

-python3 - "$ART" "$provider_id" "$batch_id" <<'PY'
+python3 - "$ART" "$provider_id" "$batch_id" "$subscription_group_id" "$model_name" <<'PY'
 import json, pathlib, sys
 art=pathlib.Path(sys.argv[1])
 provider_id=sys.argv[2]
 batch_id=int(sys.argv[3])
+subscription_group_id=sys.argv[4]
+expected_model=sys.argv[5]
 import_obj=json.loads((art/'03-import.body.json').read_text())
-access_status=json.loads((art/'12-access-status.json').read_text())
-preview=json.loads((art/'13-access-preview.json').read_text())
-chat_headers=(art/'09-chat.headers.txt').read_text()
-group_state=json.loads((art/'08-subscription-group-state.json').read_text())
+models_obj=json.loads((art/'10-models.body.json').read_text())
+access_status=json.loads((art/'14-access-status.json').read_text())
+preview=json.loads((art/'15-access-preview.json').read_text())
+models_headers=(art/'09-models.headers.txt').read_text()
+chat_headers=(art/'11-chat.headers.txt').read_text()
+models=[]
+for item in models_obj.get('data') or []:
+    model_id = item.get('id')
+    if isinstance(model_id, str) and model_id:
+        models.append(model_id)
 summary={
  'artifact_dir': str(art),
  'provider_id': provider_id,
@@ -221,13 +342,15 @@ summary={
  'batch_status': import_obj.get('batch_status'),
  'access_status_from_import': import_obj.get('access_status'),
  'provider_status_from_import': import_obj.get('provider_status'),
+  'direct_models_http200': '200 OK' in models_headers,
+  'direct_models_has_expected_model': expected_model in models,
+  'direct_models': models,
  'direct_chat_http200': '200 OK' in chat_headers,
  'latest_access_status': access_status.get('latest_access_status') or access_status.get('batch_access_status'),
  'preview_available': preview.get('available'),
  'accepted_keys_count': import_obj.get('accepted_keys_count'),
-  'subscription_group_id': group_state.get('group_id'),
-  'key_group_id': (group_state.get('key') or {}).get('group_id'),
-  'subscription_status': (group_state.get('subscription') or {}).get('status'),
+  'subscription_group_id': subscription_group_id,
+  'import_group_id': (import_obj.get('group') or {}).get('id'),
 }
 print(json.dumps(summary, ensure_ascii=False))
 PY
--- a/scripts/test_real_host_scripts.sh
+++ b/scripts/test_real_host_scripts.sh
@@ -35,6 +35,14 @@ run_test_build_subscription_access_prep_sql() {
  local quoted_sql
  quoted_sql="$(build_bind_api_key_group_sql "sk-o'reilly" 7)"
  assert_contains "$quoted_sql" "WHERE key = 'sk-o''reilly'"
+
+  local auth_cache_key balance_cache_key subscription_cache_key
+  auth_cache_key="$(build_api_key_auth_cache_key 'user-key')"
+  balance_cache_key="$(build_user_balance_cache_key 42)"
+  subscription_cache_key="$(build_subscription_billing_cache_key 42 7)"
+  assert_contains "$auth_cache_key" "apikey:auth:"
+  assert_contains "$balance_cache_key" "billing:balance:42"
+  assert_contains "$subscription_cache_key" "billing:sub:42:7"
 }

 run_test_real_host_acceptance_after_import_hook() {
@@ -157,8 +165,8 @@ case "$cmd" in
  *"select k.key from users u join api_keys k on k.user_id=u.id"*)
    printf '%s\n' 'user-key'
    ;;
-  *"curl -sS -D /tmp/import_headers.txt"*"/api/providers/deepseek/import"*)
-    printf '%s\n' '{"batch_id":123,"batch_status":"partially_succeeded","access_status":"broken"}' > /tmp/import_body.json
+  *"/api/providers/deepseek/import"*)
+    printf '%s\n' '{"batch_id":123,"batch_status":"partially_succeeded","access_status":"broken","group":{"id":"7","name":"DeepSeek 默认分组"}}' > /tmp/import_body.json
    printf '%s\n' 'HTTP/1.1 200 OK' > /tmp/import_headers.txt
    ;;
  "cat /tmp/import_headers.txt")
@@ -167,8 +175,18 @@ case "$cmd" in
  "cat /tmp/import_body.json")
    cat /tmp/import_body.json
    ;;
-  *"curl -sS -H 'Authorization: Bearer *** http://127.0.0.1:18088/api/import-batches/123"*)
-    printf '%s\n' '{"managed_resources":[{"ResourceType":"group","HostResourceID":"7","ResourceName":"DeepSeek 默认分组"}]}'
+  *"/api/import-batches/123"*)
+    printf '%s\n' '{"managed_resources":[{"ResourceType":"account","HostResourceID":"8","ResourceName":"deepseek-01"}]}'
+    ;;
+  *"curl -sS -D /tmp/models_headers.txt"*)
+    printf '%s\n' 'HTTP/1.1 200 OK' > /tmp/models_headers.txt
+    printf '%s\n' '{"data":[{"id":"gpt-4"},{"id":"gpt-4.1"}]}' > /tmp/models_body.json
+    ;;
+  "cat /tmp/models_headers.txt")
+    cat /tmp/models_headers.txt
+    ;;
+  "cat /tmp/models_body.json")
+    cat /tmp/models_body.json
    ;;
  *"curl -sS -D /tmp/chat_headers.txt"*)
    printf '%s\n' 'HTTP/1.1 200 OK' > /tmp/chat_headers.txt
@@ -180,16 +198,16 @@ case "$cmd" in
  "cat /tmp/chat_body.json")
    cat /tmp/chat_body.json
    ;;
-  *"curl -sS -H 'Authorization: Bearer *** http://127.0.0.1:18088/api/providers/deepseek/status"*)
+  *"/api/providers/deepseek/status"*)
    printf '%s\n' '{"status":"ready"}'
    ;;
-  *"curl -sS -H 'Authorization: Bearer *** http://127.0.0.1:18088/api/providers/deepseek/access/status"*)
+  *"/api/providers/deepseek/access/status"*)
    printf '%s\n' '{"latest_access_status":"subscription_ready"}'
    ;;
-  *"curl -sS -X POST -H 'Authorization: Bearer *** -H 'Content-Type: application/json' http://127.0.0.1:18088/api/providers/deepseek/access/preview"*)
+  *"/api/providers/deepseek/access/preview"*)
    printf '%s\n' '{"available":true}'
    ;;
-  *"curl -sS -H 'Authorization: Bearer *** http://127.0.0.1:18088/api/providers/deepseek/reconcile"*)
+  *"/api/providers/deepseek/reconcile"*)
    printf '%s\n' '{"status":"in_sync"}'
    ;;
  *"sudo -n docker exec -i sub2api-relaymgr-pg psql -U sub2api -d sub2api -At -F ''"*)
@@ -197,17 +215,26 @@ case "$cmd" in
    ;;
  *"sudo -n docker exec -i sub2api-relaymgr-pg psql -U sub2api -d sub2api"*)
    CMD="$cmd" LOG_DIR="$log_dir" python3 - <<'PY'
-import base64, os, re, pathlib
+import base64, os, re, pathlib, sys
 cmd = os.environ['CMD']
 log_dir = pathlib.Path(os.environ['LOG_DIR'])
 match = re.search(r"printf '%s' '([^']+)' \| base64 -d", cmd)
 if not match:
    raise SystemExit(f'failed to extract base64 payload from: {cmd}')
-log_dir.joinpath('prep.sql').write_bytes(base64.b64decode(match.group(1)))
+sql = base64.b64decode(match.group(1)).decode()
+log_dir.joinpath('prep.sql').write_text(sql, encoding='utf-8')
+if "select id from users where email like 'relay-sub-%@sub2api.local' and not exists" in sql:
+    print('')
+elif "select k.key from users u join api_keys k on k.user_id=u.id" in sql and "not exists" in sql:
+    print('')
+elif "INSERT INTO users" in sql and "INSERT INTO api_keys" in sql:
+    print('84\tuser-key-fresh')
+else:
+    print('')
 PY
    ;;
-  *"sudo -n docker exec sub2api-relaymgr-redis redis-cli FLUSHDB"*)
-    printf '%s\n' 'OK'
+  *"sudo -n docker exec sub2api-relaymgr-redis redis-cli DEL apikey:auth:"*" billing:balance:"*" billing:sub:"*":7"*)
+    printf '%s\n' '3'
    ;;
  *)
    echo "unexpected ssh command: $cmd" >&2
@@ -223,6 +250,7 @@ EOF
  REMOTE="fake@host" \
  CRM_BASE="http://127.0.0.1:18088" \
  HOST_BASE="http://127.0.0.1:18087" \
+  CRM_HOST_BASE="http://127.0.0.1:18093" \
  ROOT="$artifact_dir/root" \
  ART="$artifact_dir/run" \
  PACK_PATH="/tmp/openai-pack" \
@@ -239,6 +267,18 @@ EOF
  assert_contains "$prep_sql" "UPDATE api_keys"
  assert_contains "$prep_sql" "INSERT INTO user_subscriptions"
  assert_contains "$prep_sql" "group_id = 7"
+  local runtime_context invalidation_log
+  runtime_context="$(cat "$artifact_dir/run/01-runtime-context.json")"
+  assert_contains "$runtime_context" '"crm_host_base": "http://127.0.0.1:18093"'
+  invalidation_log="$(cat "$artifact_dir/run/07-redis-targeted-invalidation.txt")"
+  assert_contains "$invalidation_log" "auth_cache_key=apikey:auth:"
+  assert_contains "$invalidation_log" "balance_cache_key=billing:balance:84"
+  assert_contains "$invalidation_log" "subscription_cache_key=billing:sub:84:7"
+  local models_body chat_body
+  models_body="$(cat "$artifact_dir/run/10-models.body.json")"
+  chat_body="$(cat "$artifact_dir/run/12-chat.body.json")"
+  assert_contains "$models_body" '"id":"gpt-4"'
+  assert_contains "$chat_body" '"content":"pong"'
  [[ -s "$ssh_log" ]] || fail "ssh log was empty"
 }