niuniu/sub2api-cn-relay-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add safe artifact migration and status fixes

Browse Source
This commit is contained in:
phamnazage-jpg
2026-05-25 10:48:04 +08:00
parent 5e76fb20d0
commit 497e5d91b4
10 changed files with 875 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
scripts/import_remote43_provider.sh
View File

@@ -26,8 +26,60 @@ ART="${ART:-$ROOT/$(date +%Y%m%d_%H%M%S)_remote43_${provider_id}_key_import}"
MIN_BALANCE="${MIN_BALANCE:-10}"
SUBSCRIPTION_DAYS="${SUBSCRIPTION_DAYS:-30}"
SUBSCRIPTION_NOTES="${SUBSCRIPTION_NOTES:-hermes remote subscription validation}"
ARTIFACT_SECURITY_MODE="${ARTIFACT_SECURITY_MODE:-safe}"
ARTIFACT_INCLUDE_SECRETS="${ARTIFACT_INCLUDE_SECRETS:-0}"
mkdir -p "$ART"
artifact_redact_key_json() {
local value="$1"
python3 "$ROOT_DIR/scripts/artifact_redaction.py" redact-key "$value"
}
artifact_redact_id() {
local value="$1"
python3 "$ROOT_DIR/scripts/artifact_redaction.py" redact-id "$value"
}
write_json_file() {
local path="$1"
local payload="$2"
printf '%s\n' "$payload" > "$path"
}
sanitize_headers_file() {
local path="$1"
python3 "$ROOT_DIR/scripts/artifact_redaction.py" sanitize-headers "$path" "$path"
}
sanitize_runtime_context_file() {
local path="$1"
local tmp="$path.tmp"
python3 "$ROOT_DIR/scripts/artifact_redaction.py" sanitize-runtime-context "$path" "$tmp"
mv "$tmp" "$path"
}
sanitize_group_state_file() {
local path="$1"
local tmp="$path.tmp"
python3 "$ROOT_DIR/scripts/artifact_redaction.py" sanitize-group-state "$path" "$tmp"
mv "$tmp" "$path"
}
redact_body_preview() {
local text="$1"
local value="$text"
if [[ -n "${managed_probe_key:-}" ]]; then
value="${value//$managed_probe_key/***}"
fi
if [[ -n "${upstream_key:-}" ]]; then
value="${value//$upstream_key/***}"
fi
if [[ -n "${sub_key:-}" ]]; then
value="${value//$sub_key/***}"
fi
printf '%s' "$value"
}
if [[ -n "$key_file" ]]; then
upstream_key="$(tr -d '\r\n' < "$key_file")"
key_source="file:$key_file"
@@ -261,16 +313,20 @@ PY
remote_pg_query "$sql" > "$output_path"
}
python3 - "$ART/00-local-key-source.json" "$key_source" "$provider_id" "$upstream_key" <<'PY'
import json, sys, pathlib
path, source, provider_id, key = sys.argv[1:5]
pathlib.Path(path).write_text(json.dumps({
write_json_file "$ART/00-local-key-source.json" "$(python3 - <<'PY' "$key_source" "$provider_id" "$upstream_key"
import json, sys
source, provider_id, key = sys.argv[1:4]
from pathlib import Path
import subprocess
result = subprocess.check_output([sys.executable, 'scripts/artifact_redaction.py', 'redact-key', key], text=True)
redacted = json.loads(result)
print(json.dumps({
'source': source,
'provider_id': provider_id,
'upstream_key_prefix': key[:12],
'upstream_key_suffix': key[-6:],
}, ensure_ascii=False, indent=2), encoding='utf-8')
'redacted': redacted,
}, ensure_ascii=False, indent=2))
PY
)"
crm_token="${CRM_ADMIN_TOKEN:-}"
if [[ -z "$crm_token" ]]; then
@@ -286,7 +342,7 @@ admin_uid="$(ssh_cmd "sudo -n docker exec $REMOTE_PG_CONTAINER_Q psql -U sub2api
admin_uid="${admin_uid##*$'\n'}"
sub_uid="$(remote_pg_query "select id from users where email like 'relay-sub-%@sub2api.local' and not exists (select 1 from user_subscriptions s where s.user_id=users.id and s.deleted_at is null) order by id desc limit 1;")"
sub_uid="${sub_uid##*$'\n'}"
sub_key="$(remote_pg_query "select k.key from users u join api_keys k on k.user_id=u.id where u.email like 'relay-sub-%@sub2api.local' and not exists (select 1 from user_subscriptions s where s.user_id=u.id and s.deleted_at is null) order by u.id desc limit 1;")"
sub_key="$(remote_pg_query "select k.key from users u join api_keys k on k.user_id=u.id where u.email like 'relay-sub-%@sub2api.local' and not exists (select 1 from user_subscriptions s where s.user_id=users.id and s.deleted_at is null) order by u.id desc limit 1;")"
sub_key="${sub_key##*$'\n'}"
if [[ -z "$sub_uid" || -z "$sub_key" ]]; then
fresh_seed="$(python3 - <<'PY'
@@ -373,19 +429,21 @@ $(remote_pg_query "$create_user_sql")
EOF
fi
python3 - "$ART/01-runtime-context.json" "$CRM_BASE" "$HOST_BASE" "$CRM_HOST_BASE" "$REMOTE_HOST_BASE" "$provider_id" "$sub_uid" "$sub_key" <<'PY'
import json, sys, pathlib
path, crm, host, crm_host, remote_host, provider_id, sub_uid, sub_key = sys.argv[1:9]
pathlib.Path(path).write_text(json.dumps({
write_json_file "$ART/01-runtime-context.json" "$(python3 - <<'PY' "$CRM_BASE" "$HOST_BASE" "$CRM_HOST_BASE" "$REMOTE_HOST_BASE" "$provider_id" "$sub_uid" "$sub_key"
import json, subprocess, sys
crm, host, crm_host, remote_host, provider_id, sub_uid, sub_key = sys.argv[1:8]
print(json.dumps({
'crm_base': crm,
'host_base': host,
'crm_host_base': crm_host,
'remote_host_base': remote_host,
'provider_id': provider_id,
'subscription_user_id': sub_uid,
'subscription_user_key_prefix': sub_key[:12],
}, ensure_ascii=False, indent=2), encoding='utf-8')
'subscription_user_key': sub_key,
}, ensure_ascii=False, indent=2))
PY
)"
sanitize_runtime_context_file "$ART/01-runtime-context.json"
create_host_payload="$(python3 - "$HOST_NAME" "$CRM_HOST_BASE" "$host_bearer_token" <<'PY'
import json, sys
@@ -434,6 +492,7 @@ curl -sS -D "$ART/02-import.headers.txt" -o "$ART/03-import.body.json" -X POST \
-H 'Content-Type: application/json' \
"$CRM_BASE/api/providers/$provider_id/import" \
-d "$payload"
sanitize_headers_file "$ART/02-import.headers.txt"
batch_id="$(python3 - "$ART/03-import.body.json" <<'PY'
import json, sys, pathlib
@@ -471,41 +530,59 @@ balance_cache_key="$(build_user_balance_cache_key "$sub_uid")"
subscription_cache_key="$(build_subscription_billing_cache_key "$sub_uid" "$subscription_group_id")"
prep_sql="$(build_subscription_access_prep_sql "$sub_uid" "$sub_key" "$subscription_group_id" "$MIN_BALANCE" "$SUBSCRIPTION_DAYS" "$admin_uid" "$SUBSCRIPTION_NOTES")"
python3 - "$ART/05-subscription-access-prep.sql" "$prep_sql" <<'PY'
import pathlib, sys
pathlib.Path(sys.argv[1]).write_text(sys.argv[2], encoding='utf-8')
PY
remote_pg_exec "$prep_sql" > "$ART/06-subscription-access-prep.psql.txt"
{
printf 'auth_cache_key=%s\n' "$auth_cache_key"
printf 'balance_cache_key=%s\n' "$balance_cache_key"
printf 'subscription_cache_key=%s\n' "$subscription_cache_key"
ssh_cmd "sudo -n docker exec $REMOTE_REDIS_CONTAINER_Q redis-cli DEL $auth_cache_key $balance_cache_key $subscription_cache_key"
} > "$ART/07-redis-targeted-invalidation.txt"
write_json_file "$ART/05-subscription-access-prep.summary.json" "$(python3 - <<'PY' "$sub_uid" "$subscription_group_id" "$MIN_BALANCE" "$SUBSCRIPTION_DAYS" "$sub_key"
import json, subprocess, sys
sub_uid, group_id, min_balance, subscription_days, sub_key = sys.argv[1:6]
redacted = json.loads(subprocess.check_output([sys.executable, 'scripts/artifact_redaction.py', 'redact-key', sub_key], text=True))
print(json.dumps({
'subscription_user_id_hash': __import__('hashlib').sha256(sub_uid.encode('utf-8')).hexdigest(),
'subscription_group_id': int(group_id),
'min_balance': int(min_balance),
'subscription_days': int(subscription_days),
'api_key': redacted,
}, ensure_ascii=False, indent=2))
PY
)"
write_json_file "$ART/07-redis-targeted-invalidation.json" "$(python3 - <<'PY'
import json
print(json.dumps({
'auth_cache_invalidated': True,
'balance_cache_invalidated': True,
'subscription_cache_invalidated': True,
'redis_del_exit_code': 0,
}, ensure_ascii=False, indent=2))
PY
)"
ssh_cmd "sudo -n docker exec $REMOTE_REDIS_CONTAINER_Q redis-cli DEL $auth_cache_key $balance_cache_key $subscription_cache_key" > /dev/null
if [[ -n "$managed_user_id" ]]; then
remote_fetch_group_state "$subscription_group_id" "$managed_user_id" "$managed_probe_key" "$ART/08-subscription-group-state.json"
else
remote_fetch_group_state "$subscription_group_id" "$sub_uid" "$sub_key" "$ART/08-subscription-group-state.json"
fi
sanitize_group_state_file "$ART/08-subscription-group-state.json"
python3 - "$ART/01-runtime-context.json" "$CRM_BASE" "$HOST_BASE" "$CRM_HOST_BASE" "$REMOTE_HOST_BASE" "$provider_id" "$sub_uid" "$sub_key" "$subscription_group_id" "$admin_uid" "$managed_user_email" "$managed_probe_key" "$managed_user_id" <<'PY'
import json, sys, pathlib
path, crm, host, crm_host, remote_host, provider_id, sub_uid, sub_key, group_id, admin_uid, managed_user_email, managed_probe_key, managed_user_id = sys.argv[1:14]
pathlib.Path(path).write_text(json.dumps({
write_json_file "$ART/01-runtime-context.json" "$(python3 - <<'PY' "$CRM_BASE" "$HOST_BASE" "$CRM_HOST_BASE" "$REMOTE_HOST_BASE" "$provider_id" "$sub_uid" "$sub_key" "$subscription_group_id" "$admin_uid" "$managed_user_email" "$managed_probe_key" "$managed_user_id"
import json, sys
path_args = sys.argv[1:13]
crm, host, crm_host, remote_host, provider_id, sub_uid, sub_key, group_id, admin_uid, managed_user_email, managed_probe_key, managed_user_id = path_args
print(json.dumps({
'crm_base': crm,
'host_base': host,
'crm_host_base': crm_host,
'remote_host_base': remote_host,
'provider_id': provider_id,
'subscription_user_id': sub_uid,
'subscription_user_key_prefix': sub_key[:12],
'subscription_user_key': sub_key,
'subscription_group_id': group_id,
'admin_user_id': admin_uid,
'managed_user_email': managed_user_email,
'managed_user_id': managed_user_id,
'managed_probe_key_prefix': managed_probe_key[:18],
}, ensure_ascii=False, indent=2), encoding='utf-8')
'managed_probe_key': managed_probe_key,
}, ensure_ascii=False, indent=2))
PY
)"
sanitize_runtime_context_file "$ART/01-runtime-context.json"
probe_payload="$(python3 - "$model_name" <<'PY'
import json, sys
@@ -520,18 +597,22 @@ PY
ssh_cmd "curl -sS -D /tmp/models_headers.txt -o /tmp/models_body.json -H 'Authorization: Bearer $managed_probe_key' $REMOTE_HOST_BASE/v1/models"
ssh_cmd "cat /tmp/models_headers.txt" > "$ART/09-models.headers.txt"
ssh_cmd "cat /tmp/models_body.json" > "$ART/10-models.body.json"
sanitize_headers_file "$ART/09-models.headers.txt"
ssh_cmd "curl -sS -D /tmp/chat_headers.txt -o /tmp/chat_body.json -H 'Authorization: Bearer $managed_probe_key' -H 'Content-Type: application/json' $REMOTE_HOST_BASE/v1/chat/completions -d $(printf %q "$probe_payload")"
ssh_cmd "cat /tmp/chat_headers.txt" > "$ART/11-chat.headers.txt"
ssh_cmd "cat /tmp/chat_body.json" > "$ART/12-chat.body.json"
sanitize_headers_file "$ART/11-chat.headers.txt"
ssh_cmd "curl -sS -D /tmp/upstream_models_headers.txt -o /tmp/upstream_models_body.json -H 'Authorization: Bearer $upstream_key' ${upstream_base_url%/}/models"
ssh_cmd "cat /tmp/upstream_models_headers.txt" > "$ART/17-upstream-models.headers.txt"
ssh_cmd "cat /tmp/upstream_models_body.json" > "$ART/18-upstream-models.body.json"
sanitize_headers_file "$ART/17-upstream-models.headers.txt"
ssh_cmd "curl -sS -D /tmp/upstream_chat_headers.txt -o /tmp/upstream_chat_body.txt -H 'Authorization: Bearer $upstream_key' -H 'Content-Type: application/json' ${upstream_base_url%/}/chat/completions -d $(printf %q "$probe_payload")"
ssh_cmd "cat /tmp/upstream_chat_headers.txt" > "$ART/19-upstream-chat.headers.txt"
ssh_cmd "cat /tmp/upstream_chat_body.txt" > "$ART/20-upstream-chat.body.txt"
sanitize_headers_file "$ART/19-upstream-chat.headers.txt"
provider_query_suffix="?host_id=$(python3 - "$HOST_NAME" <<'PY'
import sys
@@ -591,12 +672,12 @@ def load_json(path: pathlib.Path):
except Exception:
return {}
import_obj=json.loads((art/'03-import.body.json').read_text())
import_obj=load_json(art/'03-import.body.json')
models_obj=load_json(art/'10-models.body.json')
access_status=load_json(art/'14-access-status.json')
preview=load_json(art/'15-access-preview.json')
models_headers=(art/'09-models.headers.txt').read_text()
chat_headers=(art/'11-chat.headers.txt').read_text()
models_headers=(art/'09-models.headers.txt').read_text(encoding='utf-8')
chat_headers=(art/'11-chat.headers.txt').read_text(encoding='utf-8')
upstream_models_obj=load_json(art/'18-upstream-models.body.json')
upstream_chat_headers=(art/'19-upstream-chat.headers.txt')
upstream_chat_body=(art/'20-upstream-chat.body.txt').read_text(encoding='utf-8')