refactor: clean up project structure
- Remove old review reports (keep latest only) - Move docs/ to deploy/docs-backup/ - Move performance-testing/ to deploy/ - Clean up test output files - Organize root directory
This commit is contained in:
Developer
50
deploy/monitoring/.env.example
Normal file
50
deploy/monitoring/.env.example
Normal file
@@ -0,0 +1,50 @@
|
||||
# =============================================================================
|
||||
# Sub2API 监控栈环境变量配置
|
||||
# =============================================================================
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Grafana 配置
|
||||
# -----------------------------------------------------------------------------
|
||||
GRAFANA_ADMIN_USER=admin
|
||||
GRAFANA_ADMIN_PASSWORD=changeme
|
||||
GRAFANA_ROOT_URL=http://localhost:3000
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 告警通知配置
|
||||
# -----------------------------------------------------------------------------
|
||||
# Slack Webhook URL (用于告警通知)
|
||||
SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK
|
||||
|
||||
# PagerDuty Webhook URL (用于 Critical 告警)
|
||||
PAGERDUTY_WEBHOOK_URL=https://events.pagerduty.com/integration/YOUR/KEY/enqueue
|
||||
|
||||
# SMTP 配置 (用于邮件告警)
|
||||
SMTP_HOST=smtp.gmail.com
|
||||
SMTP_PORT=587
|
||||
SMTP_USER=your-email@gmail.com
|
||||
SMTP_PASSWORD=your-app-password
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 数据保留配置
|
||||
# -----------------------------------------------------------------------------
|
||||
# Prometheus 数据保留时间
|
||||
PROMETHEUS_RETENTION_TIME=30d
|
||||
PROMETHEUS_RETENTION_SIZE=50GB
|
||||
|
||||
# Loki 数据保留时间
|
||||
LOKI_RETENTION_TIME=168h
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 外部服务配置
|
||||
# -----------------------------------------------------------------------------
|
||||
# Sub2API 服务地址 (用于抓取指标)
|
||||
SUB2API_HOST=sub2api
|
||||
SUB2API_PORT=8080
|
||||
|
||||
# PostgreSQL Exporter 配置
|
||||
POSTGRES_EXPORTER_HOST=postgres-exporter
|
||||
POSTGRES_EXPORTER_PORT=9187
|
||||
|
||||
# Redis Exporter 配置
|
||||
REDIS_EXPORTER_HOST=redis-exporter
|
||||
REDIS_EXPORTER_PORT=9121
|
||||
140
deploy/monitoring/QUICKSTART.md
Normal file
140
deploy/monitoring/QUICKSTART.md
Normal file
@@ -0,0 +1,140 @@
|
||||
# Sub2API 监控快速启动指南
|
||||
|
||||
## 单机部署 (2核4G)
|
||||
|
||||
### 1. 前置检查
|
||||
|
||||
```bash
|
||||
# 检查可用资源
|
||||
free -h
|
||||
df -h
|
||||
|
||||
# 确保有 300MB+ 可用内存和 2GB+ 磁盘空间
|
||||
```
|
||||
|
||||
### 2. 启动监控栈
|
||||
|
||||
```bash
|
||||
cd deploy/monitoring
|
||||
|
||||
# 创建数据目录
|
||||
mkdir -p prometheus-data grafana-data
|
||||
|
||||
# 启动 (单机优化版)
|
||||
docker-compose -f docker-compose.single.yml up -d
|
||||
|
||||
# 查看状态
|
||||
docker-compose -f docker-compose.single.yml ps
|
||||
```
|
||||
|
||||
### 3. 访问服务
|
||||
|
||||
| 服务 | 地址 | 默认账号 |
|
||||
|------|------|----------|
|
||||
| Grafana | http://localhost:3000 | admin/admin |
|
||||
| Prometheus | http://localhost:9090 | - |
|
||||
|
||||
### 4. 验证资源占用
|
||||
|
||||
```bash
|
||||
# 查看容器资源使用
|
||||
docker stats --no-stream
|
||||
|
||||
# 预期输出:
|
||||
# CONTAINER CPU % MEM USAGE / LIMIT
|
||||
# sub2api-prometheus 15% 85MiB / 128MiB
|
||||
# sub2api-grafana 5% 45MiB / 128MiB
|
||||
# sub2api-node-exp 1% 15MiB / 32MiB
|
||||
```
|
||||
|
||||
### 5. 配置应用指标端点
|
||||
|
||||
确保 Sub2API 应用已集成 Prometheus 指标中间件，暴露 `/metrics` 端点。
|
||||
|
||||
---
|
||||
|
||||
## 扩展到多机
|
||||
|
||||
### 阶段 1: 部署中心节点
|
||||
|
||||
在 4核8G+ 机器上:
|
||||
|
||||
```bash
|
||||
# 启动中心监控栈
|
||||
docker-compose -f docker-compose.central.yml up -d
|
||||
```
|
||||
|
||||
### 阶段 2: 配置各节点
|
||||
|
||||
在每个应用节点:
|
||||
|
||||
```bash
|
||||
# 设置环境变量
|
||||
export CLUSTER_NAME=sub2api-prod
|
||||
export INSTANCE_NAME=node-1
|
||||
export REGION=cn-north
|
||||
export VM_INSERT_URL=http://central-server:8480
|
||||
|
||||
# 使用多机节点配置启动 Prometheus
|
||||
docker-compose -f docker-compose.single.yml \
|
||||
-f docker-compose.multi-node.override.yml up -d
|
||||
```
|
||||
|
||||
### 阶段 3: 验证数据汇聚
|
||||
|
||||
在中心 Grafana 查看多实例数据。
|
||||
|
||||
---
|
||||
|
||||
## 故障排查
|
||||
|
||||
### Prometheus OOM
|
||||
|
||||
```bash
|
||||
# 检查 WAL 大小
|
||||
du -sh prometheus-data/wal
|
||||
|
||||
# 清理 (会丢失未写入数据)
|
||||
docker-compose -f docker-compose.single.yml stop prometheus
|
||||
rm -rf prometheus-data/wal/*
|
||||
docker-compose -f docker-compose.single.yml start prometheus
|
||||
```
|
||||
|
||||
### 磁盘写满
|
||||
|
||||
```bash
|
||||
# 检查 Prometheus 数据大小
|
||||
du -sh prometheus-data/
|
||||
|
||||
# 手动清理旧数据 (保留最近7天)
|
||||
curl -X POST http://localhost:9090/api/v1/admin/tsdb/clean_tombstones
|
||||
```
|
||||
|
||||
### Grafana 无法启动
|
||||
|
||||
```bash
|
||||
# 检查日志
|
||||
docker logs sub2api-grafana
|
||||
|
||||
# 重置数据库 (会丢失配置)
|
||||
rm -rf grafana-data/grafana.db
|
||||
docker-compose -f docker-compose.single.yml restart grafana
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 常用操作
|
||||
|
||||
```bash
|
||||
# 查看 Prometheus 目标状态
|
||||
curl http://localhost:9090/api/v1/targets | jq
|
||||
|
||||
# 手动触发告警测试
|
||||
curl -X POST http://localhost:9090/-/reload
|
||||
|
||||
# 备份 Grafana 配置
|
||||
tar czvf grafana-backup-$(date +%Y%m%d).tar.gz grafana-data/
|
||||
|
||||
# 查看容器日志
|
||||
docker-compose -f docker-compose.single.yml logs -f prometheus
|
||||
```
|
||||
264
deploy/monitoring/README.md
Normal file
264
deploy/monitoring/README.md
Normal file
@@ -0,0 +1,264 @@
|
||||
# Sub2API 监控栈
|
||||
|
||||
基于 Prometheus + Grafana + Loki + Jaeger 的完整可观测性解决方案。
|
||||
|
||||
## 架构概览
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ 监控栈架构 │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
|
||||
│ │ Prometheus │ │ Loki │ │ Jaeger │ │
|
||||
│ │ (指标存储) │ │ (日志存储) │ │ (追踪存储) │ │
|
||||
│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │
|
||||
│ │ │ │ │
|
||||
│ └────────────────┼────────────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌─────────────┐ │
|
||||
│ │ Grafana │ │
|
||||
│ │ (可视化) │ │
|
||||
│ └─────────────┘ │
|
||||
│ │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
|
||||
│ │ Promtail │ │ Node Exp │ │ cAdvisor │ │
|
||||
│ │ (日志收集) │ │ (主机指标) │ │ (容器指标) │ │
|
||||
│ └─────────────┘ └─────────────┘ └─────────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 准备工作
|
||||
|
||||
```bash
|
||||
cd deploy/monitoring
|
||||
|
||||
# 创建数据目录
|
||||
mkdir -p prometheus-data grafana-data loki-data alertmanager-data
|
||||
|
||||
# 复制环境变量配置
|
||||
cp .env.example .env
|
||||
|
||||
# 编辑 .env 文件，配置你的告警通知方式
|
||||
vim .env
|
||||
```
|
||||
|
||||
### 2. 启动监控栈
|
||||
|
||||
```bash
|
||||
docker-compose -f docker-compose.monitoring.yml up -d
|
||||
```
|
||||
|
||||
### 3. 访问服务
|
||||
|
||||
| 服务 | URL | 默认账号 |
|
||||
|------|-----|----------|
|
||||
| Grafana | http://localhost:3000 | admin/admin |
|
||||
| Prometheus | http://localhost:9090 | - |
|
||||
| Jaeger UI | http://localhost:16686 | - |
|
||||
| Alertmanager | http://localhost:9093 | - |
|
||||
|
||||
### 4. 配置 Sub2API 应用指标暴露
|
||||
|
||||
在 Sub2API 后端代码中集成 Prometheus 指标：
|
||||
|
||||
```go
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
// 在路由中添加 metrics 端点
|
||||
router.GET("/metrics", gin.WrapH(promhttp.Handler()))
|
||||
```
|
||||
|
||||
### 5. 查看 Dashboard
|
||||
|
||||
1. 登录 Grafana (http://localhost:3000)
|
||||
2. 导航到 Dashboards -> Sub2API
|
||||
3. 查看以下预设 Dashboard：
|
||||
- **系统概览** - 整体健康度和关键指标
|
||||
- **SLO 监控** - 服务水平目标达成情况
|
||||
- **网关性能** - API 网关详细性能指标
|
||||
- **上游健康** - 上游 AI 服务状态
|
||||
|
||||
## 配置说明
|
||||
|
||||
### Prometheus 配置
|
||||
|
||||
配置文件：`prometheus/prometheus.yml`
|
||||
|
||||
主要配置项：
|
||||
- 抓取目标（Sub2API、Node Exporter、cAdvisor）
|
||||
- 告警规则路径
|
||||
- Alertmanager 地址
|
||||
- 数据保留策略
|
||||
|
||||
### 告警规则
|
||||
|
||||
配置文件：`prometheus/rules/sub2api-alerts.yml`
|
||||
|
||||
预设告警：
|
||||
- **SLO 告警** - 基于错误预算燃烧率的多窗口告警
|
||||
- **延迟告警** - P99/P95 延迟阈值告警
|
||||
- **错误率告警** - HTTP 5xx 错误率告警
|
||||
- **基础设施告警** - CPU、内存、磁盘、连接池
|
||||
- **业务告警** - QPS、账号切换频率
|
||||
|
||||
### Alertmanager 配置
|
||||
|
||||
配置文件：`alertmanager/config.yml`
|
||||
|
||||
支持的通知渠道：
|
||||
- Email
|
||||
- Slack
|
||||
- PagerDuty
|
||||
- Webhook
|
||||
|
||||
### Grafana 配置
|
||||
|
||||
- **数据源**：Prometheus、Loki、Jaeger
|
||||
- **Dashboard**：系统概览、SLO 监控、错误分析
|
||||
- **告警**：可视化告警配置
|
||||
|
||||
### Loki 日志配置
|
||||
|
||||
配置文件：`loki/loki-config.yaml`
|
||||
|
||||
- 日志收集：Promtail 从 Docker 容器和文件收集
|
||||
- 日志查询：Grafana Explore 中查询日志
|
||||
- 日志告警：基于日志内容的告警
|
||||
|
||||
## 常用查询
|
||||
|
||||
### Prometheus 查询示例
|
||||
|
||||
```promql
|
||||
# 当前可用性
|
||||
1 - (sum(rate(sub2api_http_requests_total{status=~"5.."}[5m])) / sum(rate(sub2api_http_requests_total[5m])))
|
||||
|
||||
# P99 延迟
|
||||
histogram_quantile(0.99, sum(rate(sub2api_http_request_duration_seconds_bucket[5m])) by (le))
|
||||
|
||||
# QPS
|
||||
sum(rate(sub2api_http_requests_total[1m]))
|
||||
|
||||
# 错误预算燃烧率
|
||||
(sum(rate(sub2api_http_requests_total{status=~"5.."}[1h])) / sum(rate(sub2api_http_requests_total[1h]))) / 0.0005
|
||||
```
|
||||
|
||||
### Loki 日志查询示例
|
||||
|
||||
```logql
|
||||
# 查看错误日志
|
||||
{job="sub2api"} |= "error" | json
|
||||
|
||||
# 查看特定请求的日志
|
||||
{job="sub2api"} |= "trace_id=\"xxx\""
|
||||
|
||||
# 按错误类型统计
|
||||
sum by (error_type) (rate({job="sub2api"} |= "error" | json [5m]))
|
||||
```
|
||||
|
||||
## 故障排查
|
||||
|
||||
### 检查服务状态
|
||||
|
||||
```bash
|
||||
# 查看所有服务状态
|
||||
docker-compose -f docker-compose.monitoring.yml ps
|
||||
|
||||
# 查看服务日志
|
||||
docker-compose -f docker-compose.monitoring.yml logs -f prometheus
|
||||
docker-compose -f docker-compose.monitoring.yml logs -f grafana
|
||||
docker-compose -f docker-compose.monitoring.yml logs -f loki
|
||||
```
|
||||
|
||||
### 常见问题
|
||||
|
||||
1. **Prometheus 无法抓取指标**
|
||||
- 检查 Sub2API `/metrics` 端点是否可访问
|
||||
- 检查网络连通性
|
||||
- 查看 Prometheus Targets 页面
|
||||
|
||||
2. **Grafana 无法显示数据**
|
||||
- 检查数据源配置
|
||||
- 验证 Prometheus 查询
|
||||
- 查看浏览器开发者工具
|
||||
|
||||
3. **告警不触发**
|
||||
- 检查告警规则语法
|
||||
- 验证 Alertmanager 配置
|
||||
- 查看 Alertmanager 状态页面
|
||||
|
||||
## 扩展配置
|
||||
|
||||
### 添加自定义 Dashboard
|
||||
|
||||
1. 在 Grafana 中创建 Dashboard
|
||||
2. 导出 JSON 文件
|
||||
3. 保存到 `grafana/dashboards/` 目录
|
||||
4. 重启 Grafana 服务
|
||||
|
||||
### 添加自定义告警规则
|
||||
|
||||
1. 编辑 `prometheus/rules/sub2api-alerts.yml`
|
||||
2. 添加新的告警规则
|
||||
3. 重新加载 Prometheus 配置：
|
||||
```bash
|
||||
curl -X POST http://localhost:9090/-/reload
|
||||
```
|
||||
|
||||
### 集成外部服务
|
||||
|
||||
- **PagerDuty**: 配置 `PAGERDUTY_WEBHOOK_URL`
|
||||
- **Slack**: 配置 `SLACK_WEBHOOK_URL`
|
||||
- **自定义 Webhook**: 修改 `alertmanager/config.yml`
|
||||
|
||||
## 维护操作
|
||||
|
||||
### 备份数据
|
||||
|
||||
```bash
|
||||
# 备份 Prometheus 数据
|
||||
tar czf prometheus-backup-$(date +%Y%m%d).tar.gz prometheus-data/
|
||||
|
||||
# 备份 Grafana 配置
|
||||
tar czf grafana-backup-$(date +%Y%m%d).tar.gz grafana-data/
|
||||
```
|
||||
|
||||
### 清理旧数据
|
||||
|
||||
```bash
|
||||
# Prometheus 数据清理 (保留30天)
|
||||
curl -X POST -g 'http://localhost:9090/api/v1/admin/tsdb/delete_series?match[]={__name__=~".+"}'
|
||||
|
||||
# Loki 数据自动清理 (按配置保留)
|
||||
```
|
||||
|
||||
### 升级组件
|
||||
|
||||
```bash
|
||||
# 更新镜像版本
|
||||
vim docker-compose.monitoring.yml
|
||||
|
||||
# 重新部署
|
||||
docker-compose -f docker-compose.monitoring.yml pull
|
||||
docker-compose -f docker-compose.monitoring.yml up -d
|
||||
```
|
||||
|
||||
## 参考文档
|
||||
|
||||
- [Prometheus 文档](https://prometheus.io/docs/)
|
||||
- [Grafana 文档](https://grafana.com/docs/)
|
||||
- [Loki 文档](https://grafana.com/docs/loki/)
|
||||
- [Jaeger 文档](https://www.jaegertracing.io/docs/)
|
||||
- [Alertmanager 文档](https://prometheus.io/docs/alerting/latest/alertmanager/)
|
||||
|
||||
## 支持
|
||||
|
||||
如有问题，请联系：
|
||||
- SRE Team: sre@sub2api.org
|
||||
- On-Call: oncall@sub2api.org
|
||||
145
deploy/monitoring/alertmanager/config.yml
Normal file
145
deploy/monitoring/alertmanager/config.yml
Normal file
@@ -0,0 +1,145 @@
|
||||
# =============================================================================
|
||||
# Alertmanager 配置文件
|
||||
# =============================================================================
|
||||
|
||||
global:
|
||||
smtp_smarthost: 'localhost:587'
|
||||
smtp_from: 'alerts@sub2api.org'
|
||||
smtp_auth_username: ''
|
||||
smtp_auth_password: ''
|
||||
|
||||
# 告警分组
|
||||
slack_api_url: '${SLACK_WEBHOOK_URL}'
|
||||
|
||||
# 解决通知等待时间
|
||||
resolve_timeout: 5m
|
||||
|
||||
# 路由树
|
||||
templates:
|
||||
- '/etc/alertmanager/templates/*.tmpl'
|
||||
|
||||
route:
|
||||
group_by: ['alertname', 'cluster', 'service']
|
||||
group_wait: 10s
|
||||
group_interval: 10s
|
||||
repeat_interval: 12h
|
||||
receiver: 'default'
|
||||
|
||||
routes:
|
||||
# Critical 告警 - 立即通知
|
||||
- match:
|
||||
severity: critical
|
||||
receiver: 'critical'
|
||||
group_wait: 0s
|
||||
repeat_interval: 5m
|
||||
continue: true
|
||||
|
||||
# High 告警 - 快速通知
|
||||
- match:
|
||||
severity: high
|
||||
receiver: 'high'
|
||||
group_wait: 30s
|
||||
repeat_interval: 30m
|
||||
continue: true
|
||||
|
||||
# SLO 相关告警
|
||||
- match:
|
||||
slo: api-availability
|
||||
receiver: 'slo-team'
|
||||
continue: true
|
||||
|
||||
# 基础设施告警
|
||||
- match_re:
|
||||
alertname: Database.*|Redis.*|HighCPU.*|HighMemory.*
|
||||
receiver: 'infra-team'
|
||||
continue: true
|
||||
|
||||
# 所有告警都桥接回内置 ops_alert_events（末尾，continue: false）
|
||||
- receiver: 'ops-bridge'
|
||||
|
||||
# 接收器配置
|
||||
receivers:
|
||||
- name: 'default'
|
||||
email_configs:
|
||||
- to: 'oncall@sub2api.org'
|
||||
send_resolved: true
|
||||
headers:
|
||||
Subject: '[Alert] {{ .GroupLabels.alertname }}'
|
||||
|
||||
- name: 'critical'
|
||||
email_configs:
|
||||
- to: 'sre-lead@sub2api.org'
|
||||
send_resolved: true
|
||||
slack_configs:
|
||||
- channel: '#alerts-critical'
|
||||
send_resolved: true
|
||||
title: '🔴 CRITICAL: {{ .GroupLabels.alertname }}'
|
||||
text: |
|
||||
{{ range .Alerts }}
|
||||
*Summary:* {{ .Annotations.summary }}
|
||||
*Description:* {{ .Annotations.description }}
|
||||
*Runbook:* {{ .Annotations.runbook_url }}
|
||||
{{ end }}
|
||||
webhook_configs:
|
||||
- url: '${PAGERDUTY_WEBHOOK_URL}'
|
||||
send_resolved: true
|
||||
|
||||
- name: 'high'
|
||||
email_configs:
|
||||
- to: 'oncall@sub2api.org'
|
||||
send_resolved: true
|
||||
slack_configs:
|
||||
- channel: '#alerts-high'
|
||||
send_resolved: true
|
||||
title: '🟠 HIGH: {{ .GroupLabels.alertname }}'
|
||||
text: |
|
||||
{{ range .Alerts }}
|
||||
*Summary:* {{ .Annotations.summary }}
|
||||
*Description:* {{ .Annotations.description }}
|
||||
{{ end }}
|
||||
|
||||
- name: 'slo-team'
|
||||
email_configs:
|
||||
- to: 'slo-team@sub2api.org'
|
||||
send_resolved: true
|
||||
|
||||
- name: 'infra-team'
|
||||
slack_configs:
|
||||
- channel: '#infra-alerts'
|
||||
send_resolved: true
|
||||
title: '🔧 INFRA: {{ .GroupLabels.alertname }}'
|
||||
|
||||
# ops-bridge: 将 Prometheus 告警写回内置 ops_alert_events 表
|
||||
# 这使得运维人员可在现有 Ops Dashboard 中统一查看所有告警
|
||||
# 需要通过环境变量注入 bearer token:
|
||||
# ALERTMANAGER_INTERNAL_TOKEN=<same value as app INTERNAL_WEBHOOK_TOKEN>
|
||||
- name: 'ops-bridge'
|
||||
webhook_configs:
|
||||
- url: 'http://host.docker.internal:8080/admin/ops/prometheus-alerts'
|
||||
send_resolved: true
|
||||
max_alerts: 50
|
||||
http_config:
|
||||
bearer_token: '${ALERTMANAGER_INTERNAL_TOKEN}'
|
||||
|
||||
# 抑制规则
|
||||
inhibit_rules:
|
||||
# 高严重级别抑制低严重级别
|
||||
- source_match:
|
||||
severity: 'critical'
|
||||
target_match:
|
||||
severity: 'high'
|
||||
equal: ['alertname', 'cluster', 'service']
|
||||
|
||||
# 相同告警抑制重复通知
|
||||
- source_match:
|
||||
severity: 'high'
|
||||
target_match:
|
||||
severity: 'medium'
|
||||
equal: ['alertname', 'cluster', 'service']
|
||||
|
||||
# 应用宕机时抑制所有应用层告警（避免告警风暴）
|
||||
- source_match:
|
||||
alertname: 'Sub2APIDown'
|
||||
target_match_re:
|
||||
alertname: 'HighErrorRate|HighLatency.*|SLOErrorBudget.*'
|
||||
equal: ['job']
|
||||
35
deploy/monitoring/blackbox/config.yml
Normal file
35
deploy/monitoring/blackbox/config.yml
Normal file
@@ -0,0 +1,35 @@
|
||||
modules:
|
||||
# HTTP 200 探测 (含 TLS 证书检查)
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
|
||||
valid_status_codes: [200, 204]
|
||||
method: GET
|
||||
no_follow_redirects: false
|
||||
fail_if_ssl: false
|
||||
fail_if_not_ssl: false
|
||||
tls_config:
|
||||
insecure_skip_verify: false
|
||||
preferred_ip_protocol: "ip4"
|
||||
|
||||
# TCP + TLS 证书专项检查 (只验证证书，不关心 HTTP 响应)
|
||||
tcp_tls:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
tls: true
|
||||
tls_config:
|
||||
insecure_skip_verify: false
|
||||
|
||||
# 内网 HTTP 探测 (不验证 TLS，用于内部服务)
|
||||
http_internal:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http:
|
||||
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
|
||||
valid_status_codes: [200]
|
||||
method: GET
|
||||
tls_config:
|
||||
insecure_skip_verify: true
|
||||
147
deploy/monitoring/docker-compose.central.yml
Normal file
147
deploy/monitoring/docker-compose.central.yml
Normal file
@@ -0,0 +1,147 @@
|
||||
# Sub2API 多机统一运维 - 中心监控节点
|
||||
# 建议配置: 4核8G+ 或云服务器
|
||||
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
# VictoriaMetrics 单节点版 (可扩展为集群)
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.93.0
|
||||
container_name: sub2api-vm
|
||||
restart: unless-stopped
|
||||
mem_limit: 2g
|
||||
cpus: '1.0'
|
||||
command:
|
||||
- '--storageDataPath=/storage'
|
||||
- '--retentionPeriod=90d'
|
||||
- '--httpListenAddr=:8428'
|
||||
- '--maxConcurrentInserts=32'
|
||||
- '--maxInsertRequestSize=32MB'
|
||||
- '--search.maxQueryDuration=2m'
|
||||
- '--search.maxPointsPerTimeseries=30000'
|
||||
- '--dedup.minScrapeInterval=15s'
|
||||
volumes:
|
||||
- vm-storage:/storage
|
||||
ports:
|
||||
- "8428:8428"
|
||||
networks:
|
||||
- monitoring-central
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8428/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# vminsert - 数据写入层 (集群模式启用)
|
||||
vminsert:
|
||||
image: victoriametrics/vminsert:v1.93.0
|
||||
container_name: sub2api-vm-insert
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
cpus: '0.5'
|
||||
command:
|
||||
- '--storageNode=victoriametrics:8428'
|
||||
- '--maxConcurrentInserts=16'
|
||||
- '--maxInsertRequestSize=32MB'
|
||||
ports:
|
||||
- "8480:8480"
|
||||
networks:
|
||||
- monitoring-central
|
||||
depends_on:
|
||||
- victoriametrics
|
||||
|
||||
# vmselect - 查询层 (集群模式启用)
|
||||
vmselect:
|
||||
image: victoriametrics/vmselect:v1.93.0
|
||||
container_name: sub2api-vm-select
|
||||
restart: unless-stopped
|
||||
mem_limit: 1g
|
||||
cpus: '0.5'
|
||||
command:
|
||||
- '--storageNode=victoriametrics:8428'
|
||||
- '--search.maxQueryDuration=2m'
|
||||
- '--search.maxPointsPerTimeseries=30000'
|
||||
ports:
|
||||
- "8481:8481"
|
||||
networks:
|
||||
- monitoring-central
|
||||
depends_on:
|
||||
- victoriametrics
|
||||
|
||||
# 中心 Grafana
|
||||
grafana-central:
|
||||
image: grafana/grafana:10.3.0
|
||||
container_name: sub2api-grafana-central
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
cpus: '0.5'
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
|
||||
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin}
|
||||
- GF_DATABASE_TYPE=postgres
|
||||
- GF_DATABASE_HOST=postgres:5432
|
||||
- GF_DATABASE_NAME=grafana
|
||||
- GF_DATABASE_USER=grafana
|
||||
- GF_DATABASE_PASSWORD=${GRAFANA_DB_PASSWORD:-grafana}
|
||||
- GF_DATABASE_SSL_MODE=disable
|
||||
- GF_ANALYTICS_REPORTING_ENABLED=false
|
||||
- GF_LOG_LEVEL=info
|
||||
volumes:
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana-central-data:/var/lib/grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
networks:
|
||||
- monitoring-central
|
||||
depends_on:
|
||||
- postgres
|
||||
|
||||
# Alertmanager - 统一告警管理
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.26.0
|
||||
container_name: sub2api-alertmanager
|
||||
restart: unless-stopped
|
||||
mem_limit: 256m
|
||||
cpus: '0.2'
|
||||
command:
|
||||
- '--config.file=/etc/alertmanager/config.yml'
|
||||
- '--storage.path=/alertmanager'
|
||||
- '--web.external-url=http://localhost:9093'
|
||||
volumes:
|
||||
- ./alertmanager/config-central.yml:/etc/alertmanager/config.yml:ro
|
||||
- alertmanager-data:/alertmanager
|
||||
ports:
|
||||
- "9093:9093"
|
||||
networks:
|
||||
- monitoring-central
|
||||
|
||||
# Grafana 数据库
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
container_name: sub2api-grafana-db
|
||||
restart: unless-stopped
|
||||
mem_limit: 256m
|
||||
cpus: '0.2'
|
||||
environment:
|
||||
- POSTGRES_USER=grafana
|
||||
- POSTGRES_PASSWORD=${GRAFANA_DB_PASSWORD:-grafana}
|
||||
- POSTGRES_DB=grafana
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- monitoring-central
|
||||
|
||||
volumes:
|
||||
vm-storage:
|
||||
driver: local
|
||||
grafana-central-data:
|
||||
driver: local
|
||||
alertmanager-data:
|
||||
driver: local
|
||||
postgres-data:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
monitoring-central:
|
||||
driver: bridge
|
||||
201
deploy/monitoring/docker-compose.monitoring.yml
Normal file
201
deploy/monitoring/docker-compose.monitoring.yml
Normal file
@@ -0,0 +1,201 @@
|
||||
# =============================================================================
|
||||
# Sub2API 监控栈 - Prometheus + Grafana + Loki + Jaeger
|
||||
# =============================================================================
|
||||
# 使用方法:
|
||||
# 1. 创建监控目录: mkdir -p monitoring/{prometheus-data,grafana-data,loki-data}
|
||||
# 2. 启动: docker-compose -f docker-compose.monitoring.yml up -d
|
||||
# 3. 访问 Grafana: http://localhost:3000 (admin/admin)
|
||||
# =============================================================================
|
||||
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
# ===========================================================================
|
||||
# Prometheus - 时序数据库
|
||||
# ===========================================================================
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.50.0
|
||||
container_name: sub2api-prometheus
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--storage.tsdb.retention.time=30d'
|
||||
- '--storage.tsdb.retention.size=50GB'
|
||||
- '--web.enable-lifecycle'
|
||||
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
|
||||
- '--web.console.templates=/usr/share/prometheus/consoles'
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./prometheus/rules:/etc/prometheus/rules:ro
|
||||
- ./prometheus-data:/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
networks:
|
||||
- monitoring-network
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:9090/-/healthy"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# ===========================================================================
|
||||
# Grafana - 可视化平台
|
||||
# ===========================================================================
|
||||
grafana:
|
||||
image: grafana/grafana:10.3.1
|
||||
container_name: sub2api-grafana
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
|
||||
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin}
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_SERVER_ROOT_URL=${GRAFANA_ROOT_URL:-http://localhost:3000}
|
||||
- GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource
|
||||
volumes:
|
||||
- ./grafana-data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
ports:
|
||||
- "3000:3000"
|
||||
networks:
|
||||
- monitoring-network
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- jaeger
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:3000/api/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# ===========================================================================
|
||||
# Loki - 日志聚合
|
||||
# ===========================================================================
|
||||
loki:
|
||||
image: grafana/loki:2.9.4
|
||||
container_name: sub2api-loki
|
||||
restart: unless-stopped
|
||||
command: -config.file=/etc/loki/local-config.yaml
|
||||
volumes:
|
||||
- ./loki/loki-config.yaml:/etc/loki/local-config.yaml:ro
|
||||
- ./loki-data:/loki
|
||||
ports:
|
||||
- "3100:3100"
|
||||
networks:
|
||||
- monitoring-network
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:3100/ready"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# ===========================================================================
|
||||
# Promtail - 日志收集器
|
||||
# ===========================================================================
|
||||
promtail:
|
||||
image: grafana/promtail:2.9.4
|
||||
container_name: sub2api-promtail
|
||||
restart: unless-stopped
|
||||
command: -config.file=/etc/promtail/config.yml
|
||||
volumes:
|
||||
- ./promtail/promtail-config.yml:/etc/promtail/config.yml:ro
|
||||
- /var/log:/var/log:ro
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
networks:
|
||||
- monitoring-network
|
||||
depends_on:
|
||||
- loki
|
||||
|
||||
# ===========================================================================
|
||||
# Jaeger - 分布式追踪
|
||||
# ===========================================================================
|
||||
jaeger:
|
||||
image: jaegertracing/all-in-one:1.54
|
||||
container_name: sub2api-jaeger
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- COLLECTOR_OTLP_ENABLED=true
|
||||
ports:
|
||||
- "16686:16686" # UI
|
||||
- "4317:4317" # OTLP gRPC
|
||||
- "4318:4318" # OTLP HTTP
|
||||
- "14268:14268" # Jaeger Thrift
|
||||
networks:
|
||||
- monitoring-network
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:16686"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# ===========================================================================
|
||||
# Node Exporter - 主机指标
|
||||
# ===========================================================================
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.7.0
|
||||
container_name: sub2api-node-exporter
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- '--path.procfs=/host/proc'
|
||||
- '--path.rootfs=/rootfs'
|
||||
- '--path.sysfs=/host/sys'
|
||||
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
ports:
|
||||
- "9100:9100"
|
||||
networks:
|
||||
- monitoring-network
|
||||
|
||||
# ===========================================================================
|
||||
# cAdvisor - 容器指标
|
||||
# ===========================================================================
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:v0.47.2
|
||||
container_name: sub2api-cadvisor
|
||||
restart: unless-stopped
|
||||
privileged: true
|
||||
devices:
|
||||
- /dev/kmsg:/dev/kmsg
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker:/var/lib/docker:ro
|
||||
- /dev/disk:/dev/disk:ro
|
||||
ports:
|
||||
- "8081:8080"
|
||||
networks:
|
||||
- monitoring-network
|
||||
|
||||
# ===========================================================================
|
||||
# Alertmanager - 告警管理
|
||||
# ===========================================================================
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.27.0
|
||||
container_name: sub2api-alertmanager
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- '--config.file=/etc/alertmanager/config.yml'
|
||||
- '--storage.path=/alertmanager'
|
||||
- '--web.external-url=http://localhost:9093'
|
||||
volumes:
|
||||
- ./alertmanager/config.yml:/etc/alertmanager/config.yml:ro
|
||||
- ./alertmanager-data:/alertmanager
|
||||
ports:
|
||||
- "9093:9093"
|
||||
networks:
|
||||
- monitoring-network
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:9093/-/healthy"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
networks:
|
||||
monitoring-network:
|
||||
driver: bridge
|
||||
135
deploy/monitoring/docker-compose.single.yml
Normal file
135
deploy/monitoring/docker-compose.single.yml
Normal file
@@ -0,0 +1,135 @@
|
||||
# Sub2API 单机监控栈 (2核4G优化版)
|
||||
# 资源限制: Prometheus 128MB, Grafana 128MB, Node Exporter 32MB
|
||||
# 总内存占用: ~300MB
|
||||
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.50.0
|
||||
container_name: sub2api-prometheus
|
||||
restart: unless-stopped
|
||||
mem_limit: 128m
|
||||
cpus: '0.2'
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.retention.time=15d'
|
||||
- '--storage.tsdb.retention.size=2GB'
|
||||
- '--storage.tsdb.wal-compression'
|
||||
- '--storage.tsdb.min-block-duration=2h'
|
||||
- '--storage.tsdb.max-block-duration=2h'
|
||||
- '--query.max-samples=50000000'
|
||||
- '--query.timeout=2m'
|
||||
- '--web.enable-lifecycle'
|
||||
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
|
||||
- '--web.console.templates=/usr/share/prometheus/consoles'
|
||||
volumes:
|
||||
- ./prometheus/prometheus-single.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./prometheus/rules:/etc/prometheus/rules:ro
|
||||
- prometheus-data:/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
networks:
|
||||
- monitoring
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:10.3.0
|
||||
container_name: sub2api-grafana
|
||||
restart: unless-stopped
|
||||
mem_limit: 128m
|
||||
cpus: '0.1'
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
|
||||
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin}
|
||||
- GF_INSTALL_PLUGINS=grafana-clock-panel
|
||||
- GF_ANALYTICS_REPORTING_ENABLED=false
|
||||
- GF_ANALYTICS_CHECK_FOR_UPDATES=false
|
||||
- GF_LOG_LEVEL=warn
|
||||
- GF_LOG_MODE=console
|
||||
- GF_DATABASE_TYPE=sqlite3
|
||||
- GF_DATABASE_PATH=/var/lib/grafana/grafana.db
|
||||
- GF_SESSION_PROVIDER=memory
|
||||
- GF_METRICS_ENABLED=false
|
||||
- GF_TRACING_ENABLED=false
|
||||
volumes:
|
||||
- ./grafana/grafana-single.ini:/etc/grafana/grafana.ini:ro
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
networks:
|
||||
- monitoring
|
||||
depends_on:
|
||||
- prometheus
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.7.0
|
||||
container_name: sub2api-node-exporter
|
||||
restart: unless-stopped
|
||||
mem_limit: 32m
|
||||
cpus: '0.05'
|
||||
command:
|
||||
- '--path.procfs=/host/proc'
|
||||
- '--path.rootfs=/rootfs'
|
||||
- '--path.sysfs=/host/sys'
|
||||
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
|
||||
- '--collector.cpu.info'
|
||||
- '--collector.meminfo'
|
||||
- '--no-collector.wifi'
|
||||
- '--no-collector.hwmon'
|
||||
- '--no-collector.btrfs'
|
||||
# 暴露 textfile_collector 目录，供备份脚本写入心跳指标
|
||||
- '--collector.textfile.directory=/var/lib/node_exporter/textfile_collector'
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
- /var/lib/node_exporter:/var/lib/node_exporter:ro
|
||||
networks:
|
||||
- monitoring
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9100/metrics"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# Blackbox Exporter - HTTP/TLS 探测 & 证书过期检查
|
||||
# 内存占用 ~16MB，可放心加入单机部署
|
||||
blackbox-exporter:
|
||||
image: prom/blackbox-exporter:v0.24.0
|
||||
container_name: sub2api-blackbox
|
||||
restart: unless-stopped
|
||||
mem_limit: 32m
|
||||
cpus: '0.05'
|
||||
command:
|
||||
- '--config.file=/etc/blackbox_exporter/config.yml'
|
||||
volumes:
|
||||
- ./blackbox/config.yml:/etc/blackbox_exporter/config.yml:ro
|
||||
networks:
|
||||
- monitoring
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9115/-/healthy"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
driver: local
|
||||
grafana-data:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
monitoring:
|
||||
driver: bridge
|
||||
679
deploy/monitoring/grafana/dashboards/sub2api-overview.json
Normal file
679
deploy/monitoring/grafana/dashboards/sub2api-overview.json
Normal file
@@ -0,0 +1,679 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Sub2API 系统概览 Dashboard",
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"panels": [],
|
||||
"title": "SLO 概览",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.999
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0.9995
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - (sum(rate(sub2api_http_requests_total{status=~\"5..\"}[24h])) / sum(rate(sub2api_http_requests_total[24h])))",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "24h 可用性",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 2
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 6
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 1
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(sub2api_http_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "P99 延迟 (秒)",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 1
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(sub2api_http_requests_total{status=~\"5..\"}[5m])) / sum(rate(sub2api_http_requests_total[5m]))",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "当前错误率",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 6
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 14.4
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 1
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(sum(rate(sub2api_http_requests_total{status=~\"5..\"}[1h])) / sum(rate(sub2api_http_requests_total[1h]))) / 0.0005",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "错误预算燃烧率",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"id": 6,
|
||||
"panels": [],
|
||||
"title": "流量与性能",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 7,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "10.3.1",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(sub2api_http_requests_total[5m]))",
|
||||
"legendFormat": "QPS",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "请求速率 (QPS)",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 6
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 8,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "10.3.1",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(sub2api_http_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "P99",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(sub2api_http_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "P95",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(sub2api_http_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "P50",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "请求延迟分布",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "s",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 14
|
||||
},
|
||||
"id": 9,
|
||||
"panels": [],
|
||||
"title": "错误分析",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 10,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "10.3.1",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(sub2api_http_requests_total{status=~\"5..\"}[5m])) by (status)",
|
||||
"legendFormat": "{{status}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "错误率趋势 (按状态码)",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": true,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 11,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": false,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "10.3.1",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(sub2api_http_requests_total[5m])) by (status)",
|
||||
"legendFormat": "{{status}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "HTTP 状态码分布",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "series",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": [
|
||||
"current"
|
||||
]
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 27,
|
||||
"style": "dark",
|
||||
"tags": ["sub2api", "overview"],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Sub2API - 系统概览",
|
||||
"uid": "sub2api-overview",
|
||||
"version": 1
|
||||
}
|
||||
102
deploy/monitoring/grafana/grafana-single.ini
Normal file
102
deploy/monitoring/grafana/grafana-single.ini
Normal file
@@ -0,0 +1,102 @@
|
||||
# Sub2API 单机版 Grafana 配置
|
||||
# 优化目标: 内存 < 128MB, SQLite 数据库, 禁用非必要功能
|
||||
|
||||
[paths]
|
||||
data = /var/lib/grafana
|
||||
temp_data_lifetime = 24h
|
||||
|
||||
[server]
|
||||
http_port = 3000
|
||||
domain = localhost
|
||||
root_url = %(protocol)s://%(domain)s:%(http_port)s/
|
||||
serve_from_sub_path = false
|
||||
|
||||
# 数据库配置 - 使用 SQLite 减少资源占用
|
||||
[database]
|
||||
type = sqlite3
|
||||
path = grafana.db
|
||||
cache_mode = shared
|
||||
|
||||
# 会话配置 - 使用内存存储
|
||||
[session]
|
||||
provider = memory
|
||||
provider_config =
|
||||
cookie_name = grafana_sess
|
||||
cookie_secure = false
|
||||
cookie_samesite = lax
|
||||
|
||||
# 禁用分析和更新检查
|
||||
[analytics]
|
||||
reporting_enabled = false
|
||||
check_for_updates = false
|
||||
feedback_links_enabled = false
|
||||
|
||||
# 日志配置 - 只输出到控制台，级别 warn
|
||||
[log]
|
||||
mode = console
|
||||
level = warn
|
||||
filters =
|
||||
|
||||
[log.console]
|
||||
level = warn
|
||||
format = text
|
||||
|
||||
# 禁用指标和追踪
|
||||
[metrics]
|
||||
enabled = false
|
||||
basic_auth_username =
|
||||
basic_auth_password =
|
||||
|
||||
[metrics.graphite]
|
||||
address =
|
||||
prefix = prod.grafana.%(instance_name)s.
|
||||
|
||||
[tracing.jaeger]
|
||||
address =
|
||||
always_included_tag =
|
||||
sampler_type = const
|
||||
sampler_param = 1
|
||||
|
||||
# 安全设置
|
||||
[security]
|
||||
admin_user = admin
|
||||
# admin_password = 从环境变量读取
|
||||
cookie_secure = false
|
||||
cookie_samesite = lax
|
||||
allow_embedding = false
|
||||
strict_transport_security = false
|
||||
content_security_policy = true
|
||||
|
||||
# 用户设置
|
||||
[users]
|
||||
allow_sign_up = false
|
||||
allow_org_create = false
|
||||
auto_assign_org = true
|
||||
auto_assign_org_role = Viewer
|
||||
|
||||
# 匿名访问 - 生产环境建议关闭
|
||||
[auth.anonymous]
|
||||
enabled = false
|
||||
|
||||
# 渲染设置 - 单机禁用
|
||||
[rendering]
|
||||
server_url =
|
||||
callback_url =
|
||||
|
||||
# 面板设置
|
||||
[panels]
|
||||
disable_sanitize_html = false
|
||||
|
||||
# 插件设置
|
||||
[plugins]
|
||||
enable_alpha = false
|
||||
app_tls_skip_verify_insecure = false
|
||||
|
||||
# 实时设置
|
||||
[live]
|
||||
max_connections = 100
|
||||
allowed_origins =
|
||||
|
||||
# 快照设置
|
||||
[snapshots]
|
||||
external_enabled = false
|
||||
@@ -0,0 +1,16 @@
|
||||
# =============================================================================
|
||||
# Grafana Dashboard 配置
|
||||
# =============================================================================
|
||||
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'Sub2API Dashboards'
|
||||
orgId: 1
|
||||
folder: 'Sub2API'
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 10
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,67 @@
|
||||
# =============================================================================
|
||||
# Grafana 数据源配置
|
||||
# =============================================================================
|
||||
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prometheus - 指标数据源
|
||||
# ---------------------------------------------------------------------------
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: "15s"
|
||||
httpMethod: POST
|
||||
manageAlerts: true
|
||||
alertmanagerUid: alertmanager
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Loki - 日志数据源
|
||||
# ---------------------------------------------------------------------------
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
derivedFields:
|
||||
- name: "TraceID"
|
||||
matcherRegex: '"trace_id":"([^"]+)"'
|
||||
url: "http://localhost:16686/trace/$${__value.raw}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Jaeger - 追踪数据源
|
||||
# ---------------------------------------------------------------------------
|
||||
- name: Jaeger
|
||||
type: jaeger
|
||||
access: proxy
|
||||
url: http://jaeger:16686
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogs:
|
||||
datasourceUid: loki
|
||||
tags: ["service", "pod"]
|
||||
mappedTags: [{ key: "service", value: "service_name" }]
|
||||
mapTagNamesEnabled: false
|
||||
spanStartTimeShift: "1h"
|
||||
spanEndTimeShift: "1h"
|
||||
filterByTraceID: true
|
||||
filterBySpanID: true
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Alertmanager
|
||||
# ---------------------------------------------------------------------------
|
||||
- name: Alertmanager
|
||||
uid: alertmanager
|
||||
type: alertmanager
|
||||
access: proxy
|
||||
url: http://alertmanager:9093
|
||||
editable: false
|
||||
jsonData:
|
||||
implementation: prometheus
|
||||
67
deploy/monitoring/loki/loki-config.yaml
Normal file
67
deploy/monitoring/loki/loki-config.yaml
Normal file
@@ -0,0 +1,67 @@
|
||||
# =============================================================================
|
||||
# Loki 配置文件
|
||||
# =============================================================================
|
||||
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
instance_addr: 127.0.0.1
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
query_range:
|
||||
results_cache:
|
||||
cache:
|
||||
embedded_cache:
|
||||
enabled: true
|
||||
max_size_mb: 100
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2020-10-24
|
||||
store: boltdb-shipper
|
||||
object_store: filesystem
|
||||
schema: v11
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
ruler:
|
||||
alertmanager_url: http://alertmanager:9093
|
||||
|
||||
# 限制配置
|
||||
limits_config:
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
ingestion_rate_mb: 10
|
||||
ingestion_burst_size_mb: 20
|
||||
per_stream_rate_limit: 3MB
|
||||
per_stream_rate_limit_burst: 15MB
|
||||
|
||||
# 前端配置
|
||||
frontend:
|
||||
max_outstanding_per_tenant: 2048
|
||||
compress_responses: true
|
||||
|
||||
# 查询调度器
|
||||
query_scheduler:
|
||||
max_outstanding_requests_per_tenant: 2048
|
||||
|
||||
# 索引网关
|
||||
index_gateway:
|
||||
mode: simple
|
||||
|
||||
# 分析器
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
55
deploy/monitoring/prometheus/prometheus-multi-node.yml
Normal file
55
deploy/monitoring/prometheus/prometheus-multi-node.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
# Sub2API 多机节点 Prometheus 配置
|
||||
# 本地保留3天，同时 remote_write 到中心存储
|
||||
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
cluster: '${CLUSTER_NAME:-sub2api}'
|
||||
replica: '${INSTANCE_NAME:-node-1}'
|
||||
region: '${REGION:-default}'
|
||||
|
||||
# 本地只保留3天数据
|
||||
storage:
|
||||
tsdb:
|
||||
retention.time: 3d
|
||||
retention.size: 500MB
|
||||
wal-compression: true
|
||||
|
||||
# 发送到中心存储
|
||||
remote_write:
|
||||
- url: '${VM_INSERT_URL:-http://localhost:8480}/insert/0/prometheus/api/v1/write'
|
||||
queue_config:
|
||||
capacity: 10000
|
||||
max_samples_per_send: 2000
|
||||
batch_send_deadline: 5s
|
||||
min_shards: 1
|
||||
max_shards: 2
|
||||
min_backoff: 30ms
|
||||
max_backoff: 5s
|
||||
metadata_config:
|
||||
send: true
|
||||
max_samples_per_send: 2000
|
||||
|
||||
rule_files:
|
||||
- 'rules/sub2api-alerts-light.yml'
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'sub2api-app'
|
||||
static_configs:
|
||||
- targets: ['localhost:8080']
|
||||
labels:
|
||||
service: 'sub2api'
|
||||
tier: 'backend'
|
||||
metrics_path: '/metrics'
|
||||
scrape_interval: 15s
|
||||
|
||||
- job_name: 'node-exporter'
|
||||
static_configs:
|
||||
- targets: ['localhost:9100']
|
||||
scrape_interval: 15s
|
||||
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
scrape_interval: 15s
|
||||
97
deploy/monitoring/prometheus/prometheus-single.yml
Normal file
97
deploy/monitoring/prometheus/prometheus-single.yml
Normal file
@@ -0,0 +1,97 @@
|
||||
# Sub2API 单机版 Prometheus 配置
|
||||
# 优化目标: 内存 < 128MB, 存储 < 2GB, 保留 15天
|
||||
|
||||
global:
|
||||
scrape_interval: 30s # 单机放宽抓取间隔
|
||||
evaluation_interval: 30s
|
||||
external_labels:
|
||||
cluster: 'sub2api-single'
|
||||
replica: 'single'
|
||||
|
||||
# 告警规则
|
||||
rule_files:
|
||||
- 'rules/sub2api-alerts-light.yml'
|
||||
|
||||
# 抓取配置
|
||||
scrape_configs:
|
||||
# Sub2API 应用指标
|
||||
- job_name: 'sub2api-app'
|
||||
static_configs:
|
||||
- targets: ['host.docker.internal:8080']
|
||||
labels:
|
||||
service: 'sub2api'
|
||||
tier: 'backend'
|
||||
metrics_path: '/metrics'
|
||||
scrape_interval: 30s
|
||||
scrape_timeout: 10s
|
||||
# 只抓取关键指标，减少数据量
|
||||
params:
|
||||
collect[]:
|
||||
- 'http'
|
||||
- 'runtime'
|
||||
- 'database'
|
||||
|
||||
# Node Exporter - 系统指标
|
||||
- job_name: 'node-exporter'
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
labels:
|
||||
instance: 'sub2api-server'
|
||||
scrape_interval: 30s
|
||||
scrape_timeout: 10s
|
||||
|
||||
# Prometheus 自身指标
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
labels:
|
||||
instance: 'prometheus'
|
||||
scrape_interval: 30s
|
||||
|
||||
# Blackbox Exporter - TLS 证书检查 & 端点可用性探测
|
||||
# 需要在 docker-compose.single.yml 中添加 blackbox-exporter 容器
|
||||
# 参考: deploy/monitoring/docker-compose.single.yml 中的 blackbox-exporter service
|
||||
- job_name: 'blackbox-https'
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
# TODO: 替换为实际域名
|
||||
- https://sub2api.example.com/health
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
|
||||
# TLS 证书过期专项检查 (TCP 模式，只检查证书)
|
||||
- job_name: 'blackbox-tls-cert'
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [tcp_tls]
|
||||
static_configs:
|
||||
- targets:
|
||||
# TODO: 替换为实际域名:端口
|
||||
- sub2api.example.com:443
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
scrape_interval: 300s # 5 分钟检查一次，节省资源
|
||||
|
||||
# 启用 Alertmanager 集成 (取消注释以启用)
|
||||
# Phase 1: 暂用现有 ops 告警系统
|
||||
# Phase 2 开始启用 Alertmanager，同时通过 ops-bridge webhook 回写 ops_alert_events
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets: [] # Phase 2: 替换为 ['alertmanager:9093']
|
||||
# 生产环境建议配置 TLS:
|
||||
# tls_config:
|
||||
# ca_file: /etc/prometheus/certs/ca.crt
|
||||
68
deploy/monitoring/prometheus/prometheus.yml
Normal file
68
deploy/monitoring/prometheus/prometheus.yml
Normal file
@@ -0,0 +1,68 @@
|
||||
# =============================================================================
|
||||
# Prometheus 配置文件
|
||||
# =============================================================================
|
||||
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
cluster: 'sub2api'
|
||||
replica: '{{.ExternalURL}}'
|
||||
|
||||
# 告警规则文件
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
# Alertmanager 配置
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
# 抓取配置
|
||||
scrape_configs:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prometheus 自身监控
|
||||
# ---------------------------------------------------------------------------
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sub2API 应用指标
|
||||
# ---------------------------------------------------------------------------
|
||||
- job_name: 'sub2api'
|
||||
static_configs:
|
||||
- targets: ['sub2api:8080']
|
||||
metrics_path: /metrics
|
||||
scrape_interval: 15s
|
||||
scrape_timeout: 10s
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Node Exporter - 主机指标
|
||||
# ---------------------------------------------------------------------------
|
||||
- job_name: 'node-exporter'
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cAdvisor - 容器指标
|
||||
# ---------------------------------------------------------------------------
|
||||
- job_name: 'cadvisor'
|
||||
static_configs:
|
||||
- targets: ['cadvisor:8080']
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PostgreSQL Exporter
|
||||
# ---------------------------------------------------------------------------
|
||||
- job_name: 'postgres'
|
||||
static_configs:
|
||||
- targets: ['postgres-exporter:9187']
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Redis Exporter
|
||||
# ---------------------------------------------------------------------------
|
||||
- job_name: 'redis'
|
||||
static_configs:
|
||||
- targets: ['redis-exporter:9121']
|
||||
462
deploy/monitoring/prometheus/rules/sub2api-alerts-light.yml
Normal file
462
deploy/monitoring/prometheus/rules/sub2api-alerts-light.yml
Normal file
@@ -0,0 +1,462 @@
|
||||
# Sub2API 单机版轻量告警规则
|
||||
# 针对 2核4G 环境优化，避免告警风暴
|
||||
|
||||
groups:
|
||||
# ==================== 系统资源告警 ====================
|
||||
- name: system-alerts
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: HighCPUUsage
|
||||
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: infrastructure
|
||||
annotations:
|
||||
summary: "CPU 使用率过高"
|
||||
description: "实例 {{ $labels.instance }} CPU 使用率超过 80%，当前值: {{ $value }}%"
|
||||
|
||||
- alert: HighMemoryUsage
|
||||
expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100 > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: infrastructure
|
||||
annotations:
|
||||
summary: "内存使用率过高"
|
||||
description: "实例 {{ $labels.instance }} 内存使用率超过 85%，当前值: {{ $value }}%"
|
||||
|
||||
- alert: DiskSpaceLow
|
||||
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
category: infrastructure
|
||||
annotations:
|
||||
summary: "磁盘空间不足"
|
||||
description: "实例 {{ $labels.instance }} 磁盘剩余空间不足 10%，当前值: {{ $value }}%"
|
||||
|
||||
- alert: DiskWillFillIn24Hours
|
||||
expr: predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[6h], 24*3600) < 0
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
category: infrastructure
|
||||
annotations:
|
||||
summary: "磁盘预计24小时内写满"
|
||||
description: "实例 {{ $labels.instance }} 按当前趋势，磁盘将在24小时内写满"
|
||||
|
||||
# ==================== 应用健康告警 ====================
|
||||
- name: application-alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: Sub2APIDown
|
||||
expr: up{job="sub2api-app"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
category: availability
|
||||
annotations:
|
||||
summary: "Sub2API 服务不可用"
|
||||
description: "Sub2API 应用实例 {{ $labels.instance }} 已宕机超过 1 分钟"
|
||||
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(sub2api_http_requests_total{job="sub2api-app",status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(sub2api_http_requests_total{job="sub2api-app"}[5m]))
|
||||
) > 0.05
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
category: availability
|
||||
annotations:
|
||||
summary: "错误率过高"
|
||||
description: "API 错误率超过 5%，当前值: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: HighLatencyP99
|
||||
expr: histogram_quantile(0.99, sum(rate(sub2api_http_request_duration_seconds_bucket{job="sub2api-app"}[5m])) by (le)) > 5
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
category: performance
|
||||
annotations:
|
||||
summary: "P99 延迟过高"
|
||||
description: "API P99 延迟超过 5 秒，当前值: {{ $value }}s"
|
||||
|
||||
- alert: HighLatencyP95
|
||||
expr: histogram_quantile(0.95, sum(rate(sub2api_http_request_duration_seconds_bucket{job="sub2api-app"}[5m])) by (le)) > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: info
|
||||
category: performance
|
||||
annotations:
|
||||
summary: "P95 延迟升高"
|
||||
description: "API P95 延迟超过 2 秒，当前值: {{ $value }}s"
|
||||
|
||||
# ==================== 数据库告警 ====================
|
||||
- name: database-alerts
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: DatabaseConnectionsHigh
|
||||
expr: |
|
||||
(
|
||||
sub2api_db_connections{state="active"} / sub2api_db_connections{state="max"}
|
||||
) > 0.8
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: database
|
||||
annotations:
|
||||
summary: "数据库连接池使用率过高"
|
||||
description: "数据库连接池使用率超过 80%，当前值: {{ $value | humanizePercentage }}"
|
||||
|
||||
# ==================== 业务指标告警 ====================
|
||||
- name: business-alerts
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: LowUpstreamSuccessRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(sub2api_upstream_requests_total{status="success"}[10m]))
|
||||
/
|
||||
sum(rate(sub2api_upstream_requests_total[10m]))
|
||||
) < 0.95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: business
|
||||
annotations:
|
||||
summary: "上游服务成功率下降"
|
||||
description: "上游服务成功率低于 95%，当前值: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: HighRateLimitHits
|
||||
expr: rate(sub2api_rate_limit_hits_total[5m]) > 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: info
|
||||
category: business
|
||||
annotations:
|
||||
summary: "限流触发频繁"
|
||||
description: "限流触发频率过高，当前: {{ $value }}/s"
|
||||
|
||||
# ==================== Prometheus 自身监控告警 ====================
|
||||
- name: prometheus-alerts
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: PrometheusTargetMissing
|
||||
expr: up == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "Prometheus 抓取目标丢失"
|
||||
description: "{{ $labels.job }} 实例 {{ $labels.instance }} 无法访问"
|
||||
|
||||
- alert: PrometheusHighMemoryUsage
|
||||
expr: |
|
||||
(
|
||||
process_resident_memory_bytes{job="prometheus"} / 1024 / 1024
|
||||
) > 100
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "Prometheus 内存使用过高"
|
||||
description: "Prometheus 内存使用超过 100MB，当前: {{ $value }}MB"
|
||||
|
||||
- alert: PrometheusTSDBReloadsFailing
|
||||
expr: rate(prometheus_tsdb_reloads_failures_total[5m]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "Prometheus TSDB 重载失败"
|
||||
description: "Prometheus 时序数据库重载失败，可能需要人工介入"
|
||||
|
||||
- alert: PrometheusRuleEvaluationFailures
|
||||
expr: rate(prometheus_rule_evaluation_failures_total[5m]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "Prometheus 规则评估失败"
|
||||
description: "告警规则评估失败，请检查规则配置"
|
||||
|
||||
- alert: PrometheusDiskSpaceLow
|
||||
expr: |
|
||||
(
|
||||
prometheus_tsdb_storage_blocks_bytes / 1024 / 1024 / 1024
|
||||
) > 1.8
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "Prometheus 磁盘空间不足"
|
||||
description: "Prometheus 数据占用超过 1.8GB，接近 2GB 限制"
|
||||
|
||||
# ==================== 证书过期告警 ====================
|
||||
- name: certificate-alerts
|
||||
interval: 3600s # 每小时检查一次
|
||||
rules:
|
||||
- alert: TLSCertificateExpiringSoon7Days
|
||||
expr: |
|
||||
(
|
||||
sub2api_tls_certificate_expiry_timestamp - time()
|
||||
) / 86400 < 7
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
category: security
|
||||
annotations:
|
||||
summary: "TLS 证书即将过期 (7天内)"
|
||||
description: "{{ $labels.domain }} 证书将在 {{ $value | humanizeDuration }} 后过期"
|
||||
|
||||
- alert: TLSCertificateExpiringSoon3Days
|
||||
expr: |
|
||||
(
|
||||
sub2api_tls_certificate_expiry_timestamp - time()
|
||||
) / 86400 < 3
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
category: security
|
||||
annotations:
|
||||
summary: "TLS 证书即将过期 (3天内)"
|
||||
description: "{{ $labels.domain }} 证书将在 {{ $value | humanizeDuration }} 后过期，请立即续期"
|
||||
|
||||
- alert: TLSCertificateExpired
|
||||
expr: sub2api_tls_certificate_expiry_timestamp - time() < 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
category: security
|
||||
annotations:
|
||||
summary: "TLS 证书已过期"
|
||||
description: "{{ $labels.domain }} 证书已过期，服务可能无法正常访问"
|
||||
|
||||
# ==================== 备份任务告警 ====================
|
||||
- name: backup-alerts
|
||||
interval: 3600s # 每小时检查一次
|
||||
rules:
|
||||
- alert: DatabaseBackupFailed
|
||||
expr: |
|
||||
time() - sub2api_job_heartbeat_last_success_timestamp{job_name="database_backup"} > 90000
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
category: backup
|
||||
annotations:
|
||||
summary: "数据库备份失败"
|
||||
description: "数据库备份已超过 25 小时未成功执行"
|
||||
|
||||
- alert: DatabaseBackupMissing
|
||||
expr: absent(sub2api_job_heartbeat_last_success_timestamp{job_name="database_backup"})
|
||||
for: 1h
|
||||
labels:
|
||||
severity: info
|
||||
category: backup
|
||||
annotations:
|
||||
summary: "数据库备份监控未配置"
|
||||
description: "未检测到数据库备份心跳指标，请检查备份脚本是否上报"
|
||||
|
||||
- alert: GrafanaConfigBackupFailed
|
||||
expr: |
|
||||
time() - sub2api_job_heartbeat_last_success_timestamp{job_name="grafana_backup"} > 172800
|
||||
for: 1h
|
||||
labels:
|
||||
severity: info
|
||||
category: backup
|
||||
annotations:
|
||||
summary: "Grafana 配置备份失败"
|
||||
description: "Grafana 配置备份已超过 48 小时未成功执行"
|
||||
|
||||
# ==================== SLO 基础告警 (简化版) ====================
|
||||
- name: slo-alerts
|
||||
interval: 60s
|
||||
rules:
|
||||
# 可用性 SLO: 99.9% (30天窗口)
|
||||
# 快速燃烧率: 2% 错误预算在1小时内消耗完
|
||||
- alert: SLOErrorBudgetBurnRateFast
|
||||
expr: |
|
||||
(
|
||||
sum(rate(sub2api_http_requests_total{job="sub2api-app",status=~"5.."}[1h]))
|
||||
/
|
||||
sum(rate(sub2api_http_requests_total{job="sub2api-app"}[1h]))
|
||||
) > 0.0144 * 14.4
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
category: slo
|
||||
slo: availability
|
||||
annotations:
|
||||
summary: "SLO 错误预算快速燃烧"
|
||||
description: "可用性错误预算正在快速消耗，可能在1小时内耗尽"
|
||||
|
||||
# 慢速燃烧率: 5% 错误预算在6小时内消耗完
|
||||
- alert: SLOErrorBudgetBurnRateSlow
|
||||
expr: |
|
||||
(
|
||||
sum(rate(sub2api_http_requests_total{job="sub2api-app",status=~"5.."}[6h]))
|
||||
/
|
||||
sum(rate(sub2api_http_requests_total{job="sub2api-app"}[6h]))
|
||||
) > 0.0144 * 6
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
category: slo
|
||||
slo: availability
|
||||
annotations:
|
||||
summary: "SLO 错误预算慢速燃烧"
|
||||
description: "可用性错误预算正在持续消耗，可能在6小时内耗尽"
|
||||
|
||||
# ==================== Prometheus 自身监控 ====================
|
||||
- name: prometheus-self-monitoring
|
||||
interval: 60s
|
||||
rules:
|
||||
# 采集目标下线
|
||||
- alert: PrometheusTargetMissing
|
||||
expr: up == 0
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "监控采集目标下线"
|
||||
description: "目标 {{ $labels.job }}/{{ $labels.instance }} 已无法采集超过 3 分钟"
|
||||
|
||||
# 采集耗时过长 (可能影响数据精度)
|
||||
- alert: PrometheusScrapeDurationHigh
|
||||
expr: scrape_duration_seconds > 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "Prometheus 采集耗时过长"
|
||||
description: "目标 {{ $labels.job }} 采集耗时 {{ $value }}s，超过 10s"
|
||||
|
||||
# TSDB 重载失败
|
||||
- alert: PrometheusTSDBReloadFailing
|
||||
expr: increase(prometheus_tsdb_reloads_failures_total[1h]) > 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "Prometheus TSDB 重载失败"
|
||||
description: "过去 1 小时内 TSDB 重载出现失败，可能影响数据持久化"
|
||||
|
||||
# WAL 截断失败
|
||||
- alert: PrometheusWALCorruption
|
||||
expr: increase(prometheus_tsdb_wal_corruptions_total[5m]) > 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "Prometheus WAL 损坏"
|
||||
description: "Prometheus WAL 出现损坏，请立即检查数据完整性"
|
||||
|
||||
# 告警规则评估失败
|
||||
- alert: PrometheusRuleEvaluationFailing
|
||||
expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "告警规则评估失败"
|
||||
description: "Prometheus 有告警规则评估失败，可能导致漏报"
|
||||
|
||||
# Prometheus 存储接近容量上限
|
||||
- alert: PrometheusStorageFull
|
||||
expr: |
|
||||
(prometheus_tsdb_storage_blocks_bytes / 1024 / 1024 / 1024) > 1.8
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "Prometheus 存储接近上限"
|
||||
description: "Prometheus 存储已用 {{ $value | humanize }}GB，接近 2GB 上限，请及时清理"
|
||||
|
||||
# ==================== 证书过期告警 ====================
|
||||
- name: certificate-alerts
|
||||
interval: 300s # 5分钟检查一次，不需要高频
|
||||
rules:
|
||||
# 证书 7 天内过期
|
||||
- alert: TLSCertificateExpiringSoon
|
||||
expr: (probe_ssl_earliest_cert_expiry - time()) / 86400 < 7
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
category: security
|
||||
annotations:
|
||||
summary: "TLS 证书即将过期"
|
||||
description: "{{ $labels.instance }} 的证书将在 {{ $value | humanizeDuration }} 后过期，请及时续期"
|
||||
|
||||
# 证书 3 天内过期 - 提升为 critical
|
||||
- alert: TLSCertificateExpiringCritical
|
||||
expr: (probe_ssl_earliest_cert_expiry - time()) / 86400 < 3
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
category: security
|
||||
annotations:
|
||||
summary: "TLS 证书紧急过期警告"
|
||||
description: "{{ $labels.instance }} 的证书将在 {{ $value | humanizeDuration }} 后过期！"
|
||||
|
||||
# 证书已过期
|
||||
- alert: TLSCertificateExpired
|
||||
expr: probe_ssl_earliest_cert_expiry - time() <= 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
category: security
|
||||
annotations:
|
||||
summary: "TLS 证书已过期！"
|
||||
description: "{{ $labels.instance }} 的 TLS 证书已过期，请立即续期"
|
||||
|
||||
# ==================== 备份与定时任务监控 ====================
|
||||
- name: backup-and-job-alerts
|
||||
interval: 300s
|
||||
rules:
|
||||
# 数据库备份超过 25 小时未成功 (允许 1 小时误差)
|
||||
- alert: DatabaseBackupMissing
|
||||
expr: time() - sub2api_job_heartbeat_last_success_timestamp{job_name="db-backup"} > 90000
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
category: backup
|
||||
annotations:
|
||||
summary: "数据库备份超时未完成"
|
||||
description: "距上次成功备份已超过 25 小时，请检查备份任务"
|
||||
|
||||
# 数据库备份超过 49 小时 - 严重
|
||||
- alert: DatabaseBackupCriticallyMissing
|
||||
expr: time() - sub2api_job_heartbeat_last_success_timestamp{job_name="db-backup"} > 176400
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
category: backup
|
||||
annotations:
|
||||
summary: "数据库备份连续 2 天未完成！"
|
||||
description: "距上次成功备份已超过 49 小时，存在数据丢失风险"
|
||||
|
||||
# OpsMetricsCollector 定时任务停止心跳
|
||||
- alert: OpsCollectorJobStale
|
||||
expr: time() - sub2api_job_heartbeat_last_success_timestamp{job_name="ops-collector"} > 180
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
category: monitoring
|
||||
annotations:
|
||||
summary: "OpsMetricsCollector 心跳超时"
|
||||
description: "OpsMetricsCollector 定时任务已超过 3 分钟未上报心跳，可能已停止"
|
||||
237
deploy/monitoring/prometheus/rules/sub2api-alerts.yml
Normal file
237
deploy/monitoring/prometheus/rules/sub2api-alerts.yml
Normal file
@@ -0,0 +1,237 @@
|
||||
# =============================================================================
|
||||
# Sub2API 告警规则
|
||||
# =============================================================================
|
||||
|
||||
groups:
|
||||
# ===========================================================================
|
||||
# SLO 相关告警 - 基于错误预算燃烧率
|
||||
# ===========================================================================
|
||||
- name: slo_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
# 错误预算快速燃烧 (Critical) - 2% 预算在1小时内耗尽
|
||||
- alert: ErrorBudgetBurnRateCritical
|
||||
expr: |
|
||||
(
|
||||
sum(rate(sub2api_http_requests_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(sub2api_http_requests_total[5m]))
|
||||
) > 0.0005 * 14.4
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
slo: api-availability
|
||||
team: sre
|
||||
annotations:
|
||||
summary: "错误预算快速燃烧 (Critical)"
|
||||
description: "过去5分钟错误率是30天预算的 {{ $value | humanizePercentage }}，预计1小时内耗尽2%错误预算"
|
||||
runbook_url: "https://wiki.internal/runbooks/error-budget-burn"
|
||||
dashboard: "http://localhost:3000/d/slo-dashboard"
|
||||
|
||||
# 错误预算燃烧加速 (Warning) - 5% 预算在6小时内耗尽
|
||||
- alert: ErrorBudgetBurnRateWarning
|
||||
expr: |
|
||||
(
|
||||
sum(rate(sub2api_http_requests_total{status=~"5.."}[30m]))
|
||||
/
|
||||
sum(rate(sub2api_http_requests_total[30m]))
|
||||
) > 0.0005 * 6
|
||||
for: 5m
|
||||
labels:
|
||||
severity: high
|
||||
slo: api-availability
|
||||
team: sre
|
||||
annotations:
|
||||
summary: "错误预算燃烧加速 (Warning)"
|
||||
description: "过去30分钟错误率是30天预算的 {{ $value | humanizePercentage }}"
|
||||
runbook_url: "https://wiki.internal/runbooks/error-budget-burn"
|
||||
|
||||
# SLO 即将违约告警
|
||||
- alert: SLOViolationImminent
|
||||
expr: |
|
||||
(
|
||||
1 - (
|
||||
sum(rate(sub2api_http_requests_total{status=~"5.."}[24h]))
|
||||
/
|
||||
sum(rate(sub2api_http_requests_total[24h]))
|
||||
)
|
||||
) < 0.9995
|
||||
for: 5m
|
||||
labels:
|
||||
severity: high
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: "SLO 即将违约"
|
||||
description: "过去24小时可用性为 {{ $value | humanizePercentage }}，低于目标 99.95%"
|
||||
|
||||
# ===========================================================================
|
||||
# 延迟告警
|
||||
# ===========================================================================
|
||||
- name: latency_alerts
|
||||
rules:
|
||||
- alert: HighLatencyP99
|
||||
expr: |
|
||||
histogram_quantile(0.99,
|
||||
sum(rate(sub2api_http_request_duration_seconds_bucket[5m])) by (le)
|
||||
) > 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: high
|
||||
annotations:
|
||||
summary: "P99 延迟过高"
|
||||
description: "P99 延迟: {{ $value }}s，超过阈值 5s"
|
||||
|
||||
- alert: HighLatencyP95
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum(rate(sub2api_http_request_duration_seconds_bucket[5m])) by (le)
|
||||
) > 2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: medium
|
||||
annotations:
|
||||
summary: "P95 延迟过高"
|
||||
description: "P95 延迟: {{ $value }}s，超过阈值 2s"
|
||||
|
||||
- alert: HighTTFT
|
||||
expr: |
|
||||
histogram_quantile(0.99,
|
||||
sum(rate(sub2api_gateway_ttft_seconds_bucket[5m])) by (le)
|
||||
) > 3
|
||||
for: 5m
|
||||
labels:
|
||||
severity: high
|
||||
annotations:
|
||||
summary: "首 Token 延迟过高"
|
||||
description: "TTFT P99: {{ $value }}s，超过阈值 3s"
|
||||
|
||||
# ===========================================================================
|
||||
# 错误率告警
|
||||
# ===========================================================================
|
||||
- name: error_alerts
|
||||
rules:
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
sum(rate(sub2api_http_requests_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(sub2api_http_requests_total[5m])) > 0.05
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "错误率过高 (Critical)"
|
||||
description: "错误率: {{ $value | humanizePercentage }}，超过 5%"
|
||||
|
||||
- alert: ElevatedErrorRate
|
||||
expr: |
|
||||
sum(rate(sub2api_http_requests_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(sub2api_http_requests_total[5m])) > 0.01
|
||||
for: 5m
|
||||
labels:
|
||||
severity: medium
|
||||
annotations:
|
||||
summary: "错误率升高"
|
||||
description: "错误率: {{ $value | humanizePercentage }}，超过 1%"
|
||||
|
||||
- alert: UpstreamErrorRateHigh
|
||||
expr: |
|
||||
sum(rate(sub2api_gateway_upstream_errors_total[5m]))
|
||||
/
|
||||
sum(rate(sub2api_gateway_requests_total[5m])) > 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: high
|
||||
annotations:
|
||||
summary: "上游错误率过高"
|
||||
description: "上游错误率: {{ $value | humanizePercentage }}"
|
||||
|
||||
# ===========================================================================
|
||||
# 基础设施告警
|
||||
# ===========================================================================
|
||||
- name: infrastructure_alerts
|
||||
rules:
|
||||
- alert: DatabaseConnectionsHigh
|
||||
expr: |
|
||||
sub2api_db_connections{state="active"} / sub2api_db_connections{state="max"} > 0.8
|
||||
for: 5m
|
||||
labels:
|
||||
severity: high
|
||||
annotations:
|
||||
summary: "数据库连接池使用率过高"
|
||||
description: "数据库连接池使用率: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: RedisMemoryHigh
|
||||
expr: |
|
||||
redis_memory_used_bytes / redis_memory_max_bytes > 0.9
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Redis 内存使用率过高"
|
||||
description: "Redis 内存使用率: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: HighCPUUsage
|
||||
expr: |
|
||||
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
||||
for: 10m
|
||||
labels:
|
||||
severity: medium
|
||||
annotations:
|
||||
summary: "CPU 使用率过高"
|
||||
description: "实例 {{ $labels.instance }} CPU 使用率: {{ $value }}%"
|
||||
|
||||
- alert: HighMemoryUsage
|
||||
expr: |
|
||||
(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes > 0.85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: high
|
||||
annotations:
|
||||
summary: "内存使用率过高"
|
||||
description: "实例 {{ $labels.instance }} 内存使用率: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: DiskSpaceLow
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.15
|
||||
for: 5m
|
||||
labels:
|
||||
severity: high
|
||||
annotations:
|
||||
summary: "磁盘空间不足"
|
||||
description: "实例 {{ $labels.instance }} 磁盘剩余空间: {{ $value | humanizePercentage }}"
|
||||
|
||||
# ===========================================================================
|
||||
# 业务告警
|
||||
# ===========================================================================
|
||||
- name: business_alerts
|
||||
rules:
|
||||
- alert: LowQPS
|
||||
expr: |
|
||||
sum(rate(sub2api_http_requests_total[5m])) < 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: medium
|
||||
annotations:
|
||||
summary: "请求量异常低"
|
||||
description: "当前 QPS: {{ $value }}，可能存在流量异常"
|
||||
|
||||
- alert: AccountSwitchRateHigh
|
||||
expr: |
|
||||
sum(rate(sub2api_account_switches_total[5m])) > 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "账号切换频率过高"
|
||||
description: "账号切换率: {{ $value }}/s，可能存在上游不稳定"
|
||||
|
||||
- alert: JobHeartbeatStale
|
||||
expr: |
|
||||
time() - sub2api_job_heartbeat_last_success_timestamp > 300
|
||||
for: 2m
|
||||
labels:
|
||||
severity: high
|
||||
annotations:
|
||||
summary: "后台任务心跳超时"
|
||||
description: "任务 {{ $labels.job_name }} 超过5分钟未报告成功"
|
||||
98
deploy/monitoring/promtail/promtail-config.yml
Normal file
98
deploy/monitoring/promtail/promtail-config.yml
Normal file
@@ -0,0 +1,98 @@
|
||||
# =============================================================================
|
||||
# Promtail 配置文件
|
||||
# =============================================================================
|
||||
|
||||
server:
|
||||
http_listen_port: 9080
|
||||
grpc_listen_port: 0
|
||||
|
||||
positions:
|
||||
filename: /tmp/positions.yaml
|
||||
|
||||
clients:
|
||||
- url: http://loki:3100/loki/api/v1/push
|
||||
|
||||
scrape_configs:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sub2API 应用日志
|
||||
# ---------------------------------------------------------------------------
|
||||
- job_name: sub2api
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: sub2api
|
||||
service: sub2api
|
||||
__path__: /var/log/sub2api/*.log
|
||||
pipeline_stages:
|
||||
- json:
|
||||
expressions:
|
||||
level: level
|
||||
component: component
|
||||
trace_id: trace_id
|
||||
request_id: request_id
|
||||
- labels:
|
||||
level:
|
||||
component:
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Docker 容器日志
|
||||
# ---------------------------------------------------------------------------
|
||||
- job_name: docker
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: docker
|
||||
__path__: /var/lib/docker/containers/*/*.log
|
||||
pipeline_stages:
|
||||
- json:
|
||||
expressions:
|
||||
output: log
|
||||
stream: stream
|
||||
attrs:
|
||||
- json:
|
||||
source: attrs
|
||||
expressions:
|
||||
tag:
|
||||
- regex:
|
||||
source: tag
|
||||
expression: (?P<container_name>(?:[^|])*[^|])
|
||||
- timestamp:
|
||||
source: time
|
||||
format: RFC3339Nano
|
||||
- labels:
|
||||
stream:
|
||||
container_name:
|
||||
- output:
|
||||
source: output
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 系统日志
|
||||
# ---------------------------------------------------------------------------
|
||||
- job_name: syslog
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: syslog
|
||||
__path__: /var/log/syslog
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Nginx 访问日志 (如果有)
|
||||
# ---------------------------------------------------------------------------
|
||||
- job_name: nginx
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: nginx
|
||||
__path__: /var/log/nginx/access.log
|
||||
pipeline_stages:
|
||||
- regex:
|
||||
expression: '^(?P<remote_addr>\S+) - (?P<remote_user>\S+) \[(?P<time_local>[^\]]+)\] "(?P<request>[^"]+)" (?P<status>\d+) (?P<body_bytes_sent>\d+) "(?P<http_referer>[^"]+)" "(?P<http_user_agent>[^"]+)"'
|
||||
- labels:
|
||||
status:
|
||||
- timestamp:
|
||||
source: time_local
|
||||
format: 02/Jan/2006:15:04:05 -0700
|
||||
Reference in New Issue
Block a user