🔥 第22课：Prometheus监控

cat > prometheus.yml << 'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels: {cluster: 'prod'}

rule_files: ["alert_rules.yml"]

alerting:
  alertmanagers:
  - static_configs: [{targets: ['alertmanager:9093']}]

scrape_configs:
  - job_name: 'prometheus'
    static_configs: [{targets: ['localhost:9090']}]

  - job_name: 'node-exporter'
    static_configs: [{targets: ['node1:9100', 'node2:9100', 'node3:9100']}]

  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs: [{role: pod}]
    relabel_configs:
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
      action: keep
      regex: true
    - source_labels: [__meta_kubernetes_namespace]
      target_label: namespace

  - job_name: 'blackbox-http'
    metrics_path: /probe
    params: {module: [http_2xx]}
    static_configs: [{targets: ['https://app.example.com/health']}]
    relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - target_label: __address__
      replacement: blackbox-exporter:9115
EOF

docker run -d --name prometheus -p 9090:9090 \
  -v ./prometheus.yml:/etc/prometheus/prometheus.yml \
  prom/prometheus:v2.51.0

cat > alert_rules.yml << 'EOF'
groups:
- name: app-alerts
  rules:
  - alert: HighErrorRate
    expr: |
      sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
      / sum(rate(http_requests_total[5m])) by (service) > 0.01
    for: 5m
    labels: {severity: critical}
    annotations:
      summary: "High error rate on {{ $labels.service }}"
      description: "Error rate is {{ $value | humanizePercentage }}"

  - alert: HighLatencyP99
    expr: |
      histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)) > 1
    for: 10m
    labels: {severity: warning}
    annotations:
      summary: "High P99 latency on {{ $labels.service }}"

  - alert: PodCrashLooping
    expr: rate(kube_pod_container_status_restarts_total[15m]) > 0
    for: 5m
    labels: {severity: critical}

  - alert: DiskSpaceLow
    expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes) < 0.15
    for: 5m
    labels: {severity: warning}

  - alert: SSLExpirySoon
    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 14
    for: 1h
    labels: {severity: warning}
EOF
promtool check rules alert_rules.yml

# 请求速率(QPS)
sum(rate(http_requests_total[5m])) by (service)

# 错误率
sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
/ sum(rate(http_requests_total[5m])) by (service)

# P99延迟
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))

# CPU使用率
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 内存使用率
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

# 预测6小时后磁盘是否满
predict_linear(node_filesystem_avail_bytes[1h], 6*3600) < 0

# Top 10最慢请求
topk(10, histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)))

# Thanos Sidecar模式(长期存储+全局查询)
cat > docker-compose-thanos.yml << 'EOF'
version: '3.8'
services:
  prometheus:
    image: prom/prometheus:v2.51.0
    ports: ["9091:9090"]
    volumes: [./prometheus.yml:/etc/prometheus/prometheus.yml, prom-data:/prometheus]
    command: [--config.file=/etc/prometheus/prometheus.yml, --storage.tsdb.retention.time=2h]

  thanos-sidecar:
    image: thanosio/thanos:v0.34.0
    volumes: [prom-data:/prometheus]
    command: [sidecar, --tsdb.path=/prometheus, --prometheus.url=http://prometheus:9090, --objstore.config-file=/etc/thanos/objstore.yml]

  thanos-query:
    image: thanosio/thanos:v0.34.0
    ports: ["19192:19192"]
    command: [query, --http-address=0.0.0.0:19192, --store=thanos-sidecar:10901]

volumes: {prom-data:}
EOF
docker compose -f docker-compose-thanos.yml up -d