🔔 第24课：告警体系

cat > alertmanager.yml << 'EOF'
global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/xxx'

route:
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default'
  routes:
  - match: {severity: critical}
    receiver: 'critical'
    group_wait: 10s
    repeat_interval: 1h
  - match: {severity: warning}
    receiver: 'warning'

receivers:
- name: 'default'
  slack_configs:
  - channel: '#devops-alerts'
    title: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}'

- name: 'critical'
  slack_configs:
  - channel: '#incidents'
    title: '🚨 CRITICAL: {{ .CommonLabels.alertname }}'
  webhook_configs:
  - url: 'http://pagerduty-webhook:8080/alert'

- name: 'warning'
  slack_configs:
  - channel: '#devops-warnings'

inhibit_rules:
- source_match: {severity: 'critical'}
  target_match: {severity: 'warning'}
  equal: ['alertname', 'cluster', 'service']
EOF

docker run -d --name alertmanager -p 9093:9093 \
  -v ./alertmanager.yml:/etc/alertmanager/alertmanager.yml \
  prom/alertmanager:v0.27.0

# 创建静默(维护窗口期)
curl -X POST http://localhost:9093/api/v2/silences \
  -H "Content-Type: application/json" \
  -d '{
    "matchers": [{"name": "namespace", "value": "staging", "isRegex": false}],
    "startsAt": "2026-05-19T22:00:00.000Z",
    "endsAt": "2026-05-20T06:00:00.000Z",
    "createdBy": "devops@example.com",
    "comment": "Planned maintenance"
  }'

# 查看活跃静默
curl http://localhost:9093/api/v2/silences | jq '.[] | select(.status.state=="active")'

#!/bin/bash
# alert-quality.sh
AM_URL="http://localhost:9093"
echo "╔════════════════════════════════════════╗"
echo "║         告警质量报告                    ║"
echo "╠════════════════════════════════════════╣"

TOTAL=$(curl -s "$AM_URL/api/v2/alerts" | jq 'length')
printf "║ 总活跃告警: %-28s║
" "$TOTAL"

curl -s "$AM_URL/api/v2/alerts" | jq -r '.[].labels.severity' | sort | uniq -c | sort -rn | while read count sev; do
    printf "║   %-8s: %-28s║
" "$sev" "$count"
done

echo "╠════════════════════════════════════════╣"
echo "║ 改进建议:                              ║"
echo "║ 1. 重复告警→添加group_by聚合           ║"
echo "║ 2. 低价值告警→调整阈值或降级            ║"
echo "║ 3. 频繁抖动→增加for持续时间            ║"
echo "╚════════════════════════════════════════╝"