🚨 第27课：事件管理

# On-Call轮值数据
cat > oncall-schedule.json << 'EOF'
[
  {"week": "2026-W21", "name": "Alice", "phone": "+86-138-xxxx-0001"},
  {"week": "2026-W22", "name": "Bob",   "phone": "+86-138-xxxx-0002"},
  {"week": "2026-W23", "name": "Carol", "phone": "+86-138-xxxx-0003"},
  {"week": "2026-W24", "name": "Dave",  "phone": "+86-138-xxxx-0004"}
]
EOF

# 每周自动提醒(K8s CronJob)
cat > oncall-reminder.yaml << 'EOF'
apiVersion: batch/v1
kind: CronJob
metadata: {name: oncall-reminder}
spec:
  schedule: "0 9 * * 1"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: reminder
            image: curlimages/curl
            command: ["/bin/sh", "-c", "curl -X POST $SLACK_WEBHOOK -d '{"text":"📋 本周On-Call: ..."}'"]
          restartPolicy: OnFailure
EOF
kubectl apply -f oncall-reminder.yaml

cat > runbook-high-error-rate.md << 'RUNBOOK'
# Runbook: HighErrorRate

## 告警信息
- 严重级别: P1
- SLA: 15分钟内响应

## 前置检查
```bash
# 确认错误率
curl -s 'http://prometheus:9090/api/v1/query' \
  --data-urlencode 'query=sum(rate(http_requests_total{status=~"5.."}[5m]))/sum(rate(http_requests_total[5m]))'
# 查看最近部署
kubectl rollout history deployment/myapp | tail -5
# 查看Pod状态
kubectl get pods -l app=myapp
# 查看错误日志
kubectl logs -l app=myapp --tail=100 | grep ERROR
```

## 止血方案
### 方案A：回滚(优先)
```bash
kubectl rollout undo deployment/myapp
kubectl rollout status deployment/myapp
```

### 方案B：扩容
```bash
kubectl scale deployment/myapp --replicas=6
```

### 方案C：Feature Flag关闭
```bash
curl -X PATCH "$FLAG_API/new-feature" -d '{"enabled": false}'
```

## 验证恢复
```bash
curl -s 'http://prometheus:9090/api/v1/query' \
  --data-urlencode 'query=sum(rate(http_requests_total{status=~"5.."}[5m]))/sum(rate(http_requests_total[5m]))'
# 应 < 0.001
```

## 升级路径
- 15分钟未恢复 → 通知Tech Lead
- 30分钟未恢复 → 通知VP Engineering
RUNBOOK

#!/bin/bash
# incident-manager.sh
INCIDENT_DIR="/var/incidents"
mkdir -p "$INCIDENT_DIR"

case "${1:?start|update|resolve}" in
  start)
    ID="INC-$(date +%Y%m%d-%H%M%S)"
    cat > "$INCIDENT_DIR/$ID.yaml" << INC
id: $ID
severity: ${2:-P2}
started_at: $(date -Is)
status: investigating
commander: $(whoami)
INC
    curl -X POST "$SLACK_WEBHOOK" -d "{"text":"🚨 INCIDENT: $ID [$2]"}"
    echo "Started: $ID"
    ;;
  update)
    echo "  - time: $(date -Is)" >> "$INCIDENT_DIR/${2}.yaml"
    echo "    event: "$3"" >> "$INCIDENT_DIR/${2}.yaml"
    curl -X POST "$SLACK_WEBHOOK" -d "{"text":"📝 UPDATE: $2 - $3"}"
    ;;
  resolve)
    sed -i "s/status: investigating/status: resolved/" "$INCIDENT_DIR/${2}.yaml"
    curl -X POST "$SLACK_WEBHOOK" -d "{"text":"✅ RESOLVED: $2"}"
    ;;
esac