好的告警系统应该：只告真正需要人介入的问题。告警太多→疲劳→忽略→遗漏真正故障。Google SRE的黄金法则：每条告警都应该可执行。
┌──────────── 告警设计原则 ────────────┐
│ │
│ ❌ 坏告警： │
│ • CPU > 80% │
│ → 所以呢？需要我做什么？ │
│ • 内存使用率高 │
│ → 可能是正常的，无法直接行动 │
│ │
│ ✅ 好告警： │
│ • API P99延迟 > 500ms 持续5分钟 │
│ → 用户受到影响，需要排查 │
│ • 错误率 > 1% 持续3分钟 │
│ → 服务异常，需要修复 │
│ • SLO预算消耗速度异常 │
│ → 需要关注可靠性 │
│ │
│ 核心原则： │
│ 🔹 基于症状而非原因 │
│ 🔹 基于用户影响而非系统指标 │
│ 🔹 每条告警都必须有明确的行动指南 │
│ 🔹 使用SLO驱动告警 │
└────────────────────────────────────────┘# alerting-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: k8s-alerts
namespace: monitoring
spec:
groups:
- name: k8s-alerts
rules:
# Pod不可用告警
- alert: PodCrashLooping
expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 0
for: 15m
labels:
severity: warning
annotations:
summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping"
runbook: "https://runbook.example.com/PodCrashLooping"
# Deployment副本数不足
- alert: DeploymentReplicasMismatch
expr: |
kube_deployment_spec_replicas
!= kube_deployment_status_available_replicas
for: 10m
labels:
severity: warning
annotations:
summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has insufficient replicas"
# 节点NotReady
- alert: NodeNotReady
expr: kube_node_status_condition{condition="Ready",status="true"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: "Node {{ $labels.node }} is not ready"
# PVC即将写满
- alert: PVCAlmostFull
expr: |
kubelet_volume_stats_used_bytes / kubelet_volume_stats_capacity_bytes > 0.85
for: 5m
labels:
severity: warning
annotations:
summary: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is {{ $value | humanizePercentage }} full"
# API错误率
- alert: HighAPIErrorRate
expr: |
sum(rate(http_requests_total{status=~"5.."}[5m]))
/ sum(rate(http_requests_total[5m])) > 0.01
for: 3m
labels:
severity: critical
annotations:
summary: "API error rate is {{ $value | humanizePercentage }}"
# ✅ 验证通过
kubectl apply -f alerting-rules.yaml
kubectl get prometheusrule -n monitoring
# NAME AGE
# k8s-alerts 5s# alertmanager-config.yaml
apiVersion: v1
kind: Secret
metadata:
name: alertmanager-monitoring-kube-prometheus-alertmanager
namespace: monitoring
type: Opaque
stringData:
alertmanager.yaml: |
# 全局配置
global:
resolve_timeout: 5m
# 路由树
route:
group_by: ['namespace', 'alertname']
group_wait: 30s # 同组告警等待时间
group_interval: 5m # 同组新告警间隔
repeat_interval: 4h # 重复告警间隔
receiver: 'default'
routes:
- match:
severity: critical
receiver: 'critical'
repeat_interval: 1h
- match:
severity: warning
receiver: 'warning'
repeat_interval: 6h
# 抑制规则：critical告警抑制同组warning
inhibit_rules:
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
equal: ['namespace', 'alertname']
# 接收器
receivers:
- name: 'default'
webhook_configs:
- url: 'http://alertmanager-webhook:8080/alerts'
- name: 'critical'
# 企业微信/钉钉
webhook_configs:
- url: 'http://webhook-adapter:8080/critical'
send_resolved: true
# 邮件
email_configs:
- to: 'oncall@example.com'
from: 'alertmanager@example.com'
smarthost: 'smtp.example.com:587'
auth_username: 'alertmanager@example.com'
auth_password: '<password>'
- name: 'warning'
slack_configs:
- api_url: 'https://hooks.slack.com/services/xxx'
channel: '#alerts-warning'
title: '{{ .GroupLabels.alertname }}'
text: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}"
# ✅ 验证通过 - 查看告警
kubectl port-forward svc/monitoring-kube-prometheus-alertmanager 9093:9093 -n monitoring
# 浏览器打开 http://localhost:9093┌─────── SLO核心概念 ───────┐
│ │
│ SLI（指标） │
│ → 衡量服务的量化指标 │
│ 例：可用性、延迟、错误率 │
│ │
│ SLO（目标） │
│ → SLI的目标值 │
│ 例：99.9%可用性 │
│ │
│ SLA（协议） │
│ → 对客户的正式承诺 │
│ 例：低于99.9%则赔偿 │
│ │
│ 错误预算 = 1 - SLO │
│ 99.9% SLO → 0.1%错误预算 │
│ = 每月43.2分钟不可用 │
└─────────────────────────────┘# slo-alerts.yaml - 错误预算消耗告警
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: slo-alerts
namespace: monitoring
spec:
groups:
- name: slo-alerts
rules:
# 计算SLO可用性（30天窗口）
- record: slo:availability:ratio
expr: |
(
sum(rate(http_requests_total{status!~"5.."}[30d]))
/
sum(rate(http_requests_total[30d]))
)
# 错误预算消耗速率
- record: slo:error_budget:burn_rate
expr: |
(
1 - slo:availability:ratio
)
/
(1 - 0.999) # 99.9% SLO
# 错误预算快速消耗告警（1小时消耗2.5%预算 → 约6天内耗尽）
- alert: SLOErrorBudgetBurningFast
expr: slo:error_budget:burn_rate > 14.4
for: 1h
labels:
severity: critical
annotations:
summary: "SLO error budget burning fast for {{ $labels.service }}"
description: "Error budget burn rate is {{ $value | printf \"%.1f\" }}x, budget will exhaust in ~{{ 720 | divf $value | printf \"%.0f\" }} hours"
# 错误预算缓慢消耗告警（3天消耗2.5%预算 → 约30天内耗尽）
- alert: SLOErrorBudgetBurningSlow
expr: slo:error_budget:burn_rate > 1
for: 3d
labels:
severity: warning
annotations:
summary: "SLO error budget burning for {{ $labels.service }}"
# 可用性低于SLO
- alert: SLOViolation
expr: slo:availability:ratio < 0.999
for: 5m
labels:
severity: critical
annotations:
summary: "Availability {{ $value | humanizePercentage }} is below 99.9% SLO"# Grafana错误预算Dashboard关键查询
# 当前可用性（30天）
slo:availability:ratio * 100
# 剩余错误预算（百分比）
(1 - (1 - slo:availability:ratio) / (1 - 0.999)) * 100
# 错误预算消耗趋势
1 - slo:availability:ratio
# 每日请求量与错误量
sum(rate(http_requests_total[1d])) by (service)
sum(rate(http_requests_total{status=~"5.."}[1d])) by (service)# On-Call轮值配置
# 1. 告警分级
# P0 - 立即响应（5分钟内）
# P1 - 30分钟内响应
# P2 - 工作时间处理
# P3 - 下一工作日处理
# 2. 告警降噪
# - 合理设置for持续时间
# - 使用group_by合并相关告警
# - 抑制规则：高级别抑制低级别
# - 静默：计划维护期间静默告警
# 3. Runbook
# 每条告警必须有对应Runbook：
# - 告警含义
# - 影响范围
# - 排查步骤
# - 修复方案
# - 升级路径# 现象：同时收到大量告警
# 解决：
# 1. 紧急静默
amtool silence add --author="admin" --comment="investigating" \
alertname=PodCrashLooping namespace=default
# 2. 优化group_by
route:
group_by: ['namespace', 'alertname', 'cluster']
# 3. 增大for持续时间
# for: 5m → for: 15m
# 4. 检查是否是级联故障
# 核心服务故障→依赖服务全部异常→大量告警# 检查告警规则是否生效
kubectl get prometheusrule -A
# 在Prometheus UI查看Alerts
# Status: Inactive → 条件不满足
# Status: Pending → 满足条件但for时间未到
# Status: Firing → 正在告警
# 检查Alertmanager路由
amtool config routes --config-file=alertmanager.yaml下一课预告：第16课深入Helm包管理——K8s应用打包与分发。
15-告警与slo补充要点：K8s生产实践扩展
🔹 资源配额(ResourceQuota)：限制命名空间总资源
apiVersion: v1
kind: ResourceQuota
metadata:
name: compute-quota
namespace: production
spec:
hard:
requests.cpu: "20"
requests.memory: 40Gi
limits.cpu: "40"
limits.memory: 80Gi
pods: "50"
services: "10"
🔹 LimitRange：设置默认资源限制
apiVersion: v1
kind: LimitRange
metadata:
name: default-limits
spec:
limits:
- type: Container
default:
cpu: "200m"
memory: 256Mi
defaultRequest:
cpu: "100m"
memory: 128Mi
max:
cpu: "2"
memory: 4Gi
🔹 Pod优先级与抢占
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: high-priority
value: 1000000
globalDefault: false
---
spec:
preemptionPolicy: PreemptLowerPriority
🔹 优雅处理Pod中断
• PDB保证最小可用副本
• preStop钩子处理连接排空
• terminationGracePeriodSeconds充足
• 应用必须处理SIGTERM信号
🔹 生产环境Checklist
✅ 设置resources requests/limits
✅ 配置liveness/readiness探针
✅ 使用PDB保护关键服务
✅ 实现优雅关闭（SIGTERM）
✅ 配置HPA自动伸缩
✅ 使用NetworkPolicy隔离
✅ 开启RBAC最小权限
✅ 日志结构化输出
✅ 指标暴露/metrics端点
✅ 配置PVC数据备份