金丝雀发布将新版本先推送给一小部分用户(1-5%),观察指标无异常后逐步扩大流量比例。相比蓝绿部署,渐进暴露风险,问题影响面小。
时间线: T0 T1 T2 T3 T4
流量: 1% → 5% → 25% → 50% → 100%
│ │ │ │ │
[观察] [观察] [观察] [观察] [完成]
指标OK? 指标OK? 指标OK? 指标OK?
是→推进 是→推进 是→推进 是→全量
否→回滚 否→回滚 否→回滚 否→回滚| 指标类别 | 具体指标 | 阈值示例 | 数据源 |
|---|---|---|---|
| 错误率 | HTTP 5xx比例 | < 0.1% | Prometheus |
| 延迟 | P99延迟 | < 200ms | Prometheus |
| 业务指标 | 订单转化率 | 不降5%+ | 业务埋点 |
| 资源 | CPU/Memory | < 80% | cAdvisor |
| 日志 | ERROR日志数 | 0增长 | ELK/Loki |
# 金丝雀Nginx配置
cat > /etc/nginx/conf.d/canary.conf << 'EOF'
upstream stable {{
server 10.0.1.10:8080 weight=99;
server 10.0.1.11:8080 weight=99;
}}
upstream canary {{
server 10.0.2.10:8080 weight=1;
}}
# 基于权重分流(1% canary)
split_clients "${{remote_addr}}" $canary_backend {{
1% canary;
* stable;
}}
# 基于Cookie精确控制
map $cookie_canary $canary_backend {{
"true" canary;
"false" stable;
default $canary_backend;
}}
server {{
listen 80;
server_name app.example.com;
location / {{
proxy_pass http://$canary_backend;
proxy_set_header Host $host;
add_header X-Backend $upstream_addr;
}}
}}
EOF
nginx -t && nginx -s reload
# 手动进入金丝雀组
curl -b "canary=true" http://app.example.com/# 部署stable和canary版本
cat > deploy-canary.yaml << 'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {{name: myapp-stable}}
spec:
replicas: 3
selector: {{matchLabels: {{app: myapp, version: v1}}}}
template:
metadata: {{labels: {{app: myapp, version: v1}}}}
spec:
containers: [{{name: myapp, image: myregistry/myapp:v1.0, ports: [{{containerPort: 8080}}]}}]
---
apiVersion: apps/v1
kind: Deployment
metadata: {{name: myapp-canary}}
spec:
replicas: 1
selector: {{matchLabels: {{app: myapp, version: v2}}}}
template:
metadata: {{labels: {{app: myapp, version: v2}}}}
spec:
containers: [{{name: myapp, image: myregistry/myapp:v2.0, ports: [{{containerPort: 8080}}]}}]
EOF
kubectl apply -f deploy-canary.yaml
# DestinationRule
cat > dr.yaml << 'EOF'
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata: {{name: myapp}}
spec:
host: myapp
subsets:
- name: v1
labels: {{version: v1}}
- name: v2
labels: {{version: v2}}
EOF
# VirtualService——1% canary
cat > vs-canary.yaml << 'EOF'
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata: {{name: myapp}}
spec:
hosts: [myapp]
http:
- route:
- destination: {{host: myapp, subset: v1}}
weight: 99
- destination: {{host: myapp, subset: v2}}
weight: 1
retries: {{attempts: 3, perTryTimeout: 2s}}
EOF
kubectl apply -f dr.yaml -f vs-canary.yaml
# 逐步推进到5%
sed -i 's/weight: 99/weight: 95/;s/weight: 1$/weight: 5/' vs-canary.yaml
kubectl apply -f vs-canary.yaml# 安装Argo Rollouts
kubectl create namespace argo-rollouts
kubectl apply -n argo-rollouts -f https://github.com/argoproj/argo-rollouts/releases/latest/download/install.yaml
# Rollout资源
cat > rollout.yaml << 'EOF'
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata: {{name: myapp-rollout}}
spec:
replicas: 5
strategy:
canary:
canaryService: myapp-canary
stableService: myapp-stable
trafficRouting:
istio:
virtualServices:
- name: myapp-vsvc
routes: [primary]
steps:
- setWeight: 5
- pause: {{duration: 5m}}
- setWeight: 20
- pause: {{duration: 10m}}
- setWeight: 50
- pause: {{duration: 10m}}
analysis:
templates:
- templateName: success-rate
selector: {{matchLabels: {{app: myapp}}}}
template:
metadata: {{labels: {{app: myapp}}}}
spec:
containers: [{{name: myapp, image: myregistry/myapp:v1.0, ports: [{{containerPort: 8080}}]}}]
EOF
# AnalysisTemplate——Prometheus指标自动分析
cat > analysis.yaml << 'EOF'
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata: {{name: success-rate}}
spec:
metrics:
- name: success-rate
interval: 30s
successLimit: 3
failureLimit: 2
provider:
prometheus:
address: http://prometheus-server:9090
query: |
sum(rate(http_requests_total{{service="myapp-canary",status!~"5.."}}[1m]))
/
sum(rate(http_requests_total{{service="myapp-canary"}}[1m]))
successCondition: result[0] >= 0.99
EOF
kubectl apply -f analysis.yaml -f rollout.yaml
# 触发更新
kubectl argo rollouts set image myapp-rollout myapp=myregistry/myapp:v2.0
# 查看状态
kubectl argo rollouts get rollout myapp-rollout
# 手动推进
kubectl argo rollouts promote myapp-rollout
# 紧急回滚
kubectl argo rollouts abort myapp-rollout
┌────────────┐ ┌──────────────┐ ┌──────────────┐
│ Client │───▶│ Istio Envoy │───▶│ VirtualService│
└────────────┘ │ Gateway │ │ weight:95/5 │
└──────────────┘ └───────┬──────┘
┌────────────────┼──────────┐
▼ ▼
┌──────────────────┐ ┌──────────────────┐
│ myapp-stable │ │ myapp-canary │
│ v1 (95%流量) │ │ v2 (5%流量) │
└──────────────────┘ └────────┬─────────┘
│
┌────────▼─────────┐
│ AnalysisTemplate │
│ Prometheus查询 │
│ 成功率 >= 99%? │
│ → 自动推进/回滚 │
└──────────────────┘# Istio立即回滚
cat > vs-rollback.yaml << 'EOF'
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata: {{name: myapp}}
spec:
hosts: [myapp]
http:
- route:
- destination: {{host: myapp, subset: v1}}
weight: 100
EOF
kubectl apply -f vs-rollback.yaml
# Argo Rollouts紧急中止
kubectl argo rollouts abort myapp-rollout
# 查看canary错误
kubectl logs -l version=v2 --tail=100 | grep -i error
# Prometheus查询错误率
curl -s 'http://prometheus:9090/api/v1/query' \
--data-urlencode 'query=sum(rate(http_requests_total{{version="v2",status=~"5.."}}[5m]))/sum(rate(http_requests_total{{version="v2"}}[5m]))'掌握Nginx权重分流、Istio流量管理、Argo Rollouts自动化分析——让每次发布都有试金石保护