Grafana是开源可视化平台，连接多种数据源，创建仪表盘实时监控应用、基础设施和业务指标。
Prometheus → 指标监控 Loki → 日志查询 Tempo → 分布式追踪 Elasticsearch → 全文搜索 InfluxDB → 时序数据 MySQL/PG → 业务数据
docker run -d --name grafana -p 3000:3000 \
-v grafana-data:/var/lib/grafana \
-e GF_SECURITY_ADMIN_PASSWORD=admin123 \
grafana/grafana:10.4.0
# API配置数据源
curl -u admin:admin123 -X POST http://localhost:3000/api/datasources \
-H "Content-Type: application/json" \
-d '{"name":"Prometheus","type":"prometheus","url":"http://prometheus:9090","access":"proxy","isDefault":true}'
curl -u admin:admin123 -X POST http://localhost:3000/api/datasources \
-H "Content-Type: application/json" \
-d '{"name":"Loki","type":"loki","url":"http://loki:3100","access":"proxy"}'cat > app-dashboard.json << 'DASHBOARD'
{
"dashboard": {
"title": "Application Monitoring",
"tags": ["app", "devops"],
"panels": [
{
"id": 1, "title": "Request Rate (QPS)", "type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"targets": [{"expr": "sum(rate(http_requests_total[5m])) by (service)", "legendFormat": "{{service}}"}],
"fieldConfig": {"defaults": {"unit": "reqps"}}
},
{
"id": 2, "title": "Error Rate (%)", "type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
"targets": [{"expr": "sum(rate(http_requests_total{status=~'5..'}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service) * 100"}],
"fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [{"value": 0, "color": "green"}, {"value": 1, "color": "yellow"}, {"value": 5, "color": "red"}]}}}
},
{
"id": 3, "title": "P99 Latency", "type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
"targets": [{"expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))"}],
"fieldConfig": {"defaults": {"unit": "s"}}
}
],
"templating": {"list": [
{"name": "namespace", "type": "query", "query": "label_values(kube_pod_info, namespace)", "refresh": 2},
{"name": "service", "type": "query", "query": "label_values(http_requests_total, service)", "refresh": 2}
]}
},
"overwrite": true
}
DASHBOARD
curl -u admin:admin123 -X POST http://localhost:3000/api/dashboards/db \
-H "Content-Type: application/json" -d @app-dashboard.jsoncat > provisioning/datasources/prometheus.yml << 'EOF'
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
url: http://prometheus:9090
isDefault: true
editable: false
- name: Loki
type: loki
url: http://loki:3100
editable: false
EOF
cat > provisioning/dashboards/dashboard.yml << 'EOF'
apiVersion: 1
providers:
- name: default
type: file
options: {path: /var/lib/grafana/dashboards}
EOF
docker run -d --name grafana -p 3000:3000 \
-v ./provisioning:/etc/grafana/provisioning \
-v ./dashboards:/var/lib/grafana/dashboards \
-e GF_SECURITY_ADMIN_PASSWORD=admin123 \
grafana/grafana:10.4.0# Slack通知
curl -u admin:admin123 -X POST http://localhost:3000/api/alert-notifications \
-H "Content-Type: application/json" \
-d '{"name":"Slack DevOps","type":"slack","settings":{"url":"https://hooks.slack.com/services/xxx","recipient":"#devops-alerts"}}'
# 钉钉通知
curl -u admin:admin123 -X POST http://localhost:3000/api/alert-notifications \
-H "Content-Type: application/json" \
-d '{"name":"DingTalk","type":"webhook","settings":{"url":"https://oapi.dingtalk.com/robot/send?access_token=xxx"}}'┌──────────────────────────────────────────┐
│ Grafana Dashboard │
│ QPS │ Error Rate │ P99 │ CPU │ Logs │
└──────────────────┬───────────────────────┘
┌──────────────┼──────────────┐
▼ ▼ ▼
┌────────┐ ┌──────────┐ ┌──────────┐
│ Prom │ │ Loki │ │ Tempo │
│ (指标) │ │ (日志) │ │ (追踪) │
└────────┘ └──────────┘ └──────────┘curl -u admin:admin123 http://localhost:3000/api/datasources/proxy/1/api/v1/query?query=up
# 常见：数据源URL错误/PromQL语法错误/标签不匹配/时间范围问题掌握Dashboard JSON、Provisioning、多数据源联动——可视化让数据说话
| ID | 名称 | 用途 |
|---|---|---|
| 1860 | Node Exporter Full | Linux主机监控 |
| 15760 | Kubernetes Views | K8s集群概览 |
| 11558 | Blackbox Exporter | HTTP探测监控 |
| 15762 | Kubernetes Pod | Pod资源详情 |
| 11190 | NGINX Dashboard | Nginx监控 |
| 12486 | PostgreSQL | 数据库监控 |
# 安装常用插件
grafana-cli plugins install grafana-clock-panel
grafana-cli plugins install grafana-piechart-panel
grafana-cli plugins install marcusolsson-treemap-panel
grafana-cli plugins install alexandra-trackmap-panel
# 重启Grafana
docker restart grafana
# 导入社区仪表盘(通过ID)
# Node Exporter Full: 1860
# Kubernetes Views: 15760
# PostgreSQL: 12486
# Redis: 763
# Nginx: 11190
# Blackbox: 11558# Grafana统一告警（支持多数据源）
curl -u admin:admin123 -X POST \
http://localhost:3000/api/v1/provisioning/alert-rules \
-H "Content-Type: application/json" \
-d '{
"uid": "high-error-rate",
"title": "High Error Rate",
"condition": "B",
"data": [{
"refId": "A",
"relativeTimeRange": {"from": 300, "to": 0},
"datasourceUid": "prometheus",
"model": {
"expr": "sum(rate(http_requests_total{status=~"5.."}[5m]))/sum(rate(http_requests_total[5m]))"
}
}],
"noDataState": "NoData",
"execErrState": "Alerting",
"for": "5m"
}'