1.首先创建配置目录

mkdir -p /opt/prometheus /opt/prometheus/data
mkdir -p /opt/alertmanager /opt/alertmanager/config /opt/alertmanager/data
mkdir -p /opt/grafana/data

2.创建 Prometheus 配置 prometheus.yml

cat > /opt/prometheus/prometheus.yml << 'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['10.100.1.30:9093']

rule_files:
  - "alert.rules.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node'
    static_configs:
      - targets: ['10.100.1.30:9100']

  - job_name: 'mysql-primary'
    static_configs:
      - targets: ['10.100.1.30:9104']
        labels:
          instance: 'mysql-primary'
EOF

3.创建监控服务器的告警规则 alert.rules.yml

cat > /opt/prometheus/alert.rules.yml<< 'EOF'

groups:
  - name: system_alerts
    rules:
      # 内存使用率 > 85%
      - alert: HighMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage on {{ $labels.instance }}"
          description: "{{ $labels.instance }} memory usage is {{ $value | printf \"%.2f\" }}%"

      # CPU 使用率 > 90%
      - alert: HighCPUUsage  
        expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage on {{ $labels.instance }}"
          description: "{{ $labels.instance }} CPU usage is {{ $value | printf \"%.2f\" }}%"

      # 磁盘空间 < 10%
      - alert: LowDiskSpace
        expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs"}) * 100 < 10
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Low disk space on {{ $labels.instance }}"
          description: "{{ $labels.instance }} root partition has only {{ $value | printf \"%.2f\" }}% space left"

      # Node Exporter Down
      - alert: NodeExporterDown
        expr: up{job="node"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Node Exporter down on {{ $labels.instance }}"
          description: "Node Exporter on {{ $labels.instance }} is not responding"

      # Prometheus Down
      - alert: PrometheusDown
        expr: up{job="prometheus"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Prometheus down"
          description: "Prometheus server is not responding"
      - alert: MySQLMaxConnectionsReached
        expr: mysql_global_status_threads_connected > 3  # 只要有 4 个连接就告警
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "MySQL 连接数过高"
          description: "当前连接数 {{ $value }} 超过阈值 3"
EOF

4.在  /opt/alertmanager/templates目录下 邮件装饰

vim email.tmpl

{{ define "email.tmpl" }}
{{- if gt (len .Alerts.Firing) 0 }}
=========故障告警==========<br>
{{ range .Alerts.Firing }}
告警项目: {{ .Labels.project }} <br>
告警程序: prometheus_alert <br>
告警级别: {{ .Labels.severity }} 级 <br>
告警类型: {{ .Labels.alertname }} <br>
故障主机: {{ .Labels.instance }} <br>
告警主题: {{ .Annotations.summary }} <br>
告警详情: {{ .Annotations.description }} <br>
触发时间: {{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }} <br>
=========end==========<br>
{{ end }}
{{- end }}

{{- if gt (len .Alerts.Resolved) 0 }}
=========恢复通知==========<br>
{{ range .Alerts.Resolved }}
告警项目: {{ .Labels.project }} <br>
告警程序: prometheus_alert <br>
告警类型: {{ .Labels.alertname }} <br>
故障主机: {{ .Labels.instance }} <br>
告警状态: 已恢复 <br>
恢复时间: {{ (.EndsAt.Add 28800e9).Format "2006-01-02 15:04:05" }} <br>
=========end==========<br>
{{ end }}
{{- end }}
{{ end }}

5.创建 Alertmanager 邮箱配置 alertmanager.yml

global:
  resolve_timeout: 5m
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_from: '958250198@qq.com'
  smtp_auth_username: '958250198@qq.com'
  smtp_auth_password: 'orqiamutpkksbegh'
  smtp_require_tls: true

templates:
  - '/etc/alertmanager/email.tmpl'
route:
  group_by: ['alertname', 'cluster']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 168h
  receiver: email

receivers:
- name: 'email'
  email_configs:
  - to: 'wangfuchao@bjklb.com'
    send_resolved: true
    html: '{{ template "email.tmpl" . }}'

6.创建启动脚本 deploy-monitoring.sh   也可以提前拉取镜像 ：👇

prometheus:latest

prom/alertmanager:latest

grafana:latest

node-exporter:latest

vim deploy-monitoring.sh

#!/bin/bash

echo "=== 启动 Node Exporter ==="
docker run -d \
  --name=node-exporter \
  --restart=always \
  -p 9100:9100 \
  --net="host" \
  prom/node-exporter:latest

sleep 2

echo "=== 启动 Alertmanager ==="
docker run -d \
  --name=alertmanager \
  --restart=always \
  -p 9093:9093 \
  -v /opt/alertmanager/config:/etc/alertmanager \
  -v /opt/alertmanager/data:/alertmanager \
  prom/alertmanager:latest \
  --config.file=/etc/alertmanager/alertmanager.yml \
  --storage.path=/alertmanager

sleep 2

echo "=== 启动 Prometheus ==="
docker run -d \
  --name=prometheus \
  --restart=always \
  -p 9090:9090 \
  -v /opt/prometheus:/etc/prometheus \
  -v /opt/prometheus/data:/prometheus \
  prom/prometheus:latest \
  --config.file=/etc/prometheus/prometheus.yml \
  --storage.tsdb.path=/prometheus \
  --web.enable-lifecycle

sleep 2

echo "=== 启动 Grafana ==="
docker run -d \
  --name=grafana \
  --restart=always \
  -p 3000:3000 \
  -v /opt/grafana/data:/var/lib/grafana \
  grafana/grafana:latest

echo "=== 等待服务启动 ==="
sleep 10

echo "=== 检查容器状态 ==="
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"

echo "=== 测试服务健康状态 ==="
services=("9090" "9093" "9100" "3000")
for port in "${services[@]}"; do
  echo -n "Port $port: "
  if curl -s --connect-timeout 2 http://localhost:$port >/dev/null 2>&1; then
    echo "✅ UP"
  else
    echo "❌ DOWN"
  fi
done

echo ""
echo "=== 访问地址 ==="
echo "📊 Prometheus: http://10.100.1.30:9090"
echo "🔔 Alertmanager: http://10.100.1.30:9093"
echo "🖥️  Node Exporter: http://10.100.1.30:9100"
echo "📈 Grafana: http://10.100.1.30:3000 (默认账号: admin/admin)"

chmod +x /opt/alertmanager/deploy-monitoring.sh

7.启动脚本

cd /opt/alertmanager
bash deploy-monitoring.sh

8.执行命令查看容器 

docker ps | grep -E "prometheus|grafana|alertmanager|node-exporter"

全是up状态说明启动了服务

http://10.100.1.30:9090/alerts   查看监控node节点

9.打开浏览器《本机ip》:3000  grafana面板

导入数据：

1）

2）

3）

10).最后展示得页面