本文章使用 CentOS 7 系统进行部署，基于 Prometheus、grafana、alertmanager 对主机进行数据获取、网页展示、告警等操作。也提供了相关的压缩包路径，需自行下载。在告警的部分要手动填自己的邮箱和授权码，授权码的获取方式已提供在相应的位置，根据提供的图片进行操作就行
 

相关组件获取链接：

1. Prometheus
2. alertmanager
3. grafana
4. node_exporter
    

本文章使用的组件压缩包的名称：

1. alertmanager-0.32.1.linux-amd64.tar.gz
2. prometheus-3.11.3.linux-amd64.tar.gz
3. grafana-enterprise_13.0.1_24542347077_linux_amd64.tar.gz
4. node_exporter-1.11.1.linux-amd64.tar.gz

 

server	192.168.188.150
node1	192.168.188.151
node2	192.168.188.152

 

监控宿主机相关部署

prometheus 负责抓取、存放指标数据
grafana 会将 prometheus 采集的数据进行页面的展示
alertmanager 处理 prometheus 触发的告警
 

所有机子执行

systemctl disable firewalld --now # 关闭防火墙并永久关闭
setenforce 0 # 关闭 selinux
sed -i 's/enforcing/disabled/' /etc/selinux/config # 永久关闭 selinux
curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo # 获取阿里源
yum -y install ntp net-tools
ntpdate time1.aliyun.com # 校准时间
hwclock -w # 将当前的系统时间写入硬件时钟

 

server 执行

tar -zxf alertmanager-0.32.1.linux-amd64.tar.gz 
mv alertmanager-0.32.1.linux-amd64 /usr/local/alertmanager
ls /usr/local/alertmanager/ # alertmanager.yml 为配置文件


# 配置 alertmanager 启停相关文件
cat > /usr/lib/systemd/system/alertmanager.service << EOF
[Unit]
Description=Alertmanager Daemon
After=network.target
[Service]
Restart=on-failure
ExecStart=/usr/local/alertmanager/alertmanager --config.file=/usr/local/alertmanager/alertmanager.yml
[Install]
WantedBy=multi-user.target
EOF


systemctl daemon-reload 
systemctl enable alertmanager.service --now
systemctl status alertmanager.service
# 此时可访问 http://192.168.188.150:9093/ 页面

tar -zxf prometheus-3.11.3.linux-amd64.tar.gz
mv prometheus-3.11.3.linux-amd64 /usr/local/prometheus
ls /usr/local/prometheus/ # prometheus.yml 为配置文件


# 配置 prometheus 启停相关文件
cat > /usr/lib/systemd/system/prometheus.service << EOF
[Unit]
Description=Prometheus Daemon
[Service]
Restart=on-failure
ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml
[Install]
WantedBy=multi-user.target
EOF


systemctl daemon-reload 
systemctl enable prometheus.service --now
systemctl status prometheus
# 此时可访问 http://192.168.188.150:9090/ 页面

tar -zxf grafana-enterprise_13.0.1_24542347077_linux_amd64.tar.gz 
mv grafana-13.0.1/ /usr/local/grafana


# 配置 grafana 启停相关文件
cat > /usr/lib/systemd/system/grafana.service << EOF
[Unit]
Description=Grafana Daemon
[Service]
ExecStart=/usr/local/grafana/bin/grafana server --homepath=/usr/local/grafana
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF


systemctl daemon-reload 
systemctl enable grafana.service --now
systemctl status grafana
# 此时可访问 http://192.168.188.150:3000/ 页面，账号、密码都是 admin

 

node1-2 执行

tar -zxf node_exporter-1.11.1.linux-amd64.tar.gz
mv node_exporter-1.11.1.linux-amd64 /usr/local/node_exporter


# 配置 node_exporter 启停相关文件
cat > /usr/lib/systemd/system/node_exporter.service << EOF
[Unit]
Description=Node_Exporter Daemon
[Service]
Restart=on-failure
ExecStart=/usr/local/node_exporter/node_exporter
[Install]
WantedBy=default.target
EOF


systemctl daemon-reload 
systemctl enable node_exporter --now
systemctl status node_exporter
# 此时可以访问被监控端的9100端口相关的网页
# node_exporter 采集设备的 CPU、内存、磁盘、网络等基础指标，暴露给 Prometheus 抓取

 

server 执行

cd /usr/local/prometheus/
mkdir rules # 存储告警规则
cp -p prometheus.yml prometheus.yml.bak # 备份配置文件


cat > prometheus.yml << EOF
global:
  scrape_interval: 15s # 采集数据的时间间隔
  evaluation_interval: 15s # 评估告警规则的时间间隔，每间隔指定时间评估一次告警规则
alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - 192.168.188.150:9093 # 告警推送地址（Alertmanager 的 IP 和端口）
rule_files: # 告警规则配置文件列表
  - "rules/*.yml" # 在 prometheus 安装目录下创建 rules 目录存放告警规则
scrape_configs: # 被监控端相关配置
  - job_name: "prometheus" # 作业名称：监控 Prometheus 自身
    static_configs:
      - targets: ["localhost:9090"] # 默认监控自己
  - job_name: "nodes" # 作业名称：监控节点（服务器）
    static_configs:
      - targets: # 被监控端的地址和端口（node_exporter 默认端口 9100）
          - "192.168.188.151:9100"
          - "192.168.188.152:9100"
EOF


cat > rules/node.yml << EOF
groups:
  - name: node_instance_rules
    rules:
      - alert: InstanceDown
        expr: up == 0
        for: 1m
        labels:
          severity: error
        annotations:
          summary: "Instance {{ .Labels.instance }} 停止工作"
          description: "{{ .Labels.instance }} (job={{ .Labels.job }}) 已经停止工作超过 1 分钟"

  - name: node_disk_rules
    rules:
      - alert: NodeFilesystemUsage
        expr: |
          100 - (node_filesystem_free_bytes{fstype=~"ext4|xfs"} / node_filesystem_size_bytes{fstype=~"ext4|xfs"} * 100) > 80
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "{{ .Labels.instance }}: {{ .Labels.mountpoint }} 分区使用过高"
          description: "{{ .Labels.instance }}: {{ .Labels.mountpoint }} 分区使用大于 80% (当前值: {{ .Value }}%)"

      - alert: NodeOutOfInodes
        expr: |
          node_filesystem_files_free{fstype=~"ext4|xfs", mountpoint!~".*tmp|.*boot"} /
          node_filesystem_files{fstype=~"ext4|xfs", mountpoint!~".*tmp|.*boot"} * 100 < 10
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "主机分区 Inode 节点不足"
          description: "主机 {{ .Labels.instance }} {{ .Labels.mountpoint }} 分区 Inode 可用 < 10% (当前值: {{ .Value }}%)"

      - alert: HostUnusualDiskReadLatency
        expr: |
          rate(node_disk_read_time_seconds_total{device=~"sd.*|vd.*"}[1m]) /
          rate(node_disk_reads_completed_total{device=~"sd.*|vd.*"}[1m]) > 0.1
          and rate(node_disk_reads_completed_total{device=~"sd.*|vd.*"}[1m]) > 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "主机磁盘读延迟过高"
          description: "主机 {{ .Labels.instance }}, 磁盘 {{ .Labels.device }} 读延迟 > 100ms (当前值: {{ .Value }}ms)"

      - alert: HostUnusualDiskWriteLatency
        expr: |
          rate(node_disk_write_time_seconds_total{device=~"sd.*|vd.*"}[1m]) /
          rate(node_disk_writes_completed_total{device=~"sd.*|vd.*"}[1m]) > 0.1
          and rate(node_disk_writes_completed_total{device=~"sd.*|vd.*"}[1m]) > 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "主机磁盘写延迟过高"
          description: "主机 {{ .Labels.instance }}, 磁盘 {{ .Labels.device }} 写延迟 > 100ms (当前值: {{ .Value }}ms)"

      - alert: NodeDiskReadRate
        expr: |
          sum by (instance, device) (rate(node_disk_read_bytes_total{device=~"sd.*|vd.*"}[2m])) / 1024 / 1024 > 50
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "主机磁盘读取速率过高"
          description: "主机 {{ .Labels.instance }}, 磁盘 {{ .Labels.device }} 读取速度 > 50 MB/s (当前值: {{ .Value }} MB/s)"

      - alert: NodeDiskWriteRate
        expr: |
          sum by (instance, device) (rate(node_disk_written_bytes_total{device=~"sd.*|vd.*"}[2m])) / 1024 / 1024 > 50
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "主机磁盘写入速率过高"
          description: "主机 {{ .Labels.instance }}, 磁盘 {{ .Labels.device }} 写入速度 > 50 MB/s (当前值: {{ .Value }} MB/s)"

  - name: node_memory_cpu_rules
    rules:
      - alert: NodeMemoryUsage
        expr: |
          (1 - (node_memory_MemFree_bytes + node_memory_Cached_bytes + node_memory_Buffers_bytes) / node_memory_MemTotal_bytes) * 100 > 80
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "{{ .Labels.instance }}: 内存使用过高"
          description: "{{ .Labels.instance }}: 内存使用大于 80% (当前值: {{ .Value }}%)"

      - alert: NodeCPUUsage
        expr: |
          100 - (avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance) * 100) > 80
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "{{ .Labels.instance }}: CPU 使用过高"
          description: "{{ .Labels.instance }}: CPU 使用大于 80% (当前值: {{ .Value }}%)"

      - alert: NodeCpuLoadHigh
        expr: sum(node_load5) by (instance) > 10
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "5分钟内 CPU 负载过高"
          description: "{{ .Labels.instance }}: 5分钟内 CPU 负载超过 10 (当前值: {{ .Value }})"

      - alert: NodeIoWait
        expr: |
          (sum(increase(node_cpu_seconds_total{mode="iowait"}[5m])) by (instance) /
           sum(increase(node_cpu_seconds_total[5m])) by (instance)) * 100 > 10
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "5分钟内磁盘 I/O Wait 负载过高"
          description: "{{ .Labels.instance }}: 5分钟内磁盘 I/O Wait 超过 10% (当前值: {{ .Value }}%)"

  - name: node_network_rules
    rules:
      - alert: NodeNetworkConnectionEstablished
        expr: sum(node_netstat_Tcp_CurrEstab) by (instance) > 1000
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "主机 ESTABLISHED 连接数过高"
          description: "主机 {{ .Labels.instance }}: TCP 连接数超过 1000 (当前值: {{ .Value }})"

      - alert: NodeNetworkThroughputIn
        expr: |
          sum by (instance, device) (rate(node_network_receive_bytes_total{device=~"ens.*|eth.*"}[2m])) / 1024 / 1024 > 100
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "主机网卡入口流量过高"
          description: "主机 {{ .Labels.instance }}, 网卡 {{ .Labels.device }} 入口流量 > 100 MB/s (当前值: {{ .Value }} MB/s)"

      - alert: NodeNetworkThroughputTransmit
        expr: |
          sum by (instance, device) (rate(node_network_transmit_bytes_total{device=~"ens.*|eth.*"}[2m])) / 1024 / 1024 > 100
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "主机网卡出口流量过高"
          description: "主机 {{ .Labels.instance }}, 网卡 {{ .Labels.device }} 出口流量 > 100 MB/s (当前值: {{ .Value }} MB/s)"
EOF


./promtool check config prometheus.yml # 检测配置是否有误
systemctl restart prometheus.service
systemctl status prometheus.service
# 此时访问 192.168.188.150:9090 可以获取到被监控端的数据

 

配置 node 的 grafana 页面

访问 192.168.188.150:3000 页面，grafana 默认的账号、密码都是 admin
设置中文：右上角头像 --> Profile --> Preferences --> Language --> 中文 --> Save preferences
添加数据源： 连接 --> 数据源 --> 添加新数据源 --> Promethues --> 输入IP地址、端口 --> 保存并测试

导入仪表盘：仪表盘 --> 新建 --> 导入 --> 输入：1860 --> import


 

告警配置

获取授权码：QQ邮箱 --> 设置 --> 账号 --> 继续获取授权码
授权码：自己的授权码

 

server 执行

cd /usr/local/alertmanager/
mkdir templates
cp alertmanager.yml alertmanager.yml.bak


# 注意，自己修改要4处（内容为 xxx 的地方是要改的地方），修改为自己邮箱相关的配置，因为是用于测试，发件人和收件人都可以为自己的邮箱
cat > alertmanager.yml << EOF
global: # 邮箱配置
  resolve_timeout: 1m # 告警解除后等待 1 分钟才标记为已解决
  smtp_smarthost: 'smtp.qq.com:465' # QQ 邮箱 SMTP 服务器地址和端口
  smtp_from: 'xxx@qq.com' # 发件人邮箱地址
  smtp_auth_username: 'xxx@qq.com' # SMTP 认证用户名（发件人邮箱）
  smtp_auth_password: 'xxx' # QQ 邮箱授权码
  smtp_require_tls: false # 是否启用 TLS 加密
templates: # 告警模板配置
  - '/usr/local/alertmanager/templates/*.tmpl' # 模板文件存放路径
route: # 告警路由配置
  group_by: ['alertname'] # 按告警名称分组
  group_wait: 10s # 分组内第一个告警等待时间（秒）
  group_interval: 10s # 同一分组告警发送间隔（秒）
  repeat_interval: 1h # 相同告警重复发送间隔（小时）
  receiver: 'mail' # 默认接收器名称
receivers: # 接收人配置
- name: 'mail' # 接收器名称
  email_configs:
  - to: 'xxx@qq.com' # 收件人邮箱地址
    send_resolved: true # 告警恢复时也发送通知
    html: '{{ template "email.template.tmpl" . }}'
EOF


# 告警模版
cat > templates/email.template.tmpl << EOF
{{ define "email.template.tmpl" }}
{{- if gt (len .Alerts.Firing) 0 -}}{{ range .Alerts }}

==================== 告警通知 ====================<br>
告警名称：{{ .Labels.alertname }}<br>
实例名：{{ .Labels.instance }}<br>
摘要：{{ .Annotations.summary }}<br>
详情：{{ .Annotations.description }}<br>
级别：{{ .Labels.severity }}<br>
开始时间：{{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}<br>

{{ end }}{{- end }}
{{- if gt (len .Alerts.Resolved) 0 -}}
{{ range .Alerts }}

==================== 告警恢复 ====================<br>
告警名称：{{ .Labels.alertname }}<br>
实例名：{{ .Labels.instance }}<br>
摘要：{{ .Annotations.summary }}<br>
详情：{{ .Annotations.description }}<br>
级别：{{ .Labels.severity }}<br>
开始时间：{{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}<br>
恢复时间：{{ (.EndsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}<br>

{{ end }}{{- end }}
{{- end }}
EOF


/usr/local/alertmanager/amtool check-config /usr/local/alertmanager/alertmanager.yml
systemctl restart alertmanager
systemctl status alertmanager

# 模拟告警
vim +17 /usr/local/prometheus/rules/node.yml # 将 17 行的大于改为小于，手动触发告警
systemctl restart prometheus.service
systemctl status prometheus.service

等到变红后，过一会就会触发告警

 

server 执行

vim +17 /usr/local/prometheus/rules/node.yml # 恢复，将 17 行的小于改为大于
systemctl restart prometheus.service
systemctl status prometheus.service