labels:

severity: warning

annotations:

summary: "{{$labels.instance}}: CPU使用过高"

description: "{{$labels.instance}}: CPU使用大于 80% (当前值: {{ $value }})"

# cat rules/reload.yml

groups:

- name: prometheus.rules

rules:

- alert: AlertmanagerReloadFailed

expr: alertmanager_config_last_reload_successful == 0

for: 10m

labels:

severity: warning

annotations:

summary: "{{$labels.instance}}: Alertmanager配置重新加载失败"

description: "{{$labels.instance}}: Alertmanager配置重新加载失败"

- alert: PrometheusReloadFailed

expr: prometheus_config_last_reload_successful == 0

for: 10m

labels:

severity: warning

annotations:

summary: "{{$labels.instance}}: Prometheus配置重新加载失败"

description: "{{$labels.instance}}: Prometheus配置重新加载失败"

在主节点进行操作

（ 1）安装 Alertmanager

[root@controlnode tools]# tar -xzf alertmanager-0.21.0-rc.0.linux-amd64.tar.gz

[root@controlnode tools]# mv alertmanager-0.21.0-rc.0.linux-amd64 /usr/local/

[root@controlnode tools]# cd /usr/local/

[root@controlnode local]# ln -s /usr/local/alertmanager-0.21.0-rc.0.linux-amd64/ /usr/local/alertmanager

（ 2）alert manager.yml 配置文件如下 (发送邮箱和接收者 )

[root@controlnode local]# vim alertmanager/alertmanager.yml

global:

resolve_timeout: 5m

# 解析的超时时间

smtp_smarthost: 'smtp.163.com:25'

smtp_from: 'hyjy2504164765@163.com'

smtp_auth_username: 'hyjy2504164765'

smtp_auth_password: 'linux123'

smtp_require_tls: false

# 配置邮件发送的smtp服务器

route:

# 路由规则

group_by: ['alertname']

# 报警分组依据,以报警名称为分组依据

group_wait: 10s

# 为一个组发送通知的初始等待时间

group_interval: 10s

# 在发送新告警前的等待时间

repeat_interval: 1h

# 发送重复告警的周期

receiver: 'mail'

# 邮件接收者

receivers:

# 报警接收者

- name: 'mail'

email_configs:

- to: 'hyjy2504164765@163.com'

# 给谁发送邮件

#inhibit_rules:

#  - source_match:

#      severity: 'critical'

#    target_match:

#      severity: 'warning'

#    equal: ['alertname', 'dev', 'instance']

# inhibit_rules: 表示报警抑制规则

（ 3）检查alertmanager配置文件

[root@controlnode local]# alertmanager/amtool check-config alertmanager/alertmanager.yml

Checking 'alertmanager/alertmanager.yml'  SUCCESS

Found:

- global config

- route

- 0 inhibit rules

- 1 receivers

- 0 templates

（ 4 ）将 alertmanager 加入 systemd

[root@controlnode local]# cd /usr/lib/systemd/system

[root@controlnode system]# vim alertmanager.service

[Unit]

Description=https://prometheus.io

[Service]

Restart=on-failure

ExecStart=/usr/local/alertmanager/alertmanager --config.file=/usr/local/alertmanager/alertmanager.yml

[Install]

WantedBy=multi-user.target

#重新加载 systemd

[root@controlnode system]# systemctl daemon-reload

（ 5 ）启动 alertmanager

[root@controlnode system]# systemctl start alertmanager.service

[root@controlnode system]# systemctl enable alertmanager.service

[root@controlnode system]# netstat -tunlp | grep alertmanager

tcp6       0      0 :::9093                 :::*                    LISTEN      3864/alertmanager

#查看版本信息

[root@controlnode system]# /usr/local/alertmanager/alertmanager --version

alertmanager, version 0.21.0-rc.0 (branch: HEAD, revision: 4346e1a51cf3d1226d3c44e28c7b25ff48f7b6b2)

build user:       root@a89e5ee3c12c

build date:       20200605-11:03:01

go version:       go1.14.4

（ 6 ）配置 prometheus与alertmanager通信、prometheus报警规则、alertmanager加入监控

#在 alerting: 、 rule_files: 、 scrape_configs 修改如下内容

[root@controlnode system]# cd /usr/local/prometheus

[root@controlnode prometheus]# vim prometheus.yml

# Alertmanager configuration

alerting:

alertmanagers:

- static_configs:

- targets:

- 127.0.0.1:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.

rule_files: