1
3)创建目录并写入配置
2
[root@node3 ~]# mkdir -p /etc/alertmanager /var/lib/alertmanager
3
[root@node3 ~]# cat > /etc/alertmanager/alertmanager.yml <<'EOF'
4
global:
5
  resolve_timeout: 5m          # 告警被视为"已解决"的等待时长, 超时仍无更新才标记 resolved
6
  # ===== 邮件发送方(发件人)配置 =====
7
  smtp_from: 'jiuhao_linux@163.com'          # 发件人邮箱
8
  smtp_smarthost: 'smtp.163.com:465'         # 邮件服务器:端口(465=SSL)
9
  smtp_auth_username: 'jiuhao_linux@163.com' # 登录账号(一般同发件人)
10
  smtp_auth_password: 'AZeQaZRg4hPWt3dK'     # 邮箱授权码(不是登录密码)
11
  smtp_require_tls: false      # 不校验服务端 TLS 证书(内网/测试用, 生产建议 true)
12
  smtp_hello: '163.com'        # EHLO 握手用的域名
13

14
route:
15
  group_by: ['alertname']      # 分组: 同名告警合并成一条通知
16
  group_wait: 5s               # 组内首条告警到达后等 5s 再发(攒批, 防轰炸)
17
  group_interval: 10s          # 同一组再次发送通知的间隔
18
  repeat_interval: 5m          # 告警持续未恢复时, 重复提醒的间隔
19
  receiver: 'kpyun_system'     # 顶层默认接收者(兜底: 子路由都没命中时走这里)
20
  # 配置子路由: 按 job 名正则匹配后分发给对应小组
21
  routes:
22
    - receiver: 'kpyun_dba'
23
      match_re:
24
        job: kpyun_dba_exporter
25
      # continue=true: 匹配后不中断, 继续向下匹配, 让消息最终也发给系统组
26
      continue: true
27
    - receiver: 'kpyun_k8s'
28
      match_re:
29
        job: kpyun_k8s_exporter
30
      continue: true
31
    - receiver: 'kpyun_system'
32
      match_re:
33
        job: .*                # .* 匹配任意 job, 即所有告警都再发给系统组
34
      continue: true
35

36
# dc = datacenter(数据中心/机房),它不是 Prometheus 内置的,而是我们自己在告警规则里打的一个自定义标签
37
# 告警抑制: 同一 dc 下, 若出现 critical 级告警, 则抑制(不发)同 dc 的 warning 级告警, 减少噪音
38
# dc 就是"机房标签，用来区分"这条告警属于哪个机房"；保证"只在同一机房内才抑制",避免把北京机房的 critical 拿去压制深圳机房的 warning
39
inhibit_rules:
40
  - source_match:              # 触发抑制的"源"条件
41
      severity: critical
42
    target_match:              # 被抑制的"目标"条件
43
      severity: warning
44
    equal:
45
      - dc                     # 源与目标 dc 标签值相同才抑制
46

47
receivers:
48
- name: 'kpyun_dba'            # 接收组(名字被 route 引用)
49
  email_configs:
50
  - to: 'jiuzhao996@qq.com'    # 该组接收邮箱
51
    send_resolved: true        # 告警恢复后仍发一封"已恢复"邮件
52
    headers: { Subject: "[WARN] kpyun linux 报警邮件" }  # 邮件主题
53
- name: 'kpyun_k8s'
54
  email_configs:
55
  - to: 'jiuzhao996@qq.com'
56
    send_resolved: true
57
    headers: { Subject: "[WARN] kpyun linux 报警邮件" }
58
- name: 'kpyun_system'
59
  email_configs:
60
  - to: 'jiuzhao996@qq.com'
61
    send_resolved: true
62
    headers: { Subject: "[WARN] kpyun linux 报警邮件" }
63
EOF
64

65
4)语法检查
66
[root@node3 ~]# amtool check-config /etc/alertmanager/alertmanager.yml
67
Checking '/etc/alertmanager/alertmanager.yml'  SUCCESS
68
Found:
69
 - global config
70
 - route
71
 - 1 inhibit rules
72
 - 3 receivers
73
 - 0 templates
74
✅️ 路由 + 子路由 + 抑制规则 校验通过
75

76
5)写 systemd 服务文件
77
[root@node3 ~]# cat > /usr/lib/systemd/system/alertmanager.service <<'EOF'
78
[Unit]
79
Description=kpyun Alertmanager
80
After=network.target
81

82
[Service]
83
Type=simple
84
ExecStart=/usr/local/bin/alertmanager \
85
  --config.file=/etc/alertmanager/alertmanager.yml \
86
  --storage.path=/var/lib/alertmanager \
87
  # Alertmanager 的 /-/reload 默认开启, 无需 --web.enable-lifecycle(那是 Prometheus 的参数)
88
  --web.listen-address=0.0.0.0:9093
89
Restart=on-failure
90
RestartSec=5
91

92
[Install]
93
WantedBy=multi-user.target
94
EOF
95

96
6)启动并验证
97
[root@node3 ~]# systemctl daemon-reload && systemctl enable --now alertmanager.service
98
[root@node3 ~]# systemctl is-active alertmanager.service
99
active
100
[root@node3 ~]# ss -lnt | grep 9093
101
LISTEN 0      4096               *:9093            *:*
102
✅️ Alertmanager 已在 node3:9093 监听
103

104
http://10.0.0.12:9093/#/status

1
`修改 Prometheus 配置(路由到 Alertmanager)`
2
1)Prom 上开启 alerting + rule_files
3
root@Prom ~# vim /etc/prometheus/prometheus.yml
4
root@Prom ~# egrep -v '^$|^[[:space:]]*#' /etc/prometheus/prometheus.yml
5
global:
6
  # 每 15 秒从所有配置的目标（targets）拉取（抓取）一次监控指标数据（控制采集数据的频率）
7
  scrape_interval: 15s
8
  # 每 5 秒评估一次告警规则和记录规则（控制检查规则的频率）
9
  evaluation_interval: 5s ✅
10
alerting:
11
  alertmanagers:
12
    - static_configs:
13
        - targets:
14
          # 告警目标地址，告诉 Prometheus "告警发到哪里"
15
          - 10.0.0.12:9093 ✅
16

17
rule_files:
18
  # 告警规则文件，告诉 Prometheus "告警规则从哪里读"
19
  - "/etc/prometheus/kpyun-linux-rules.yml" ✅
20
  - "/etc/prometheus/kpyun-linux-rules-inhibit.yml" ✅
21
scrape_configs:
22
  - job_name: "prometheus"
23
    scheme: 'https'
24
    tls_config:
25
      insecure_skip_verify: true
26
    basic_auth:
27
      username: kpyun
28
      password: kpyun123
29
    static_configs:
30
      - targets: ["10.0.0.10:9090"]
31
        labels:
32
          app: "prometheus"
33
`注释其余配置`
34

35
# 追加三个被监控 job(分别对应 dba/k8s/bigdata 分组)
36
root@Prom ~# cat >> /etc/prometheus/prometheus.yml << 'EOF'
37
  - job_name: "kpyun_dba_exporter"
38
    static_configs:
39
      - targets: ["10.0.0.2:9100"]
40
  - job_name: "kpyun_k8s_exporter"
41
    static_configs:
42
      - targets: ["10.0.0.11:9100"]
43
  - job_name: "kpyun_bigdata_exporter"
44
    static_configs:
45
      - targets: ["10.0.0.12:9100"]
46
EOF

1
1)基础规则: up==0 实例宕机即触发
2
root@Prom ~# cat > /etc/prometheus/kpyun-linux-rules.yml <<'EOF'
3
groups:
4
- name: kpyun-linux-rules-alert          # 规则组名, 一个文件可含多个 group
5
  rules:
6
  - alert: kpyun-dba_exporter-alert      # 告警名(唯一标识, Alertmanager 用它分组)
7
    expr: up{job="kpyun_dba_exporter"} == 0   # 触发条件: 该 job 实例 up=0(宕机)
8
    for: 1s        # 条件需持续为真 1s，告警才从 pending 变 firing（for 决定真正 firing 延迟）
9
    labels:             # 附加标签, 供路由/分组/抑制匹配
10
      school: kpyun
11
      apps: dba
12
    annotations:        # 告警内容(支持 Go 模板变量)
13
      summary: "{{ $labels.instance }} 数据库实例已停止运行！"  # $labels.instance 取实例名
14
      value: "{{ $value }}"             # $value 取触发时表达式的值(此处恒为 0)
15
  - alert: kpyun-k8s_exporter-alert
16
    expr: up{job="kpyun_k8s_exporter"} == 0
17
    for: 1s
18
    labels:
19
      school: kpyun
20
      apps: k8s
21
    annotations:
22
      summary: "{{ $labels.instance }} K8S服务器已停止运行！"
23
      value: "{{ $value }}"
24
  - alert: kpyun-bigdata_exporter-alert
25
    expr: up{job="kpyun_bigdata_exporter"} == 0
26
    for: 1s
27
    labels:
28
      school: kpyun
29
      apps: bigdata
30
    annotations:
31
      summary: "{{ $labels.instance }} 大数据服务器已停止运行！"
32
      value: "{{ $value }}"
33
EOF
34

35
2)抑制配套规则: 打上 severity/critical|warning + dc 标签
36
root@Prom ~# cat > /etc/prometheus/kpyun-linux-rules-inhibit.yml <<'EOF'
37
groups:
38
- name: kpyun-linux-rules-alert-inhibit  # 抑制配套规则组(结构与第一套完全一致)
39
  rules:
40
  # 相比第一套: 仅额外打了 severity / dc 两个标签, 供 Alertmanager inhibit_rules 按级别+机房匹配
41
  - alert: kpyun-dba_exporter-alert-xixi
42
    expr: up{job="kpyun_dba_exporter"} == 0
43
    for: 3s       # 比第一套稍长, 让 critical 与 warning 时间重叠, 抑制才生效
44
    labels:
45
      apps: dba
46
      severity: critical  # 严重级别: 关键(抑制的"源")
47
      dc: beijing      # 机房标签: 北京
48
    annotations:
49
      summary: "{{ $labels.instance }} 数据库实例已停止运行超过 3s！"
50
      value: "{{ $value }}"
51
  - alert: kpyun-k8s_exporter-alert-haha
52
    expr: up{job="kpyun_k8s_exporter"} == 0
53
    for: 3s
54
    labels:
55
      apps: k8s
56
      severity: warning    # 警告(被抑制的"目标")
57
      dc: beijing      # 与上方 critical 同 dc → 满足 inhibit 的 equal:[dc]
58
    annotations:
59
      summary: "{{ $labels.instance }} K8S服务器已停止运行超过 3s！"
60
      value: "{{ $value }}"
61
  - alert: kpyun-bigdata_exporter-alert-hehe
62
    expr: up{job="kpyun_bigdata_exporter"} == 0
63
    for: 5s
64
    labels:
65
      apps: bigdata
66
      severity: warning
67
      dc: shenzhen      # 不同 dc → 不被上面 beijing 的 critical 抑制
68
    annotations:
69
      summary: "{{ $labels.instance }} 大数据服务器已停止运行超过 5s！"
70
      value: "{{ $value }}"
71
EOF
72

73
3)语法检查
74
root@Prom ~# promtool check config /etc/prometheus/prometheus.yml
75
Checking prometheus.yml
76
  SUCCESS: 2 rule files found
77
 SUCCESS: prometheus.yml is valid prometheus config file syntax
78
Checking /etc/prometheus/kpyun-linux-rules.yml
79
  SUCCESS: 3 rules found
80
Checking /etc/prometheus/kpyun-linux-rules-inhibit.yml
81
  SUCCESS: 3 rules found
82
✅️ 2 个规则文件，共 6 条规则
83

84
4)热加载(Prometheus 已带 --web.enable-lifecycle)
85
root@Prom ~# curl -s -k -u kpyun:kpyun123 -X POST https://10.0.0.10:9090/-/reload
86
'Prometheus 已把警报发给 Alertmanager'
87

88
https://10.0.0.10:9090/alerts

1
1)写模板
2
[root@node3 ~]# mkdir -p /etc/alertmanager/tmpl
3
[root@node3 ~]# cat > /etc/alertmanager/tmpl/kpyun.html <<'EOF'
4
{{ define "kpyun-email" }}
5
<!DOCTYPE html>
6
<html>
7
<head>
8
  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
9
  <title>{{ if eq .Status "firing" }}&#128680; 告警触发{{ else }}&#9989; 告警恢复{{ end }}</title>
10
  <style>
11
    @font-face { font-family: "EmojiFont"; src: local("Apple Color Emoji"), local("Segoe UI Emoji"), local("Noto Color Emoji"); }
12
    body { font-family: 'Segoe UI', system-ui, sans-serif, "EmojiFont"; line-height: 1.6; color: #333; max-width: 800px; margin: 20px auto; padding: 0 20px; background-color: #f9f9f9; }
13
    .header { text-align: center; padding: 24px; border-radius: 12px; margin-bottom: 24px; background: {{ if eq .Status "firing" }}#fff0f0{{ else }}#f0fff4{{ end }}; border: 2px solid {{ if eq .Status "firing" }}#FF5252{{ else }}#4CAF50{{ end }}; }
14
    .header h1 { margin: 0; font-size: 22px; }
15
    .alert-card { border-radius: 8px; padding: 20px; margin-bottom: 20px; background: {{ if eq .Status "firing" }}linear-gradient(135deg, #FFF6F6 0%, #FFEBEB 100%){{ else }}linear-gradient(135deg, #F6FFF6 0%, #EBFFEB 100%){{ end }}; border-left: 5px solid {{ if eq .Status "firing" }}#FF5252{{ else }}#4CAF50{{ end }}; box-shadow: 0 2px 10px rgba(0,0,0,0.1); }
16
    .alert-field { margin-bottom: 8px; display: flex; }
17
    .field-label { font-weight: bold; min-width: 95px; color: #555; }
18
    .field-value { flex: 1; }
19
    .severity-critical { color: #FF5252; font-weight: bold; }
20
    .severity-warning { color: #FFBB33; font-weight: bold; }
21
    .divider { height: 1px; background: #eee; margin: 12px 0; }
22
    .footer { text-align: center; color: #888; font-size: 13px; margin-top: 20px; }
23
  </style>
24
</head>
25
<body>
26
  <div class="header">
27
    <h1>{{ if eq .Status "firing" }}<span class="emoji">&#128680;</span> 告警触发通知{{ else }}<span class="emoji">&#9989;</span> 告警恢复通知{{ end }}</h1>
28
  </div>
29

30
  {{ range .Alerts }}
31
  <div class="alert-card">
32
    <div class="alert-field"><span class="field-label">&#128227; 告警名称</span><span class="field-value">{{ .Labels.alertname }}</span></div>
33
    <div class="alert-field"><span class="field-label">&#9888;&#65039; 告警级别</span><span class="field-value severity-{{ .Labels.severity }}">{{ if .Labels.severity }}{{ .Labels.severity }}{{ else }}无{{ end }}</span></div>
34
    <div class="alert-field"><span class="field-label">&#128346; 目标机器</span><span class="field-value">{{ .Labels.instance }}</span></div>
35
    <div class="alert-field"><span class="field-label">&#128221; 告警摘要</span><span class="field-value">{{ .Annotations.summary }}</span></div>
36
    {{ if eq .Status "firing" }}
37
    <div class="alert-field"><span class="field-label">&#128337; 触发时间</span><span class="field-value">{{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}</span></div>
38
    {{ else }}
39
    <div class="alert-field"><span class="field-label">&#9203; 持续时间</span><span class="field-value">{{ .StartsAt.Format "15:04:05" }} - {{ .EndsAt.Format "15:04:05" }} ({{ .EndsAt.Sub .StartsAt }})</span></div>
40
    <div class="alert-field"><span class="field-label">&#9989; 恢复时间</span><span class="field-value">{{ (.EndsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}</span></div>
41
    {{ end }}
42
    {{- if .Annotations.description }}
43
    <div class="divider"></div>
44
    <div class="alert-field"><span class="field-label">&#128209; 详细描述</span><span class="field-value">{{ .Annotations.description }}</span></div>
45
    {{- end }}
46
  </div>
47
  {{ end }}
48

49
  <div class="footer">Kpyun 自动化运维 &middot; 本次共 {{ len .Alerts }} 条告警</div>
50
</body>
51
</html>
52
{{ end }}
53
EOF
54

55
2)alertmanager.yml 挂接模板(两处改动)
56
[root@node3 ~]# vim /etc/alertmanager/alertmanager.yml
57
# 改动一: 顶层新增 templates 指向模板文件
58
templates:
59
  - '/etc/alertmanager/tmpl/kpyun.html'
60

61
# 改动二: receivers 的 kpyun_dba 下加 html 引用
62
- name: 'kpyun_dba'
63
  email_configs:
64
  - to: 'jiuzhao996@qq.com'
65
    send_resolved: true
66
    html: '{{ template "kpyun-email" . }}'   # 用自定义 HTML 模板渲染邮件
67
    headers: { Subject: "[WARN] kpyun linux 报警邮件" }
68

69
3)校验+热重载
70
[root@node3 ~]# amtool check-config /etc/alertmanager/alertmanager.yml
71
 - 1 inhibit rules
72
 - 3 receivers
73
 - 1 templates
74
✅️ 模板已加载(1 templates), 发往 dba 组的邮件会用该 HTML 渲染
75
# 校验通过只是第一步, 配置必须重新加载才生效
76
# Alertmanager 的 /-/reload 默认开启, 无需 --web.enable-lifecycle, 该 flag 是 Prometheus 的参数
77
[root@node3 ~]# curl -s -X POST http://localhost:9093/-/reload

1
`部署钉钉插件(node2)`
2
1)解包
3
jiuzhao@Ubuntu ~$ scp /home/jiuzhao/下载/prometheus-webhook-dingtalk-2.1.0.linux-amd64.tar.gz node2:/server/pkgs/
4
[root@node2 ~]# tar xf /server/pkgs/prometheus-webhook-dingtalk-2.1.0.linux-amd64.tar.gz -C /usr/local/bin prometheus-webhook-dingtalk-2.1.0.linux-amd64/prometheus-webhook-dingtalk --strip-components=1
5
# 只需要解压这个二进制工具
6

7
2)配置(占位，换成你自己的机器人)
8
[root@node2 ~]# mkdir -p /etc/dingtalk
9
[root@node2 ~]# cat > /etc/dingtalk/config.yml <<'EOF'
10
targets:
11
  kpyun-linux:
12
    # 对应的是dingding的webhook
13
    url: https://oapi.dingtalk.com/robot/send?access_token=f9c739ae3516301eea13f55a2bcafa8a9397c3d82ca69c6610209ed75c6b888d
14
    # 对应的是"加签"的值，复制过来即可
15
    secret: "SEC97c0de0400bdc90469546381f381d38c1b35a612101dcfef4ab0a7204764d554"
16
EOF

1
3)systemd + 启动
2
[root@node2 ~]# cat > /usr/lib/systemd/system/prometheus-webhook-dingtalk.service <<'EOF'
3
[Unit]
4
Description=kpyun Prometheus Webhook Dingtalk
5
After=network.target
6

7
[Service]
8
Type=simple
9
ExecStart=/usr/local/bin/prometheus-webhook-dingtalk --web.listen-address=0.0.0.0:8060 --config.file=/etc/dingtalk/config.yml
10
Restart=on-failure
11
RestartSec=5
12

13
[Install]
14
WantedBy=multi-user.target
15
EOF
16
[root@node2 ~]# systemctl daemon-reload && systemctl enable --now prometheus-webhook-dingtalk.service
17

18
4)验证
19
[root@node2 ~]# ss -lnt | grep 8060
20
LISTEN 0      4096               *:8060            *:*
21
✅️ 钉钉插件已在 node2:8060 监听
22
[root@node2 ~]# journalctl -u prometheus-webhook-dingtalk.service | tail -8 | egrep 'Started|urls'
23
Aug 22 15:24:53 node2 systemd[1]: Started prometheus-webhook-dingtalk.service - kpyun Prometheus Webhook Dingtalk.
24
Aug 22 15:24:53 node2 prometheus-webhook-dingtalk[2436]: ts=2026-08-22T07:24:53.842Z caller=main.go:113 component=configuration msg="Webhook urls for prometheus alertmanager" urls=http://0.0.0.0:8060/dingtalk/kpyun-linux/send
25
`💡 就是上面这个地址！【可以直接复制】`
26

27
5)联合 Alertmanager（真正把告警推到钉钉）
28
[root@node3 ~]# vim /etc/alertmanager/alertmanager.yml
29
......  # 修改Alertmanager的配置文件
30
`这段要放在 alertmanager.yml 的某个 receiver 下`
31
- name: 'kpyun_system'
32
#  email_configs:
33
#  - to: 'jiuzhao996@qq.com'
34
#    send_resolved: true
35
#    html: '{{ template "kpyun-email" . }}'   # 用自定义 HTML 模板渲染邮件
36
#    headers: { Subject: "[WARN] kpyun linux 报警邮件" }
37
  webhook_configs:
38
    # 指向的是Prometheus的钉钉插件发送端地址
39
  - url: 'http://10.0.0.11:8060/dingtalk/kpyun-linux/send'
40
    # 空字典=用默认 HTTP 设置(不配代理/TLS/认证); 跨网或需鉴权时在此填写
41
    http_config: {}
42
    # 单次 webhook 最多携带多少条告警, 0 = 不限制(全部带上)
43
    max_alerts: 0
44
    # 告警恢复后也向钉钉发一条"已恢复"消息
45
    send_resolved: true
46

47
6)校验+热重载
48
[root@node3 ~]# amtool check-config /etc/alertmanager/alertmanager.yml
49
Checking '/etc/alertmanager/alertmanager.yml' ✅️ SUCCESS
50
[root@node3 ~]# curl -s -X POST http://localhost:9093/-/reload
51

52
7)模拟故障
53
[root@node1 ~]# systemctl stop node-exporter.service
54
[root@node1 ~]# systemctl start node-exporter.service

1
1)拷贝安装
2
jiuzhao@Ubuntu 下载$ scp /home/jiuzhao/下载/etcd-v3.7.1-linux-amd64.tar.gz node1:/server/pkgs/
3

4
[root@node1 ~]# tar -xf /server/pkgs/etcd-v3.7.1-linux-amd64.tar.gz -C /usr/local/bin etcd-v3.7.1-linux-amd64/etcd{,utl,ctl}  --strip-components=1
5
[root@node1 ~]# ll /usr/local/bin/etcd*
6
-rwxr-xr-x 1 1000 1000 24801442 Jul 24 03:32 /usr/local/bin/etcd
7
-rwxr-xr-x 1 1000 1000 16789666 Jul 24 03:32 /usr/local/bin/etcdctl
8
-rwxr-xr-x 1 1000 1000 16478370 Jul 24 03:32 /usr/local/bin/etcdutl
9
`只有这三个工具对我们有用！`
10
[root@node1 ~]# etcdctl version
11
etcdctl version: 3.7.1
12
API version: 3.7
13

14
2)分发其他节点
15
[root@node1 ~]# scp /usr/local/bin/etcd* 10.0.0.11:/usr/local/bin
16
[root@node1 ~]# scp /usr/local/bin/etcd* 10.0.0.12:/usr/local/bin
17
[root@node3 ~]# etcdctl version
18
etcdctl version: 3.7.1
19
API version: 3.7

1
3)在 Prom 的既有 CA 目录下签 etcd 证书
2
root@Prom ~# cd /etc/prometheus/certs
3
root@Prom ~# openssl genrsa -out private/etcd.kpyun.com.key 4096
4
root@Prom ~# openssl req -new -key private/etcd.kpyun.com.key -out etcd.kpyun.com.csr \
5
  -subj "/C=CN/ST=Beijing/L=Beijing/O=kpyun/OU=Etcd/CN=etcd.kpyun.com"
6
root@Prom ~# cat > etcd-server.ext <<'EOF'
7
authorityKeyIdentifier=keyid,issuer
8
basicConstraints=CA:FALSE
9
keyUsage=critical,digitalSignature,keyEncipherment
10
extendedKeyUsage=serverAuth,clientAuth
11
subjectAltName=@alt_names
12
[alt_names]
13
DNS.1=node1
14
DNS.2=node2
15
DNS.3=node3
16
IP.1=10.0.0.2
17
IP.2=10.0.0.11
18
IP.3=10.0.0.12
19
IP.4=127.0.0.1
20
EOF
21
root@Prom ~# openssl x509 -req -in etcd.kpyun.com.csr -CA certs/intermediate.crt \
22
  -CAkey private/intermediate.key -CAcreateserial -out certs/etcd.kpyun.com.crt \
23
  -days 365 -sha256 -extfile etcd-server.ext
24
root@Prom ~# cat certs/rootCA.crt certs/intermediate.crt > certs/etcd-ca-bundle.pem
25
root@Prom ~# openssl verify -CAfile certs/rootCA.crt -untrusted certs/intermediate.crt certs/etcd.kpyun.com.crt
26
certs/etcd.kpyun.com.crt: OK
27
✅️ Root CA → Intermediate CA → etcd server 证书 验证通过
28

29
4)分发三件套到三节点 /etc/etcd/certs 与 Prom(监控用)
30
# 三件套 = 证书(etcd.kpyun.com.crt→etcd-server.pem) / 私钥(etcd.kpyun.com.key→etcd-server-key.pem) / CA 链(etcd-ca-bundle.pem→etcd-ca.pem)
31
# /etc/etcd 与 /etc/etcd/certs 必须提前建
32
# /var/lib/etcd 由 etcd 首次启动自动创建, 无需手动建(本例 etcd 以 root 运行, 有创建权限)
33
# 前置(Prom 一次性): 装 sshpass 免交互输密码; 全机密码统一为 1
34
root@Prom ~# apt-get install -y sshpass
35
# sshpass -p '1' 自动填密码; -o StrictHostKeyChecking=accept-new 首次自动接受主机密钥并写入 known_hosts(跳过 yes, 重跑不再提示; 实验室环境专用, 生产改用 SSH key)
36
root@Prom ~# for N in 10.0.0.2 10.0.0.11 10.0.0.12; do
37
  sshpass -p '1' ssh -o StrictHostKeyChecking=accept-new root@$N 'mkdir -p /etc/etcd/certs'
38
  sshpass -p '1' scp -o StrictHostKeyChecking=accept-new /etc/prometheus/certs/certs/etcd.kpyun.com.crt   root@$N:/etc/etcd/certs/etcd-server.pem
39
  sshpass -p '1' scp -o StrictHostKeyChecking=accept-new /etc/prometheus/certs/private/etcd.kpyun.com.key root@$N:/etc/etcd/certs/etcd-server-key.pem
40
  sshpass -p '1' scp -o StrictHostKeyChecking=accept-new /etc/prometheus/certs/certs/etcd-ca-bundle.pem   root@$N:/etc/etcd/certs/etcd-ca.pem
41
done
42
# Prom 自身也要一份(给 Prometheus 抓 etcd /metrics 走 mTLS)
43
root@Prom ~# mkdir -p /etc/prometheus/certs/etcd
44
root@Prom ~# cp /etc/prometheus/certs/certs/etcd.kpyun.com.crt   /etc/prometheus/certs/etcd/etcd-server.pem
45
root@Prom ~# cp /etc/prometheus/certs/private/etcd.kpyun.com.key /etc/prometheus/certs/etcd/etcd-server-key.pem
46
root@Prom ~# cp /etc/prometheus/certs/certs/etcd-ca-bundle.pem   /etc/prometheus/certs/etcd/etcd-ca.pem
47
✅️ 三节点 + Prom 各就位三件套(证书/私钥/CA)

1
5)node1 的配置文件(其余节点改 name / IP 即可)
2
[root@node1 ~]# cat > /etc/etcd/etcd.config.yml <<'EOF'
3
name: 'node1'
4
data-dir: /var/lib/etcd
5
snapshot-count: 5000
6
heartbeat-interval: 100
7
election-timeout: 1000
8
quota-backend-bytes: 0
9
listen-peer-urls: 'https://10.0.0.2:2380'
10
listen-client-urls: 'https://10.0.0.2:2379,http://127.0.0.1:2379'
11
max-snapshots: 3
12
max-wals: 5
13
initial-advertise-peer-urls: 'https://10.0.0.2:2380'
14
advertise-client-urls: 'https://10.0.0.2:2379'
15
initial-cluster: 'node1=https://10.0.0.2:2380,node2=https://10.0.0.11:2380,node3=https://10.0.0.12:2380'
16
initial-cluster-token: 'etcd-kpyun-cluster'
17
initial-cluster-state: 'new'
18
strict-reconfig-check: false
19
enable-pprof: true
20
proxy: 'off'
21
client-transport-security:
22
  cert-file: '/etc/etcd/certs/etcd-server.pem'
23
  key-file: '/etc/etcd/certs/etcd-server-key.pem'
24
  client-cert-auth: true
25
  trusted-ca-file: '/etc/etcd/certs/etcd-ca.pem'
26
  auto-tls: true
27
peer-transport-security:
28
  cert-file: '/etc/etcd/certs/etcd-server.pem'
29
  key-file: '/etc/etcd/certs/etcd-server-key.pem'
30
  peer-client-cert-auth: true
31
  trusted-ca-file: '/etc/etcd/certs/etcd-ca.pem'
32
  auto-tls: true
33
debug: false
34
log-outputs: [default]
35
EOF
36
# 三节点 mutual TLS: client-cert-auth + peer-client-cert-auth 都用同一张 etcd 证书
37
======================================================================
38
# 把 node1 配置分发到 node2 / node3: 用 sed 改 name 与本机 IP, 省去每台手敲
39
# initial-cluster: 它列举【全部三节点】, 每个节点必须一字不差, 绝不能改, 所以要跳过该行
40
# 只改本机相关的 4 处 URL(listen-peer / listen-client / initial-advertise-peer / advertise-client)
41
# node1 也装 sshpass(它要主动 scp 给 node2/node3); 同样跳过 yes
42
[root@node1 ~]# dnf install -y sshpass
43
[root@node1 ~]# sed -e '/initial-cluster:/!s#https://10.0.0.2:#https://10.0.0.11:#g' \
44
  -e "s/^name: 'node1'/name: 'node2'/" \
45
  /etc/etcd/etcd.config.yml > /tmp/etcd.config.node2.yml
46
[root@node1 ~]# echo -e '10.0.0.11    node2\n10.0.0.12    node3' >> /etc/hosts
47
`hosts解析`
48
[root@node1 ~]# sshpass -p '1' scp -o StrictHostKeyChecking=accept-new /tmp/etcd.config.node2.yml root@node2:/etc/etcd/etcd.config.yml
49

50
[root@node1 ~]# sed -e '/initial-cluster:/!s#https://10.0.0.2:#https://10.0.0.12:#g' \
51
  -e "s/^name: 'node1'/name: 'node3'/" \
52
  /etc/etcd/etcd.config.yml > /tmp/etcd.config.node3.yml
53
[root@node1 ~]# sshpass -p '1' scp -o StrictHostKeyChecking=accept-new /tmp/etcd.config.node3.yml root@node3:/etc/etcd/etcd.config.yml
54
✅️ node2 / node3 配置就位(name 与 IP 已对应替换, initial-cluster 三节点保持一致)

1
6)写 unit(三节点相同)
2
[root@node1 ~]# cat > /usr/lib/systemd/system/etcd.service <<'EOF'
3
[Unit]
4
Description=kpyun Etcd
5
After=network.target
6

7
[Service]
8
# simple = 进程起了就算活；notify = 服务真能干活了才算活
9
Type=notify
10
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
11
Restart=on-failure
12
RestartSec=10
13
LimitNOFILE=65536
14

15
[Install]
16
WantedBy=multi-user.target
17
EOF
18
[root@node1 ~]# systemctl daemon-reload && systemctl enable --now etcd.service
19
# node2 / node3 同样操作，然后三台一起启动
20
[root@node3 ~]# systemctl is-active etcd.service
21
active
22

23
7)查看集群状态
24
[root@node1 ~]# etcdctl --endpoints="https://10.0.0.2:2379,https://10.0.0.11:2379,https://10.0.0.12:2379" \
25
  --cacert=/etc/etcd/certs/etcd-ca.pem --cert=/etc/etcd/certs/etcd-server.pem \
26
  --key=/etc/etcd/certs/etcd-server-key.pem endpoint status --write-out=table
27
# 实际输出列较多，下面只抽取关键列：
28
✅️ 三节点 3.7.1 集群成立，node1(10.0.0.2) 为 Leader（RAFT TERM 2）

1
1)添加别名(免敲长证书参数)
2
[root@node1 ~]# cat >> /root/.bashrc <<'EOF'
3

4
# kpyun etcd 集群快捷别名
5
alias etcdctl='etcdctl --endpoints="https://10.0.0.2:2379,https://10.0.0.11:2379,https://10.0.0.12:2379" --cacert=/etc/etcd/certs/etcd-ca.pem --cert=/etc/etcd/certs/etcd-server.pem --key=/etc/etcd/certs/etcd-server-key.pem'
6
EOF
7
[root@node1 ~]# source /root/.bashrc
8

9
2)查看集群成员列表
10
[root@node1 ~]# etcdctl member list
11
ID               状态      名称    Peer URL(节点间通信)    Client URL(客户端访问)
12
217112283068c8a, started, node1, https://10.0.0.2:2380, https://10.0.0.2:2379, false
13
787c4515ec48fd16, started, node2, https://10.0.0.11:2380, https://10.0.0.11:2379, false
14
9c5a59fd7ae10a75, started, node3, https://10.0.0.12:2380, https://10.0.0.12:2379, false
15
# 最后一列 `false` 是「是否为 learner 成员」（学习者节点）标记，不是是否 Leader！
16
'三行都为 `false` 表示三个节点都是正式投票成员（非 learner）'
17
# 想看谁是主节点，依靠 `member list` 是看不出来的
18

19

20
3)查看各节点健康状态和详细信息
21
[root@node1 ~]# etcdctl endpoint status
22
https://10.0.0.2:2379, 217112283068c8a, 3.7.1, 3.7.0, 20 kB, 16 kB, 20%, 2.1 GB, false, false, 3, 16, 16, , , false
23
https://10.0.0.11:2379, 787c4515ec48fd16, 3.7.1, 3.7.0, 20 kB, 16 kB, 20%, 2.1 GB, true, false, 3, 16, 16, , , false
24
https://10.0.0.12:2379, 9c5a59fd7ae10a75, 3.7.1, 3.7.0, 20 kB, 16 kB, 20%, 2.1 GB, false, false, 3, 16, 16, , , false
25

26

27
4)写入键值对（增 / 改）
28
[root@node1 ~]# etcdctl put school kpyun
29
OK
30
[root@node1 ~]# etcdctl put /class linux
31
OK
32
# "/" 只是普通字符，etcd 本身不分子目录，靠 --prefix 做"伪目录"范围查询
33
# 改 = 同一个键再 put 一次即覆盖旧值（etcd 按 revision 记录每次写，旧值仍能在历史里读到）
34
[root@node1 ~]# etcdctl put school kpyun-v2
35
OK
36
[root@node1 ~]# etcdctl get school
37
# 默认 键key和值value 都会打印
38
school
39
kpyun-v2
40

41
5)查询键值（查）
42
[root@node1 ~]# etcdctl get school --print-value-only
43
--print-value-only    # 只打印值
44
kpyun-v2
45

46
[root@node1 ~]# etcdctl get "" --prefix --keys-only
47
--prefix         # 前缀匹配（空串=匹配全部键）
48
--keys-only       # 只打印键，不打印值（和 --print-value-only 正好相反）
49
/class
50

51
school
52

53
6)查看某键当前 revision
54
[root@node1 ~]# etcdctl get school -w json | jq
55
{
56
  "header": {
57
    "revision": 6,      # 是当前集群总版本号
58
    "raft_term": 3      # Raft 任期（每次重新选主 +1）
59
  },
60
  "kvs": [
61
    {
62
      "key": "c2Nob29s",  # 解码= school
63
      "create_revision": 3,  # 首次写入时的 revision（= 第一次 put school）
64
      "mod_revision": 5,  # 最近一次修改的 revision（= 改成 kpyun-v2 那次）
65
      "version": 2,      # 该键被改过几次（2 次 put）
66
      "value": "a3B5dW4tdjI="  # 解码= kpyun-v2
67
    }
68
  ],
69
  "count": 1
70
}
71
`注意 key/value 是 base64 编码，需解码才可读`
72
[root@node1 ~]# echo 'c2Nob29s' | base64 -d ;echo
73
school
74
[root@node1 ~]# echo 'a3B5dW4tdjI=' | base64 -d ;echo
75
kpyun-v2
76
`etcd 是 revision 模型，每次写都产生新 revision`
77
[root@node1 ~]# etcdctl put school kpyun-v3
78
OK
79
# get -w json 只给首尾 revision（创建时和最近一次修改）
80

81
7)重放历史事件
82
`etcdctl watch <key> --rev=<create_revision> -w json | jq`
83
[root@node1 ~]# etcdctl watch school --rev=3 -w json | jq
84
...
85
"Events": [
86
    {
87
      "kv": {
88
        "key": "c2Nob29s",
89
        "create_revision": 3,
90
        "mod_revision": 3,
91
        "version": 1,
92
        "value": "a3B5dW4="
93
      }
94
    },
95
    {
96
      "kv": {
97
        "key": "c2Nob29s",
98
        "create_revision": 3,
99
        "mod_revision": 5,
100
        "version": 2,
101
        "value": "a3B5dW4tdjI="
102
      }
103
    },
104
    {
105
      "kv": {
106
        "key": "c2Nob29s",
107
        "create_revision": 3,
108
        "mod_revision": 7,
109
        "version": 3,
110
        "value": "a3B5dW4tdjM="
111
      }
112
    }
113
...    # 每条事件的 mod_revision 就是各版本的落点
114
# key 被覆盖后旧值不立即消失，可用 --rev 回看历史
115
[root@node1 ~]# etcdctl get school --rev=3
116
school
117
kpyun  # 覆盖前：school 当时还是 kpyun
118
[root@node1 ~]# etcdctl get school --rev=5
119
school
120
kpyun-v2
121
[root@node1 ~]# etcdctl get school --rev=7
122
school
123
kpyun-v3
124

125
8)删除键值（删）
126
[root@node1 ~]# etcdctl del /class
127
1
128
# 删除同样支持范围删除，但极危险！下面这条等于清空整个 keyspace，生产千万别手滑：
129
# etcdctl del "" --prefix
130
[root@node1 ~]# etcdctl get "" --prefix
131
school
132
kpyun-v3
133
# ✅️ 增删改查与 Redis/Zookeeper 类似，都是键值对；
134
# 区别：etcd 是 revision 模型（每次写都留历史，可 --rev 回看）

1
1)先写点数据便于验证
2
[root@node1 ~]# etcdctl put /kpyun/backup/demo "restore-demo"
3
OK
4

5
2)对单节点(10.0.0.2)做快照
6
# 相当于备份数据
7
`注意快照只能对 单节点 发起，不能一次指定多 endpoint`
8
# 我们之前有别名alias，别名中指定的是多服务端点地址，\etcdctl 取消别名
9
[root@node1 ~]# \etcdctl snapshot save /tmp/kpyun-etcd-`date +%F`.backup
10
Snapshot saved at /tmp/kpyun-etcd-2026-08-23.backup
11
Server version 3.7.0
12

13
3)查看快照信息
14
`这里用的是 etcdutl`
15
[root@node1 ~]# etcdutl snapshot --help
16
用法：
17
  etcdutl snapshot [命令]
18

19
可用命令：
20
  restore     将 etcd 成员快照恢复到 etcd 目录
21
  status      获取给定文件的后端快照状态
22

23
[root@node1 ~]# etcdutl snapshot status /tmp/kpyun-etcd-`date +%F`.backup -w table
24
┌──────────┬──────────┬────────────┬────────────┬─────────┐
25
│   HASH   │ REVISION │ TOTAL KEYS │ TOTAL SIZE │ VERSION │
26
├──────────┼──────────┼────────────┼────────────┼─────────┤
27
│ 8c95d9f6 │       11 │          1 │      20 kB │   3.7.0 │
28
└──────────┴──────────┴────────────┴────────────┴─────────┘
29
✅️ 1 个 key，revision 11，来自 3.7.0 集群
30

31
4)拷贝备份数据
32
[root@node1 ~]# sshpass -p '1' scp -o StrictHostKeyChecking=accept-new /tmp/kpyun-etcd-`date +%F`.backup node2:/tmp
33
[root@node1 ~]# sshpass -p '1' scp -o StrictHostKeyChecking=accept-new /tmp/kpyun-etcd-`date +%F`.backup node3:/tmp

1
5)三台停服并清空旧 restore 目录
2
[root@node1 ~]# systemctl stop etcd.service
3
# node2/node3 同理
4
[root@node1 ~]# rm -rf /var/lib/etcd
5
# 之前的数据都在 /var/lib/etcd（etcd 首次启动自动创建）
6

7
6)每台用自己 --name 恢复(以 node1 为例)
8
`重新指定了一个存储目录 /var/lib/etcd-restore`
9
[root@node1 ~]# etcdutl snapshot restore /tmp/kpyun-etcd-`date +%F`.backup \
10
  --data-dir=/var/lib/etcd-restore --name=node1 \
11
  --initial-cluster='node1=https://10.0.0.2:2380,node2=https://10.0.0.11:2380,node3=https://10.0.0.12:2380' \
12
  --initial-advertise-peer-urls=https://10.0.0.2:2380
13
# node2/node3 同理，改 --name / --initial-advertise-peer-urls 即可
14
successfully ✅
15
  "path": "/tmp/kpyun-etcd-2026-08-23.backup",
16
  "wal-dir": "/var/lib/etcd-restore/member/wal",
17
  "data-dir": "/var/lib/etcd-restore",
18
  "snap-dir": "/var/lib/etcd-restore/member/snap",
19
[root@node1 ~]# tree /var/lib/etcd-restore
20
/var/lib/etcd-restore
21
└── member
22
    ├── snap
23
    │   ├── 0000000000000001-0000000000000003.snap
24
    │   └── db
25
    └── wal
26
        └── 0000000000000000-0000000000000000.wal
27

28
7)把配置的 数据目录 指向 恢复目录，三台一起启动
29
data-dir: /var/lib/etcd
30
[root@node1 ~]# sed -i 's#/var/lib/etcd#/var/lib/etcd-restore#g' /etc/etcd/etcd.config.yml
31
[root@node1 ~]# systemctl restart etcd.service
32
# node2/node3 同时
33
[root@node1 ~]# etcdctl endpoint status --write-out=table
34
# 三节点重新 healthy，node1 再次成为 Leader  ✅
35

36
8)验证数据恢复
37
[root@node1 ~]# etcdctl get /kpyun/backup/demo
38
/kpyun/backup/demo
39
restore-demo
40
✅️ 数据从快照完整恢复

1
1)Docker 虚拟机拉镜像(xuanyuan 镜像源，不用配代理)
2
root@Docker ~# docker pull docker.xuanyuan.run/tzfun/etcd-workbench:1.1.4
3
root@Docker ~# docker tag docker.xuanyuan.run/tzfun/etcd-workbench:1.1.4 etcd-workbench:1.1.4
4
root@Docker ~# docker rmi docker.xuanyuan.run/tzfun/etcd-workbench:1.1.4
5
root@Docker ~# docker images | grep etcd
6
etcd-workbench:1.1.4  c58de0e1b96e    479MB          137MB
7

8
2)准备配置(账号 admin:kpyun123)并运行
9
root@Docker ~# cat > /root/etcd-workbench.conf <<'EOF'
10
[server]
11
# 服务监听端口
12
port = 8002
13
# etcd 操作超时时间（毫秒）
14
etcdExecuteTimeoutMillis = 3000
15
# 数据存储目录
16
dataDir = ./data
17

18
[auth]
19
# 是否启用认证
20
enable = true
21
# 登录用户名和密码（格式：用户名:密码）
22
user = admin:kpyun123
23

24
[log]
25
# 日志级别：DEBUG / INFO / WARN / ERROR
26
level = INFO
27
# 日志文件存储目录
28
file = ./logs
29
# 日志文件基础名称（实际文件会追加时间戳或序号）
30
fileName = etcd-workbench
31
# 单个日志文件大小上限（MB），超过后自动滚动切割
32
fileLimitSize = 100
33
# 日志输出位置：std（终端）和 file（文件），逗号分隔可同时启用
34
printers = std,file
35
EOF
36

37
root@Docker ~# docker run -d \
38
  -v /root/etcd-workbench.conf:/usr/tzfun/etcd-workbench/etcd-workbench.conf \
39
  -p 8002:8002 \
40
  --name etcd-workbench \
41
  etcd-workbench:1.1.4
42
root@Docker ~# ss -ntl | grep 8002
43
LISTEN 0      4096         0.0.0.0:8002      0.0.0.0:*
44
✅️ workbench 已在 Docker(10.0.0.9):8002 监听
45

46
3)浏览访问
47
http://10.0.0.9:8002/
48
账号：admin
49
密码：kpyun123
50
# 登录后连接集群