1
1)下载软件包（官方源，宿主机）
2
jiuzhao@Ubuntu ~$ wget -P /home/jiuzhao/下载 https://github.com/prometheus/pushgateway/releases/download/v1.11.3/pushgateway-1.11.3.linux-amd64.tar.gz
3
jiuzhao@Ubuntu ~$ scp /home/jiuzhao/下载/pushgateway-1.11.3.linux-amd64.tar.gz node2:/tmp/
4

5
2)解压二进制到 /usr/local/bin/
6
[root@node2 ~]# tar tf /tmp/pushgateway-1.11.3.linux-amd64.tar.gz
7
pushgateway-1.11.3.linux-amd64/
8
pushgateway-1.11.3.linux-amd64/LICENSE
9
pushgateway-1.11.3.linux-amd64/NOTICE
10
pushgateway-1.11.3.linux-amd64/pushgateway
11
[root@node2 ~]# tar xf /tmp/pushgateway-1.11.3.linux-amd64.tar.gz -C /usr/local/bin/ pushgateway-1.11.3.linux-amd64/pushgateway --strip-components=1
12
# 只解压这一个二进制，其余 LICENSE/NOTICE 不要
13
[root@node2 ~]# /usr/local/bin/pushgateway --version
14
pushgateway, version 1.11.3
15
✅️ 二进制可执行，版本 1.11.3
16

17
3)编写 systemd 服务文件
18
[root@node2 ~]# mkdir -p /var/lib/pushgateway
19
# 数据目录放 /var/lib/（FHS 规范），pushgateway 不会自动创建父目录，必须先建
20
[root@node2 ~]# cat > /etc/systemd/system/pushgateway.service <<"EOF"
21
[Unit]
22
# 服务描述
23
Description=Kpyun pushgateway server
24
# 官方文档地址
25
Documentation=https://github.com/prometheus/pushgateway
26
# 等待网络就绪后再启动
27
After=network-online.target
28
# 主动声明依赖网络在线目标
29
Wants=network-online.target
30

31
[Service]
32
# 前台常驻进程：ExecStart 拉起即算成功（不 fork 不后台化）
33
Type=simple
34
# 异常退出时自动重启
35
Restart=on-failure
36
# 重启间隔 3 秒
37
RestartSec=3
38
# 文件描述符上限
39
LimitNOFILE=65535
40
# 启动命令（无日志重定向，直接写二进制即可，日志自动进 journald）
41
ExecStart=/usr/local/bin/pushgateway --web.telemetry-path=/metrics --web.listen-address=:9091 --web.enable-admin-api --persistence.file=/var/lib/pushgateway/pushgateway.data
42

43
[Install]
44
# 开机自启
45
WantedBy=multi-user.target
46
EOF

1
1)是查看系统里所有 TCP 套接字（socket）连接
2
[root@node1 ~]# ss -tan
3
-t    # 只看 TCP 连接
4
-a    # 显示所有状态（含 LISTEN 和已建立连接，默认只显示已建立的）
5
-n    # 用数字显示端口/IP，不做域名反解（否则会很慢）
6
State        Recv-Q  Send-Q  Local Address:Port    Peer Address:Port
7
LISTEN       0       128     0.0.0.0:9100          0.0.0.0:*
8
ESTAB        0       0       10.0.0.2:22           10.0.0.1:53210
9
TIME-WAIT    0       0       10.0.0.2:9091         10.0.0.11:52000
10
`最左边 State 那一列，就是 TCP 连接的状态`
11

12
2)编写 TCP 状态监控脚本（Prom 上，统计本机）
13
root@Prom ~# vim /usr/local/bin/tcp_status.sh
14
#!/bin/bash
15

16
url="http://10.0.0.11:9091/metrics/job/tcp_status/instance/10.0.0.10"
17

18
echo """
19
# HELP tcp_connections Number of TCP state connections
20
# TYPE tcp_connections gauge""" > /tmp/tcp.txt
21

22
for i in SYN-SENT SYN-RECV FIN-WAIT-1 FIN-WAIT-2 TIME-WAIT CLOSE CLOSE-WAIT LAST-ACK LISTEN CLOSING ESTAB UNKNOWN; do
23
  count=$(ss -tan | grep -c "$i ")
24
  echo "tcp_connections{state=\"$i\"} $count" >> /tmp/tcp.txt
25
done
26

27
cat /tmp/tcp.txt | curl -s --data-binary @- $url
28

29
3)调用脚本
30
root@Prom ~# bash /usr/local/bin/tcp_status.sh
31

32
4)验证数据
33
root@Prom ~# curl -s http://10.0.0.11:9091/metrics | grep "tcp_connections"
34
# HELP tcp_connections Number of TCP state connections
35
# TYPE tcp_connections gauge
36
tcp_connections{instance="10.0.0.10",job="tcp_status",state="CLOSE"} 0
37
tcp_connections{instance="10.0.0.10",job="tcp_status",state="CLOSE-WAIT"} 0
38
tcp_connections{instance="10.0.0.10",job="tcp_status",state="CLOSING"} 0
39
tcp_connections{instance="10.0.0.10",job="tcp_status",state="ESTAB"} 10
40
tcp_connections{instance="10.0.0.10",job="tcp_status",state="FIN-WAIT-1"} 0
41
tcp_connections{instance="10.0.0.10",job="tcp_status",state="FIN-WAIT-2"} 0
42
tcp_connections{instance="10.0.0.10",job="tcp_status",state="LAST-ACK"} 0
43
tcp_connections{instance="10.0.0.10",job="tcp_status",state="LISTEN"} 6
44
tcp_connections{instance="10.0.0.10",job="tcp_status",state="SYN-RECV"} 0
45
tcp_connections{instance="10.0.0.10",job="tcp_status",state="SYN-SENT"} 2
46
tcp_connections{instance="10.0.0.10",job="tcp_status",state="TIME-WAIT"} 1
47
tcp_connections{instance="10.0.0.10",job="tcp_status",state="UNKNOWN"} 0
48
✅️ 12 种 TCP 状态全部采集

1
1)编写 exporter 代码（本机，/home/jiuzhao/go/src/ 下）
2
jiuzhao@Ubuntu ~$ cd /home/jiuzhao/go/src/
3
jiuzhao@Ubuntu src$ ls
4
hello
5
# 创建项目目录
6
jiuzhao@Ubuntu src$ mkdir -p ./kpyun-go-exporter && cd ./kpyun-go-exporter
7
# 再创建项目骨架
8
jiuzhao@Ubuntu kpyun-go-exporter$ go mod init kpyun-go-exporter
9
jiuzhao@Ubuntu kpyun-go-exporter$ ls
10
go.mod  # 生成模块配置文件
11
jiuzhao@Ubuntu ~$ vim main.go
12
package main
13

14
import (
15
  "encoding/json"
16
  "log"
17
  "net/http"
18
  "time"
19

20
  "github.com/prometheus/client_golang/prometheus"
21
  "github.com/prometheus/client_golang/prometheus/promauto"
22
  "github.com/prometheus/client_golang/prometheus/promhttp"
23
)
24

25
var (
26
  requestTime = promauto.NewSummary(prometheus.SummaryOpts{
27
    Name: "request_processing_seconds",
28
    Help: "Time spent processing request",
29
  })
30
  requestCount = promauto.NewCounter(prometheus.CounterOpts{
31
    Name: "request_count_total",
32
    Help: "Total request count of the host",
33
  })
34
)
35

36
func appsHandler(w http.ResponseWriter, r *http.Request) {
37
  start := time.Now()
38
  defer requestTime.Observe(time.Since(start).Seconds())
39
  requestCount.Inc()
40

41
  w.Header().Set("Content-Type", "application/json")
42
  resp := []map[string]string{
43
    {"office": "https://www.kpyun.com"},
44
    {"auther": "jiuzhao"},
45
  }
46
  b, _ := json.Marshal(resp)
47
  w.Write(b)
48
}
49

50
func main() {
51
  // 指标服务 :8000
52
  go func() {
53
    http.Handle("/metrics", promhttp.Handler())
54
    log.Fatal(http.ListenAndServe(":8000", nil))
55
  }()
56

57
  // 业务服务 :8001
58
  mux := http.NewServeMux()
59
  mux.HandleFunc("/apps", appsHandler)
60
  log.Println("启动kpyun程序: kpyun-go-exporter, 业务 http://0.0.0.0:8001/apps, 指标 http://0.0.0.0:8000")
61
  log.Fatal(http.ListenAndServe(":8001", mux))
62
}
63

64
2)拉取依赖并静态编译
65
jiuzhao@Ubuntu kpyun-go-exporter$ go mod tidy
66
# 拉取 github.com/prometheus/client_golang（GOPROXY 已配 goproxy.cn）
67
jiuzhao@Ubuntu kpyun-go-exporter$ CGO_ENABLED=0 go build -ldflags="-s -w" -o /home/jiuzhao/下载/kpyun-go-exporter $(pwd)
68
# CGO_ENABLED=0 静态编译，-ldflags="-s -w" 去掉调试信息减小体积
69
jiuzhao@Ubuntu ~$ ls -lh /home/jiuzhao/下载/kpyun-go-exporter
70
-rwxrwxr-x 1 jiuzhao jiuzhao 9.9M Aug 16 20:13 /home/jiuzhao/下载/kpyun-go-exporter
71
# 可执行文件
72

73
3)拷贝二进制到 node3 并启动（单个静态二进制，目标机零依赖）
74
jiuzhao@Ubuntu ~$ scp /home/jiuzhao/下载/kpyun-go-exporter node3:/usr/local/bin/
75
[root@node3 ~]# nohup /usr/local/bin/kpyun-go-exporter > /var/log/kpyun-go-exporter.log 2>&1 &
76

77
4)验证两个端口
78
[root@node3 ~]# ss -ntl | egrep ":8000|:8001"
79
LISTEN 0      4096               *:8000            *:*
80
LISTEN 0      4096               *:8001            *:*
81
# 8000 = 指标端口（client_golang 的 promhttp）
82
# 8001 = 业务端口（net/http 的 /apps）
83

84
5)客户端测试
85
[root@node3 ~]# curl -s http://10.0.0.12:8001/apps
86
[{"office":"https://www.kpyun.com"},{"auther":"jiuzhao"}]
87
# 实际业务，对外提供的服务
88
[root@node3 ~]# curl -s http://10.0.0.12:8000/metrics | grep -E "request_count|request_processing_seconds"
89
# HELP request_count_total Total request count of the host
90
# TYPE request_count_total counter
91
request_count_total 1
92
# HELP request_processing_seconds Time spent processing request
93
# TYPE request_processing_seconds summary
94
request_processing_seconds_sum 6.52e-07
95
request_processing_seconds_count 1

1
1)修改 prometheus.yml 追加 Go exporter 的 job（Prom）
2
root@Prom ~# vim /etc/prometheus/prometheus.yml
3
...
4
  - job_name: "kpyun-go-exporter"
5
    metrics_path: "/metrics"
6
    scheme: "http"
7
    static_configs:
8
      - targets:
9
        - 10.0.0.12:8000
10

11
2)检查语法 + 热加载
12
root@Prom ~# promtool check config /etc/prometheus/prometheus.yml
13
Checking /etc/prometheus/prometheus.yml
14
 SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax
15
root@Prom ~# curl -X POST http://localhost:9090/-/reload
16

17
3)验证采集（等 15s 抓取一轮后）
18
root@Prom ~# curl -s "http://localhost:9090/api/v1/query" --data-urlencode "query=request_count_total" | jq -r ".data.result[] | \"\(.metric.job) | \(.value[1])\""
19
kpyun-go-exporter | 160
20
✅️ 请求总数已达 160（压测生效）
21

22
4)Grafana 出图
23
# 请求总数（累计值，单调递增）
24
request_count_total
25

26
# 每分钟请求增量（次数）
27
increase(request_count_total[1m])
28

29
# 每秒请求速率 QPS（平均）
30
rate(request_count_total[1m])
31
`面板右侧 ---> 标准选项 ---> 单位 ---> 吞吐量 ---> 请求/秒钟 (rps)`
32

33
# 每秒瞬时速率（敏感，看突发尖峰）
34
irate(request_count_total[1m])
35
`吞吐量 ---> 操作/秒钟 (ops) 或  ✅ 正确`
36

37
# 累计平均耗时（总耗时/总次数）
38
request_processing_seconds_sum / request_processing_seconds_count
39
`右侧 Standard options（标准选项）→ Unit（单位）→ Time（时间）→ microseconds (微秒μs)`

1
1)修改 prometheus.yml 追加 file_sd job（Prom）
2
root@Prom ~# vim /etc/prometheus/prometheus.yml
3
...
4
  - job_name: "kpyun-file-sd"
5
    metrics_path: "/metrics"
6
    scheme: "http"
7
    file_sd_configs:
8
      - files:
9
          - /tmp/xixi.json
10
          - /tmp/haha.yaml
11

12
2)检查语法（此时文件还没创建，会有 WARNING，属正常）
13
root@Prom ~# promtool check config /etc/prometheus/prometheus.yml
14
Checking /etc/prometheus/prometheus.yml
15
  WARNING: file "/tmp/xixi.json" for file_sd in scrape job "kpyun-file-sd" does not exist
16
  WARNING: file "/tmp/haha.yaml" for file_sd in scrape job "kpyun-file-sd" does not exist
17
 SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax
18

19
3)热加载配置
20
root@Prom ~# curl -X POST http://localhost:9090/-/reload
21

22
4)编写 json 格式的 target 文件
23
root@Prom ~# cat > /tmp/xixi.json <<"EOF"
24
[
25
  {
26
    "targets": [ "10.0.0.2:9100" ],
27
    "labels": {
28
      "school": "kpyun",
29
      "class": "C413"
30
    }
31
  }
32
]
33
EOF
34

35
5)验证自动发现（无需 reload！）
36
root@Prom ~# curl -s "http://localhost:9090/api/v1/targets" | jq -r ".data.activeTargets[] | select(.labels.job==\"kpyun-file-sd\") | \"\(.labels.instance) | school=\(.labels.school // \"-\")\""
37
10.0.0.2:9100 | school=kpyun
38
✅️ 自动发现 json 里的目标
39

40
6)再写一个 yaml 格式的文件
41
root@Prom ~# cat > /tmp/haha.yaml <<"EOF"
42
- targets:
43
  - "10.0.0.11:9100"
44
  - "10.0.0.12:9100"
45
  labels:
46
    address: Tianjin
47
    class: C413
48
EOF
49

50
7)验证（无需 reload，自动发现 2 个新目标）
51
root@Prom ~# curl -s "http://localhost:9090/api/v1/targets" | jq -r ".data.activeTargets[] | select(.labels.job==\"kpyun-file-sd\") | \"\(.labels.instance) | \(.health)\""
52
10.0.0.2:9100 | up
53
10.0.0.11:9100 | up
54
10.0.0.12:9100 | up
55
✅️ 三个目标全部自动发现
56

57
# WebUI
58
http://10.0.0.10:9090/targets?search=kpyun-file-sd

1
1)下载并解压 consul（三台都装，官方源）
2
jiuzhao@Ubuntu ~$ wget -P /home/jiuzhao/下载 https://releases.hashicorp.com/consul/2.0.3/consul_2.0.3_linux_amd64.zip
3
jiuzhao@Ubuntu ~$ scp /home/jiuzhao/下载/consul_2.0.3_linux_amd64.zip node1:/tmp/
4
# node2/node3 同样拷贝
5
[root@node1 ~]# unzip -l /tmp/consul_2.0.3_linux_amd64.zip
6
Archive:  /tmp/consul_2.0.3_linux_amd64.zip
7
  Length      Date    Time    Name
8
---------  ---------- -----   ----
9
     4943  08-07-2026 22:52   LICENSE.txt ❌
10
189911975  08-07-2026 22:52   consul ✅
11
---------                     -------
12
189916918                     2 files
13
[root@node1 ~]# unzip /tmp/consul_2.0.3_linux_amd64.zip consul -d /usr/local/bin/
14
Archive:  /tmp/consul_2.0.3_linux_amd64.zip
15
  inflating: /usr/local/bin/consul   `只解压 consul 文件`
16
[root@node1 ~]# consul version | head -1
17
Consul v2.0.3
18
# 三台都解压
19

20
2)创建数据目录（三台，FHS 规范放 /var/lib/）
21
[root@node1 ~]# mkdir -p /var/lib/consul
22
[root@node2 ~]# mkdir -p /var/lib/consul
23
[root@node3 ~]# mkdir -p /var/lib/consul
24

25
3)编写 systemd 服务文件（三台，用 systemd 托管替代 nohup 裸跑）
26
# node1 是 bootstrap 引导节点，参数里多一个 -bootstrap；node2/node3 用 -retry-join 加入集群
27
[root@node1 ~]# cat > /etc/systemd/system/consul.service <<"EOF"
28
[Unit]
29
# 服务描述
30
Description=HashiCorp Consul Agent
31
# 官方文档地址
32
Documentation=https://developer.hashicorp.com/consul/docs
33
# 等待网络就绪后再启动
34
After=network-online.target
35
# 主动声明依赖网络在线目标
36
Wants=network-online.target
37

38
[Service]
39
# 前台常驻进程：consul agent 不 fork，拉起即算成功
40
Type=simple
41
# 异常退出时自动重启
42
Restart=on-failure
43
# 重启间隔 3 秒
44
RestartSec=3
45
# 文件描述符上限
46
LimitNOFILE=65536
47
# 启动命令（bootstrap 引导，首个节点）
48
ExecStart=/usr/local/bin/consul agent -server -bootstrap -bind=10.0.0.2 -data-dir=/var/lib/consul -client=10.0.0.2 -ui
49

50
[Install]
51
# 开机自启
52
WantedBy=multi-user.target
53
EOF
54

55
# node2 / node3 的服务文件与 node1 区别：去掉 -bootstrap，加上 -retry-join=10.0.0.2
56
[root@node2 ~]# cat > /etc/systemd/system/consul.service <<"EOF"
57
[Unit]
58
Description=HashiCorp Consul Agent
59
Documentation=https://developer.hashicorp.com/consul/docs
60
After=network-online.target
61
Wants=network-online.target
62

63
[Service]
64
Type=simple
65
Restart=on-failure
66
RestartSec=3
67
LimitNOFILE=65536
68
# 启动命令（客户端，retry-join 加入 node1 集群）
69
ExecStart=/usr/local/bin/consul agent -server -bind=10.0.0.11 -data-dir=/var/lib/consul -client=10.0.0.11 -ui -retry-join=10.0.0.2
70

71
[Install]
72
WantedBy=multi-user.target
73
EOF
74

75
# node3 同理（bind/client 换成 10.0.0.12）
76
[root@node3 ~]# sed -i 's/10.0.0.11/10.0.0.12/g' /etc/systemd/system/consul.service
77
[root@node3 ~]# grep ExecStart /etc/systemd/system/consul.service
78
ExecStart=/usr/local/bin/consul agent -server -bind=10.0.0.12 -data-dir=/var/lib/consul -client=10.0.0.12 -ui -retry-join=10.0.0.2
79

80
4)启动 consul 集群（三台）
81
[root@node1 ~]# systemctl daemon-reload && systemctl enable --now consul
82
[root@node2 ~]# systemctl daemon-reload && systemctl enable --now consul
83
[root@node3 ~]# systemctl daemon-reload && systemctl enable --now consul
84

85
5)验证端口
86
[root@node1 ~]# ss -lnt | grep -E ":8300|:8500"
87
LISTEN 0      4096       10.0.0.2:8300       0.0.0.0:*
88
LISTEN 0      4096       10.0.0.2:8500       0.0.0.0:*
89
`8300 = RPC，8500 = HTTP/WebUI`
90
# 三台都验证
91

92
6)查看集群成员（注意：consul 默认连 localhost:8500，client 绑了具体 IP 需指定）
93
[root@node1 ~]# export CONSUL_HTTP_ADDR=http://10.0.0.2:8500
94
[root@node1 ~]# consul members
95
Node   Address         Status  Type    Build   Protocol  DC   Partition  Segment
96
node1  10.0.0.2:8301   alive   server  2.0.3  2         dc1  default    <all>
97
node2  10.0.0.11:8301  alive   server  2.0.3  2         dc1  default    <all>
98
node3  10.0.0.12:8301  alive   server  2.0.3  2         dc1  default    <all>
99
✅️ 3 节点 consul 集群就绪
100

101
7)访问 WebUI
102
http://10.0.0.2:8500/ui/dc1/nodes

1
1)修改 prometheus.yml 追加 consul_sd job（Prom）
2
root@Prom ~# vim /etc/prometheus/prometheus.yml
3
...
4
  - job_name: "kpyun-consul-service-discovery"
5
    consul_sd_configs:
6
      - server: 10.0.0.2:8500
7
      - server: 10.0.0.11:8500
8
      - server: 10.0.0.12:8500
9
    # 重写/重处理标签
10
    relabel_configs:
11
      # 匹配 consul 的源标签字段（服务名）
12
      - source_labels: [__meta_consul_service]
13
        # 正则匹配 consul 自身服务
14
        regex: consul
15
        # 执行 drop 动作，把 consul 自己从监控目标里剔除
16
        action: drop
17

18
2)检查语法 + 热加载
19
root@Prom ~# promtool check config /etc/prometheus/prometheus.yml
20
 SUCCESS: /etc/prometheus/prometheus.yml is valid prometheus config file syntax
21
root@Prom ~# curl -X POST http://localhost:9090/-/reload

1
2)编写 systemd 服务文件（systemd 托管）
2
[root@node2 ~]# cat > /etc/systemd/system/consul_exporter.service <<"EOF"
3
[Unit]
4
# 服务描述
5
Description=Consul Exporter
6
# 等待网络就绪后再启动
7
After=network-online.target
8
# 主动声明依赖网络在线目标
9
Wants=network-online.target
10

11
[Service]
12
# 前台常驻进程
13
Type=simple
14
# 异常退出时自动重启
15
Restart=on-failure
16
# 重启间隔 3 秒
17
RestartSec=3
18
# 启动命令（--consul.server 指定任意一个 consul 节点即可）
19
ExecStart=/usr/local/bin/consul_exporter --consul.server="http://10.0.0.11:8500" --web.telemetry-path="/metrics" --web.listen-address=:9107
20

21
[Install]
22
# 开机自启
23
WantedBy=multi-user.target
24
EOF
25
[root@node2 ~]# systemctl daemon-reload && systemctl enable --now consul_exporter
26

27
3)验证指标
28
[root@node2 ~]# curl -s http://10.0.0.11:9107/metrics | grep -E "^consul_up|^consul_raft_peers"
29
consul_raft_peers 3
30
consul_up 1
31
✅️ consul_exporter 正常，看到 3 个 raft peers
32
# WebUI 验证
33

34

35
4)注册 consul_exporter 服务到 consul（让 Prometheus 自动发现）
36
[root@node2 ~]# curl -X PUT -d '{"id":"consul-server","name":"consul-cluster","address":"10.0.0.11","port":9107,"tags":["consul-cluster"],"checks":[{"http":"http://10.0.0.11:9107","interval":"5s"}]}' http://10.0.0.2:8500/v1/agent/service/register
37

38
5)验证 Prometheus 自动发现 consul_exporter
39
root@Prom ~# curl -s "http://localhost:9090/api/v1/targets" | jq -r ".data.activeTargets[] | select(.scrapePool==\"kpyun-consul-service-discovery\") | \"\(.labels.instance) | \(.health)\""
40
10.0.0.12:9100 | up
41
10.0.0.11:9100 | up
42
10.0.0.11:9107 | up
43
✅️ consul_exporter 被自动发现，无需改配置
44

45
6)Grafana 导入模板 ID
46
12049