nginx接入prometheus

发表于 更新于 阅读次数： 本文字数： 24k 阅读时长 ≈ 22 分钟

方案1

openresty + lua脚本

优点:

  • openresty直接集成lua库
  • 如果nginx已支持lua，只需添加配置文件即可
1
2
3
4
yum install yum-utils
sudo yum-config-manager --add-repo https://openresty.org/package/centos/openresty.repo
yum install openresty
git clone https://github.com/knyar/nginx-lua-prometheus
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# HELP nginx_http_connections Number of HTTP connections
# TYPE nginx_http_connections gauge
nginx_http_connections{state="reading"} 0
nginx_http_connections{state="waiting"} 5
nginx_http_connections{state="writing"} 1
# HELP nginx_http_request_duration_seconds HTTP request latency
# TYPE nginx_http_request_duration_seconds histogram
nginx_http_request_duration_seconds_bucket{host="",le="0.005"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.01"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.02"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.03"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.05"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.075"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.1"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.2"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.3"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.4"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.5"} 100
nginx_http_request_duration_seconds_bucket{host="",le="0.75"} 100
nginx_http_request_duration_seconds_bucket{host="",le="1"} 100
nginx_http_request_duration_seconds_bucket{host="",le="1.5"} 100
nginx_http_request_duration_seconds_bucket{host="",le="2"} 100
nginx_http_request_duration_seconds_bucket{host="",le="3"} 100
nginx_http_request_duration_seconds_bucket{host="",le="4"} 100
nginx_http_request_duration_seconds_bucket{host="",le="5"} 100
nginx_http_request_duration_seconds_bucket{host="",le="10"} 100
nginx_http_request_duration_seconds_bucket{host="",le="+Inf"} 100
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.005"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.01"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.02"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.03"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.05"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.075"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.1"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.2"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.3"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.4"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.5"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="0.75"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="1"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="1.5"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="2"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="3"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="4"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="5"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="10"} 66
nginx_http_request_duration_seconds_bucket{host="localhost",le="+Inf"} 66
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.005"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.01"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.02"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.03"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.05"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.075"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.1"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.2"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.3"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.4"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.5"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="0.75"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="1"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="1.5"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="2"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="3"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="4"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="5"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="10"} 11
nginx_http_request_duration_seconds_bucket{host="test.nbugs.com",le="+Inf"} 11
nginx_http_request_duration_seconds_count{host=""} 100
nginx_http_request_duration_seconds_count{host="localhost"} 66
nginx_http_request_duration_seconds_count{host="test.nbugs.com"} 11
nginx_http_request_duration_seconds_sum{host=""} 0
nginx_http_request_duration_seconds_sum{host="localhost"} 0
nginx_http_request_duration_seconds_sum{host="test.nbugs.com"} 0
# HELP nginx_http_requests_total Number of HTTP requests
# TYPE nginx_http_requests_total counter
nginx_http_requests_total{host="",status="200"} 99
nginx_http_requests_total{host="",status="404"} 1
nginx_http_requests_total{host="localhost",status="200"} 48
nginx_http_requests_total{host="localhost",status="304"} 8
nginx_http_requests_total{host="localhost",status="404"} 10
nginx_http_requests_total{host="test.nbugs.com",status="304"} 5
nginx_http_requests_total{host="test.nbugs.com",status="404"} 6
# HELP nginx_metric_errors_total Number of nginx-lua-prometheus errors
# TYPE nginx_metric_errors_total counter
nginx_metric_errors_total 0

方案2

nginx status + nginx-prometheus-exporter

开启status页面

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
server {
listen 8080;
server_name localhost;
  gzip on;.
  gzip_http_version 1.1;
  gzip_vary on;
  gzip_comp_level 6;
  gzip_proxied any;
  gzip_types text/plain text/css application/json application/javascript application/x-javascript text/javascript;
  location / {
    root /usr/share/nginx/html;
    index index.html index.htm;
  }
location = /stub_status {
    stub_status;
  }
}

使用nginx-prometheus-exporter

1
2
3
4
#二进制
./nginx-prometheus-exporter -nginx.scrape-uri http://localhost:8080/stub_status -web.listen-address 0.0.0.0:9000  &
#docker
docker run -p 9113:9113 --net=host nginx/nginx-prometheus-exporter:0.8.0 -nginx.scrape-uri http://localhost:8080/stub_status

方案3

nginx-vts

vts源码 https://github.com/vozlt/nginx-module-vts
exporter源码 https://github.com/hnlq715/nginx-vts-exporter

1
2
3
4
5
yum -y install gcc redhat-rpm-config.noarch pcre-devel openssl openssl-devel libxml2 libxml2-devel libxslt-devel zlib-devel gd-devel perl-devel perl-ExtUtils-Embed gperftools GeoIP GeoIP-devel

./configure --add-module=/root/nginx-module-vts-master --prefix=/usr/share/nginx --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib64/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --http-client-body-temp-path=/var/lib/nginx/tmp/client_body --http-proxy-temp-path=/var/lib/nginx/tmp/proxy --http-fastcgi-temp-path=/var/lib/nginx/tmp/fastcgi --http-uwsgi-temp-path=/var/lib/nginx/tmp/uwsgi --http-scgi-temp-path=/var/lib/nginx/tmp/scgi --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx --user=nginx --group=nginx --with-file-aio --with-ipv6 --with-http_ssl_module --with-http_v2_module --with-http_realip_module --with-stream_ssl_preread_module --with-http_addition_module --with-http_xslt_module=dynamic --with-http_image_filter_module=dynamic --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_random_index_module --with-http_secure_link_module --with-http_degradation_module --with-http_slice_module --with-http_stub_status_module --with-http_perl_module=dynamic --with-http_auth_request_module --with-mail=dynamic --with-mail_ssl_module --with-pcre --with-pcre-jit --with-stream=dynamic --with-stream_ssl_module --with-google_perftools_module --with-debug --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -m64 -mtune=generic' --with-ld-opt='-Wl,-z,relro -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -Wl,-E'

make && make install

nginx.conf加入以下配置

1
2
3
4
5
6
7
8
9
10
http {
    vhost_traffic_status_zone;
    server {
        listen 8088;
        location /status {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format html;
        }
    }
}

需要监控uri情况的域名配置文件中写入该行配置vhost_traffic_status_filter_by_set_key $uri uri::$server_name; 

1
2
3
4
5
6
7
8
9
# 示例
server {
listen 80;
server_name test.nbugs.com;
vhost_traffic_status_filter_by_set_key $uri uri::$server_name;   #这行用于显示uri信息
location / {
    proxy_pass http://www.baidu.com;
   }
}

新版直接提供prometheus接口

1
http://nginxlua.test.com:8088/status/format/prometheus

下载vts exporter 编译为二进制无法连接外网使用docker

1
docker run -ti --net=host --rm --env NGINX_STATUS="http://localhost:8088/status/format/json" sophos/nginx-vts-exporter

grafana模板2949

访问

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
curl http://172.12.1.111:9913/metrics
# HELP nginx_filter_bytes request/response bytes
# TYPE nginx_filter_bytes counter
nginx_filter_bytes{direction="in",filter="uri::test.nbugs.com",filterName="/"} 539
nginx_filter_bytes{direction="out",filter="uri::test.nbugs.com",filterName="/"} 1131
# HELP nginx_filter_requestMsec average of request processing times in milliseconds
# TYPE nginx_filter_requestMsec gauge
nginx_filter_requestMsec{filter="uri::test.nbugs.com",filterName="/"} 0
# HELP nginx_filter_requests requests counter
# TYPE nginx_filter_requests counter
nginx_filter_requests{code="1xx",filter="uri::test.nbugs.com",filterName="/"} 0
nginx_filter_requests{code="2xx",filter="uri::test.nbugs.com",filterName="/"} 0
nginx_filter_requests{code="3xx",filter="uri::test.nbugs.com",filterName="/"} 1
nginx_filter_requests{code="4xx",filter="uri::test.nbugs.com",filterName="/"} 0
nginx_filter_requests{code="5xx",filter="uri::test.nbugs.com",filterName="/"} 0
nginx_filter_requests{code="total",filter="uri::test.nbugs.com",filterName="/"} 1
# HELP nginx_filter_responseMsec average of only upstream/backend response processing times in milliseconds
# TYPE nginx_filter_responseMsec gauge
nginx_filter_responseMsec{filter="uri::test.nbugs.com",filterName="/"} 0
# HELP nginx_server_bytes request/response bytes
# TYPE nginx_server_bytes counter
nginx_server_bytes{direction="in",host="*"} 128355
nginx_server_bytes{direction="in",host="_"} 127816
nginx_server_bytes{direction="in",host="test.nbugs.com"} 539
nginx_server_bytes{direction="out",host="*"} 3.263892e+06
nginx_server_bytes{direction="out",host="_"} 3.262761e+06
nginx_server_bytes{direction="out",host="test.nbugs.com"} 1131
# HELP nginx_server_cache cache counter
# TYPE nginx_server_cache counter
nginx_server_cache{host="*",status="bypass"} 0
nginx_server_cache{host="*",status="expired"} 0
nginx_server_cache{host="*",status="hit"} 0
nginx_server_cache{host="*",status="miss"} 0
nginx_server_cache{host="*",status="revalidated"} 0
nginx_server_cache{host="*",status="scarce"} 0
nginx_server_cache{host="*",status="stale"} 0
nginx_server_cache{host="*",status="updating"} 0
nginx_server_cache{host="_",status="bypass"} 0
nginx_server_cache{host="_",status="expired"} 0
nginx_server_cache{host="_",status="hit"} 0
nginx_server_cache{host="_",status="miss"} 0
nginx_server_cache{host="_",status="revalidated"} 0
nginx_server_cache{host="_",status="scarce"} 0
nginx_server_cache{host="_",status="stale"} 0
nginx_server_cache{host="_",status="updating"} 0
nginx_server_cache{host="test.nbugs.com",status="bypass"} 0
nginx_server_cache{host="test.nbugs.com",status="expired"} 0
nginx_server_cache{host="test.nbugs.com",status="hit"} 0
nginx_server_cache{host="test.nbugs.com",status="miss"} 0
nginx_server_cache{host="test.nbugs.com",status="revalidated"} 0
nginx_server_cache{host="test.nbugs.com",status="scarce"} 0
nginx_server_cache{host="test.nbugs.com",status="stale"} 0
nginx_server_cache{host="test.nbugs.com",status="updating"} 0
# HELP nginx_server_connections nginx connections
# TYPE nginx_server_connections gauge
nginx_server_connections{status="accepted"} 18
nginx_server_connections{status="active"} 1
nginx_server_connections{status="handled"} 18
nginx_server_connections{status="reading"} 0
nginx_server_connections{status="requests"} 383
nginx_server_connections{status="waiting"} 0
nginx_server_connections{status="writing"} 1
# HELP nginx_server_info nginx info
# TYPE nginx_server_info gauge
nginx_server_info{hostName="python",nginxVersion="1.16.1"} 651
# HELP nginx_server_requestMsec average of request processing times in milliseconds
# TYPE nginx_server_requestMsec gauge
nginx_server_requestMsec{host="*"} 0
nginx_server_requestMsec{host="_"} 0
nginx_server_requestMsec{host="test.nbugs.com"} 0
# HELP nginx_server_requests requests counter
# TYPE nginx_server_requests counter
nginx_server_requests{code="1xx",host="*"} 0
nginx_server_requests{code="1xx",host="_"} 0
nginx_server_requests{code="1xx",host="test.nbugs.com"} 0
nginx_server_requests{code="2xx",host="*"} 369
nginx_server_requests{code="2xx",host="_"} 369
nginx_server_requests{code="2xx",host="test.nbugs.com"} 0
nginx_server_requests{code="3xx",host="*"} 11
nginx_server_requests{code="3xx",host="_"} 10
nginx_server_requests{code="3xx",host="test.nbugs.com"} 1
nginx_server_requests{code="4xx",host="*"} 2
nginx_server_requests{code="4xx",host="_"} 2
nginx_server_requests{code="4xx",host="test.nbugs.com"} 0
nginx_server_requests{code="5xx",host="*"} 0
nginx_server_requests{code="5xx",host="_"} 0
nginx_server_requests{code="5xx",host="test.nbugs.com"} 0
nginx_server_requests{code="total",host="*"} 382
nginx_server_requests{code="total",host="_"} 381
nginx_server_requests{code="total",host="test.nbugs.com"} 1
# HELP nginx_server_sharedzones vts module shared memory metrics
# TYPE nginx_server_sharedzones gauge
nginx_server_sharedzones{memstat="maxsize",name="ngx_http_vhost_traffic_status"} 1.048575e+06
nginx_server_sharedzones{memstat="usednode",name="ngx_http_vhost_traffic_status"} 4
nginx_server_sharedzones{memstat="usedsize",name="ngx_http_vhost_traffic_status"} 14088
# HELP nginx_upstream_bytes request/response bytes
# TYPE nginx_upstream_bytes counter
nginx_upstream_bytes{backend="180.101.49.12:80",direction="in",upstream="::nogroups"} 539
nginx_upstream_bytes{backend="180.101.49.12:80",direction="out",upstream="::nogroups"} 1131
# HELP nginx_upstream_requestMsec average of request processing times in milliseconds
# TYPE nginx_upstream_requestMsec gauge
nginx_upstream_requestMsec{backend="180.101.49.12:80",upstream="::nogroups"} 0
# HELP nginx_upstream_requests requests counter
# TYPE nginx_upstream_requests counter
nginx_upstream_requests{backend="180.101.49.12:80",code="1xx",upstream="::nogroups"} 0
nginx_upstream_requests{backend="180.101.49.12:80",code="2xx",upstream="::nogroups"} 0
nginx_upstream_requests{backend="180.101.49.12:80",code="3xx",upstream="::nogroups"} 1
nginx_upstream_requests{backend="180.101.49.12:80",code="4xx",upstream="::nogroups"} 0
nginx_upstream_requests{backend="180.101.49.12:80",code="5xx",upstream="::nogroups"} 0
nginx_upstream_requests{backend="180.101.49.12:80",code="total",upstream="::nogroups"} 1
# HELP nginx_upstream_responseMsec average of only upstream/backend response processing times in milliseconds
# TYPE nginx_upstream_responseMsec gauge
nginx_upstream_responseMsec{backend="180.101.49.12:80",upstream="::nogroups"} 0
# HELP nginx_vts_exporter_build_info A metric with a constant '1' value labeled by version, revision, branch, and goversion from which nginx_vts_exporter was built.
# TYPE nginx_vts_exporter_build_info gauge
nginx_vts_exporter_build_info{branch="",goversion="go1.13.6",revision="",version=""} 1
# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
# TYPE promhttp_metric_handler_requests_in_flight gauge
promhttp_metric_handler_requests_in_flight 1
# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 13
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0

方案4

nginx添加lua支持 + lua脚本

Nginx安装lua支持

1、编译安装lua

1
2
3
4
5
wget  http://luajit.org/download/LuaJIT-2.0.4.tar.gz
tar xf LuaJIT-2.0.4.tar.gz
cd LuaJIT-2.0.4
make PREFIX=/usr/local/luajit
make install PREFIX=/usr/local/luajit

2、下载扩展模块

1
2
3
4
5
cd /usr/local/src/
wget https://github.com/simpl/ngx_devel_kit/archive/v0.3.0.tar.gz
wget https://github.com/openresty/lua-nginx-module/archive/v0.10.9.tar.gz
tar xf lua-nginx-module-0.10.9.tar.gz
tar xf ngx_devel_kit-0.3.0.tar.gz

3、查看之前编译的参数

1
./nginx -V

4、开始编译安装nginx并且添加需要的模块

1
2
3
4
5
6
7
8
9
10
11
12
13
//设置环境变量
export LUAJIT_LIB=/usr/local/luajit/lib
export LUAJIT_INC=/usr/local/luajit/include/luajit-2.0
设置完成后记得source profile

//下载nginx
wget http://nginx.org/download/nginx-1.10.3.tar.gz

//编译
./configure --add-module=/usr/local/src/ngx_devel_kit-0.3.0 --add-module=/usr/local/src/lua-nginx-module-0.10.9 --add-module=/root/nginx-module-vts-master --prefix=/usr/share/nginx --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib64/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --http-client-body-temp-path=/var/lib/nginx/tmp/client_body --http-proxy-temp-path=/var/lib/nginx/tmp/proxy --http-fastcgi-temp-path=/var/lib/nginx/tmp/fastcgi --http-uwsgi-temp-path=/var/lib/nginx/tmp/uwsgi --http-scgi-temp-path=/var/lib/nginx/tmp/scgi --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx --user=nginx --group=nginx --with-file-aio --with-ipv6 --with-http_ssl_module --with-http_v2_module --with-http_realip_module --with-stream_ssl_preread_module --with-http_addition_module --with-http_xslt_module=dynamic --with-http_image_filter_module=dynamic --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_random_index_module --with-http_secure_link_module --with-http_degradation_module --with-http_slice_module --with-http_stub_status_module --with-http_perl_module=dynamic --with-http_auth_request_module --with-mail=dynamic --with-mail_ssl_module --with-pcre --with-pcre-jit --with-stream=dynamic --with-stream_ssl_module --with-google_perftools_module --with-debug --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -m64 -mtune=generic' --with-ld-opt='-Wl,-z,relro -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -Wl,-E'

//安装
make && make install

5、验证

1
2
3
4
5
ln -s /usr/local/luajit/lib/libluajit-5.1.so.2 /lib64
location /lua {
            default_type 'text/html';
            content_by_lua_file conf/lua/test.lua; #相对于nginx安装目录
}

nginx版本1.16.1 要使用20181120版本的nginx-lua-prometheus-master

https://github.com/jialj/nginx-lua-prometheus

1
mv <nginx-lua-prometheus-master_file> /usr/local/luajit/share/luajit-2.0.4/