k8s告警规则

发表于 更新于 分类于 阅读次数： 本文字数： 112k 阅读时长 ≈ 1:42

聚合规则

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
groups:
- name: node-exporter.rules
  rules:
  - expr: |
      count without (cpu) (
        count without (mode) (
          node_cpu_seconds_total{job="node-exporter"}
        )
      )
    record: instance:node_num_cpu:sum
  - expr: |
      1 - avg without (cpu, mode) (
        rate(node_cpu_seconds_total{job="node-exporter", mode="idle"}[1m])
      )
    record: instance:node_cpu_utilisation:rate1m
  - expr: |
      (
        node_load1{job="node-exporter"}
      /
        instance:node_num_cpu:sum{job="node-exporter"}
      )
    record: instance:node_load1_per_cpu:ratio
  - expr: |
      1 - (
        node_memory_MemAvailable_bytes{job="node-exporter"}
      /
        node_memory_MemTotal_bytes{job="node-exporter"}
      )
    record: instance:node_memory_utilisation:ratio
  - expr: |
      rate(node_vmstat_pgmajfault{job="node-exporter"}[1m])
    record: instance:node_vmstat_pgmajfault:rate1m
  - expr: |
      rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m])
    record: instance_device:node_disk_io_time_seconds:rate1m
  - expr: |
      rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m])
    record: instance_device:node_disk_io_time_weighted_seconds:rate1m
  - expr: |
      sum without (device) (
        rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[1m])
      )
    record: instance:node_network_receive_bytes_excluding_lo:rate1m
  - expr: |
      sum without (device) (
        rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[1m])
      )
    record: instance:node_network_transmit_bytes_excluding_lo:rate1m
  - expr: |
      sum without (device) (
        rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[1m])
      )
    record: instance:node_network_receive_drop_excluding_lo:rate1m
  - expr: |
      sum without (device) (
        rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[1m])
      )
    record: instance:node_network_transmit_drop_excluding_lo:rate1m
- name: kube-apiserver.rules
  rules:
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[1d]))
          -
          (
            (
              sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[1d]))
              or
              vector(0)
            )
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[1d]))
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[1d]))
          )
        )
        +
        # errors
        sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d]))
    labels:
      verb: read
    record: apiserver_request:burnrate1d
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[1h]))
          -
          (
            (
              sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[1h]))
              or
              vector(0)
            )
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[1h]))
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[1h]))
          )
        )
        +
        # errors
        sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h]))
    labels:
      verb: read
    record: apiserver_request:burnrate1h
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[2h]))
          -
          (
            (
              sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[2h]))
              or
              vector(0)
            )
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[2h]))
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[2h]))
          )
        )
        +
        # errors
        sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h]))
    labels:
      verb: read
    record: apiserver_request:burnrate2h
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30m]))
          -
          (
            (
              sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30m]))
              or
              vector(0)
            )
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30m]))
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30m]))
          )
        )
        +
        # errors
        sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m]))
    labels:
      verb: read
    record: apiserver_request:burnrate30m
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[3d]))
          -
          (
            (
              sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[3d]))
              or
              vector(0)
            )
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[3d]))
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[3d]))
          )
        )
        +
        # errors
        sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d]))
    labels:
      verb: read
    record: apiserver_request:burnrate3d
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[5m]))
          -
          (
            (
              sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[5m]))
              or
              vector(0)
            )
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[5m]))
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[5m]))
          )
        )
        +
        # errors
        sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m]))
    labels:
      verb: read
    record: apiserver_request:burnrate5m
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[6h]))
          -
          (
            (
              sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[6h]))
              or
              vector(0)
            )
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[6h]))
            +
            sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[6h]))
          )
        )
        +
        # errors
        sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h]))
    labels:
      verb: read
    record: apiserver_request:burnrate6h
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d]))
          -
          sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[1d]))
        )
        +
        sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d]))
    labels:
      verb: write
    record: apiserver_request:burnrate1d
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))
          -
          sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[1h]))
        )
        +
        sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))
    labels:
      verb: write
    record: apiserver_request:burnrate1h
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h]))
          -
          sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[2h]))
        )
        +
        sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h]))
    labels:
      verb: write
    record: apiserver_request:burnrate2h
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))
          -
          sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[30m]))
        )
        +
        sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))
    labels:
      verb: write
    record: apiserver_request:burnrate30m
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d]))
          -
          sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[3d]))
        )
        +
        sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d]))
    labels:
      verb: write
    record: apiserver_request:burnrate3d
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
          -
          sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[5m]))
        )
        +
        sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
    labels:
      verb: write
    record: apiserver_request:burnrate5m
  - expr: |
      (
        (
          # too slow
          sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))
          -
          sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[6h]))
        )
        +
        sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h]))
      )
      /
      sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))
    labels:
      verb: write
    record: apiserver_request:burnrate6h
  - expr: |
      sum by (code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m]))
    labels:
      verb: read
    record: code_resource:apiserver_request_total:rate5m
  - expr: |
      sum by (code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
    labels:
      verb: write
    record: code_resource:apiserver_request_total:rate5m
  - expr: |
      histogram_quantile(0.99, sum by (le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET"}[5m]))) > 0
    labels:
      quantile: "0.99"
      verb: read
    record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.99, sum by (le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))) > 0
    labels:
      quantile: "0.99"
      verb: write
    record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod))
    labels:
      quantile: "0.99"
    record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod))
    labels:
      quantile: "0.9"
    record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod))
    labels:
      quantile: "0.5"
    record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
- name: kube-apiserver-availability.rules
  interval: 3m
  rules:
  - expr: |
      1 - (
        (
          # write too slow
          sum(increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d]))
          -
          sum(increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d]))
        ) +
        (
          # read too slow
          sum(increase(apiserver_request_duration_seconds_count{verb=~"LIST|GET"}[30d]))
          -
          (
            (
              sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30d]))
              or
              vector(0)
            )
            +
            sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="namespace",le="0.5"}[30d]))
            +
            sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="cluster",le="5"}[30d]))
          )
        ) +
        # errors
        sum(code:apiserver_request_total:increase30d{code=~"5.."} or vector(0))
      )
      /
      sum(code:apiserver_request_total:increase30d)
    labels:
      verb: all
    record: apiserver_request:availability30d
  - expr: |
      1 - (
        sum(increase(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30d]))
        -
        (
          # too slow
          (
            sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30d]))
            or
            vector(0)
          )
          +
          sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30d]))
          +
          sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30d]))
        )
        +
        # errors
        sum(code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0))
      )
      /
      sum(code:apiserver_request_total:increase30d{verb="read"})
    labels:
      verb: read
    record: apiserver_request:availability30d
  - expr: |
      1 - (
        (
          # too slow
          sum(increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d]))
          -
          sum(increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d]))
        )
        +
        # errors
        sum(code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0))
      )
      /
      sum(code:apiserver_request_total:increase30d{verb="write"})
    labels:
      verb: write
    record: apiserver_request:availability30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"2.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"2.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"2.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"2.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"2.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"2.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"3.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"3.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"3.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"3.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"3.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"3.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"4.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"4.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"4.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"4.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"4.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"4.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"5.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"5.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"5.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"5.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"5.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"5.."}[30d]))
    record: code_verb:apiserver_request_total:increase30d
  - expr: |
      sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"})
    labels:
      verb: read
    record: code:apiserver_request_total:increase30d
  - expr: |
      sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
    labels:
      verb: write
    record: code:apiserver_request_total:increase30d
- name: k8s.rules
  rules:
  - expr: |
      sum by (cluster, namespace, pod, container) (
        rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m])
      ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
        1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""})
      )
    record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate
  - expr: |
      container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
      * on (namespace, pod) group_left(node) topk by(namespace, pod) (1,
        max by(namespace, pod, node) (kube_pod_info{node!=""})
      )
    record: node_namespace_pod_container:container_memory_working_set_bytes
  - expr: |
      container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
      * on (namespace, pod) group_left(node) topk by(namespace, pod) (1,
        max by(namespace, pod, node) (kube_pod_info{node!=""})
      )
    record: node_namespace_pod_container:container_memory_rss
  - expr: |
      container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
      * on (namespace, pod) group_left(node) topk by(namespace, pod) (1,
        max by(namespace, pod, node) (kube_pod_info{node!=""})
      )
    record: node_namespace_pod_container:container_memory_cache
  - expr: |
      container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
      * on (namespace, pod) group_left(node) topk by(namespace, pod) (1,
        max by(namespace, pod, node) (kube_pod_info{node!=""})
      )
    record: node_namespace_pod_container:container_memory_swap
  - expr: |
      sum by (namespace) (
          sum by (namespace, pod) (
              max by (namespace, pod, container) (
                  kube_pod_container_resource_requests_memory_bytes{job="kube-state-metrics"}
              ) * on(namespace, pod) group_left() max by (namespace, pod) (
                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
              )
          )
      )
    record: namespace:kube_pod_container_resource_requests_memory_bytes:sum
  - expr: |
      sum by (namespace) (
          sum by (namespace, pod) (
              max by (namespace, pod, container) (
                  kube_pod_container_resource_requests_cpu_cores{job="kube-state-metrics"}
              ) * on(namespace, pod) group_left() max by (namespace, pod) (
                kube_pod_status_phase{phase=~"Pending|Running"} == 1
              )
          )
      )
    record: namespace:kube_pod_container_resource_requests_cpu_cores:sum
  - expr: |
      max by (cluster, namespace, workload, pod) (
        label_replace(
          label_replace(
            kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"},
            "replicaset", "$1", "owner_name", "(.*)"
          ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (
            1, max by (replicaset, namespace, owner_name) (
              kube_replicaset_owner{job="kube-state-metrics"}
            )
          ),
          "workload", "$1", "owner_name", "(.*)"
        )
      )
    labels:
      workload_type: deployment
    record: namespace_workload_pod:kube_pod_owner:relabel
  - expr: |
      max by (cluster, namespace, workload, pod) (
        label_replace(
          kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"},
          "workload", "$1", "owner_name", "(.*)"
        )
      )
    labels:
      workload_type: daemonset
    record: namespace_workload_pod:kube_pod_owner:relabel
  - expr: |
      max by (cluster, namespace, workload, pod) (
        label_replace(
          kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"},
          "workload", "$1", "owner_name", "(.*)"
        )
      )
    labels:
      workload_type: statefulset
    record: namespace_workload_pod:kube_pod_owner:relabel
- name: kube-scheduler.rules
  rules:
  - expr: |
      histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
    labels:
      quantile: "0.99"
    record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
    labels:
      quantile: "0.99"
    record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
    labels:
      quantile: "0.99"
    record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
    labels:
      quantile: "0.9"
    record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
    labels:
      quantile: "0.9"
    record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
    labels:
      quantile: "0.9"
    record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
    labels:
      quantile: "0.5"
    record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
    labels:
      quantile: "0.5"
    record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
    labels:
      quantile: "0.5"
    record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
- name: node.rules
  rules:
  - expr: |
      topk by(namespace, pod) (1,
        max by (node, namespace, pod) (
          label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)")
      ))
    record: 'node_namespace_pod:kube_pod_info:'
  - expr: |
      count by (cluster, node) (sum by (node, cpu) (
        node_cpu_seconds_total{job="node-exporter"}
      * on (namespace, pod) group_left(node)
        node_namespace_pod:kube_pod_info:
      ))
    record: node:node_num_cpu:sum
  - expr: |
      sum(
        node_memory_MemAvailable_bytes{job="node-exporter"} or
        (
          node_memory_Buffers_bytes{job="node-exporter"} +
          node_memory_Cached_bytes{job="node-exporter"} +
          node_memory_MemFree_bytes{job="node-exporter"} +
          node_memory_Slab_bytes{job="node-exporter"}
        )
      ) by (cluster)
    record: :node_memory_MemAvailable_bytes:sum
- name: kubelet.rules
  rules:
  - expr: |
      histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})
    labels:
      quantile: "0.99"
    record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})
    labels:
      quantile: "0.9"
    record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})
    labels:
      quantile: "0.5"
    record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
- name: kube-prometheus-node-recording.rules
  rules:
  - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance)
    record: instance:node_cpu:rate:sum
  - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance)
    record: instance:node_network_receive_bytes:rate:sum
  - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance)
    record: instance:node_network_transmit_bytes:rate:sum
  - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance)
    record: instance:node_cpu:ratio
  - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m]))
    record: cluster:node_cpu:sum_rate5m
  - expr: cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))
    record: cluster:node_cpu:ratio
- name: kube-prometheus-general.rules
  rules:
  - expr: count without(instance, pod, node) (up == 1)
    record: count:up1
  - expr: count without(instance, pod, node) (up == 0)
    record: count:up0

kubelet

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
groups:
- name: kubernetes-system-kubelet
  rules:
  - alert: Node状态异常
    annotations:
      description: '{{ $labels.node }} has been unready for more than 15 minutes.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubenodenotready
      summary: Node is not ready.
    expr: |
      kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
    for: 15m
    labels:
      severity: warning
  - alert: Node不可达
    annotations:
      description: '{{ $labels.node }} is unreachable and some workloads may be rescheduled.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubenodeunreachable
      summary: Node is unreachable.
    expr: |
      (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
    for: 15m
    labels:
      severity: warning
  - alert: Node运行太多Pod
    annotations:
      description: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubelettoomanypods
      summary: Kubelet is running at capacity.
    expr: |
      count by(node) (
        (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
      )
      /
      max by(node) (
        kube_node_status_capacity_pods{job="kube-state-metrics"} != 1
      ) > 0.95
    for: 15m
    labels:
      severity: warning
  - alert: Node状态抖动
    annotations:
      description: The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubenodereadinessflapping
      summary: Node readiness status is flapping.
    expr: |
      sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2
    for: 15m
    labels:
      severity: warning
  - alert: PLEG耗时高
    annotations:
      description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletplegdurationhigh
      summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist.
    expr: |
      node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
    for: 5m
    labels:
      severity: warning
  - alert: Pod启动耗时高
    annotations:
      description: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletpodstartuplatencyhigh
      summary: Kubelet Pod startup latency is too high.
    expr: |
      histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60
    for: 15m
    labels:
      severity: warning
  - alert: 客户端证书过期
    annotations:
      description: Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletclientcertificateexpiration
      summary: Kubelet client certificate is about to expire.
    expr: |
      kubelet_certificate_manager_client_ttl_seconds < 604800
    labels:
      severity: warning
  - alert: 客户端证书过期
    annotations:
      description: Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletclientcertificateexpiration
      summary: Kubelet client certificate is about to expire.
    expr: |
      kubelet_certificate_manager_client_ttl_seconds < 86400
    labels:
      severity: critical
  - alert: 服务端证书过期
    annotations:
      description: Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletservercertificateexpiration
      summary: Kubelet server certificate is about to expire.
    expr: |
      kubelet_certificate_manager_server_ttl_seconds < 604800
    labels:
      severity: warning
  - alert: 服务端证书过期
    annotations:
      description: Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletservercertificateexpiration
      summary: Kubelet server certificate is about to expire.
    expr: |
      kubelet_certificate_manager_server_ttl_seconds < 86400
    labels:
      severity: critical
  - alert: 客户端证书续签错误
    annotations:
      description: Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletclientcertificaterenewalerrors
      summary: Kubelet has failed to renew its client certificate.
    expr: |
      increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
    for: 15m
    labels:
      severity: warning
  - alert: 服务证书续签错误
    annotations:
      description: Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletservercertificaterenewalerrors
      summary: Kubelet has failed to renew its server certificate.
    expr: |
      increase(kubelet_server_expiration_renew_errors[5m]) > 0
    for: 15m
    labels:
      severity: warning
  - alert: kubelet故障
    annotations:
      description: Kubelet has disappeared from Prometheus target discovery.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletdown
      summary: Target disappeared from Prometheus target discovery.
    expr: |
      absent(up{job="kubelet"} == 1)
    for: 15m
    labels:
      severity: critical

prometheus

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
groups:
- name: prometheus
  rules:
  - alert: Prometheus配置错误
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to reload its configuration.
      summary: Failed Prometheus configuration reload.
    expr: |
      # Without max_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      max_over_time(prometheus_config_last_reload_successful{job="prometheus-k8s",namespace="monitoring"}[5m]) == 0
    for: 10m
    labels:
      severity: critical
  - alert: Prometheus通知队列已满
    annotations:
      description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} is running full.
      summary: Prometheus alert notification queue predicted to run full in less than 30m.
    expr: |
      # Without min_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      (
        predict_linear(prometheus_notifications_queue_length{job="prometheus-k8s",namespace="monitoring"}[5m], 60 * 30)
      >
        min_over_time(prometheus_notifications_queue_capacity{job="prometheus-k8s",namespace="monitoring"}[5m])
      )
    for: 15m
    labels:
      severity: warning
  - alert: Prometheus向一些Alertmanager发送错误警报
    annotations:
      description: '{{ printf "%.1f" $value }}% errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}.'
      summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager.
    expr: |
      (
        rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m])
      /
        rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring"}[5m])
      )
      * 100
      > 1
    for: 15m
    labels:
      severity: warning
  - alert: Prometheus未连接到Alertmanager
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected to any Alertmanagers.
      summary: Prometheus is not connected to any Alertmanagers.
    expr: |
      # Without max_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus-k8s",namespace="monitoring"}[5m]) < 1
    for: 10m
    labels:
      severity: warning
  - alert: PrometheusTSDB 重新加载失败
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} reload failures over the last 3h.
      summary: Prometheus has issues reloading blocks from disk.
    expr: |
      increase(prometheus_tsdb_reloads_failures_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
    for: 4h
    labels:
      severity: warning
  - alert: PrometheusTSDB 压缩失败
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} compaction failures over the last 3h.
      summary: Prometheus has issues compacting blocks.
    expr: |
      increase(prometheus_tsdb_compactions_failed_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
    for: 4h
    labels:
      severity: warning
  - alert: Prometheus不摄取样本
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting samples.
      summary: Prometheus is not ingesting samples.
    expr: |
      (
        rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-k8s",namespace="monitoring"}[5m]) <= 0
      and
        (
          sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus-k8s",namespace="monitoring"}) > 0
        or
          sum without(rule_group) (prometheus_rule_group_rules{job="prometheus-k8s",namespace="monitoring"}) > 0
        )
      )
    for: 10m
    labels:
      severity: warning
  - alert: Prometheus重复时间戳
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with different values but duplicated timestamp.
      summary: Prometheus is dropping samples with duplicate timestamps.
    expr: |
      rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
    for: 10m
    labels:
      severity: warning
  - alert: Prometheus乱序时间戳
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with timestamps arriving out of order.
      summary: Prometheus drops samples with out-of-order timestamps.
    expr: |
      rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
    for: 10m
    labels:
      severity: warning
  - alert: Prometheus远程存储故障
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}
      summary: Prometheus fails to send samples to remote storage.
    expr: |
      (
        rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m])
      /
        (
          rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m])
        +
          rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m])
        )
      )
      * 100
      > 1
    for: 15m
    labels:
      severity: critical
  - alert: Prometheus远程写入落后
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}.
      summary: Prometheus remote write is behind.
    expr: |
      # Without max_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      (
        max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus-k8s",namespace="monitoring"}[5m])
      - ignoring(remote_name, url) group_right
        max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus-k8s",namespace="monitoring"}[5m])
      )
      > 120
    for: 15m
    labels:
      severity: critical
  - alert: Prometheus远程写入所需的分片
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="prometheus-k8s",namespace="monitoring"}` $labels.instance | query | first | value }}.
      summary: Prometheus remote write desired shards calculation wants to run more than configured max shards.
    expr: |
      # Without max_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      (
        max_over_time(prometheus_remote_storage_shards_desired{job="prometheus-k8s",namespace="monitoring"}[5m])
      >
        max_over_time(prometheus_remote_storage_shards_max{job="prometheus-k8s",namespace="monitoring"}[5m])
      )
    for: 15m
    labels:
      severity: warning
  - alert: Prometheus规则失败
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf "%.0f" $value }} rules in the last 5m.
      summary: Prometheus is failing rule evaluations.
    expr: |
      increase(prometheus_rule_evaluation_failures_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
    for: 15m
    labels:
      severity: critical
  - alert: Prometheus缺少规则评估
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ printf "%.0f" $value }} rule group evaluations in the last 5m.
      summary: Prometheus is missing rule evaluations due to slow rule group evaluation.
    expr: |
      increase(prometheus_rule_group_iterations_missed_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
    for: 15m
    labels:
      severity: warning
  - alert: Prometheus目标限制命中
    annotations:
      description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because the number of targets exceeded the configured target_limit.
      summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit.
    expr: |
      increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
    for: 15m
    labels:
      severity: warning
  - alert: Prometheus向所有Alertmanager发送错误警报
    annotations:
      description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.'
      summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager.
    expr: |
      min without (alertmanager) (
        rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m])
      /
        rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring"}[5m])
      )
      * 100
      > 3
    for: 15m
    labels:
      severity: critical

Node

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
groups:
- name: 节点
  rules:
  - alert: 主机内存不足
    expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 节点内存不足 (instance {{ $labels.instance }})
      description: "节点内存已满 (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机内存有压力
    expr: rate(node_vmstat_pgmajfault[1m]) > 1000
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 节点内存压力大 (instance {{ $labels.instance }})
      description: "节点内存压力大. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机入口网络吞吐量异常
    expr: sum by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: 主机异常网络吞吐量 入 (instance {{ $labels.instance }})
      description: "主机网络接口接收了太多数据 (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机出口网络吞吐量异常
    expr: sum by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: 主机异常网络吞吐量 出 (instance {{ $labels.instance }})
      description: "主机网络接口发送了太多数据 (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机磁盘读取速率异常
    expr: sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: 主机异常磁盘读取率 (instance {{ $labels.instance }})
      description: "磁盘正在读取太多数据 (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机磁盘写入速率异常
    expr: sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机异常磁盘写入率 (instance {{ $labels.instance }})
      description: "磁盘正在写入太多数据 (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Please add ignored mountpoints in node_exporter parameters like
  # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)".
  # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users.
  - alert: 主机磁盘空间不足
    expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机磁盘空间不足 (instance {{ $labels.instance }})
      description: "磁盘快满了 (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Please add ignored mountpoints in node_exporter parameters like
  # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)".
  # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users.
  - alert: 主机磁盘将在 24 小时内填满
    expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机磁盘将在 24 小时内填满 (instance {{ $labels.instance }})
      description: "以当前写入速率，预计文件系统将在未来 24 小时内耗尽空间\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 文件系统inode使用率高
    expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机不足 inode (instance {{ $labels.instance }})
      description: "磁盘几乎用完了可用的 inode (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 文件系统inode将在24小时内填满
    expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and predict_linear(node_filesystem_files_free{mountpoint="/rootfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机 inode 将在 24 小时内填满 (instance {{ $labels.instance }})
      description: "文件系统预计将在未来 24 小时内以当前写入速率耗尽 inode\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机磁盘读取延迟异常
    expr: rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机异常磁盘读取延迟 (instance {{ $labels.instance }})
      description: "磁盘延迟越来越大 (read operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机磁盘写入延迟异常
    expr: rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机异常磁盘写入延迟 (instance {{ $labels.instance }})
      description: "磁盘延迟越来越大 (write operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 节点cpu负载高
    expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: 主机 CPU 负载高 (instance {{ $labels.instance }})
      description: "CPU load is > 80%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 有其他云主机窃取cpu
    expr: avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
      description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # 1000 context switches is an arbitrary number.
  # Alert threshold depends on nature of application.
  # Please read: https://github.com/samber/awesome-prometheus-alerts/issues/58
  - alert: 主机上下文切换异常
    expr: (rate(node_context_switches_total[5m])) / (count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1000
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: 主机上下文切换 (instance {{ $labels.instance }})
      description: "节点上的上下文切换正在增长 (> 1000 / s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机交换内存快满了
    expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机交换内存已满 (instance {{ $labels.instance }})
      description: "交换内存已满 (>80%)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机系统服务崩溃
    expr: node_systemd_unit_state{state="failed"} == 1
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: 主机 systemd 服务崩溃 (instance {{ $labels.instance }})
      description: "systemd 服务崩溃\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机内核版本偏差
    expr: count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 1
    for: 6h
    labels:
      severity: warning
    annotations:
      summary: 主机内核版本偏差 (instance {{ $labels.instance }})
      description: "不同的内核版本正在运行\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 检测到 HostOomKill
    expr: increase(node_vmstat_oom_kill[1m]) > 0
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: 检测到主机 OOM 终止 (instance {{ $labels.instance }})
      description: "检测到 OOM\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机网络接收错误
    expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机网络接收错误 (instance {{ $labels.instance }})
      description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机网络出口错误
    expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机网络传输错误 (instance {{ $labels.instance }})
      description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机网络接口饱和
    expr: (rate(node_network_receive_bytes_total{device!~"^tap.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*"} > 0.8 < 10000
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: 主机网络接口饱和 (instance {{ $labels.instance }})
      description: "The network interface \"{{ $labels.device }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: conntrack数量接近极限
    expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: 主机连接限制 (instance {{ $labels.instance }})
      description: "conntrack数量接近极限\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机时钟偏差
    expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机时钟偏差 (instance {{ $labels.instance }})
      description: "检测到时钟偏差。 时钟不同步。\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: 主机时钟不同步
    expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: 主机时钟不同步 (instance {{ $labels.instance }})
      description: "时钟不同步。\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
  - alert: 节点文件系统空间填满
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/nodefilesystemspacefillingup
      summary: Filesystem is predicted to run out of space within the next 24 hours.
    expr: |
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 40
      and
        predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0
      and
        node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
      )
    for: 1h
    labels:
      severity: warning
  - alert: 节点文件系统空间填满
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up fast.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/nodefilesystemspacefillingup
      summary: Filesystem is predicted to run out of space within the next 4 hours.
    expr: |
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 15
      and
        predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0
      and
        node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
      )
    for: 1h
    labels:
      severity: critical
  - alert: 节点磁盘快满了
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/nodefilesystemalmostoutofspace
      summary: Filesystem has less than 5% space left.
    expr: |
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 5
      and
        node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
      )
    for: 1h
    labels:
      severity: warning
  - alert: 节点磁盘快满了
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/nodefilesystemalmostoutofspace
      summary: Filesystem has less than 3% space left.
    expr: |
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 3
      and
        node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
      )
    for: 1h
    labels:
      severity: critical
  - alert: 节点磁盘快满了
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/nodefilesystemfilesfillingup
      summary: Filesystem is predicted to run out of inodes within the next 24 hours.
    expr: |
      (
        node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 40
      and
        predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0
      and
        node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
      )
    for: 1h
    labels:
      severity: warning
  - alert: 节点磁盘快满了
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/nodefilesystemfilesfillingup
      summary: Filesystem is predicted to run out of inodes within the next 4 hours.
    expr: |
      (
        node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 20
      and
        predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0
      and
        node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
      )
    for: 1h
    labels:
      severity: critical
  - alert: 节点磁盘快满了
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/nodefilesystemalmostoutoffiles
      summary: Filesystem has less than 5% inodes left.
    expr: |
      (
        node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 5
      and
        node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
      )
    for: 1h
    labels:
      severity: warning
  - alert: 节点磁盘快满了
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/nodefilesystemalmostoutoffiles
      summary: Filesystem has less than 3% inodes left.
    expr: |
      (
        node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 3
      and
        node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
      )
    for: 1h
    labels:
      severity: critical
  - alert: 使用的节点Conntrack条目数量高
    annotations:
      description: '{{ $value | humanizePercentage }} of conntrack entries are used.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/nodehighnumberconntrackentriesused
      summary: Number of conntrack are getting close to the limit.
    expr: |
      (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75
    labels:
      severity: warning
  - alert: 节点文本文件收集器抓取错误
    annotations:
      description: Node Exporter text file collector failed to scrape.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/nodetextfilecollectorscrapeerror
      summary: Node Exporter text file collector failed to scrape.
    expr: |
      node_textfile_scrape_error{job="node-exporter"} == 1
    labels:
      severity: warning
  - alert: 节点网络接口抖动
    annotations:
      message: Network interface "{{ $labels.device }}" changing it's up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}"
    expr: |
      changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2
    for: 2m
    labels:
      severity: warning

kubelet

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
groups:
- name: kubernetes-system-kubelet
  rules:
  - alert: Node状态异常
    annotations:
      description: '{{ $labels.node }} 超过 15 分钟未准备好.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubenodenotready
      summary: 节点未就绪.
    expr: |
      kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
    for: 15m
    labels:
      severity: warning
  - alert: Node不可达
    annotations:
      description: '{{ $labels.node }} 无法访问，并且可能会重新安排某些工作负载.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubenodeunreachable
      summary: Node不可达.
    expr: |
      (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
    for: 15m
    labels:
      severity: warning
  - alert: Node运行太多Pod
    annotations:
      description: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubelettoomanypods
      summary: Node运行太多Pod.
    expr: |
      count by(node) (
        (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
      )
      /
      max by(node) (
        kube_node_status_capacity_pods{job="kube-state-metrics"} != 1
      ) > 0.95
    for: 15m
    labels:
      severity: warning
  - alert: Node状态抖动
    annotations:
      description: 节点 {{ $labels.node }} 的就绪状态在过去 15 分钟内更改了 {{ $value }}次.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubenodereadinessflapping
      summary: Node状态抖动
    expr: |
      sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2
    for: 15m
    labels:
      severity: warning
  - alert: PLEG耗时高
    annotations:
      description: Kubelet Pod 生命周期事件生成器在节点 {{ $labels.node }} 上的第 99 个百分位持续时间为 {{ $value }} 秒。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletplegdurationhigh
      summary: Kubelet Pod 生命周期事件生成器需要很长时间才能重新列出。
    expr: |
      node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
    for: 5m
    labels:
      severity: warning
  - alert: Pod启动耗时高
    annotations:
      description: Kubelet Pod 启动第 99 个百分位延迟是节点 {{ $labels.node }} 上的 {{ $value }} 秒。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletpodstartuplatencyhigh
      summary: Kubelet Pod 启动延迟太高.
    expr: |
      histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60
    for: 15m
    labels:
      severity: warning
  - alert: 客户端证书过期
    annotations:
      description: Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletclientcertificateexpiration
      summary: Kubelet client certificate is about to expire.
    expr: |
      kubelet_certificate_manager_client_ttl_seconds < 604800
    labels:
      severity: warning
  - alert: 客户端证书过期
    annotations:
      description: Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletclientcertificateexpiration
      summary: Kubelet client certificate is about to expire.
    expr: |
      kubelet_certificate_manager_client_ttl_seconds < 86400
    labels:
      severity: critical
  - alert: 服务端证书过期
    annotations:
      description: Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletservercertificateexpiration
      summary: Kubelet server certificate is about to expire.
    expr: |
      kubelet_certificate_manager_server_ttl_seconds < 604800
    labels:
      severity: warning
  - alert: 服务端证书过期
    annotations:
      description: Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletservercertificateexpiration
      summary: Kubelet server certificate is about to expire.
    expr: |
      kubelet_certificate_manager_server_ttl_seconds < 86400
    labels:
      severity: critical
  - alert: 客户端证书续签错误
    annotations:
      description: Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletclientcertificaterenewalerrors
      summary: Kubelet has failed to renew its client certificate.
    expr: |
      increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
    for: 15m
    labels:
      severity: warning
  - alert: 服务证书续签错误
    annotations:
      description: Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletservercertificaterenewalerrors
      summary: Kubelet has failed to renew its server certificate.
    expr: |
      increase(kubelet_server_expiration_renew_errors[5m]) > 0
    for: 15m
    labels:
      severity: warning
  - alert: kubelet故障
    annotations:
      description: Kubelet 已从 Prometheus 目标发现中消失。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeletdown
      summary: 目标从 Prometheus 目标发现中消失.
    expr: |
      absent(up{job="kubelet"} == 1)
    for: 15m
    labels:
      severity: critical

apiserver

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
groups:
- name: kubernetes-system-apiserver
  rules:
  - alert: KubeClientCertificateExpiration
    annotations:
      description: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeclientcertificateexpiration
      summary: Client certificate is about to expire.
    expr: |
      apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800
    labels:
      severity: warning
  - alert: KubeClientCertificateExpiration
    annotations:
      description: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeclientcertificateexpiration
      summary: Client certificate is about to expire.
    expr: |
      apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400
    labels:
      severity: critical
  - alert: AggregatedAPIErrors
    annotations:
      description: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. The number of errors have increased for it in the past five minutes. High values indicate that the availability of the service changes too often.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/aggregatedapierrors
      summary: An aggregated API has reported errors.
    expr: |
      sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[5m])) > 2
    labels:
      severity: warning
  - alert: AggregatedAPIDown
    annotations:
      description: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/aggregatedapidown
      summary: An aggregated API is down.
    expr: |
      (1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85
    for: 5m
    labels:
      severity: warning
  - alert: KubeAPIDown
    annotations:
      description: KubeAPI has disappeared from Prometheus target discovery.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeapidown
      summary: Target disappeared from Prometheus target discovery.
    expr: |
      absent(up{job="apiserver"} == 1)
    for: 15m
    labels:
      severity: critical
  - alert: KubeAPIErrorBudgetBurn
    annotations:
      description: The API server is burning too much error budget.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeapierrorbudgetburn
      summary: The API server is burning too much error budget.
    expr: |
      sum(apiserver_request:burnrate1h) > (14.40 * 0.01000)
      and
      sum(apiserver_request:burnrate5m) > (14.40 * 0.01000)
    for: 2m
    labels:
      long: 1h
      severity: critical
      short: 5m
  - alert: KubeAPIErrorBudgetBurn
    annotations:
      description: The API server is burning too much error budget.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeapierrorbudgetburn
      summary: The API server is burning too much error budget.
    expr: |
      sum(apiserver_request:burnrate6h) > (6.00 * 0.01000)
      and
      sum(apiserver_request:burnrate30m) > (6.00 * 0.01000)
    for: 15m
    labels:
      long: 6h
      severity: critical
      short: 30m
  - alert: KubeAPIErrorBudgetBurn
    annotations:
      description: The API server is burning too much error budget.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeapierrorbudgetburn
      summary: The API server is burning too much error budget.
    expr: |
      sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
      and
      sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
    for: 1h
    labels:
      long: 1d
      severity: warning
      short: 2h
  - alert: KubeAPIErrorBudgetBurn
    annotations:
      description: The API server is burning too much error budget.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeapierrorbudgetburn
      summary: The API server is burning too much error budget.
    expr: |
      sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
      and
      sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
    for: 3h
    labels:
      long: 3d
      severity: warning
      short: 6h

kubernetes-system

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
groups:
- name: kubernetes-system
  rules:
  - alert: KubeVersion Mis match
    annotations:
      description: There are {{ $value }} different semantic versions of Kubernetes components running.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeversionmismatch
      summary: 运行的 Kubernetes 组件版本不同.
    expr: |
      count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*).*"))) > 1
    for: 15m
    labels:
      severity: warning
  - alert: KubeClientErrors
    annotations:
      description: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubeclienterrors
      summary: Kubernetes API 服务器客户端遇到错误。
    expr: |
      (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job)
        /
      sum(rate(rest_client_requests_total[5m])) by (instance, job))
      > 0.01
    for: 15m
    labels:
      severity: warning
  - alert: KubeStateMetrics列表错误
    annotations:
      description: kube-state-metrics 在列表操作中遇到错误的频率较高。 这可能导致它无法正确或根本无法公开有关 Kubernetes 对象的指标。
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricslisterrors
      summary: kube-state-metrics 在List操作中遇到错误。
    expr: |
      (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m]))
        /
      sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])))
      > 0.01
    for: 5m
    labels:
      severity: critical
  - alert: KubeStateMetricsWatchErrors
    annotations:
      description: kube-state-metrics 在 watch 操作中出现错误的频率越来越高。 这可能导致它无法正确或根本无法公开有关 Kubernetes 对象的指标。
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricswatcherrors
      summary: kube-state-metrics 在Watch操作中遇到错误。
    expr: |
      (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m]))
        /
      sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])))
      > 0.01
    for: 5m
    labels:
      severity: critical

kubernetes-storage

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
groups:
- name: kubernetes-storage
  rules:
  - alert: KubePersistentVolumeFillingUp
    annotations:
      description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubepersistentvolumefillingup
      summary: PersistentVolume 快满了.
    expr: |
      kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
        /
      kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
        < 0.03
    for: 1m
    labels:
      severity: critical
  - alert: KubePersistentVolumeFillingUp
    annotations:
      description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubepersistentvolumefillingup
      summary: PersistentVolume 快满了.
    expr: |
      (
        kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
          /
        kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
      ) < 0.15
      and
      predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
    for: 1h
    labels:
      severity: warning
  - alert: Kube PV 错误
    annotations:
      description: The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubepersistentvolumeerrors
      summary: PV配置错误.
    expr: |
      kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
    for: 5m
    labels:
      severity: critical
  - alert: 容器数据卷使用率
    expr: (1 - (sum(container_fs_inodes_free) BY (instance) / sum(container_fs_inodes_total) BY (instance)) * 100) > 80
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Container Volume usage (instance {{  $labels.instance $labels.namespace $labels.pod }})"
      description: "容器Volume使用率高于 80%  VALUE = {{ $value }}  LABELS: {{ $labels }}"

  - alert: 容器数据卷IO使用率
    expr: (sum(container_fs_io_current) BY (instance, name) * 100) > 80
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Container Volume IO usage (instance {{  $labels.instance $labels.namespace $labels.pod }})"
      description: "容器Volume I/O使用率高于 80%  VALUE = {{ $value }}  LABELS: {{ $labels }}"

kubernetes-resources

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
groups:
- name: kubernetes-resources
  rules:
  - alert: KubeCPUOvercommit
    annotations:
      description: 集群对 Pod 的 CPU 资源请求过多，无法容忍节点故障。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubecpuovercommit
      summary: 集群已过度使用 CPU 资源请求.
    expr: |
      sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum{})
        /
      sum(kube_node_status_allocatable_cpu_cores)
        >
      (count(kube_node_status_allocatable_cpu_cores)-1) / count(kube_node_status_allocatable_cpu_cores)
    for: 5m
    labels:
      severity: warning
  - alert: KubeMemoryOvercommit
    annotations:
      description: 集群对 Pod 的内存资源请求过多，无法容忍节点故障。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubememoryovercommit
      summary: 集群有过量的内存资源请求.
    expr: |
      sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum{})
        /
      sum(kube_node_status_allocatable_memory_bytes)
        >
      (count(kube_node_status_allocatable_memory_bytes)-1)
        /
      count(kube_node_status_allocatable_memory_bytes)
    for: 5m
    labels:
      severity: warning
  - alert: KubeCPUQuotaOvercommit
    annotations:
      description: 集群对命名空间的 CPU 资源请求过多。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubecpuquotaovercommit
      summary: 集群已过度使用 CPU 资源请求.
    expr: |
      sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"})
        /
      sum(kube_node_status_allocatable_cpu_cores)
        > 1.5
    for: 5m
    labels:
      severity: warning
  - alert: KubeMemoryQuotaOvercommit
    annotations:
      description: 集群对命名空间的内存资源请求过多。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubememoryquotaovercommit
      summary: 集群有过量的内存资源请求.
    expr: |
      sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"})
        /
      sum(kube_node_status_allocatable_memory_bytes{job="kube-state-metrics"})
        > 1.5
    for: 5m
    labels:
      severity: warning
  - alert: KubeQuotaAlmostFull
    annotations:
      description: 命名空间 {{ $labels.namespace }} 正在使用 {{ $value | humanizePercentage }} 的 {{ $labels.resource }} 配额。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubequotaalmostfull
      summary: 命名空间配额即将满.
    expr: |
      kube_resourcequota{job="kube-state-metrics", type="used"}
        / ignoring(instance, job, type)
      (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
        > 0.9 < 1
    for: 15m
    labels:
      severity: info
  - alert: KubeQuotaFullyUsed
    annotations:
      description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubequotafullyused
      summary: 命名空间配额已完全使用
    expr: |
      kube_resourcequota{job="kube-state-metrics", type="used"}
        / ignoring(instance, job, type)
      (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
        == 1
    for: 15m
    labels:
      severity: info
  - alert: K8sPod重启告警
    expr: (sum(kube_pod_container_status_restarts_total) by(pod) *on(pod) group_right() sum(kube_pod_info) by (pod,node,host_ip,pod_ip,namespace) - sum(kube_pod_container_status_restarts_total offset 3m) by(pod) *on(pod) group_right() sum(kube_pod_info) by (pod,node,host_ip,pod_ip,namespace) ) != 0
    for: 3m
    labels:
      severity: critical
    annotations:
      summary: "Kubernetes Pod Restart "
      description: "{{ $labels.pod }} 重启中，重启次数：{{ $value }}\n"
  - alert: 容器killed
    expr: time() - container_last_seen > 60
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Container killed"
      description: "容器被kill消失  VALUE = {{ $value }}  LABELS: {{ $labels }}"
  - alert: 容器cpu使用率
    #expr: sum(rate(container_cpu_usage_seconds_total{image!=""}[1m])) by (pod, namespace) / (sum(container_spec_cpu_quota{image!=""}/100000) by (pod, namespace)) * 100 > 50
    expr: sum(rate(container_cpu_usage_seconds_total{namespace!="kube-system",image!=""}[5m])) by (pod, namespace) / (sum(container_spec_cpu_quota{namespace!="kube-system",image!=""}/100000) by (pod, namespace)) * 100 > 70
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Container CPU usage"
      description: "5分钟内容器CPU使用率持续高于 70%  VALUE = {{ $value }}  LABELS: {{ $labels }}"
  - alert: 容器内存使用率
    expr: sum(container_memory_rss{image!=""}) by(pod, namespace) / sum(container_spec_memory_limit_bytes{image!=""}) by(pod, namespace) * 100 != +inf > 90
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Container Memory usage"
      description: "容器内存使用率高于 90% VALUE = {{ $value }} \n LABELS: {{ $labels }}"
  - alert: 容器cpu节流
    expr: rate(container_cpu_cfs_throttled_seconds_total[3m]) > 1
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Container high throttle rate (instance {{ $labels.instance }})"
      description: "容器被节流\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
  - alert: 容器网络出丢包
    expr: container_network_transmit_packets_dropped_total > 0
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "容器网络出丢包 (instance {{ $labels.instance }})"
      description: "容器网络出丢包\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
  - alert: 容器网络入丢包
    expr: container_network_receive_packets_dropped_total > 0
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "容器网络入丢包 (instance {{ $labels.instance }})"
      description: "容器网络入丢包\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
  - alert: 容器网络出带宽高
    expr: sum(irate(container_network_transmit_bytes_total{image!=""}[2m])) by (pod,namespace)  > 104857600
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "容器网络出带宽高 (instance {{ $labels.instance }})"
      description: "容器网络出带宽高于100Mb\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
  - alert: 容器网络入带宽高
    expr: sum(irate(container_network_receive_bytes_total{image!=""}[2m])) by (pod,namespace)  > 104857600
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "容器网络入带宽高 (instance {{ $labels.instance }})"
      description: "容器网络入带宽高于100Mb\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

kubernetes-apps

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
groups:
- name: kubernetes-apps
  rules:
  - alert: KubernetesContainerOomKiller
    expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: Kubernetes 容器 oom 了 (instance {{ $labels.instance }})
      description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
  - alert: KubePodCrashLooping
    annotations:
      description: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubepodcrashlooping
      summary: Pod正在循环崩溃
    expr: |
      rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[5m]) * 60 * 5 > 0
    for: 15m
    labels:
      severity: warning
  - alert: Pod 未就绪
    annotations:
      description: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubepodnotready
      summary: Pod 处于非就绪状态超过 15 分钟。
    expr: |
      sum by (namespace, pod) (
        max by(namespace, pod) (
          kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"}
        ) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) (
          1, max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})
        )
      ) > 0
    for: 15m
    labels:
      severity: warning
  - alert: KubeDeploymentGenerationMismatch
    annotations:
      description: 部署集{{ $labels.namespace }}/{{ $labels.deployment }} 生成匹配，这表明部署失败但尚未回滚。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubedeploymentgenerationmismatch
      summary: Deployment generation mismatch due to possible roll-back
    expr: |
      kube_deployment_status_observed_generation{job="kube-state-metrics"}
        !=
      kube_deployment_metadata_generation{job="kube-state-metrics"}
    for: 15m
    labels:
      severity: warning
  - alert: KubeDeploymentReplicasMismatch
    annotations:
      description: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} 超过15分钟未达到预期的副本数量.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubedeploymentreplicasmismatch
      summary: 部署与预期的副本数不匹配。.
    expr: |
      (
        kube_deployment_spec_replicas{job="kube-state-metrics"}
          !=
        kube_deployment_status_replicas_available{job="kube-state-metrics"}
      ) and (
        changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[5m])
          ==
        0
      )
    for: 15m
    labels:
      severity: warning
  - alert: KubeStatefulSetReplicasMismatch
    annotations:
      description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} 超过15分钟未达到预期的副本数量.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubestatefulsetreplicasmismatch
      summary: 部署与预期的副本数不匹配
    expr: |
      (
        kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
          !=
        kube_statefulset_status_replicas{job="kube-state-metrics"}
      ) and (
        changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m])
          ==
        0
      )
    for: 15m
    labels:
      severity: warning
  - alert: KubeStatefulSetGenerationMismatch
    annotations:
      description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} 生成不匹配，这表明 StatefulSet 失败但尚未回滚。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubestatefulsetgenerationmismatch
      summary: StatefulSet generation mismatch due to possible roll-back
    expr: |
      kube_statefulset_status_observed_generation{job="kube-state-metrics"}
        !=
      kube_statefulset_metadata_generation{job="kube-state-metrics"}
    for: 15m
    labels:
      severity: warning
  - alert: KubeStatefulSetUpdateNotRolledOut
    annotations:
      description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubestatefulsetupdatenotrolledout
      summary: StatefulSet update has not been rolled out.
    expr: |
      (
        max without (revision) (
          kube_statefulset_status_current_revision{job="kube-state-metrics"}
            unless
          kube_statefulset_status_update_revision{job="kube-state-metrics"}
        )
          *
        (
          kube_statefulset_replicas{job="kube-state-metrics"}
            !=
          kube_statefulset_status_replicas_updated{job="kube-state-metrics"}
        )
      )  and (
        changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m])
          ==
        0
      )
    for: 15m
    labels:
      severity: warning
  - alert: KubeDaemonSetRolloutStuck
    annotations:
      description: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubedaemonsetrolloutstuck
      summary: DaemonSet rollout 卡住.
    expr: |
      (
        (
          kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"}
           !=
          kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
        ) or (
          kube_daemonset_status_number_misscheduled{job="kube-state-metrics"}
           !=
          0
        ) or (
          kube_daemonset_updated_number_scheduled{job="kube-state-metrics"}
           !=
          kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
        ) or (
          kube_daemonset_status_number_available{job="kube-state-metrics"}
           !=
          kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
        )
      ) and (
        changes(kube_daemonset_updated_number_scheduled{job="kube-state-metrics"}[5m])
          ==
        0
      )
    for: 15m
    labels:
      severity: warning
  - alert: KubeContainerWaiting
    annotations:
      description: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubecontainerwaiting
      summary: Pod 容器等待超过 1 小时
    expr: |
      sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0
    for: 1h
    labels:
      severity: warning
  - alert: KubeDaemonSetNotScheduled
    annotations:
      description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubedaemonsetnotscheduled
      summary: DaemonSet pod 未调度
    expr: |
      kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
        -
      kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0
    for: 10m
    labels:
      severity: warning
  - alert: KubeDaemonSetMisScheduled
    annotations:
      description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubedaemonsetmisscheduled
      summary: DaemonSet pod 调度错误。
    expr: |
      kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0
    for: 15m
    labels:
      severity: warning
  - alert: KubeJobCompletion
    annotations:
      description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than 12 hours to complete.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubejobcompletion
      summary: Job 没有按时完成
    expr: |
      kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"}  > 0
    for: 12h
    labels:
      severity: warning
  - alert: KubeJobFailed
    annotations:
      description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubejobfailed
      summary: Job 运行失败.
    expr: |
      kube_job_failed{job="kube-state-metrics"}  > 0
    for: 15m
    labels:
      severity: warning
  - alert: KubeHpaReplicasMismatch
    annotations:
      description: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the desired number of replicas for longer than 15 minutes.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubehpareplicasmismatch
      summary: HPA 与预期的副本数不匹配
    expr: |
      (kube_hpa_status_desired_replicas{job="kube-state-metrics"}
        !=
      kube_hpa_status_current_replicas{job="kube-state-metrics"})
        and
      (kube_hpa_status_current_replicas{job="kube-state-metrics"}
        >
      kube_hpa_spec_min_replicas{job="kube-state-metrics"})
        and
      (kube_hpa_status_current_replicas{job="kube-state-metrics"}
        <
      kube_hpa_spec_max_replicas{job="kube-state-metrics"})
        and
      changes(kube_hpa_status_current_replicas[15m]) == 0
    for: 15m
    labels:
      severity: warning
  - alert: KubeHpaMaxedOut
    annotations:
      description: HPA {{ $labels.namespace }}/{{ $labels.hpa }} 已经以最大副本数运行超过 15 分钟。
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubehpamaxedout
      summary: HPA 以最大副本数运行
    expr: |
      kube_hpa_status_current_replicas{job="kube-state-metrics"}
        ==
      kube_hpa_spec_max_replicas{job="kube-state-metrics"}
    for: 15m
    labels:
      severity: warning

alertmanager

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
groups:
- name: alertmanager
  rules:
  - alert: AlertmanagerFailedReload
    annotations:
      description: Configuration has failed to load for {{ $labels.namespace }}/{{ $labels.pod}}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/alertmanagerfailedreload
      summary: Reloading an Alertmanager configuration has failed.
    expr: |
      # Without max_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      max_over_time(alertmanager_config_last_reload_successful{job="alertmanager-main",namespace="monitoring"}[5m]) == 0
    for: 10m
    labels:
      severity: critical
  - alert: AlertmanagerMembersInconsistent
    annotations:
      description: Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} has only found {{ $value }} members of the {{$labels.job}} cluster.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/alertmanagermembersinconsistent
      summary: A member of an Alertmanager cluster has not found all other cluster members.
    expr: |
      # Without max_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
        max_over_time(alertmanager_cluster_members{job="alertmanager-main",namespace="monitoring"}[5m])
      < on (namespace,service) group_left
        count by (namespace,service) (max_over_time(alertmanager_cluster_members{job="alertmanager-main",namespace="monitoring"}[5m]))
    for: 10m
    labels:
      severity: critical
  - alert: AlertmanagerFailedToSendAlerts
    annotations:
      description: Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} failed to send {{ $value | humanizePercentage }} of notifications to {{ $labels.integration }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/alertmanagerfailedtosendalerts
      summary: An Alertmanager instance failed to send notifications.
    expr: |
      (
        rate(alertmanager_notifications_failed_total{job="alertmanager-main",namespace="monitoring"}[5m])
      /
        rate(alertmanager_notifications_total{job="alertmanager-main",namespace="monitoring"}[5m])
      )
      > 0.01
    for: 5m
    labels:
      severity: warning
  - alert: AlertmanagerClusterFailedToSendAlerts
    annotations:
      description: The minimum notification failure rate to {{ $labels.integration }} sent from any instance in the {{$labels.job}} cluster is {{ $value | humanizePercentage }}.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/alertmanagerclusterfailedtosendalerts
      summary: All Alertmanager instances in a cluster failed to send notifications.
    expr: |
      min by (namespace,service) (
        rate(alertmanager_notifications_failed_total{job="alertmanager-main",namespace="monitoring"}[5m])
      /
        rate(alertmanager_notifications_total{job="alertmanager-main",namespace="monitoring"}[5m])
      )
      > 0.01
    for: 5m
    labels:
      severity: critical
  - alert: AlertmanagerConfigInconsistent
    annotations:
      description: Alertmanager instances within the {{$labels.job}} cluster have different configurations.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/alertmanagerconfiginconsistent
      summary: Alertmanager instances within the same cluster have different configurations.
    expr: |
      count by (namespace,service) (
        count_values by (namespace,service) ("config_hash", alertmanager_config_hash{job="alertmanager-main",namespace="monitoring"})
      )
      != 1
    for: 20m
    labels:
      severity: critical
  - alert: AlertmanagerClusterDown
    annotations:
      description: '{{ $value | humanizePercentage }} of Alertmanager instances within the {{$labels.job}} cluster have been up for less than half of the last 5m.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/alertmanagerclusterdown
      summary: Half or more of the Alertmanager instances within the same cluster are down.
    expr: |
      (
        count by (namespace,service) (
          avg_over_time(up{job="alertmanager-main",namespace="monitoring"}[5m]) < 0.5
        )
      /
        count by (namespace,service) (
          up{job="alertmanager-main",namespace="monitoring"}
        )
      )
      >= 0.5
    for: 5m
    labels:
      severity: critical
  - alert: AlertmanagerClusterCrashlooping
    annotations:
      description: '{{ $value | humanizePercentage }} of Alertmanager instances within the {{$labels.job}} cluster have restarted at least 5 times in the last 10m.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/alertmanagerclustercrashlooping
      summary: Half or more of the Alertmanager instances within the same cluster are crashlooping.
    expr: |
      (
        count by (namespace,service) (
          changes(process_start_time_seconds{job="alertmanager-main",namespace="monitoring"}[10m]) > 4
        )
      /
        count by (namespace,service) (
          up{job="alertmanager-main",namespace="monitoring"}
        )
      )
      >= 0.5
    for: 5m
    labels:
      severity: critical

prometheus-operator

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
groups:
- name: prometheus-operator
  rules:
  - alert: PrometheusOperatorListErrors
    annotations:
      description: Errors while performing List operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/prometheusoperatorlisterrors
      summary: Errors while performing list operations in controller.
    expr: |
      (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[10m])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{job="prometheus-operator",namespace="monitoring"}[10m]))) > 0.4
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusOperatorWatchErrors
    annotations:
      description: Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/prometheusoperatorwatcherrors
      summary: Errors while performing watch operations in controller.
    expr: |
      (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[10m])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[10m]))) > 0.4
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusOperatorSyncFailed
    annotations:
      description: Controller {{ $labels.controller }} in {{ $labels.namespace }} namespace fails to reconcile {{ $value }} objects.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/prometheusoperatorsyncfailed
      summary: Last controller reconciliation failed
    expr: |
      min_over_time(prometheus_operator_syncs{status="failed",job="prometheus-operator",namespace="monitoring"}[5m]) > 0
    for: 10m
    labels:
      severity: warning
  - alert: PrometheusOperatorReconcileErrors
    annotations:
      description: '{{ $value | humanizePercentage }} of reconciling operations failed for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace.'
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/prometheusoperatorreconcileerrors
      summary: Errors while reconciling controller.
    expr: |
      (sum by (controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="prometheus-operator",namespace="monitoring"}[5m]))) > 0.1
    for: 10m
    labels:
      severity: warning
  - alert: PrometheusOperatorNodeLookupErrors
    annotations:
      description: Errors while reconciling Prometheus in {{ $labels.namespace }} Namespace.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/prometheusoperatornodelookuperrors
      summary: Errors while reconciling Prometheus.
    expr: |
      rate(prometheus_operator_node_address_lookup_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]) > 0.1
    for: 10m
    labels:
      severity: warning
  - alert: PrometheusOperatorNotReady
    annotations:
      description: Prometheus operator in {{ $labels.namespace }} namespace isn't ready to reconcile {{ $labels.controller }} resources.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/prometheusoperatornotready
      summary: Prometheus operator not ready
    expr: |
      min by(namespace, controller) (max_over_time(prometheus_operator_ready{job="prometheus-operator",namespace="monitoring"}[5m]) == 0)
    for: 5m
    labels:
      severity: warning
  - alert: PrometheusOperatorRejectedResources
    annotations:
      description: Prometheus operator in {{ $labels.namespace }} namespace rejected {{ printf "%0.0f" $value }} {{ $labels.controller }}/{{ $labels.resource }} resources.
      runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/prometheusoperatorrejectedresources
      summary: Resources rejected by Prometheus operator
    expr: |
      min_over_time(prometheus_operator_managed_resources{state="rejected",job="prometheus-operator",namespace="monitoring"}[5m]) > 0
    for: 5m
    labels:
      severity: warning