fluentbit收集k8s java日志

发表于 阅读次数： 3 本文字数： 2.3k 阅读时长 ≈ 2 分钟

111

配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
config:
  service: |
    [SERVICE]
        Daemon Off
        Flush {{ .Values.flush }}
        Log_Level {{ .Values.logLevel }}
        Parsers_File /fluent-bit/etc/parsers.conf
        Parsers_File /fluent-bit/etc/conf/custom_parsers.conf
        HTTP_Server On
        HTTP_Listen 0.0.0.0
        HTTP_Port {{ .Values.metricsPort }}
        Health_Check On

1
2
3
4
5
6
7
8
9
10
11
12
13
14
inputs: |
  [Input]
      Name    tail
      Path    /var/log/containers/*.log
      Exclude_Path    /var/log/containers/*kubesphere*.log,/var/log/containers/kube-*.log,/var/log/containers/*cattle*.log,/var/log/containers/*calico*.log
      Docker_Mode On
      Docker_Mode_Flush   5 
      # 这里写的解析只管解析多行模式，不提取正则中的key
      Docker_Mode_Parser java_multi_line 
      Parser docker
      Tag kube.*
      Mem_Buf_Limit 5MB
      Skip_Long_Lines Off

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
filters: |
  [Filter]
      Name kubernetes
      Match kube.*
      # 开启收集注解
      Annotations On
      Labels Off
      Merge_Log On
      Keep_Log Off
      K8S-Logging.Parser On   
      K8S-Logging.Exclude On
  # 注解过滤（只保留有logging=java的Pod）
  [FILTER]
      Name                grep
      Match               kube.*
      Regex               $kubernetes['annotations']['logging'] ^java$
  [Filter]
      Name    nest
      Match    kube.*
      Operation    lift
      Nested_under    kubernetes
      Add_prefix    kubernetes_
  [FILTER]
      #将kubernetes_annotations 提取至根
      Name nest
      Match kube.*
      Operation lift
      Nested_under  kubernetes_annotations
      Add_prefix k8s_ann_
  [Filter]
      Name    modify
      Match    kube.*
      Remove    stream
      Remove    kubernetes_pod_id
      Remove    kubernetes_docker_id
      Remove    kubernetes_host
      Remove    kubernetes_container_hash
      Rename    kubernetes_namespace_name namespace_name
      Rename    kubernetes_pod_name pod_name
      Rename    kubernetes_container_name container_name
      Rename    kubernetes_container_image container_image
      Rename    k8s_ann_app app
      Remove_regex ^k8s_ann

  [FILTER]
      # 使用 过滤解析将level和time字段提取出来，并保留原始内容
      Name parser
      Match kube.*
      Key_Name log
      # 这里可以识别()里的字段
      Parser java_multi_line
      # 保留解析后的字段内容
      Preserve_Key true
      # 保留日志其他key
      Reserve_Data true

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
outputs: |
  [OUTPUT]
      Name http
      Match *
      URI /api/default/ds-log/_json
      Host openobserve.openobserve.svc
      Port 5080
      tls Off
      Format json
      Json_date_key    _timestamp
      Json_date_format iso8601
      HTTP_User admin@xxxx.com
      HTTP_Passwd XQHN6gqjQxDAn0HH
      compress gzip
  [OUTPUT]
      Name   stdout
      Match  *
      Format json

1
2
3
4
5
6
customParsers: |
  [PARSER]
      Name java_multi_line    
      Key_name log   
      Format regex 
      Regex    /(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}.\d{3})\s{1,3}(?<level>\w{3,6})/