pod公网出口ip自定义

发表于 更新于 阅读次数： 本文字数： 15k 阅读时长 ≈ 13 分钟

需求:

java应用访问qyapi.weixin.qq.com时根据不同组织对应不同公网出口ip

java实现部分：更新token的任务，先把token写到redis里TTL设置为4小时，再写到mysql

redis格式为

qwtoken: cropid

image-20230721134947846

image-20230721134955475

注意事项

在nginx/openresty中 企微接口proxy_pass不要直接写域名，否则会走系统解析(nginx -s reload 时解析 变化后还需要再次reload)

使用变量定义就会走resolver配置

参考文档: https://blog.csdn.net/ai2000ai/article/details/96329239

实现:

configmap

public-openresty-nginx-conf.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
kind: ConfigMap
apiVersion: v1
metadata:
  name: public-openresty-nginx-conf
  namespace: suosi-pre
data:
  nginx.conf: >-
    user  root;

    #worker_processes 1;
    # Enables the use of JIT for regular expressions to speed-up their
    processing.

    pcre_jit on;

    events {
        worker_connections  1024;
    }

    http {
        include       mime.types;
        default_type  application/octet-stream;
        
        #resolver 100.100.2.136 100.100.2.138 202.106.0.20 114.114.114.114 valid=5 ipv6=off;
        resolver 10.252.0.10 valid=5 ipv6=off;
        # 在coredns中将qyapi.weixin.qq.com rename 到 public-openresty服务 的配置为: rewrite name qyapi.weixin.qq.com.suosi-pre.svc.cluster.local public-openresty.suosi-pre.svc.cluster.local
        # java实际访问的是qyapi.weixin.qq.com.suosi-pre.svc.cluster.local  而  openresty访问的是qyapi.weixin.qq.com所以这里可以直接指定dns为coredns的svcip
        #默认连接池大小，默认30
        lua_socket_pool_size 30;
        #默认超时时间,默认60s
        lua_socket_keepalive_timeout 60s;
        # Log in JSON Format
        log_format nginxlog_json escape=json '{"timestamp":"$time_iso8601","status":"$status","upstream_status":"$upstream_status","wx_crop_id":"$wx_crop_id","request_uri":"$request","request_time":"$request_time""upstream_addr":"$upstream_addr","upstream_response_time":"$upstream_response_time","host":"$host"}';

        access_log /dev/stdout nginxlog_json;
        error_log /dev/stdout notice;

        # See Move default writable paths to a dedicated directory (#119)
        # https://github.com/openresty/docker-openresty/issues/119
        client_body_temp_path /var/run/openresty/nginx-client-body;
        proxy_temp_path       /var/run/openresty/nginx-proxy;
        fastcgi_temp_path     /var/run/openresty/nginx-fastcgi;
        uwsgi_temp_path       /var/run/openresty/nginx-uwsgi;
        scgi_temp_path        /var/run/openresty/nginx-scgi;

        sendfile        on;
        #tcp_nopush     on;

        #keepalive_timeout  0;
        keepalive_timeout  65;

        #gzip  on;

        client_header_buffer_size 16k;

        #include /etc/nginx/conf.d/*.conf;
        
        lua_shared_dict crop_id 10m;
        init_by_lua_file '/usr/local/openresty/nginx/lua/init_map.lua';

        include conf.d/*.conf;

        # Don't reveal OpenResty version to clients.
        # server_tokens off;
    }

public-openresty-nginx-confd.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
kind: ConfigMap
apiVersion: v1
metadata:
  name: public-openresty-nginx-confd
  namespace: suosi-pre
data:
  cropid_map.map: ''
  server.crt: |-
    -----BEGIN CERTIFICATE-----
    MIIDrzCCApegAwIBAgIJANAXkECaGgIgMA0GCSqGSIb3DQEBCwUAMIGAMQswCQYD
    VQQGEwJDTjELMAkGA1UECAwCU0QxCzAJBgNVBAcMAkpOMQ0wCwYDVQQKDARRRFpZ
    MRwwGgYDVQQLDBNxeWFwaS53ZWl4aW4ucXEuY29tMQswCQYDVQQDDAJDQTEdMBsG
    CSqGSIb3DQEJARYOYWRtaW5AdGVzdC5jb20wIBcNMjMwNTI0MDcxMzQ1WhgPMjIy
    ODA5MjYwNzEzNDVaMIGAMQswCQYDVQQGEwJDTjELMAkGA1UECAwCU0QxCzAJBgNV
    BAcMAkpOMQ0wCwYDVQQKDARRRFpZMRwwGgYDVQQLDBNxeWFwaS53ZWl4aW4ucXEu
    Y29tMQswCQYDVQQDDAJDQTEdMBsGCSqGSIb3DQEJARYOYWRtaW5AdGVzdC5jb20w
    ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDgr+ktRZzDmV0FejtLJ6Jk
    B9FSu95yZSAmm8AWsPKXXU9BYQE0MTTmdEnqW+KnhOLPaTpxaD1xJwEXqSIrSKsX
    nYs2rfby4+duW9+7Xe4J9UxONHvAuctzOm0i5UoaMcfKi99Yny7WDNjYvFTEgTdn
    GBSCo8VYbyqZ7RTAbU17E76uCXf6KQb0e81ziANsxGaCzdx8F+nuENNfZca+RQRj
    xnXDcThi6NihZrQxRiHlPeIGqbUbG5ZQTS33AA85LW6O8kcRQEXNPAqItSZVELci
    ov1dqmRgdI2K9uyoj3ipFu64CJ2grY1FKLAXJkO1IYp08IAkCPvKLTYUh6xL1Ou7
    AgMBAAGjKDAmMCQGA1UdEQQdMBuCE3F5YXBpLndlaXhpbi5xcS5jb22HBAAAAAAw
    DQYJKoZIhvcNAQELBQADggEBAIafbkCm+PSR3YBYl42lD+MbIkAEUps58WvqgdJO
    7UT3lX7V7V3lqfaO8mx3+Q24ruH8+PaoA064PT3LYlzlG+2EIU3Jy9/cl+6Pp2LO
    xez+6GYcJxf/XpN5kwcm1HT4gYfhXyKWw1+Bhmujqauyz7tvIG+zxE3kEJzzzxRu
    sfIDol45ZY4aPVcpB/mzh+JWkRdhMuXtyntLXEeCcMAZJRcdiIQLnILb55MiwC/q
    MMwjZWBN6OpmraXLD7zQb7BYkJ4KK+II5HwMQZiEiZDrnqYmd2Zs8Hx1EF4XpKNO
    LsTa1Ebm2Sf4xXXw+v72l0LLErfmHf8yjsaVQe+wXW+OrnM=
    -----END CERTIFICATE-----
  server.key: |-
    -----BEGIN RSA PRIVATE KEY-----
    MIIEpAIBAAKCAQEA4K/pLUWcw5ldBXo7SyeiZAfRUrvecmUgJpvAFrDyl11PQWEB
    NDE05nRJ6lvip4Tiz2k6cWg9cScBF6kiK0irF52LNq328uPnblvfu13uCfVMTjR7
    wLnLczptIuVKGjHHyovfWJ8u1gzY2LxUxIE3ZxgUgqPFWG8qme0UwG1NexO+rgl3
    +ikG9HvNc4gDbMRmgs3cfBfp7hDTX2XGvkUEY8Z1w3E4YujYoWa0MUYh5T3iBqm1
    GxuWUE0t9wAPOS1ujvJHEUBFzTwKiLUmVRC3IqL9XapkYHSNivbsqI94qRbuuAid
    oK2NRSiwFyZDtSGKdPCAJAj7yi02FIesS9TruwIDAQABAoIBAEsDb93ld8j10tCZ
    VmJpARZUZdYxUrrueCVrql3pBZTzWhqBwF0kcHzgJi1QMAOtoeuNPi3Ol3THiN3V
    YcsBn91qg6flvKSq4gE+Oxva6DX651bUvtxBK2N1Biq4Ul0ccY9100NLId/kuiDh
    /4r7ePu6Vl6nPqOfuaFaPatg0pVcCfjpStPAf5YCpUeXpJfGFt5VGAKxf65vKCkh
    grbppM9yVzs2iiVK3U/STpbPXDC9e7/2aoqtNeKlCLPrUWKGFwZpBR+SxpUHHIZ1
    3qb/JovEdeSN4Dl5xjwNHt7hd4SRAp/86NLEw5YVbvuRjkSzmMNrj4mfrMPPAZ6C
    NEH3ZcECgYEA+CLusJ0r1N1oGdQ99ALLIoXje7AqYFxwVTrcD5Y7d6m5npLr+hs7
    yIWMz189MG1zWwv/s9RM9secsry+HhnltJ1qQZTOy7NZFMtnKKAa0NUy2xBQUslE
    AtgQgBGNIvVYK4upE2nngL0NAvNG7Sgwyv9UFtMyeprcqYH5xOtcIQ0CgYEA5869
    ggoE7O+lBrbRu723MuaXj2mg8Y+UzxrIqgwX08HsMGub7P00SVyIwoybTQJNPGie
    1kz82xDwXzrFq7H1tp1iEk103kHJq1rO0ggatrgD0VM0YpA9Xltv4/qA5xaVBo4B
    RhNrpjEj8pSn549CNfhqTMzeesd3bH8KXIRfPecCgYEA9VLEHgUmQqwrse2u2sKw
    Rw+MWstO+joqLXml/BsR7Dr3c5naiEnIj3XKQ3PrsSdk900jn410EkBD4krMxEHi
    YvGHDhOraKWGmxKGiRnRqUo/n2m/oDmwbgdkONohacCbTWIk5Ta9VQCUDqirJOmp
    Y+mQH4jqzWCybTw9zrzLNzkCgYEAsLCTHqXIb1mTLnT3lOTc2T2O1M+sz7Ojt+Ew
    hv1ExDISeC3t4kx2KF0SGUjXr3FLsfoE6FAyhEB7F/tSZLb3FcUM1eqYZDk9IRHM
    h6eJxTCqKEoFqgNL47pKpTlyO7Ko0SA4tFNlQH5Aak0JVqWJ0F2TmQqnomqcCuUi
    3rY/ao0CgYBzZMeD/J6XTYzU7IU4xIpvcGPVGMXCgBgu99UOKG3IZNp8bRonupx7
    77R6S8hzqup7b2mr19yb7nGYZVcEx1QLJrE6dghULqVaONMhRaEAnTBfJ44tmHy/
    h1mlLqSyeiunl13ZY/rJ/NXwgn5hmpclLP+Z7dC4squwWyUU+TVIjA==
    -----END RSA PRIVATE KEY-----
  test.conf: |-
    upstream svc1 {
        server qywx-svc1.suosi-pre.svc.cluster.local:20000;
    }
    upstream svc2 {
        server whoami.pre.suosihulian.com;
    }

    server {
        listen 80;
        set $wx_crop_id "";
        set $wx_upstream "";
        set $wx_host "";
        location / {
            default_type text/html;
            access_by_lua_file '/usr/local/openresty/nginx/lua/getredis.lua';
            proxy_set_header Host $wx_host;
            proxy_pass $wx_upstream;
        }
        location /health {
            return 200 'ok';
        }
    }
    server {
        listen 443 ssl;
        ssl_certificate /usr/local/openresty/nginx/conf/conf.d/server.crt;
        ssl_certificate_key /usr/local/openresty/nginx/conf/conf.d/server.key;
        set $wx_crop_id "";
        set $wx_upstream "";
        set $wx_host "";
        location / {
            default_type text/html;
            access_by_lua_file '/usr/local/openresty/nginx/lua/getredis.lua';
            proxy_set_header Host $wx_host;
            proxy_pass $wx_upstream;
        }
    }

public-openresty-nginx-lua.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
kind: ConfigMap
apiVersion: v1
metadata:
  name: public-openresty-nginx-lua
  namespace: suosi-pre
data:
  getredis.lua: |
    local redis = require("resty.redis");
    local cjson = require("cjson")
    local red = redis:new();
    red:set_timeout(5000)
    local host = "xxxxx"
    local port = 6379
    local password = "xxxx"
    local access_token = ngx.var.arg_access_token

    local function ret_def()
        ngx.var.wx_upstream = "https://qyapi.weixin.qq.com"
        ngx.var.wx_host = "qyapi.weixin.qq.com"
        ngx.var.wx_crop_id = "no"
    end

    if type(access_token) == "nil" then
        ret_def()
        return
    end
    local keys = 'qwtoken:' .. access_token

    -- 连接redis
    local ok, err = red:connect(host, port)
    if not ok then
      io.stderr:write('cant connect redis\n')
      ret_def()
      return
    end

    local res, err = red:auth(password)
    if not res then
      io.stderr:write('cant auth redis\n')
      return
    end

    -- 选择db
    ok, err = red:select(3)
    if not ok then
        io.stderr:write('failed to select db\n')
        ret_def()
        return
    end

    -- 获取redis key
    local res, err = red:get(keys)
    if not res then
      io.stderr:write('cant get redis\n')
      ret_def()
      return
    end

    ok, err = red:set_keepalive(10000, 300) --线程池
    if not ok then
      io.stderr:write('cant close redis\n')
      ret_def()
      return
    end
    -- 将string转为map
    local keys = ngx.shared.crop_id:get("a")
    local my_table = cjson.decode(keys)
    local value = my_table[res]

    if value then
        ngx.var.wx_crop_id = res
        ngx.var.wx_upstream = "http://" .. value
        ngx.var.wx_host = "qyapi.weixin.qq.com"
        -- 后续的服务会加这个HOST
    else
        ret_def()
    end
  init_map.lua: >-
    local cjson = require("cjson")
      local function load_map(file_path)
          local map = {}
          local file = io.open(file_path, "r")

          if not file then
              ngx.log(ngx.ERR, "Failed to open map file: ", file_path)
              return map
          end

          for line in file:lines() do
              local key, value = string.match(line, [["(%S+)"%s+"(%S+)";]])
              if key and value then
                  map[key] = value
              end
          end

          file:close()
          return map
      end

    local map_file_path =
    "/usr/local/openresty/nginx/conf/conf.d/cropid_map.map"

    local tmp = load_map(map_file_path)

    ngx.shared.crop_id:set("a", cjson.encode(tmp))

    local keys = ngx.shared.crop_id:get("a")

    io.stderr:write("\n---------------------------------\n")

    io.stderr:write("JSON: ", keys, "\n")

svc

public-openresty.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
kind: Service
apiVersion: v1
metadata:
  name: public-openresty
  namespace: suosi-pre
spec:
  ports:
    - name: tcp-443
      protocol: TCP
      port: 443
      targetPort: 443
    - name: tcp-80
      protocol: TCP
      port: 80
      targetPort: 80
  selector:
    app: public-openresty
  type: ClusterIP
  sessionAffinity: None

qywx-svc1.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
kind: Service
apiVersion: v1
metadata:
  name: qywx-svc1
  namespace: suosi-pre
spec:
  ports:
    - name: tcp-20000
      protocol: TCP
      port: 20000
      targetPort: 20000
  selector:
    svc: qywx-svc1
  type: ClusterIP
  sessionAffinity: None

deployment

public-openresty.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
kind: Deployment
apiVersion: apps/v1
metadata:
  name: public-openresty
  namespace: suosi-pre
spec:
  replicas: 1
  selector:
    matchLabels:
      app: public-openresty
  template:
    metadata:
      labels:
        app: public-openresty
    spec:
      volumes:
        - name: public-openresty-nginx-conf
          configMap:
            name: public-openresty-nginx-conf
            defaultMode: 420
        - name: public-openresty-nginx-lua
          configMap:
            name: public-openresty-nginx-lua
            defaultMode: 420
        - name: public-openresty-nginx-confd
          configMap:
            name: public-openresty-nginx-confd
            defaultMode: 420
      containers:
        - name: openresty
          image: openresty
          ports:
            - name: http
              containerPort: 80
              protocol: TCP
            - name: https
              containerPort: 443
              protocol: TCP
          env:
            - name: TZ
              value: Asia/Shanghai
          resources:
            limits:
              cpu: '2'
              memory: 300Mi
            requests:
              cpu: 10m
              memory: 300Mi
          volumeMounts:
            - name: public-openresty-nginx-conf
              mountPath: /usr/local/openresty/nginx/conf/nginx.conf
              subPath: nginx.conf
            - name: public-openresty-nginx-lua
              mountPath: /usr/local/openresty/nginx/lua
            - name: public-openresty-nginx-confd
              mountPath: /usr/local/openresty/nginx/conf/conf.d
          readinessProbe:
            httpGet:
              path: /health
              port: 80
              scheme: HTTP
            initialDelaySeconds: 10
            timeoutSeconds: 3
            periodSeconds: 20
            successThreshold: 1
            failureThreshold: 3
          lifecycle:
            preStop:
              exec:
                command:
                  - sleep 20
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          imagePullPolicy: Always
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      dnsPolicy: ClusterFirst
      securityContext: {}
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - public-openresty
                topologyKey: kubernetes.io/hostname
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 25%
      maxSurge: 25%
  revisionHistoryLimit: 10
  progressDeadlineSeconds: 600

qywx-dep1.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
kind: Deployment
apiVersion: apps/v1
metadata:
  name: qywx-dep1
  namespace: suosi-pre
spec:
  replicas: 1
  selector:
    matchLabels:
      app: qywx-dep1
  template:
    metadata:
      labels:
        app: qywx-dep1
        svc: qywx-svc1
      annotations:
        k8s.aliyun.com/pod-eip-instanceid: # 对应的eip名称
    spec:
      volumes:
        - name: podinfo
          downwardAPI:
            items:
              - path: labels
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.labels
              - path: annotations
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.annotations
            defaultMode: 420
      initContainers:
        - name: init
          image: harbor-core.suosihulian.com/devops/busybox:1.28
          command:
            - timeout
            - '-t'
            - '60'
            - sh
            - '-c'
            - >-
              until grep -E '^k8s.aliyun.com\/allocated-eipAddress=\S?[0-9]+\S?'
              /etc/podinfo/annotations; do echo waiting for annotations; sleep
              2; done
          resources: {}
          volumeMounts:
            - name: podinfo
              mountPath: /etc/podinfo
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          imagePullPolicy: IfNotPresent
      containers:
        - name: qywx-dep1
          image: proxy-qywx
          ports:
            - name: http
              containerPort: 20000
              protocol: TCP
          env:
            - name: TZ
              value: Asia/Shanghai
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          imagePullPolicy: Always
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      dnsPolicy: ClusterFirst
      securityContext: {}
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - qywx-dep1
                topologyKey: kubernetes.io/hostname
      schedulerName: default-scheduler
  strategy:
    type: Recreate
  revisionHistoryLimit: 10
  progressDeadlineSeconds: 600

proxy-qywx镜像配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
user  root;
#worker_processes 1;

pcre_jit on;



events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;
    log_format nginxlog_json escape=json '{"timestamp":"$time_iso8601","status":"$status","upstream_status":"$upstream_status","request_uri":"$request","request_time":"$request_time""upstream_addr":"$upstream_addr","upstream_response_time":"$upstream_response_time","remote_addr":"$remote_addr"}';
    resolver 10.252.0.10 valid=10s ipv6=off;
    access_log /dev/stdout nginxlog_json;
    error_log /dev/stdout notice;
    client_body_temp_path /var/run/openresty/nginx-client-body;
    proxy_temp_path       /var/run/openresty/nginx-proxy;
    fastcgi_temp_path     /var/run/openresty/nginx-fastcgi;
    uwsgi_temp_path       /var/run/openresty/nginx-uwsgi;
    scgi_temp_path        /var/run/openresty/nginx-scgi;

    sendfile        on;

    keepalive_timeout  90;

    gzip  on;
    server {
        listen  20000;
        server_name  localhost;
        client_header_buffer_size 2046k;
        large_client_header_buffers 4 2046k;
        client_max_body_size 500M;
        client_body_buffer_size 500M;
        set $wx_upstream "https://qyapi.weixin.qq.com";
        set $wx_host "qyapi.weixin.qq.com";
        location / {
            proxy_set_header Host $wx_host;
            proxy_pass $wx_upstream;
        }
    }
    include /etc/nginx/conf.d/*.conf;
}