minio集群搭建

服务器优化

最好升级4.xx内核

系统最大文件数修改

1
2
echo "*   soft    nofile  65535" >> /etc/security/limits.conf
echo "* hard nofile 65535" >> /etc/security/limits.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
cat > /etc/sysctl.d/minio.conf <<EOF
# maximum number of open files/file descriptors
fs.file-max = 4194303
# use as little swap space as possible
vm.swappiness = 1
# prioritize application RAM against disk/swap cache
vm.vfs_cache_pressure = 50
# minimum free memory
vm.min_free_kbytes = 1000000
# follow mellanox best practices https://community.mellanox.com/s/article/linux-sysctl-tuning
# the following changes are recommended for improving IPv4 traffic performance by Mellanox
# disable the TCP timestamps option for better CPU utilization
net.ipv4.tcp_timestamps = 0
# enable the TCP selective acks option for better throughput
net.ipv4.tcp_sack = 1
# increase the maximum length of processor input queues
net.core.netdev_max_backlog = 250000
# increase the TCP maximum and default buffer sizes using setsockopt()
net.core.rmem_max = 4194304
net.core.wmem_max = 4194304
net.core.rmem_default = 4194304
net.core.wmem_default = 4194304
net.core.optmem_max = 4194304
# increase memory thresholds to prevent packet dropping:
net.ipv4.tcp_rmem = 4096 87380 4194304
net.ipv4.tcp_wmem = 4096 65536 4194304
# enable low latency mode for TCP:
net.ipv4.tcp_low_latency = 1
# the following variable is used to tell the kernel how much of the socket buffer
# space should be used for TCP window size, and how much to save for an application
# buffer. A value of 1 means the socket buffer will be divided evenly between.
# TCP windows size and application.
net.ipv4.tcp_adv_win_scale = 1
# maximum number of incoming connections
net.core.somaxconn = 65535
# maximum number of packets queued
net.core.netdev_max_backlog = 10000
# queue length of completely established sockets waiting for accept
net.ipv4.tcp_max_syn_backlog = 4096
# time to wait (seconds) for FIN packet
net.ipv4.tcp_fin_timeout = 15
# disable icmp send redirects
net.ipv4.conf.all.send_redirects = 0
# disable icmp accept redirect
net.ipv4.conf.all.accept_redirects = 0
# drop packets with LSR or SSR
net.ipv4.conf.all.accept_source_route = 0
# MTU discovery, only enable when ICMP blackhole detected
net.ipv4.tcp_mtu_probing = 1
EOF
sysctl -p /etc/sysctl.d/minio.conf

/data1/minio必须是单独挂载的数据盘

文件系统最好为xfs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
fdisk /dev/vdb
Command (m for help): n #新建分区
Partition type:
p primary (0 primary, 0 extended, 4 free)
e extended
Select (default p): p # 主分区
Partition number (1-4, default 1):
First sector (2048-104857599, default 2048):
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-104857599, default 104857599):
Using default value 104857599
Partition 1 of type Linux and of size 50 GiB is set

Command (m for help): w # 保存
The partition table has been altered!

Calling ioctl() to re-read partition table.
Syncing disks.

mkfs.xfs /dev/vdb1 # 文件系统初始化

blkid
/dev/vda1: UUID="" TYPE="ext4"
/dev/vdb1: UUID="" TYPE="xfs"
echo `blkid /dev/vdb1 | awk '{print $2}' | sed 's/\"//g'` /data1/minio xfs defaults 0 0 >> /etc/fstab

创建环境变量文件

vim /etc/default/minio

3个节点配置文件必须一样,不能带空格和中文和注释

1
2
3
4
5
MINIO_VOLUMES="http://192.168.14.24:9000/data1/minio http://192.168.14.25:9000/data1/minio http://192.168.14.26:9000/data1/minio"
MINIO_OPTS="--console-address :9001 --address :9000"
MINIO_ROOT_USER=
MINIO_ROOT_PASSWORD=
MINIO_PROMETHEUS_AUTH_TYPE="public"

创建systemd文件

vim /usr/lib/systemd/system/minio.service

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
[Unit]
Description=MinIO
Documentation=https://min.io/docs/minio/linux/index.html
Wants=network-online.target
After=network-online.target
AssertFileIsExecutable=/usr/local/bin/minio

[Service]
Type=simple
WorkingDirectory=/usr/local

User=root
Group=root

EnvironmentFile=-/etc/default/minio
ExecStartPre=/bin/bash -c "if [ -z \"${MINIO_VOLUMES}\" ]; then echo \"Variable MINIO_VOLUMES not set in /etc/default/minio\"; exit 1; fi"
ExecStart=/usr/local/bin/minio server $MINIO_OPTS $MINIO_VOLUMES

Restart=always

LimitNOFILE=65536

TasksMax=infinity

TimeoutStopSec=infinity
SendSIGKILL=no

[Install]
WantedBy=multi-user.target

启动服务

1
2
3
4
systemctl daemon-reload
systemctl enable minio
systemctl restart minio
systemctl status minio

minio压测

50G磁盘 PL1性能ESDD ecs.r6a.large

集群单节点压测

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
[root@iZbp1imzcyvws0523mzrg4Z ~]# docker run --rm --net host minio/warp:latest mixed --host 192.168.14.6:9000 --access-key= --secret-key= --obj.size=3145728 --objects=5000 --autoterm
warp: Preparing server.
Uploading 5000 objects of Random data; 3145728 bytes totalwarp: Starting benchmark in 3s...
warp: Benchmark starting...
Throughput 95.2 objects/s within 7.500000% for 12.439s. Assuming stability. Terminating benchmark.
warp: Saving benchmark data...
warp: Benchmark data written to "warp-mixed-2024-04-18[030554]-0L1p.csv.zst"

Mixed operations.
Operation: DELETE, 10%, Concurrency: 20, Ran 44s.
* Throughput: 27.63 obj/s

Operation: GET, 45%, Concurrency: 20, Ran 44s.
* Throughput: 370.83 MiB/s, 123.61 obj/s

Operation: PUT, 15%, Concurrency: 20, Ran 44s.
* Throughput: 123.91 MiB/s, 41.30 obj/s

Operation: STAT, 30%, Concurrency: 20, Ran 44s.
* Throughput: 82.49 obj/s

Cluster Total: 494.24 MiB/s, 274.61 obj/s over 45s.
warp: Starting cleanup...
Clearing Prefix "warp-benchmark-bucket/g4PyEoef/"...warp: Cleanup Done.

负载均衡压测

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@iZbp1imzcyvws0523mzrg4Z ~]# docker run --rm --net host minio/warp:latest mixed --host 192.168.14.19:9000 --access-key= --secret-key= --obj.size=3145728 --objects=5000 --autoterm
warp: Preparing server.
Uploading 5000 objects of Random data; 3145728 bytes totalwarp: Starting benchmark in 3s...
warp: Benchmark starting...
Throughput 66.5 objects/s within 7.500000% for 19.761s. Assuming stability. Terminating benchmark.
warp: Saving benchmark data...
warp: Benchmark data written to "warp-mixed-2024-04-18[031629]-btcX.csv.zst"

Mixed operations.
Operation: DELETE, 10%, Concurrency: 20, Ran 1m10s.
* Throughput: 25.53 obj/s

Operation: GET, 45%, Concurrency: 20, Ran 1m10s.
* Throughput: 344.61 MiB/s, 114.87 obj/s

Operation: PUT, 15%, Concurrency: 20, Ran 1m10s.
* Throughput: 114.97 MiB/s, 38.32 obj/s

Operation: STAT, 30%, Concurrency: 20, Ran 1m10s.
* Throughput: 76.61 obj/s

Cluster Total: 459.42 MiB/s, 255.31 obj/s over 1m11s.

MINIO_SERVER_URL配置后压测无区别

升级服务器内网带宽后压测

ecs.g7nex.xlarge 4c 16G PL1

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
[root@iZbp1imzcyvws0523mzrg4Z ~]# docker run --rm --net host minio/warp:latest mixed --host 192.168.14.19:9000 --access-key= --secret-key= --obj.size=3145728 --objects=5000 --autoterm
warp: Preparing server.
Uploading 5000 objects of Random data; 3145728 bytes totalwarp: Starting benchmark in 3s...
warp: Benchmark starting...
Throughput 100.0 objects/s within 7.500000% for 10.43s. Assuming stability. Terminating benchmark.
warp: Saving benchmark data...
warp: Benchmark data written to "warp-mixed-2024-04-18[083834]-mhZd.csv.zst"

Mixed operations.
Operation: DELETE, 10%, Concurrency: 20, Ran 37s.
* Throughput: 30.10 obj/s

Operation: GET, 45%, Concurrency: 20, Ran 37s.
* Throughput: 404.28 MiB/s, 134.76 obj/s

Operation: PUT, 15%, Concurrency: 20, Ran 37s.
* Throughput: 134.80 MiB/s, 44.93 obj/s

Operation: STAT, 30%, Concurrency: 20, Ran 37s.
* Throughput: 89.80 obj/s

Cluster Total: 537.62 MiB/s, 298.80 obj/s over 38s.
warp: Starting cleanup...

cpu 70% mem70%

优化内核参数后

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@iZbp1imzcyvws0523mzrg4Z ~]# docker run --rm --net host harbor-core.suosihulian.com/dockerhub/minio/warp:latest mixed --host 192.168.14.19:90 --access-key=Bj2RDnb4bnAey3 --secret-key=fDfRoPz5Kt2tHN23u9ej --obj.size=3145728 --objects=5000 --autoterm
warp: Preparing server.
Uploading 5000 objects of Random data; 3145728 bytes totalwarp: Starting benchmark in 3s...
warp: Benchmark starting...
Throughput 99.0 objects/s within 7.500000% for 10.381s. Assuming stability. Terminating benchmark.
warp: Saving benchmark data...
warp: Benchmark data written to "warp-mixed-2024-04-18[090030]-DukG.csv.zst"

Mixed operations.
Operation: DELETE, 10%, Concurrency: 20, Ran 37s.
* Throughput: 30.26 obj/s

Operation: GET, 45%, Concurrency: 20, Ran 37s.
* Throughput: 406.50 MiB/s, 135.50 obj/s

Operation: PUT, 15%, Concurrency: 20, Ran 37s.
* Throughput: 135.27 MiB/s, 45.09 obj/s

Operation: STAT, 30%, Concurrency: 20, Ran 37s.
* Throughput: 90.27 obj/s

Cluster Total: 540.83 MiB/s, 300.75 obj/s over 37s.