From c549c384a75970dc474f491db276fc2dee4bf622 Mon Sep 17 00:00:00 2001 From: colben Date: Wed, 17 Apr 2024 20:44:50 +0800 Subject: [PATCH] update --- content/post/minio.md | 162 ++++++++++++++++++------------------------ 1 file changed, 69 insertions(+), 93 deletions(-) diff --git a/content/post/minio.md b/content/post/minio.md index f3c1c30..497ef33 100644 --- a/content/post/minio.md +++ b/content/post/minio.md @@ -1,7 +1,7 @@ --- title: "Minio 笔记" date: 2021-10-17T00:48:00+08:00 -lastmod: 2021-10-17T00:48:00+08:00 +lastmod: 2024-04-17T20:08:00+08:00 keywords: [] tags: ["minio"] categories: ["storage"] @@ -65,38 +65,80 @@ categories: ["storage"] ## 部署多节点分布式存储 ### 环境 -自定义域名 | 网卡 IP | 硬盘槽 | 数据分区 ----- | ---- | ---- | ---- -X1.X.X | eth0: 192.168.1.11
eth1:10.0.0.11 | 8 个 | /dev/sdb1, /dev/sdc1
/dev/sdd1, /dev/sde1 -X2.X.X | eth0: 192.168.1.12
eth1:10.0.0.12 | 8 个 | /dev/sdb1, /dev/sdc1
/dev/sdd1, /dev/sde1 -X3.X.X | eth0: 192.168.1.13
eth1:10.0.0.13 | 8 个 | /dev/sdb1, /dev/sdc1
/dev/sdd1, /dev/sde1 -X4.X.X | eth0: 192.168.1.14
eth1:10.0.0.14 | 8 个 | /dev/sdb1, /dev/sdc1
/dev/sdd1, /dev/sde1 +自定义域名 | 网卡 IP | 数据盘 +---- | ---- | ---- +X1.X.X | eth0: 192.168.1.11
eth1:10.0.0.11 | /dev/sdb, /dev/sdc
/dev/sdd, /dev/sde +X2.X.X | eth0: 192.168.1.12
eth1:10.0.0.12 | /dev/sdb, /dev/sdc
/dev/sdd, /dev/sde +X3.X.X | eth0: 192.168.1.13
eth1:10.0.0.13 | /dev/sdb, /dev/sdc
/dev/sdd, /dev/sde +X4.X.X | eth0: 192.168.1.14
eth1:10.0.0.14 | /dev/sdb, /dev/sdc
/dev/sdd, /dev/sde - eth0 用于接收 api 请求 - eth1 用于集群内部通信 -- 每台服务器有 8 个硬盘槽，这里先各用 4 个组成 server pool，运行 minio 集群 -- 每台服务器剩下的 4 个硬盘槽用于后面的扩容操作 +- 操作系统：RockyLinux 9.3 -### 挂载数据分区 +### 挂载数据盘 - 在全部服务器上执行如下操作 -- 挂载硬盘 +- 格式化硬盘，设置卷标 ```bash - mkdir -p /mnt/minio{1,2,3,4} - mount /dev/sdb1 /mnt/minio1 - mount /dev/sdc1 /mnt/minio2 - mount /dev/sdd1 /mnt/minio3 - mount /dev/sde1 /mnt/minio4 + mkfs.xfs -f /dev/sdb -L MINIO1 + mkfs.xfs -f /dev/sdc -L MINIO2 + mkfs.xfs -f /dev/sdd -L MINIO3 + mkfs.xfs -f /dev/sde -L MINIO4 ``` -- 建议在 fstab 中配置开机自动挂载 +- 在 /etc/fstab 中增加挂载配置 + ``` + LABEL=MINIO1 /mnt/minio1 xfs defaults,noatime 0 2 + LABEL=MINIO2 /mnt/minio2 xfs defaults,noatime 0 2 + LABEL=MINIO3 /mnt/minio3 xfs defaults,noatime 0 2 + LABEL=MINIO4 /mnt/minio4 xfs defaults,noatime 0 2 + ``` + +- 创建挂载目录，挂载数据盘 + ```bash + systemctl daemon-reload + mkdir -p /mnt/minio{1,2,3,4} + mount -a + ``` + +- 禁用 xfs 错误重试 + ```bash + for i in sdb sdc sdd sde; do + echo 0 > /sys/fs/xfs/$i/error/metadata/EIO/max_retries + echo 0 > /sys/fs/xfs/$i/error/metadata/ENOSPC/max_retries + echo 0 > /sys/fs/xfs/$i/error/metadata/default/max_retries + done + ``` + +- 为 /etc/rc.d/rc.local 增加可执行权限 + ```bash + chmod 0755 /etc/rc.d/rc.local + ``` + +- 编辑 /etc/rc.d/rc.local，增加“禁用 xfs 错误重试”的脚本 + ```bash + for i in sdb sdc sdd sde; do + echo 0 > /sys/fs/xfs/$i/error/metadata/EIO/max_retries + echo 0 > /sys/fs/xfs/$i/error/metadata/ENOSPC/max_retries + echo 0 > /sys/fs/xfs/$i/error/metadata/default/max_retries + done + ``` ### 安装 minio - 在全部服务器上执行如下操作 -- 下载 +- 下载 minio rpm 包，安装 + ```bash + curl -L -o minio.rpm https://dl.min.io/server/minio/release/linux-amd64/archive/minio-20240406052602.0.0-1.x86_64.rpm + rpm -Uvh minio.rpm + groupadd -r minio-user + useradd -r -g minio-user minio-user + chown minio-user:minio-user /mnt/minio{1..4} + ``` + +- 下载 minio 客户端工具 mc ```bash - curl -L -o /usr/local/bin/minio https://dl.min.io/server/minio/release/linux-amd64/minio curl -L -o /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc - chmod 0755 /usr/local/bin/{minio,mc} + chmod 0755 /usr/local/bin/mc ``` - 修改 hosts，增加自定义域名解析，**注意这里是集群通信，要用 eth1 网卡 ip** @@ -107,87 +149,21 @@ X4.X.X | eth0: 192.168.1.14
eth1:10.0.0.14 | 8 个 | /dev/sdb1, /dev/sd 10.0.0.14 X4.X.X ``` -- 创建 /etc/systemd/system/minio.service，内容如下 +- 创建 /etc/default/minio，内容如下 ```ini - [Unit] - Description=MinIO - After=network.service - Wants=network.service - - [Service] - Environment=MINIO_ROOT_USER=XXX - Environment=MINIO_ROOT_PASSWORD=XXXXXXXX - ExecStart=/usr/local/bin/minio server \ - --address ":9000" \ - --console-address ":9010" \ - http://X{1...4}.X.X:9000/mnt/minio{1...4} - #Restart=on-failure - RestartSec=10s - - [Install] - WantedBy=multi-user.target + MINIO_VOLUMES="http://X{1...4}:9000/mnt/minio{1...4}" + MINIO_OPTS="--console-address :9001" + MINIO_ROOT_USER=minio-admin + MINIO_ROOT_PASSWORD=1234567890 ``` ### 启动 minio - 在全部服务器上执行如下操作 -- 配置开机自动启动 +- 启动 minio，并配置开机自动启动 ```bash systemctl daemon-reload - systemctl enable minio - ``` - -- 启动 minio - ```bash systemctl start monio - ``` - -## 扩容多节点分布式存储 -### 加装硬盘 -- 在全部服务器上执行如下操作 -- 在每台服务器上个增加四个新硬盘，每个硬盘创建一个分区，格式化 -- 挂载新增的硬盘分区 - ```bash - mkdir -p /mnt/minio{5,6,7,8} - mount /dev/sdf1 /mnt/minio5 - mount /dev/sdg1 /mnt/minio6 - mount /dev/sdh1 /mnt/minio7 - mount /dev/sdi1 /mnt/minio8 - ``` - -- 建议在 fstab 中配置开机自动挂载 - -### 扩容 -- 在全部服务器上执行如下操作 -- 停止 minio 服务 - ```bash - systemctl stop minio - ``` - -- 修改 /etc/systemd/system/minio.service，增加新的 server pool，内容如下 - ```ini - [Unit] - Description=MinIO - After=network.service - Wants=network.service - - [Service] - Environment=MINIO_ROOT_USER=XXX - Environment=MINIO_ROOT_PASSWORD=XXXXXXXX - ExecStart=/usr/local/bin/minio server \ - --address ":9000" \ - --console-address ":9010" \ - http://X{1...4}.X.X:9000/mnt/minio{1...4} \ - http://X{1...4}.X.X:9000/mnt/minio{5...8} - #Restart=on-failure - RestartSec=10s - - [Install] - WantedBy=multi-user.target - ``` - -- 启动 minio 服务 - ```bash - systemctl start minio + systemctl enable minio ``` ## 客户端常用操作