Dokcer 部署 3 节点 MinIO 集群
资源清单
| 主机 | IP |
|---|---|
| minio1 | 10.0.0.1 |
| minio2 | 10.0.0.2 |
| minio3 | 10.0.0.3 |
| nginx | 10.0.0.4 |
| 软件 | 版本 |
|---|---|
| docker | 20.10.12 |
| minio | RELEASE.2022-02-18T01-50-10Z |
| nginx | 1.21.6 |
一、Docker 安装
1. 使用国内 yum 源
# yum install -y yum-utils device-mapper-persistent-data lvm2
# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
2. 卸载旧版本的 docker
## 如果主机上已经有docker存在且不是想要安装的版本，需要先进行卸载。
# yum remove -y docker \
docker-client \
docker-client-latest \
docker-common \
docker-latest \
docker-latest-logrotate \
docker-logrotate \
docker-selinux \
docker-engine-selinux \
docker-engine \
container*
3. 安装 Docker20.10 版本
# yum -y install docker-ce-20.10.12-3.el7 docker-ce-cli-20.10.12-3.el7
4. 设置镜像加速
# mkdir /etc/docker
# vi /etc/docker/daemon.json
{
"registry-mirrors": ["https://xxxxxxxxx.mirror.aliyuncs.com"]
}
5. 启动 docker
# systemctl start docker
# systemctl enable docker
# systemctl status docker
二、部署服务
1. 部署节点1
10.0.0.1
a | 编辑 install.sh 文件
# cat << EOF >> install.sh
#!/bin/bash
docker run -d --net=host --name=minio \
-v /data/minio/data1:/data1 \
-v /data/minio/data2:/data2 \
-e "MINIO_ACCESS_KEY=minio" \
-e "MINIO_SECRET_KEY=minio123" \
--add-host minio-1:10.0.0.1 \
--add-host minio-2:10.0.0.2 \
--add-host minio-3:10.0.0.3 \
minio/minio:RELEASE.2022-02-18T01-50-10Z \
server --address 10.0.0.1 \ # 填写 节点1 IP地址
http://minio-1/data1 http://minio-1/data2 \
http://minio-2/data1 http://minio-2/data2 \
http://minio-3/data1 http://minio-3/data2
EOF
b | 部署服务
# mkdir /data/minio/data{1,2} -pv
# bash install.sh
2. 部署节点2
10.0.0.2
a | 编辑 install.sh 文件
# cat << EOF >> install.sh
#!/bin/bash
docker run -d --net=host --name=minio \
-v /data/minio/data1:/data1 \
-v /data/minio/data2:/data2 \
-e "MINIO_ACCESS_KEY=minio" \
-e "MINIO_SECRET_KEY=minio123" \
--add-host minio-1:10.0.0.1 \
--add-host minio-2:10.0.0.2 \
--add-host minio-3:10.0.0.3 \
minio/minio:RELEASE.2022-02-18T01-50-10Z \
server --address 10.0.0.2 \ # 填写 节点2 IP地址
http://minio-1/data1 http://minio-1/data2 \
http://minio-2/data1 http://minio-2/data2 \
http://minio-3/data1 http://minio-3/data2
EOF
b | 部署服务
# mkdir /data/minio/data{1,2} -pv
# bash install.sh
3. 部署节点3
10.0.0.3
a | 编辑 install.sh 文件
# cat << EOF >> install.sh
#!/bin/bash
docker run -d --net=host --name=minio \
-v /data/minio/data1:/data1 \
-v /data/minio/data2:/data2 \
-e "MINIO_ACCESS_KEY=minio" \
-e "MINIO_SECRET_KEY=minio123" \
--add-host minio-1:10.0.0.1 \
--add-host minio-2:10.0.0.2 \
--add-host minio-3:10.0.0.3 \
minio/minio:RELEASE.2022-02-18T01-50-10Z \
server --address 10.0.0.3 \ # 填写 节点3 IP地址
http://minio-1/data1 http://minio-1/data2 \
http://minio-2/data1 http://minio-2/data2 \
http://minio-3/data1 http://minio-3/data2
EOF
b | 部署服务
# mkdir /data/minio/data{1,2} -pv
# bash install.sh
4. 部署 Nginx 实现高可用
a | 编辑 install.sh 文件
# cat << EOF >> install.sh
#/bin/bash
docker run -d --name nginx_minio \
--restart always \
-p 9000:9000 \
-p 9001:9001 \
--add-host minio1:10.0.0.1 \
--add-host minio2:10.0.0.2 \
--add-host minio3:10.0.0.3 \
-v /data/nginx/nginx.conf:/etc/nginx/nginx.conf \
-v /etc/localtime:/etc/localtime \
nginx:1.21.6
EOF
b | 提供 Nginx 配置文件
详细看 Nginx 配置文件
c | 启动 Nginx
# mkdir -pv /data/nginx/
## 负责步骤b中配置文件到nginx目录
# cp nginx.conf /data/nginx/
## 启动服务
# bash install.sh
d | 访问服务
在浏览器访问 http://10.0.0.4:9000 访问minio web 访问页面
5. Nginx 配置文件
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 4096;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
# include /etc/nginx/conf.d/*.conf;
upstream minio {
server minio1:9000;
server minio2:9000;
server minio3:9000;
}
upstream console {
ip_hash;
server minio1:9001;
server minio2:9001;
server minio3:9001;
}
server {
listen 9000;
listen [::]:9000;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio;
}
}
server {
listen 9001;
listen [::]:9001;
server_name localhost;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 0;
# To disable buffering
proxy_buffering off;
proxy_request_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
# This is necessary to pass the correct IP to be hashed
real_ip_header X-Real-IP;
proxy_connect_timeout 300;
# To support websocket
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
chunked_transfer_encoding off;
proxy_pass http://console;
}
}
}
参考文档
https://docs.min.io/docs/deploy-minio-on-docker-compose
