二、Ceph的ceph-deploy部署
系统版本：buntu 18.04.5 LTS 内核参数：4.15.0-112-generic ceph版本：pacific/16.2.5 主机分配： #部署服务器ceph-deploy 172.168.32.101/10.0.0.101 ceph-deploy #两个ceph-mgr 管理服务器 172.168.32.102/10.0.0.102 ceph-mgr01 172.168.32.103/10.0.0.103 ceph-mgr02 #三台服务器作为ceph 集群Mon 监视服务器，每台服务器可以和ceph 集群的cluster 网络通信。 172.168.32.104/10.0.0.104 ceph-mon01 ceph-mds01 172.168.32.105/10.0.0.105 ceph-mon02 ceph-mds02 172.168.32.106/10.0.0.106 ceph-mon03 ceph-mds03 #四台服务器作为ceph 集群OSD 存储服务器，每台服务器支持两个网络，public 网络针对客户端访问，cluster 网络用于集群管理及数据同步，每台三块或以上的磁盘 172.168.32.107/10.0.0.107 ceph-node01 172.168.32.108/10.0.0.108 ceph-node02 172.168.32.109/10.0.0.109 ceph-node03 172.168.32.110/10.0.0.110 ceph-node04 #磁盘划分 #/dev/sdb /dev/sdc /dev/sdd /dev/sde #50G
1)所有节点更换为清华源
cat >/etc/apt/source.list<<EOF # 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释 deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse # deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse # deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse # deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse # deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse EOF
apt install iproute2 ntpdate tcpdump telnet traceroute nfs-kernel-server nfs-common lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute gcc openssh-server lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute iotop unzip zip openjdk-8-jdk -y
cat >/etc/sysctl.conf <<EOF # Controls source route verification net.ipv4.conf.default.rp_filter = 1 net.ipv4.ip_nonlocal_bind = 1 net.ipv4.ip_forward = 1 # Do not accept source routing net.ipv4.conf.default.accept_source_route = 0 # Controls the System Request debugging functionality of the kernel kernel.sysrq = 0 # Controls whether core dumps will append the PID to the core filename. # Useful for debugging multi-threaded applications. kernel.core_uses_pid = 1 # Controls the use of TCP syncookies net.ipv4.tcp_syncookies = 1 # Disable netfilter on bridges. net.bridge.bridge-nf-call-ip6tables = 0 net.bridge.bridge-nf-call-iptables = 0 net.bridge.bridge-nf-call-arptables = 0 # Controls the default maxmimum size of a mesage queue kernel.msgmnb = 65536 # # Controls the maximum size of a message, in bytes kernel.msgmax = 65536 # Controls the maximum shared segment size, in bytes kernel.shmmax = 68719476736 # # Controls the maximum number of shared memory segments, in pages kernel.shmall = 4294967296 # TCP kernel paramater net.ipv4.tcp_mem = 786432 1048576 1572864 net.ipv4.tcp_rmem = 4096 87380 4194304 net.ipv4.tcp_wmem = 4096 16384 4194304 n et.ipv4.tcp_window_scaling = 1 net.ipv4.tcp_sack = 1 # socket buffer net.core.wmem_default = 8388608 net.core.rmem_default = 8388608 net.core.rmem_max = 16777216 net.core.wmem_max = 16777216 net.core.netdev_max_backlog = 262144 net.core.somaxconn = 20480 net.core.optmem_max = 81920 # TCP conn net.ipv4.tcp_max_syn_backlog = 262144 net.ipv4.tcp_syn_retries = 3 net.ipv4.tcp_retries1 = 3 net.ipv4.tcp_retries2 = 15 # tcp conn reuse net.ipv4.tcp_timestamps = 0 net.ipv4.tcp_tw_reuse = 0 net.ipv4.tcp_tw_recycle = 0 net.ipv4.tcp_fin_timeout = 1 net.ipv4.tcp_max_tw_buckets = 20000 net.ipv4.tcp_max_orphans = 3276800 net.ipv4.tcp_synack_retries = 1 net.ipv4.tcp_syncookies = 1 # keepalive conn net.ipv4.tcp_keepalive_time = 300 net.ipv4.tcp_keepalive_intvl = 30 net.ipv4.tcp_keepalive_probes = 3 net.ipv4.ip_local_port_range = 10001 65000 # swap vm.overcommit_memory = 0 vm.swappiness = 10 #net.ipv4.conf.eth1.rp_filter = 0 #net.ipv4.conf.lo.arp_ignore = 1 #net.ipv4.conf.lo.arp_announce = 2 #net.ipv4.conf.all.arp_ignore = 1 #net.ipv4.conf.all.arp_announce = 2 EOF
cat > /etc/security/limits.conf <<EOF root soft core unlimited root hard core unlimited root soft nproc 1000000 root hard nproc 1000000 root soft nofile 1000000 root hard nofile 1000000 root soft memlock 32000 root hard memlock 32000 root soft msgqueue 8192000 root hard msgqueue 8192000 * soft core unlimited * hard core unlimited * soft nproc 1000000 * hard nproc 1000000 * soft nofile 1000000 * hard nofile 1000000 * soft memlock 32000 * hard memlock 32000 * soft msgqueue 8192000 * hard msgqueue 8192000 EOF
#安装cron并启动 apt install cron -y systemctl status cron.service #同步时间 /usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w #每5分钟同步一次时间 echo "*/5 * * * * /usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w" >> /var/spool/cron/crontabs/root
cat >>/etc/hosts<<EOF
172.168.32.101 ceph-deploy
172.168.32.102 ceph-mgr01
172.168.32.103 ceph-mgr02
172.168.32.104 ceph-mon01 ceph-mds01
172.168.32.105 ceph-mon02 ceph-mds02
172.168.32.106 ceph-mon03 ceph-mds03
172.168.32.107 ceph-node01
172.168.32.108 ceph-node02
172.168.32.109 ceph-node03
172.168.32.110 ceph-node04
EOF
7)所有节点安装python2
apt install python2.7 -y ln -sv /usr/bin/python2.7 /usr/bin/python2
3、ceph部署
#配置ceph仓库
sudo echo "deb https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic main" >> /etc/apt/sources.list
#导入key
wget -q -O- 'https://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc' | sudo apt-key add -
#更新仓库源
apt update
推荐使用指定的普通用户部署和运行ceph 集群，普通用户只要能以非交互方式执行命令执行一些特权命令即可，新版的ceph-deploy 可以指定包含root 的在内只要可以执行 命令的用户，不过仍然推荐使用普通用户，比如ceph、cephuser、cephadmin 这样的用户去管理ceph 集群。
#因为前面安装的ceph-common会更改ceph用户的家目录，建议使用其它用户来部署，如cephadmin用户
groupadd -r -g 2021 cephadmin && useradd -r -m -s /bin/bash -u 2021 -g 2021 cephadmin && echo cephadmin:123456 | chpasswd
#允许ceph 用户以 执行特权命令
echo "cephadmin ALL=(ALL) NOPASSWD:ALL" >> /etc/ers
在ceph-deploy 节点配置允许以非交互的方式登录到各ceph node/mon/mgr 节点，即在ceph-deploy 节点的ceph用户生成秘钥对，然后分发公钥到各被管理节点的ceph用户。
#(1)创建ssh密钥
cephadmin@ceph-deploy:/tmp$ ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/home/ceph/.ssh/id_rsa):
Created directory '/home/ceph/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/ceph/.ssh/id_rsa.
Your public key has been saved in /home/ceph/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:Oq0Vh0Do/VUVklh3U58XNgNNkIfCIPAiXFw+ztEhbqM cephadmin@ceph-deploy
The key's randomart image is:
+---[RSA 2048]----+
| .+++ oooo=@O+|
| . oo+ + ooo=.+B|
| + o.O . .. ..o|
| o B.+.. .|
| E +S.. |
| o.o |
| o o |
| + |
| . |
+----[SHA256]-----+
#(2)安装sshpass
cephadmin@ceph-deploy:/tmp$ sudo apt install sshpass
#(3)ceph-deploy节点使用ceph用户分发密钥脚本
cat >>/tmp/ssh_fenfa.sh<<EOF
#!/bin/bash
#目标主机列表
IP="
172.168.32.101
172.168.32.102
172.168.32.103
172.168.32.104
172.168.32.105
172.168.32.106
172.168.32.107
172.168.32.108
172.168.32.109
172.168.32.110"
for node in ${IP};do
sshpass -p 123456 ssh-copy-id cephadmin@${node} -o StrictHostKeyChecking=no &> /dev/null
if [ $? -eq 0 ];then
echo "${node}----> 密钥分发success完成"
else
echo "${node}----> 密钥分发false失败"
fi
done
EOF
#(4)使用脚本分发ssh密钥
cephadmin@ceph-deploy:/tmp$ bash ssh_fenfa.sh
172.168.32.101----> 密钥分发success完成
172.168.32.102----> 密钥分发success完成
172.168.32.103----> 密钥分发success完成
172.168.32.104----> 密钥分发success完成
172.168.32.105----> 密钥分发success完成
172.168.32.106----> 密钥分发success完成
172.168.32.107----> 密钥分发success完成
172.168.32.108----> 密钥分发success完成
172.168.32.109----> 密钥分发success完成
172.168.32.110----> 密钥分发success完成
cephadmin@ceph-deploy:~# sudo apt-cache madison ceph-deploy ceph-deploy | 2.0.1 | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic/main amd64 Packages ceph-deploy | 2.0.1 | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic/main i386 Packages ceph-deploy | 1.5.38-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu bionic/universe amd64 Packages ceph-deploy | 1.5.38-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu bionic/universe i386 Packages cephadmin@ceph-deploy:~# sudo apt install ceph-deploy
在管理节点初始化mon 节点
cephadmin@ceph-deploy:~$ mkdir ceph-cluster #保存当前集群的初始化配置信息
cephadmin@ceph-deploy:~$ cd ceph-cluster/ cephadmin@ceph-deploy:~/ceph-cluster$
前期只先初始化ceph-mon01节点，ceph-mon02和ceph-mon03在集群部署完成后，再手动添加
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy new --cluster-network 10.0.0.0/16 --public-network 172.168.0.0/16 ceph-mon01
#下面内容为ceph-mon01的初始化过程
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy new --cluster-network 10.0.0.0/16 --public-network 172.168.0.0/16 ceph-mon01
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy new --cluster-network 10.0.0.0/16 --public-network 172.168.0.0/16 ceph-mon01
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f25a37afe10>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] ssh_copykey : True
[ceph_deploy.cli][INFO ] mon : ['ceph-mon01']
[ceph_deploy.cli][INFO ] func : <function new at 0x7f25a0a64ad0>
[ceph_deploy.cli][INFO ] public_network : 172.168.0.0/16
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] cluster_network : 10.0.0.0/16
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] fsid : None
[ceph_deploy.new][DEBUG ] Creating new cluster named ceph
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[ceph-mon01][DEBUG ] connected to host: ceph-deploy
[ceph-mon01][INFO ] Running command: ssh -CT -o BatchMode=yes ceph-mon01
[ceph_deploy.new][WARNIN] could not connect via SSH
[ceph_deploy.new][INFO ] will connect again with password prompt
The authenticity of host 'ceph-mon01 (172.168.32.104)' can't be established.
ECDSA key fingerprint is SHA256:AIDN3qa9QKjViElHDrtTXhJ5EpTXdWj5Sc3tiy91E4Y.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'ceph-mon01' (ECDSA) to the list of known hosts.
[ceph-mon01][DEBUG ] connected to host: ceph-mon01
[ceph-mon01][DEBUG ] detect platform information from remote host
[ceph-mon01][DEBUG ] detect machine type
[ceph_deploy.new][INFO ] adding public keys to authorized_keys
[ceph-mon01][DEBUG ] append contents to file
[ceph-mon01][DEBUG ] connection detected need for sudo
[ceph-mon01][DEBUG ] connected to host: ceph-mon01
[ceph-mon01][DEBUG ] detect platform information from remote host
[ceph-mon01][DEBUG ] detect machine type
[ceph-mon01][DEBUG ] find the location of an executable
[ceph-mon01][INFO ] Running command: sudo /bin/ip link show
[ceph-mon01][INFO ] Running command: sudo /bin/ip addr show
[ceph-mon01][DEBUG ] IP addresses found: [u'172.168.32.104', u'10.0.0.104']
[ceph_deploy.new][DEBUG ] Resolving host ceph-mon01
[ceph_deploy.new][DEBUG ] Monitor ceph-mon01 at 172.168.32.104
[ceph_deploy.new][DEBUG ] Monitor initial members are ['ceph-mon01']
[ceph_deploy.new][DEBUG ] Monitor addrs are [u'172.168.32.104']
[ceph_deploy.new][DEBUG ] Creating a random mon key...
[ceph_deploy.new][DEBUG ] Writing monitor keyring to ceph.mon.keyring...
[ceph_deploy.new][DEBUG ] Writing initial config to ceph.conf...
初始化为验证
cephadmin@ceph-deploy:~/ceph-cluster$ ll
total 16
drwxrwxr-x 2 cephadmin cephadmin 75 Aug 30 10:48 ./
drwxr-xr-x 6 cephadmin cephadmin 157 Aug 30 10:25 ../
-rw-rw-r-- 1 cephadmin cephadmin 264 Aug 30 10:48 ceph.conf #自动生成的配置文件
-rw-r--r-- 1 cephadmin cephadmin 6207 Aug 30 10:48 ceph-deploy-ceph.log #初始化日志
-rw------- 1 cephadmin cephadmin 73 Aug 30 10:48 ceph.mon.keyring #用于ceph mon 节点内部通讯认证的秘钥环文件
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf
[global]
fsid = c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb #ceph集群ID
public_network = 172.168.0.0/16
cluster_network = 10.0.0.0/16
mon_initial_members = ceph-mon01 #可以用逗号做分割添加多个mon节点
mon_host = 172.168.32.104
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
初始化ceph-node节点
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --no-adjust-repos --nogpgcheck ceph-node01 ceph-node02 ceph-node03 ceph-node04
此过程会在指定的ceph node 节点按照串行的方式逐个服务器安装epel 源和ceph 源并安装ceph所需软件
#...... [ceph-node03][DEBUG ] The following additional packages will be installed: [ceph-node03][DEBUG ] ceph-base ceph-common ceph-mgr ceph-mgr-modules-core libaio1 libbabeltrace1 [ceph-node03][DEBUG ] libcephfs2 libdw1 libgoogle-perftools4 libibverbs1 libjaeger libjs-jquery [ceph-node03][DEBUG ] libleveldb1v5 liblttng-ust-ctl4 liblttng-ust0 liblua5.3-0 libnl-route-3-200 [ceph-node03][DEBUG ] liboath0 librabbitmq4 librados2 libradosstriper1 librbd1 librdkafka1 [ceph-node03][DEBUG ] librdmacm1 librgw2 libsnappy1v5 libtcmalloc-minimal4 liburcu6 [ceph-node03][DEBUG ] python-pastedeploy-tpl python3-bcrypt python3-bs4 python3-ceph-argparse [ceph-node03][DEBUG ] python3-ceph-common python3-cephfs python3-cherrypy3 python3-dateutil [ceph-node03][DEBUG ] python3-distutils python3-jwt python3-lib2to3 python3-logutils python3-mako [ceph-node03][DEBUG ] python3-markupsafe python3-paste python3-pastedeploy python3-pecan [ceph-node03][DEBUG ] python3-prettytable python3-rados python3-rbd python3-rgw [ceph-node03][DEBUG ] python3-simplegeneric python3-singledispatch python3-tempita [ceph-node03][DEBUG ] python3-waitress python3-webob python3-webtest python3-werkzeug [ceph-node03][DEBUG ] Suggested packages: [ceph-node03][DEBUG ] python3-influxdb python3-crypto python3-beaker python-mako-doc httpd-wsgi [ceph-node03][DEBUG ] libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc [ceph-node03][DEBUG ] python-waitress-doc python-webob-doc python-webtest-doc ipython3 [ceph-node03][DEBUG ] python3-lxml python3-termcolor python3-watchdog python-werkzeug-doc [ceph-node03][DEBUG ] Recommended packages: [ceph-node03][DEBUG ] ntp | time-daemon ceph-fuse ceph-mgr-dashboard ceph-mgr-diskprediction-local [ceph-node03][DEBUG ] ceph-mgr-k8sevents ceph-mgr-cephadm nvme-cli smartmontools ibverbs-providers [ceph-node03][DEBUG ] javascript-common python3-lxml python3-routes python3-simplejson [ceph-node03][DEBUG ] python3-pastescript python3-pyinotify [ceph-node03][DEBUG ] The following NEW packages will be installed: [ceph-node03][DEBUG ] ceph ceph-base ceph-common ceph-mds ceph-mgr ceph-mgr-modules-core ceph-mon [ceph-node03][DEBUG ] ceph-osd libaio1 libbabeltrace1 libcephfs2 libdw1 libgoogle-perftools4 [ceph-node03][DEBUG ] libibverbs1 libjaeger libjs-jquery libleveldb1v5 liblttng-ust-ctl4 [ceph-node03][DEBUG ] liblttng-ust0 liblua5.3-0 libnl-route-3-200 liboath0 librabbitmq4 librados2 [ceph-node03][DEBUG ] libradosstriper1 librbd1 librdkafka1 librdmacm1 librgw2 libsnappy1v5 [ceph-node03][DEBUG ] libtcmalloc-minimal4 liburcu6 python-pastedeploy-tpl python3-bcrypt [ceph-node03][DEBUG ] python3-bs4 python3-ceph-argparse python3-ceph-common python3-cephfs [ceph-node03][DEBUG ] python3-cherrypy3 python3-dateutil python3-distutils python3-jwt [ceph-node03][DEBUG ] python3-lib2to3 python3-logutils python3-mako python3-markupsafe [ceph-node03][DEBUG ] python3-paste python3-pastedeploy python3-pecan python3-prettytable [ceph-node03][DEBUG ] python3-rados python3-rbd python3-rgw python3-simplegeneric [ceph-node03][DEBUG ] python3-singledispatch python3-tempita python3-waitress python3-webob [ceph-node03][DEBUG ] python3-webtest python3-werkzeug radosgw #......
在各mon 节点按照组件ceph-mon,并通初始化mon 节点，mon 节点ha 还可以后期横向扩容。
root@ceph-mon01:~# apt install ceph-mon root@ceph-mon02:~# apt install ceph-mon root@ceph-mon03:~# apt install ceph-mon
在ceph-deploy节点初始化mon节点
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon create-initial
#下面为mon节点初始化内容
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mon create-initial
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : create-initial
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fbee5b13fa0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] func : <function mon at 0x7fbee5af7ad0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] keyrings : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.mon][DEBUG ] Deploying mon, cluster ceph hosts ceph-mon01
[ceph_deploy.mon][DEBUG ] detecting platform for host ceph-mon01 ...
[ceph-mon01][DEBUG ] connection detected need for sudo
[ceph-mon01][DEBUG ] connected to host: ceph-mon01
[ceph-mon01][DEBUG ] detect platform information from remote host
[ceph-mon01][DEBUG ] detect machine type
[ceph-mon01][DEBUG ] find the location of an executable
[ceph_deploy.mon][INFO ] distro info: Ubuntu 18.04 bionic
[ceph-mon01][DEBUG ] determining if provided host has same hostname in remote
[ceph-mon01][DEBUG ] get remote short hostname
[ceph-mon01][DEBUG ] deploying mon to ceph-mon01
[ceph-mon01][DEBUG ] get remote short hostname
[ceph-mon01][DEBUG ] remote hostname: ceph-mon01
[ceph-mon01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph-mon01][DEBUG ] create the mon path if it does not exist
[ceph-mon01][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-ceph-mon01/done
[ceph-mon01][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-ceph-mon01/done
[ceph-mon01][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-ceph-mon01.mon.keyring
[ceph-mon01][DEBUG ] create the monitor keyring file
[ceph-mon01][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i ceph-mon01 --keyring /var/lib/ceph/tmp/ceph-ceph-mon01.mon.keyring --setuser 64045 --setgroup 64045
[ceph-mon01][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-ceph-mon01.mon.keyring
[ceph-mon01][DEBUG ] create a done file to avoid re-doing the mon deployment
[ceph-mon01][DEBUG ] create the init path if it does not exist
[ceph-mon01][INFO ] Running command: sudo systemctl enable ceph.target
[ceph-mon01][INFO ] Running command: sudo systemctl enable ceph-mon@ceph-mon01
[ceph-mon01][WARNIN] Created symlink /etc/systemd/system/ceph-mon.target.wants/ceph-mon@ceph-mon01.service → /lib/systemd/system/ceph-mon@.service.
[ceph-mon01][INFO ] Running command: sudo systemctl start ceph-mon@ceph-mon01
[ceph-mon01][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon01.asok mon_status
[ceph-mon01][DEBUG ] ********************************************************************************
[ceph-mon01][DEBUG ] status for monitor: mon.ceph-mon01
[ceph-mon01][DEBUG ] {
[ceph-mon01][DEBUG ] "election_epoch": 3,
[ceph-mon01][DEBUG ] "extra_probe_peers": [],
[ceph-mon01][DEBUG ] "feature_map": {
[ceph-mon01][DEBUG ] "mon": [
[ceph-mon01][DEBUG ] {
[ceph-mon01][DEBUG ] "features": "0x3f01cfb9fffdffff",
[ceph-mon01][DEBUG ] "num": 1,
[ceph-mon01][DEBUG ] "release": "luminous"
[ceph-mon01][DEBUG ] }
[ceph-mon01][DEBUG ] ]
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] "features": {
[ceph-mon01][DEBUG ] "quorum_con": "4540138297136906239",
[ceph-mon01][DEBUG ] "quorum_mon": [
[ceph-mon01][DEBUG ] "kraken",
[ceph-mon01][DEBUG ] "luminous",
[ceph-mon01][DEBUG ] "mimic",
[ceph-mon01][DEBUG ] "osdmap-prune",
[ceph-mon01][DEBUG ] "nautilus",
[ceph-mon01][DEBUG ] "octopus",
[ceph-mon01][DEBUG ] "pacific",
[ceph-mon01][DEBUG ] "elector-pinging"
[ceph-mon01][DEBUG ] ],
[ceph-mon01][DEBUG ] "required_con": "2449958747317026820",
[ceph-mon01][DEBUG ] "required_mon": [
[ceph-mon01][DEBUG ] "kraken",
[ceph-mon01][DEBUG ] "luminous",
[ceph-mon01][DEBUG ] "mimic",
[ceph-mon01][DEBUG ] "osdmap-prune",
[ceph-mon01][DEBUG ] "nautilus",
[ceph-mon01][DEBUG ] "octopus",
[ceph-mon01][DEBUG ] "pacific",
[ceph-mon01][DEBUG ] "elector-pinging"
[ceph-mon01][DEBUG ] ]
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] "monmap": {
[ceph-mon01][DEBUG ] "created": "2021-08-30T03:27:46.534560Z",
[ceph-mon01][DEBUG ] "disallowed_leaders: ": "",
[ceph-mon01][DEBUG ] "election_strategy": 1,
[ceph-mon01][DEBUG ] "epoch": 1,
[ceph-mon01][DEBUG ] "features": {
[ceph-mon01][DEBUG ] "optional": [],
[ceph-mon01][DEBUG ] "persistent": [
[ceph-mon01][DEBUG ] "kraken",
[ceph-mon01][DEBUG ] "luminous",
[ceph-mon01][DEBUG ] "mimic",
[ceph-mon01][DEBUG ] "osdmap-prune",
[ceph-mon01][DEBUG ] "nautilus",
[ceph-mon01][DEBUG ] "octopus",
[ceph-mon01][DEBUG ] "pacific",
[ceph-mon01][DEBUG ] "elector-pinging"
[ceph-mon01][DEBUG ] ]
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] "fsid": "c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb",
[ceph-mon01][DEBUG ] "min_mon_release": 16,
[ceph-mon01][DEBUG ] "min_mon_release_name": "pacific",
[ceph-mon01][DEBUG ] "modified": "2021-08-30T03:27:46.534560Z",
[ceph-mon01][DEBUG ] "mons": [
[ceph-mon01][DEBUG ] {
[ceph-mon01][DEBUG ] "addr": "172.168.32.104:6789/0",
[ceph-mon01][DEBUG ] "crush_location": "{}",
[ceph-mon01][DEBUG ] "name": "ceph-mon01",
[ceph-mon01][DEBUG ] "priority": 0,
[ceph-mon01][DEBUG ] "public_addr": "172.168.32.104:6789/0",
[ceph-mon01][DEBUG ] "public_addrs": {
[ceph-mon01][DEBUG ] "addrvec": [
[ceph-mon01][DEBUG ] {
[ceph-mon01][DEBUG ] "addr": "172.168.32.104:3300",
[ceph-mon01][DEBUG ] "nonce": 0,
[ceph-mon01][DEBUG ] "type": "v2"
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] {
[ceph-mon01][DEBUG ] "addr": "172.168.32.104:6789",
[ceph-mon01][DEBUG ] "nonce": 0,
[ceph-mon01][DEBUG ] "type": "v1"
[ceph-mon01][DEBUG ] }
[ceph-mon01][DEBUG ] ]
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] "rank": 0,
[ceph-mon01][DEBUG ] "weight": 0
[ceph-mon01][DEBUG ] }
[ceph-mon01][DEBUG ] ],
[ceph-mon01][DEBUG ] "stretch_mode": false
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] "name": "ceph-mon01",
[ceph-mon01][DEBUG ] "outside_quorum": [],
[ceph-mon01][DEBUG ] "quorum": [
[ceph-mon01][DEBUG ] 0
[ceph-mon01][DEBUG ] ],
[ceph-mon01][DEBUG ] "quorum_age": 2,
[ceph-mon01][DEBUG ] "rank": 0,
[ceph-mon01][DEBUG ] "state": "leader",
[ceph-mon01][DEBUG ] "stretch_mode": false,
[ceph-mon01][DEBUG ] "sync_provider": []
[ceph-mon01][DEBUG ] }
[ceph-mon01][DEBUG ] ********************************************************************************
[ceph-mon01][INFO ] monitor: mon.ceph-mon01 is running
[ceph-mon01][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon01.asok mon_status
[ceph_deploy.mon][INFO ] processing monitor mon.ceph-mon01
[ceph-mon01][DEBUG ] connection detected need for sudo
[ceph-mon01][DEBUG ] connected to host: ceph-mon01
[ceph-mon01][DEBUG ] detect platform information from remote host
[ceph-mon01][DEBUG ] detect machine type
[ceph-mon01][DEBUG ] find the location of an executable
[ceph-mon01][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon01.asok mon_status
[ceph_deploy.mon][INFO ] mon.ceph-mon01 monitor has reached quorum!
[ceph_deploy.mon][INFO ] all initial monitors are running and have formed quorum
[ceph_deploy.mon][INFO ] Running gatherkeys...
[ceph_deploy.gatherkeys][INFO ] Storing keys in temp directory /tmp/tmpCb4OB1
[ceph-mon01][DEBUG ] connection detected need for sudo
[ceph-mon01][DEBUG ] connected to host: ceph-mon01
[ceph-mon01][DEBUG ] detect platform information from remote host
[ceph-mon01][DEBUG ] detect machine type
[ceph-mon01][DEBUG ] get remote short hostname
[ceph-mon01][DEBUG ] fetch remote file
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --admin-daemon=/var/run/ceph/ceph-mon.ceph-mon01.asok mon_status
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon01/keyring auth get client.admin
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon01/keyring auth get client.bootstrap-mds
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon01/keyring auth get client.bootstrap-mgr
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon01/keyring auth get client.bootstrap-osd
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon01/keyring auth get client.bootstrap-rgw
[ceph_deploy.gatherkeys][INFO ] Storing ceph.client.admin.keyring
[ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mds.keyring
[ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mgr.keyring
[ceph_deploy.gatherkeys][INFO ] keyring 'ceph.mon.keyring' already exists
[ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-osd.keyring
[ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-rgw.keyring
[ceph_deploy.gatherkeys][INFO ] Destroy temp directory /tmp/tmpCb4OB1
验证在mon 定节点已经自动安装并启动了ceph-mon 服务，并且后期在ceph-deploy 节点初始化目录会生成一些bootstrap ceph mds/mgr/osd/rgw 等服务的keyring 认证文件，这些初始化文件拥有对ceph 集群的最高权限，所以一定要保存好。
root@ceph-mon01:~# ps -ef|grep ceph-mon ceph 8304 1 0 22:43 ? 00:00:00 /usr/bin/ceph-mon -f --cluster ceph --id ceph-mon01 --setuser ceph --setgroup ceph
在ceph-deploy 节点把配置文件和admin 密钥拷贝至Ceph 集群需要执行ceph 管理命令的节点，从而不需要后期通过ceph 命令对ceph 集群进行管理配置的时候每次都需要指定ceph-mon 节点地址和ceph.client.admin.keyring 文件,另外各ceph-mon 节点也需要同步ceph 的集群配置文件与认证文件。
如果在ceph-deploy 节点管理集群
root@ceph-deploy:~# apt install ceph-common -y #先安装ceph 的公共组件 root@ceph-node01:~# apt install ceph-common -y root@ceph-node02:~# apt install ceph-common -y root@ceph-node03:~# apt install ceph-common -y root@ceph-node04:~# apt install ceph-common -y
拷贝密钥到ceph-node01 ceph-node02 ceph-node03 ceph-node04
#在ceph-deploy上操作
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy admin ceph-node01 ceph-node02 ceph-node03 ceph-node04
#下面为运行过程
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy admin ceph-node01 ceph-node02 ceph-node03 ceph-node04
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f3066490190>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] client : ['ceph-node01', 'ceph-node02', 'ceph-node03', 'ceph-node04']
[ceph_deploy.cli][INFO ] func : <function admin at 0x7f3066d91a50>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node01
[ceph-node01][DEBUG ] connection detected need for sudo
[ceph-node01][DEBUG ] connected to host: ceph-node01
[ceph-node01][DEBUG ] detect platform information from remote host
[ceph-node01][DEBUG ] detect machine type
[ceph-node01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node02
[ceph-node02][DEBUG ] connection detected need for sudo
[ceph-node02][DEBUG ] connected to host: ceph-node02
[ceph-node02][DEBUG ] detect platform information from remote host
[ceph-node02][DEBUG ] detect machine type
[ceph-node02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node03
[ceph-node03][DEBUG ] connection detected need for sudo
[ceph-node03][DEBUG ] connected to host: ceph-node03
[ceph-node03][DEBUG ] detect platform information from remote host
[ceph-node03][DEBUG ] detect machine type
[ceph-node03][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node04
[ceph-node04][DEBUG ] connection detected need for sudo
[ceph-node04][DEBUG ] connected to host: ceph-node04
[ceph-node04][DEBUG ] detect platform information from remote host
[ceph-node04][DEBUG ] detect machine type
[ceph-node04][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
root@ceph-node01:~# ll /etc/ceph total 24 drwxr-xr-x 2 root root 87 Aug 17 22:54 ./ drwxr-xr-x 98 root root 8192 Aug 17 22:16 ../ -rw------- 1 root root 151 Aug 17 22:54 ceph.client.admin.keyring -rw-r--r-- 1 root root 264 Aug 17 22:54 ceph.conf -rw-r--r-- 1 root root 92 Jul 8 22:17 rbdmap -rw------- 1 root root 0 Aug 17 22:54 tmp8AdP3P root@ceph-node02:~# ll /etc/ceph total 24 drwxr-xr-x 2 root root 87 Aug 17 22:54 ./ drwxr-xr-x 98 root root 8192 Aug 17 22:16 ../ -rw------- 1 root root 151 Aug 17 22:54 ceph.client.admin.keyring -rw-r--r-- 1 root root 264 Aug 17 22:54 ceph.conf -rw-r--r-- 1 root root 92 Jul 8 22:17 rbdmap -rw------- 1 root root 0 Aug 17 22:54 tmp8AdP3P root@ceph-node03:~# ll /etc/ceph total 24 drwxr-xr-x 2 root root 87 Aug 17 22:54 ./ drwxr-xr-x 98 root root 8192 Aug 17 22:16 ../ -rw------- 1 root root 151 Aug 17 22:54 ceph.client.admin.keyring -rw-r--r-- 1 root root 264 Aug 17 22:54 ceph.conf -rw-r--r-- 1 root root 92 Jul 8 22:17 rbdmap -rw------- 1 root root 0 Aug 17 22:54 tmp8AdP3P root@ceph-node04:~# ll /etc/ceph total 24 drwxr-xr-x 2 root root 87 Aug 17 22:54 ./ drwxr-xr-x 98 root root 8192 Aug 17 22:16 ../ -rw------- 1 root root 151 Aug 17 22:54 ceph.client.admin.keyring -rw-r--r-- 1 root root 264 Aug 17 22:54 ceph.conf -rw-r--r-- 1 root root 92 Jul 8 22:17 rbdmap -rw------- 1 root root 0 Aug 17 22:54 tmp8AdP3P
11)在ceph-node节点对密钥授权
root@ceph-node01:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring root@ceph-node02:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring root@ceph-node03:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring root@ceph-node04:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring
(1)在ceph-mgr01和ceph-mgr02上安装ceph-mgr软件
root@ceph-mgr01:~# apt install -y ceph-mgr root@ceph-mgr02:~# apt install -y ceph-mgr
(2)在ceph-deploy上初始化ceph-mgr节点
#只初始化ceph-mgr01节点
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mgr create ceph-mgr01
#下面为过程内容
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mgr create ceph-mgr01
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] mgr : [('ceph-mgr01', 'ceph-mgr01')]
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : create
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f07c9190c30>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] func : <function mgr at 0x7f07c95f0150>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.mgr][DEBUG ] Deploying mgr, cluster ceph hosts ceph-mgr01:ceph-mgr01
The authenticity of host 'ceph-mgr01 (172.168.32.102)' can't be established.
ECDSA key fingerprint is SHA256:AIDN3qa9QKjViElHDrtTXhJ5EpTXdWj5Sc3tiy91E4Y.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'ceph-mgr01' (ECDSA) to the list of known hosts.
[ceph-mgr01][DEBUG ] connection detected need for sudo
[ceph-mgr01][DEBUG ] connected to host: ceph-mgr01
[ceph-mgr01][DEBUG ] detect platform information from remote host
[ceph-mgr01][DEBUG ] detect machine type
[ceph_deploy.mgr][INFO ] Distro info: Ubuntu 18.04 bionic
[ceph_deploy.mgr][DEBUG ] remote host will use systemd
[ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to ceph-mgr01
[ceph-mgr01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph-mgr01][WARNIN] mgr keyring does not exist yet, creating one
[ceph-mgr01][DEBUG ] create a keyring file
[ceph-mgr01][DEBUG ] create path recursively if it doesn't exist
[ceph-mgr01][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.ceph-mgr01 mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-ceph-mgr01/keyring
[ceph-mgr01][INFO ] Running command: sudo systemctl enable ceph-mgr@ceph-mgr01
[ceph-mgr01][WARNIN] Created symlink /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@ceph-mgr01.service → /lib/systemd/system/ceph-mgr@.service.
[ceph-mgr01][INFO ] Running command: sudo systemctl start ceph-mgr@ceph-mgr01
[ceph-mgr01][INFO ] Running command: sudo systemctl enable ceph.target
在ceph-mgr01上
root@ceph-mgr01:~# ps -ef|grep ceph-mgr
ceph 10771 1 16 12:05 ? 00:00:07 /usr/bin/ceph-mgr -f --cluster ceph --id ceph-mgr01 --setuser ceph --setgroup ceph
#ceph-deploy管理ceph集群环境设置
cephadmin@ceph-deploy:~/ceph-cluster$ sudo apt install -y ceph-common
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy admin ceph-deploy
cephadmin@ceph-deploy:~/ceph-cluster$ sudo setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring
#ceph-deploy管理ceph集群信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb
health: HEALTH_WARN
mon is allowing insecure global_id reclaim
OSD count 0 < osd_pool_default_size 3
services:
mon: 1 daemons, quorum ceph-mon01 (age 38m)
mgr: ceph-mgr01(active, since 65s)
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
cephadmin@ceph-deploy:~/ceph-cluster$ ceph config set mon auth_allow_insecure_global_id_reclaim false
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb
health: HEALTH_WARN
OSD count 0 < osd_pool_default_size 3
services:
mon: 1 daemons, quorum ceph-mon01 (age 39m)
mgr: ceph-mgr01(active, since 2m)
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
cephadmin@ceph-deploy:~/ceph-cluster$ ceph version
ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph versions
{
"mon": {
"ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1
},
"mgr": {
"ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1
},
"osd": {},
"mds": {},
"overall": {
"ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2
}
}
#前面执行了node节点的初始化，如下步骤，后面的操作可以不执行
ceph-deploy install --no-adjust-repos --nogpgcheck ceph-node01 ceph-node02 ceph-node03 ceph-node04
#擦除磁盘之前通过deploy 节点对node 节点执行安装ceph 基本运行环境。
#在ceph-deploy上操作
#cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node01
#cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node02
#cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node03
#cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node04
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy disk list ceph-node01
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk list ceph-node01
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] debug : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : list
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f68cc180fa0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] host : ['ceph-node01']
[ceph_deploy.cli][INFO ] func : <function disk at 0x7f68cc15c2d0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph-node01][DEBUG ] connection detected need for sudo
[ceph-node01][DEBUG ] connected to host: ceph-node01
[ceph-node01][DEBUG ] detect platform information from remote host
[ceph-node01][DEBUG ] detect machine type
[ceph-node01][DEBUG ] find the location of an executable
[ceph-node01][INFO ] Running command: sudo fdisk -l
[ceph-node01][INFO ] Disk /dev/sda: 50 GiB, 53687091200 bytes, 104857600 sectors
[ceph-node01][INFO ] Disk /dev/sdb: 50 GiB, 53687091200 bytes, 104857600 sectors
[ceph-node01][INFO ] Disk /dev/sdc: 50 GiB, 53687091200 bytes, 104857600 sectors
[ceph-node01][INFO ] Disk /dev/sdd: 50 GiB, 53687091200 bytes, 104857600 sectors
[ceph-node01][INFO ] Disk /dev/sde: 50 GiB, 53687091200 bytes, 104857600 sectors
#在ceph-deploy上操作 ceph-deploy disk zap ceph-node01 /dev/sdb ceph-deploy disk zap ceph-node01 /dev/sdc ceph-deploy disk zap ceph-node01 /dev/sdd ceph-deploy disk zap ceph-node01 /dev/sde ceph-deploy disk zap ceph-node02 /dev/sdb ceph-deploy disk zap ceph-node02 /dev/sdc ceph-deploy disk zap ceph-node02 /dev/sdd ceph-deploy disk zap ceph-node02 /dev/sde ceph-deploy disk zap ceph-node03 /dev/sdb ceph-deploy disk zap ceph-node03 /dev/sdc ceph-deploy disk zap ceph-node03 /dev/sdd ceph-deploy disk zap ceph-node03 /dev/sde ceph-deploy disk zap ceph-node04 /dev/sdb ceph-deploy disk zap ceph-node04 /dev/sdc ceph-deploy disk zap ceph-node04 /dev/sdd ceph-deploy disk zap ceph-node04 /dev/sde
#在ceph-deploy上操作 ceph-deploy osd create ceph-node01 --data /dev/sdb ceph-deploy osd create ceph-node01 --data /dev/sdc ceph-deploy osd create ceph-node01 --data /dev/sdd ceph-deploy osd create ceph-node01 --data /dev/sde ceph-deploy osd create ceph-node02 --data /dev/sdb ceph-deploy osd create ceph-node02 --data /dev/sdc ceph-deploy osd create ceph-node02 --data /dev/sdd ceph-deploy osd create ceph-node02 --data /dev/sde ceph-deploy osd create ceph-node03 --data /dev/sdb ceph-deploy osd create ceph-node03 --data /dev/sdc ceph-deploy osd create ceph-node03 --data /dev/sdd ceph-deploy osd create ceph-node03 --data /dev/sde ceph-deploy osd create ceph-node04 --data /dev/sdb ceph-deploy osd create ceph-node04 --data /dev/sdc ceph-deploy osd create ceph-node04 --data /dev/sdd ceph-deploy osd create ceph-node04 --data /dev/sde
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb
health: HEALTH_OK
services:
mon: 1 daemons, quorum ceph-mon01 (age 48m)
mgr: ceph-mgr01(active, since 10m)
osd: 16 osds: 16 up (since 6s), 16 in (since 15s)
data:
pools: 1 pools, 1 pgs
objects: 0 objects, 0 B
usage: 94 MiB used, 800 GiB / 800 GiB avail
pgs: 1 active+clean
#在ceph-node02上测试，其他node节点操作一样 root@ceph-node02:~# ps -ef|grep osd ceph 15521 1 0 00:08 ? 00:00:03 /usr/bin/ceph-osd -f --cluster ceph --id 3 --setuser ceph --setgroup ceph ceph 17199 1 0 00:09 ? 00:00:03 /usr/bin/ceph-osd -f --cluster ceph --id 4 --setuser ceph --setgroup ceph ceph 18874 1 0 00:09 ? 00:00:03 /usr/bin/ceph-osd -f --cluster ceph --id 5 --setuser ceph --setgroup ceph ceph 20546 1 0 00:09 ? 00:00:03 /usr/bin/ceph-osd -f --cluster ceph --id 6 --setuser ceph --setgroup ceph root@ceph-node02:~# systemctl enable ceph-osd@3 ceph-osd@4 ceph-osd@5 ceph-osd@6 Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@3.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@4.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@5.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@6.service → /lib/systemd/system/ceph-osd@.service.
$ ceph-deploy --help new：开始部署一个新的ceph 存储集群，并生成CLUSTER.conf 集群配置文件和keyring 认证文件。 install: 在远程主机上安装ceph 相关的软件包, 可以通过--release 指定安装的版本。 rgw：管理RGW 守护程序(RADOSGW,对象存储网关)。 mgr：管理MGR 守护程序(ceph-mgr,Ceph Manager DaemonCeph 管理器守护程序)。 mds：管理MDS 守护程序(Ceph Metadata Server，ceph 源数据服务器)。 mon：管理MON 守护程序(ceph-mon,ceph 监视器)。 gatherkeys：从指定获取提供新节点的验证keys，这些keys 会在添加新的MON/OSD/MD加入的时候使用。 disk：管理远程主机磁盘。 osd：在远程主机准备数据磁盘，即将指定远程主机的指定磁盘添加到ceph 集群作为osd 使用。 repo： 远程主机仓库管理。 admin：推送ceph 集群配置文件和client.admin 认证文件到远程主机。 config：将ceph.conf 配置文件推送到远程主机或从远程主机拷贝。 uninstall：从远端主机删除安装包。 purgedata：从/var/lib/ceph 删除ceph 数据,会删除/etc/ceph 下的内容。 purge: 删除远端主机的安装包和所有数据。 forgetkeys：从本地主机删除所有的验证keyring, 包括client.admin, monitor, bootstrap 等 认证文件。 pkg： 管理远端主机的安装包。 calamari：安装并配置一个calamari web 节点，calamari 是一个web 监控平台。
1、磁盘有数据
处理方案：清空磁盘数据，实验中的操作是关闭虚拟机，删除原磁盘，重新添加磁盘。
[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
root@ceph-node01's password:
root@ceph-node01's password:
[ceph-node01][DEBUG ] connected to host: ceph-node01
[ceph-node01][DEBUG ] detect platform information from remote host
[ceph-node01][DEBUG ] detect machine type
[ceph-node01][DEBUG ] find the location of an executable
[ceph_deploy.osd][INFO ] Distro info: Ubuntu 18.04 bionic
[ceph_deploy.osd][DEBUG ] Deploying osd to ceph-node01
[ceph-node01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph-node01][DEBUG ] find the location of an executable
[ceph-node01][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
[ceph-node01][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
[ceph-node01][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 93330566-19f1-47a7-8d5e-5d47c52b4274
[ceph-node01][WARNIN] Running command: /sbin/lvcreate --yes -l 5119 -n osd-block-93330566-19f1-47a7-8d5e-5d47c52b4274 ceph-f3745436-e965-4dda-a440-4c7fb3104c48
[ceph-node01][WARNIN] stderr:
[ceph-node01][WARNIN] stderr: Volume group "ceph-f3745436-e965-4dda-a440-4c7fb3104c48" has insufficient free space (0 extents): 5119 required.
[ceph-node01][WARNIN] stderr:
[ceph-node01][WARNIN] --> Was unable to complete a new OSD, will rollback changes
[ceph-node01][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring osd purge-new osd.16 --yes-i-really-mean-it
[ceph-node01][WARNIN] stderr: 2021-08-18T10:03:49.653+0800 7f2ffca71700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
[ceph-node01][WARNIN] stderr:
[ceph-node01][WARNIN] stderr: 2021-08-18T10:03:49.653+0800 7f2ffca71700 -1 AuthRegistry(0x7f2ff805b408) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
[ceph-node01][WARNIN] stderr:
[ceph-node01][WARNIN] stderr: purged osd.16
[ceph-node01][WARNIN] --> RuntimeError: command returned non-zero exit status: 5
[ceph-node01][ERROR ] RuntimeError: command returned non-zero exit status: 1
[ceph_deploy.osd][ERROR ] Failed to execute command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
[ceph_deploy][ERROR ] GenericError: Failed to create 1 OSDs
1)扩展ceph-mon 节点
Ceph-mon 是原生具备自选举以实现高可用机制的ceph 服务，节点数量通常是奇数。
#在ceph-mon节点安装ceph-mon root@ceph-mon02:~# apt install ceph-mon root@ceph-mon03:~# apt install ceph-mon #在ceph-deploy节点添加mon节点 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon add ceph-mon02 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon add ceph-mon03
ceph-mon结果验证
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: f0e7c394-989b-4803-86c3-5557ae25e814
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon01,ceph-mon02,ceph-mon03 (age 7s) #ceph-mon02和ceph-mon03添加完成
mgr: ceph-mgr01(active, since 15h)
osd: 20 osds: 16 up (since 22m), 15 in (since 25m)
data:
pools: 1 pools, 1 pgs
objects: 0 objects, 0 B
usage: 134 MiB used, 300 GiB / 300 GiB avail
pgs: 1 active+clean
#ceph-mon的状态
cephadmin@ceph-deploy:~/ceph-cluster$ ceph quorum_status --format json-pretty
{
"election_epoch": 12,
"quorum": [
0,
1,
2
],
"quorum_names": [
"ceph-mon01",
"ceph-mon02",
"ceph-mon03"
],
"quorum_leader_name": "ceph-mon01", #当前mon的leader
"quorum_age": 176,
"features": {
"quorum_con": "4540138297136906239",
"quorum_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus",
"octopus",
"pacific",
"elector-pinging"
]
},
"monmap": {
"epoch": 3,
"fsid": "f0e7c394-989b-4803-86c3-5557ae25e814",
"modified": "2021-08-18T06:31:20.933855Z",
"created": "2021-08-17T14:43:20.965196Z",
"min_mon_release": 16,
"min_mon_release_name": "pacific",
"election_strategy": 1,
"disallowed_leaders: ": "",
"stretch_mode": false,
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus",
"octopus",
"pacific",
"elector-pinging"
],
"optional": []
},
"mons": [
{
"rank": 0, #ceph-mon01的等级
"name": "ceph-mon01", #mon的节点名称
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.168.32.104:3300", #监控地址
"nonce": 0
},
{
"type": "v1",
"addr": "172.168.32.104:6789", #监控地址
"nonce": 0
}
]
},
"addr": "172.168.32.104:6789/0",
"public_addr": "172.168.32.104:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
},
{
"rank": 1,
"name": "ceph-mon02",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.168.32.105:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.168.32.105:6789",
"nonce": 0
}
]
},
"addr": "172.168.32.105:6789/0",
"public_addr": "172.168.32.105:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
},
{
"rank": 2,
"name": "ceph-mon03",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.168.32.106:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.168.32.106:6789",
"nonce": 0
}
]
},
"addr": "172.168.32.106:6789/0",
"public_addr": "172.168.32.106:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
}
]
}
}
#在ceph-mgr02节点上安装ceph-mgr root@ceph-mgr02:~#apt install -y ceph-mgr #在ceph-deploy上添加ceph-mgr02 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mgr create ceph-mgr02 #在ceph-deploy上同步配置文件到ceph-mgr02 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy admin ceph-mgr02
ceph-mgr验证
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: f0e7c394-989b-4803-86c3-5557ae25e814
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon01,ceph-mon02,ceph-mon03 (age 12m)
mgr: ceph-mgr01(active, since 15h), standbys: ceph-mgr02 #备用的ceph-mgr02已经添加完成
osd: 20 osds: 16 up (since 34m), 15 in (since 37m)
data:
pools: 1 pools, 1 pgs
objects: 0 objects, 0 B
usage: 134 MiB used, 300 GiB / 300 GiB avail
pgs: 1 active+clean
1)、安装ceph-mds软件包
如果要使用cephFS，需要部署cephfs 服务。MDS服务与mon服务部署在一起（最好分开部署）。
apt install ceph-mds
在ceph-deploy上操作
ceph-deploy mds create ceph-mds01 ceph-deploy mds create ceph-mds02 ceph-deploy mds create ceph-mds03
#创建cephfs-metadata cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-metadata 32 32 pool 'cephfs-metadata' created #保存metadata 的pool #创建cephfs-data cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-data 64 64 pool 'cephfs-data' created #保存数据的pool #创建一个叫mycephfs的cephFS cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs new mycephfs cephfs-metadata cephfs-data new fs with metadata pool 2 and data pool 3
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs ls
name: mycephfs, metadata pool: cephfs-metadata, data pools: [cephfs-data ]
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status mycephfs
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mds01 Reqs: 0 /s 10 13 12 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 96.0k 253G
cephfs-data data 0 253G
STANDBY MDS
ceph-mds02
ceph-mds03
MDS version: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)
#验证Cephfs的服务状态
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat
mycephfs:1 {0=ceph-mds01=up:active} 2 up:standby
5)、MDS的高可用
为了测试，我们把ceph-mgr02复用为mds
#添加第4台mds ceph-mgr02
#在ceph-mgr02上安装ceph-mds
root@ceph-mgr02:~# apt install ceph-mds
#在ceph-deploy上把mgr02添加进mds
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mds create ceph-mgr02
#现在mds的状态为1主3备
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mds01 Reqs: 0 /s 10 13 12 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 96.0k 253G
cephfs-data data 0 253G
STANDBY MDS
ceph-mds02
ceph-mds03
ceph-mgr02
MDS version: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)
设置每个Rank 的备份MDS，也就是如果此Rank 当前的MDS 出现问题马上切换到另个MDS。设置备份的方法有很多，常用选项如下。
mds_standby_replay：值为true 或false，true 表示开启replay 模式，这种模式下主MDS 内的数量将实时与从MDS 同步，如果主宕机，从可以快速的切换。如果为false 只有宕机的时候才去同步数据，这样会有一段时间的中断。 mds_standby_for_name：设置当前MDS 进程只用于备份于指定名称的MDS。 mds_standby_for_rank：设置当前MDS 进程只用于备份于哪个Rank，通常为Rank 编号。另外在存在之个CephFS 文件系统中，还可以使用mds_standby_for_fscid 参数来为指定不同的文件系统。 mds_standby_for_fscid：指定CephFS 文件系统ID，需要联合mds_standby_for_rank 生效，如果设置mds_standby_for_rank，那么就是用于指定文件系统的指定Rank，如果没有设置，就是指定文件系统的所有Rank。
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mds01 Reqs: 0 /s 10 13 12 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 96.0k 253G
cephfs-data data 0 253G
STANDBY MDS
ceph-mds02
ceph-mds03
ceph-mgr02
MDS version: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs get mycephfs
Filesystem 'mycephfs' (1)
fs_name mycephfs
epoch 6
flags 12
created 2021-08-30T14:17:05.882172+0800
modified 2021-08-30T14:17:06.888751+0800
tableserver 0
root 0
session_timeout 60
session_autoclose 300
max_file_size 1099511627776
required_client_features {}
last_failure 0
last_failure_osd_epoch 0
compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2}
max_mds 1
in 0
up {0=14122}
failed
damaged
stopped
data_pools [3]
metadata_pool 2
inline_data disabled
balancer
standby_count_wanted 1
[mds.ceph-mds01{0:14122} state up:active seq 112 addr [v2:172.168.32.104:6800/3723142001,v1:172.168.32.104:6801/3723142001]]
目前有四个mds 服务器，但是有一个主三个备，可以优化一下部署架构，设置为为两主两备。
#设置同时活跃的主mds最大值为2。
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs set mycephfs max_mds 2
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat
mycephfs:2 {0=ceph-mds01=up:active,1=ceph-mgr02=up:active} 2 up:standby
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mds01 Reqs: 0 /s 10 13 12 0
1 active ceph-mgr02 Reqs: 0 /s 10 13 11 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 168k 253G
cephfs-data data 0 253G
STANDBY MDS
ceph-mds02
ceph-mds03
MDS version: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)
目前的状态是ceph-mds01 和ceph-mgr02 分别是active 状态，ceph-mds02和ceph-mds03 分别处于standby 状态，现在可以将ceph-mds02设置为ceph-mds01 的standby，将ceph-mds03 设置为ceph-mgr02的standby，以实现每个主都有一个固定备份角色的结构，则修改配置文件如.
cephadmin@ceph-deploy:~/ceph-cluster$ vim ceph.conf [global] fsid = c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb public_network = 172.168.0.0/16 cluster_network = 10.0.0.0/16 mon_initial_members = ceph-mon01 mon_host = 172.168.32.104 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx [mds.ceph-mds02] #mds_standby_for_fscid = mycephfs mds_standby_for_name = ceph-mds01 mds_standby_replay = true [mds.ceph-mds03] mds_standby_for_name = ceph-mgr02 mds_standby_replay = true
分发配置文件并重启mds 服务:
#分发配置文件保证各mds 服务重启有效 cephadmin@ceph-deploy:/etc/ceph$ceph-deploy --overwrite-conf config push ceph-mon03 cephadmin@ceph-deploy:/etc/ceph$ceph-deploy --overwrite-conf config push ceph-mon02 cephadmin@ceph-deploy:/etc/ceph$ceph-deploy --overwrite-conf config push ceph-mon01 cephadmin@ceph-deploy:/etc/ceph$ceph-deploy --overwrite-conf config push ceph-mgr02 #重启mds节点，先重启active节点ceph-mds01和ceph-mgr02,再重启standby节点ceph-mds02和ceph-mds03 cephadmin@ceph-mon01:~$ systemctl restart ceph-mds@ceph-mds01.service cephadmin@ceph-mgr02:~$ systemctl restart ceph-mds@ceph-mgr02.service cephadmin@ceph-mon02:~$ systemctl restart ceph-mds@ceph-mds02.service cephadmin@ceph-mon03:~$ systemctl restart ceph-mds@ceph-mds03.service
验证mds
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mgr02 Reqs: 0 /s 10 13 12 0
1 active ceph-mds01 Reqs: 0 /s 10 13 11 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 168k 253G
cephfs-data data 0 253G
STANDBY MDS
ceph-mds03
ceph-mds02
查看active 和standby 对应关系：
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs get mycephfs
Filesystem 'mycephfs' (1)
fs_name mycephfs
epoch 34
flags 12
created 2021-08-30T14:17:05.882172+0800
modified 2021-08-30T15:11:24.372592+0800
tableserver 0
root 0
session_timeout 60
session_autoclose 300
max_file_size 1099511627776
required_client_features {}
last_failure 0
last_failure_osd_epoch 103
compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2}
max_mds 2
in 0,1
up {0=14176,1=4653}
failed
damaged
stopped
data_pools [3]
metadata_pool 2
inline_data disabled
balancer
standby_count_wanted 1
[mds.ceph-mgr02{0:14176} state up:active seq 10 addr [v2:172.168.32.103:6800/4136492160,v1:172.168.32.103:6801/4136492160]]
[mds.ceph-mds01{1:4653} state up:active seq 7 addr [v2:172.168.32.104:6800/3715813078,v1:172.168.32.104:6801/3715813078]]
将ceph-mgr01、ceph-mgr02 服务器部署为高可用的radosGW 服务
1)、在ceph-mgr01和ceph-mgr02上安装ceph-radosgw
root@ceph-mgr01:~# apt install radosgw -y root@ceph-mgr02:~# apt install radosgw -y
#注意：如果前面有做ceph.conf的配置变动，需要将ceph.conf重新拷贝到ceph-mgr01和cpeh-mgr02上 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy rgw create ceph-mgr01 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy rgw create ceph-mgr02
在mgr节点上查看radosgw节点信息
root@ceph-mgr01:~# ps -ef|grep radosgw ceph 13551 1 0 15:19 ? 00:00:00 /usr/bin/radosgw -f --cluster ceph --name client.rgw.ceph-mgr01 --setuser ceph --setgroup ceph root@ceph-mgr02:~# ps -ef|grep radosgw ceph 13832 1 0 15:19 ? 00:00:00 /usr/bin/radosgw -f --cluster ceph --name client.rgw.ceph-mgr02 --setuser ceph --setgroup ceph
在ceph-deploy上查看集群的radosgw信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon01,ceph-mon02,ceph-mon03 (age 3h)
mgr: ceph-mgr01(active, since 3h), standbys: ceph-mgr02
mds: 2/2 daemons up, 2 standby
osd: 16 osds: 16 up (since 3h), 16 in (since 3h)
rgw: 2 daemons active (2 hosts, 1 zones)
data:
volumes: 1/1 healthy
pools: 7 pools, 208 pgs
objects: 230 objects, 8.5 KiB
usage: 146 MiB used, 800 GiB / 800 GiB avail
pgs: 0.481% pgs not active
207 active+clean
1 peering
progress:
Global Recovery Event (0s)
[............................]
cephadmin@ceph-deploy:~/ceph-cluster$ curl http://172.168.32.102:7480 <?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>cephadmin@ceph-deploy:~/ceph-cluster$ cephadmin@ceph-deploy:~/ceph-cluster$ curl http://172.168.32.103:7480 <?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>cephadmin@ceph-deploy:~/ceph-cluster$
