ceph 部署

修改主机名

?

1	hostnamectl set-hostname services-ceph

?

1 2 3 4

cat /etc/hosts 192.168.11.21 services-ceph    reboot

 关闭防火墙

?

1 2 3 4 5 6 7

# 关闭防火樯 systemctl disable firewalld systemctl stop firewalld   # 关闭 selinux setenforce 0 sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

 SSH配置免密

?

1 2 3 4 5 6

# 给ceph单节点配置免密，一台机器也要 ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys   # 权限设置 644 chmod 644 ~/.ssh/authorized_keys

 配置yum 源

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42

# 备份现有的repo源 cp /etc/yum.repos.d/CentOS-Base.repo  /etc/yum.repos.d/CentOS-Base.repo.bak   # CentOS基础源 curl -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo   # 指定系统版本 cat /etc/yum.repos.d/ceph.repo   [ceph] name=Ceph packages for$basearch baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/$basearch enabled=1 gpgcheck=1 priority=1 gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc    [ceph-noarch] name=Ceph noarchpackages baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch enabled=1 gpgcheck=1 priority=1 type=rpm-md gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc    [ceph-source] name=Ceph sourcepackages baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/SRPMS enabled=0 gpgcheck=1 type=rpm-md gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc priority=1              # 刷新 yum yum clean all yum makecache   # 安装epel源 yum install -y epel-release

 安装ceph-deploy软件包

?

1 2 3 4 5 6 7 8

# yum方式安装 yum install -y ceph-deploy   # 检验版本 ceph-deploy --version   # 如果安装的不是2.0版本，可以直接rpm安装 yum install http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch/ceph-deploy-2.0.1-0.noarch.rpm

 创建ceph集群

?

1 2 3 4 5 6

# 创建一个目录保存ceph配置及密钥 mkdir -p /data/services/ceph   # 创建ceph cluster集群 cd /data/services/ceph ceph-deploy new services-ceph

故障问题：若遇到

?

1 2 3 4

wget https://pypi.python.org/packages/source/d/distribute/distribute-0.7.3.zip --no-check-certificate unzip distribute-0.7.3.zip cd distribute-0.7.3 python setup.py install

检查生成文件

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24

# 检查生成的文件 [root@services-ceph ceph]# ls -l total 12 -rw-r--r-- 1 root root  204 Apr  2 22:33 ceph.conf -rw-r--r-- 1 root root 3021 Apr  2 22:33 ceph-deploy-ceph.log -rw------- 1 root root   73 Apr  2 22:33 ceph.mon.keyring     # 因为是单节点部署，还需要将集群的副本数量设置为1，修改ceph.conf文件。 [root@services-ceph ceph]# echo "osd pool default size = 1" >>  ceph.conf [root@services-ceph ceph]# echo "osd pool default min size = 1" >>  ceph.conf   # 检查文件 [root@services-ceph ceph]# cat ceph.conf [global] fsid = 098f5601-a1f1-4eb4-a150-8db0090bc9d7 mon_initial_members = services-ceph mon_host = 192.168.11.21 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx   osd pool default size = 1 osd pool default min size = 1

 安装 ceph 软件

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18

# 安装L版本的软件 yum install -y ceph ceph-radosgw   # 检查安装的版本号 [root@services-ceph yum.repos.d]# rpm -qa | grep ceph libcephfs2-14.2.22-0.el7.x86_64 ceph-radosgw-14.2.22-0.el7.x86_64 python-ceph-argparse-14.2.22-0.el7.x86_64 ceph-selinux-14.2.22-0.el7.x86_64 ceph-osd-14.2.22-0.el7.x86_64 ceph-deploy-2.0.1-0.noarch python-cephfs-14.2.22-0.el7.x86_64 ceph-base-14.2.22-0.el7.x86_64 ceph-mgr-14.2.22-0.el7.x86_64 ceph-common-14.2.22-0.el7.x86_64 ceph-mon-14.2.22-0.el7.x86_64 ceph-14.2.22-0.el7.x86_64 ceph-mds-14.2.22-0.el7.x86_64

 初始化 mon

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29

## 初始化 monitor ceph-deploy mon create-initial   ## 把配置文件和密钥拷贝到管理节点和Ceph节点 ceph-deploy admin services-ceph   ## 确保对秘钥环有权限 chmod +r /etc/ceph/ceph.client.admin.keyring   cp /data/services/ceph/ceph*  /etc/ceph/ chmod +r /etc/ceph/ceph*   # 启动monitor节点后，检查ceph集群 [root@services-ceph ceph]# ceph -s   cluster:     id:     098f5601-a1f1-4eb4-a150-8db0090bc9d7     health: HEALTH_WARN             mon is allowing insecure global_id reclaim     services:     mon: 1 daemons, quorum services-ceph (age 4m)     mgr: no daemons active     osd: 0 osds: 0 up, 0 in     data:     pools:   0 pools, 0 pgs     objects: 0 objects, 0 B     usage:   0 B used, 0 B / 0 B avail     pgs:

 部署mgr

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21

## 部署mgr组件 ceph-deploy mgr create services-ceph   ## 查看mgr状态为active [root@services-ceph ceph]# ceph -s   cluster:     id:     098f5601-a1f1-4eb4-a150-8db0090bc9d7     health: HEALTH_WARN             OSD count 0 < osd_pool_default_size 1             mon is allowing insecure global_id reclaim     services:     mon: 1 daemons, quorum services-ceph (age 10m)     mgr: services-ceph(active, since 103s)     osd: 0 osds: 0 up, 0 in     data:     pools:   0 pools, 0 pgs     objects: 0 objects, 0 B     usage:   0 B used, 0 B / 0 B avail     pgs:

 部署osd数据节点

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44

## 安装lvm2 yum install -y lvm2   # 创建pv pvcreate /dev/sdb   # 创建 vg vgcreate ceph-pool /dev/sdb   # 创建lv，单个osd节点 lvcreate -n osd0 -l 100%FREE ceph-pool   # 使用xfs格式化 mkfs.xfs  /dev/ceph-pool/osd0   # 添加osd节点 ceph-deploy osd create --data /dev/ceph-pool/osd0 services-ceph   # 查看集群状态 [root@services-ceph ceph]# ceph -s   cluster:     id:     098f5601-a1f1-4eb4-a150-8db0090bc9d7     health: HEALTH_WARN             mon is allowing insecure global_id reclaim     services:     mon: 1 daemons, quorum services-ceph (age 71m)     mgr: services-ceph(active, since 63m)     osd: 1 osds: 1 up (since 5s), 1 in (since 5s)     task status:     data:     pools:   0 pools, 0 pgs     objects: 0 objects, 0 B     usage:   1.0 GiB used, 199 GiB / 200 GiB avail     pgs:   # 通过ceph osd tree 查看osd的列表情况 [root@services-ceph ceph]# ceph osd tree ID CLASS WEIGHT  TYPE NAME              STATUS REWEIGHT PRI-AFF -1       0.00870 root default -3       0.00870     host services-ceph  0   hdd 0.00290         osd.0              up  1.00000 1.00000

 部署rgw实例

?

1 2 3 4 5 6 7 8 9 10 11 12 13

## 部署 rgw 组件 ceph-deploy rgw create services-ceph   ##  验证rgb，浏览器访问 http://192.168.11.21:7480/，输出以下内容   This XML file does not appear to have any style information associated with it. The document tree is shown below. <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Owner> <ID>anonymous</ID> <DisplayName/> </Owner> <Buckets/> </ListAllMyBucketsResult>

 部署mds实例

?

1 2 3 4 5 6

## 部署 mds 组件 ceph-deploy mds create services-ceph   ##  查看 mds 状态 [root@services-ceph ceph]# ceph mds stat  1 up:standby

 创建ceph的账号

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40

[root@services-ceph ceph]# radosgw-admin user create --uid=admin --display-name=admin --admin {     "user_id": "admin",     "display_name": "admin",     "email": "",     "suspended": 0,     "max_buckets": 1000,     "subusers": [],     "keys": [         {             "user": "admin",             "access_key": "******",             "secret_key": "******"         }     ],     "swift_keys": [],     "caps": [],     "op_mask": "read, write, delete",     "admin": "true",     "default_placement": "",     "default_storage_class": "",     "placement_tags": [],     "bucket_quota": {         "enabled": false,         "check_on_raw": false,         "max_size": -1,         "max_size_kb": 0,         "max_objects": -1     },     "user_quota": {         "enabled": false,         "check_on_raw": false,         "max_size": -1,         "max_size_kb": 0,         "max_objects": -1     },     "temp_url_keys": [],     "type": "rgw",     "mfa_ids": [] }

 客户端工具

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18

## 安装s3cmd，执行如下命令： yum install -y s3cmd   ## 配置文件,把admin的密钥配置如下 [root@services-ceph ~]# cat .s3cfg [default] access_key = ****** secret_key = ****** host_base = 192.168.11.21:7480 host_bucket = 192.168.11.21:7480/%(bucket) use_https = False   ## 执行创建桶操作，如下： s3cmd mb s3://test   # 查看桶是否创建成功 [root@services-ceph ~]# s3cmd ls 2022-04-02 17:46  s3://test