ceph集群部署(基于jewel版)
环境
两个节点：ceph1、ceph2
- ceph1: mon、mds、osd.0、osd.1
- ceph2: osd.2、osd.3
网络配置：
ceph1: 管理网络，eth0,10.0.0.20
存储网络，eth1, 10.0.1.20
ceph2: 管理网络，eth0,10.0.0.21
存储网络，eth1, 10.0.1.21
安装
root@ceph1:~# wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
root@ceph1:~# echo deb http://download.ceph.com/debian-jewel/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
root@ceph1:~# apt-get update && apt-get upgrade
root@ceph1:~# apt-get install ceph-common ceph-fs-common ceph-mds ceph
配置
/etc/ceph/ceph.conf
[global]
max open files = 131072
fsid = eb3d751f-829f-4eae-8d69-0423b81f88f4
auth cluster required = none
auth service required = none
auth client required = none
osd pool default size = 2
osd pool default min size = 1
mon osd full ratio = .95
mon osd nearfull ratio = .85
[mon]
mon data = /var/lib/ceph/mon/$cluster-$name
[osd]
osd journal size = 1024
osd mkfs type = xfs
osd mkfs options xfs = -f
osd mount options xfs = rw,noatime
[mon.monster]
host = ceph1
mon addr = 10.0.1.20:6789
[osd.0]
host = ceph1
devs = /dev/sdb1
[osd.1]
host = ceph1
devs= /dev/sdc1
[osd.2]
host = ceph2
devs = /dev/sdb1
[osd.3]
host = ceph2
devs = /dev/sdc1
部署mon
root@ceph1:~# ceph-authtool /etc/ceph/ceph.mon.keyring --create-keyring --gen-key -n mon.
root@ceph1:~# ceph-mon -i monster --mkfs --keyring /etc/ceph/ceph.mon.keyring
root@ceph1:~# chown ceph:ceph -R /var/lib/ceph/mon/ /var/run/ceph
root@ceph1:~# /etc/init.d/ceph start mon.monster
部署osd
root@ceph1:~# mkfs.xfs /dev/sdb1
root@ceph1:~# mkfs.xfs /dev/sdc1
root@ceph1:~# mkdir -p /var/lib/ceph/osd/ceph-0/
root@ceph1:~# mkdir -p /var/lib/ceph/osd/ceph-1/
root@ceph1:~# mount /dev/sdb1 /var/lib/ceph/osd/ceph-0/
root@ceph1:~# mount /dev/sdc1 /var/lib/ceph/osd/ceph-1/
部署osd.0
ps：无论osd在哪台服务器上，ceph osd creat都要在mon所在服务器上执行
root@ceph1:~# ceph osd create
root@ceph1:~# ceph-osd -i 0 --mkfs --mkkey
root@ceph1:~# ceph auth add osd.0 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-0/keyring
root@ceph1:~# ceph osd crush add osd.0 0.2 root=default host=ceph1
root@ceph1:~# ceph-osd -i 0 #或者/etc/init.d/ceph start osd.0
同理部署其他osd
ps: 不同服务器时，需先复制ceph.conf到对应服务器上
root@ceph1:~# ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1 0.79999 root default
-2 0.39999 host ceph1
0 0.20000 osd.0 up 1.00000 1.00000
1 0.20000 osd.1 up 1.00000 1.00000
-3 0.39999 host ceph2
2 0.20000 osd.2 up 1.00000 1.00000
3 0.20000 osd.3 up 1.00000 1.00000
部署mds
root@ceph1:~# ceph-mds -i a -n mds.monster -c /etc/ceph/ceph.conf -m 10.0.1.20:6789
root@ceph1:~# ceph mds stat
e3:, 1 up:standby
块存储的应用
创建存储池：
root@ceph-monster:~# rados mkpool k8s-pool
successfully created pool k8s-pool
在该存储池里新建镜像：
root@ceph-monster:~# rbd create k8s-pool/k8s-image.v1 --size 4096 --image-feature layering
映射到块设备中：
root@ceph-monster:~# rbd map k8s-pool/k8s-image.v1
/dev/rbd3
root@ceph-monster:~# rbd showmapped
id pool image snap device
0 rbd myimage.v1 - /dev/rbd0
1 test_pool test-image.v3 - /dev/rbd1
2 test_pool test.img - /dev/rbd2
3 k8s-pool k8s-image.v1 - /dev/rbd3
格式化并挂载：
root@ceph-monster:~# mkfs.xfs /dev/rbd3
root@ceph-monster:~# mount /dev/rbd3 /rbdfs
rdb map的一个报错： rbd sysfs write failed
rbd: sysfs write failed
RBD image feature set mismatch. You can disable features unsupported by the kernel with "rbd feature disable".
In some cases useful info is found in syslog - try "dmesg | tail" or so.
原因：rbd镜像的一些特性，OS kernel并不支持，所以映射失败，需要把部分不支持的特性disable掉。
方法一：
直接diable这个rbd镜像的不支持的特性：
$ rbd feature disable myimage exclusive-lock object-map fast-diff deep-flatten
方法二：
创建rbd镜像时就指明需要的特性，如：
$ rbd create --size 4096 myimage --image-feature layering
方法三：
如果还想一劳永逸，那么就在执行创建rbd镜像命令的服务器中，修改Ceph配置文件/etc/ceph/ceph.conf，在global section下，增加
rbd_default_features = 1
再创建rdb镜像。
$ rbd create --size 4096 myimage
cephfs 文件系统
新建一个pool：
root@ceph1:~# rados mkpool cephfs_mdata
successfully created pool cephfs_mdata
root@ceph1:~# rados lspools
rbd
cephfs_mdata
创建cephfs:
root@ceph1:~# ceph fs new cephfs1 cephfs_metadata cephfs_mdata
new fs with metadata pool 1 and data pool 2
root@ceph1:~# ceph fs ls
name: cephfs1, metadata pool: cephfs_metadata, data pools:
[cephfs_mdata ]
- fs new 表示需要创建一个新的文件系统
- cephfs1 表示新的文件系统的名字叫做cephfs2。
- cephfs_metadata 表示文件系统元数据保存信息的存储pool
- cephfs_mdata 表示文件系统的数据保存信息的存储pool
ps: ceph集群上只能创建一个文件系统
挂载到本地：
root@ceph1:~# mkdir /cephfs
root@ceph1:~# sudo mount -t ceph 10.0.1.20:6789:/ /cephfs/
root@ceph1:~# ceph mds stat
e130: 1/1/1 up {0=a=up:active}
root@ceph1:~# ceph -s
cluster eb3d751f-829f-4eae-8d69-0423b81f88f4
health HEALTH_OK
monmap e1: 1 mons at {a=10.0.1.20:6789/0}
election epoch 4, quorum 0 a
fsmap e130: 1/1/1 up {0=a=up:active}
osdmap e158: 4 osds: 4 up, 4 in
flags sortbitwise,require_jewel_osds
pgmap v354: 80 pgs, 3 pools, 2542 bytes data, 20 objects
4530 MB used, 195 GB / 199 GB avail
80 active+clean
