最新Ceph L版与openstack Pike对接
安装Ceph luminous
实验环境
三台服务器，每台服务器都有4块硬盘，每台服务器都将自己的第一块硬盘作为系统盘，剩下的做ceph
一、在所有服务器上操作
#使用阿里源
yum install wget -y
rm -f /etc/yum.repos.d/*
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
sed -i '/aliyuncs.com/d' /etc/yum.repos.d/*.repo #删除阿里内网地址
#创建ceph源
echo '#阿里ceph源
[ceph]
name=ceph
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/
gpgcheck=0
[ceph-noarch]
name=cephnoarch
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/
gpgcheck=0
[ceph-source]
name=ceph-source
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS/
gpgcheck=0
#'>/etc/yum.repos.d/ceph.repo
yum clean all && yum makecache #生成缓存
#关闭selinux、防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
firewall-cmd --state
sed -i '/^SELINUX=.*/c SELINUX=disabled' /etc/selinux/config
sed -i 's/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g' /etc/selinux/config
grep --color=auto '^SELINUX' /etc/selinux/config
setenforce 0
二、只在node1上操作
三台服务器分别为node1 node2 node3 其中我们在node1上面安装ceph官方的自动化部署工具，也就是说我们只需要操作node1即可完成部署
#添加服务器主机名
echo '
10.0.0.11 node1
10.0.0.22 node2
10.0.0.33 node3
10.0.0.44 node4
10.0.0.55 node5
10.0.0.66 node6
'>>/etc/hosts
#创建秘钥
ssh-keygen
ssh-copy node2
ssh-copy node3
#将node1的hosts文件发送到其他服务器上面
scp /etc/hosts node2:/etc
scp /etc/hosts node3:/etc
三、开始在node1上安装ceph
#安装ceph-deploy配置工具
yum install -y ceph-deploy
yum install python-setuptools
#创建配置目录
mkdir /etc/ceph
cd /etc/ceph/
#初始化Mon配置
ceph-deploy new node1
# #修改冗余份数为2，日志大小2G
# #配置网络,单网卡忽略
echo '
public network = 10.0.0.0/24
cluster network = 10.0.0.0/24
mon_clock_drift_allowed = 2
osd_journal_size = 4086
osd_pool_default_pg_num = 128
osd_pool_default_pgp_num = 128
osd pool default size = 2
osd pool default min size = 1
rbd_default_features = 1
client_quota = true
'>>./ceph.conf
#安装Ceph
ceph-deploy install node1 node2 node3
#yum install -y ceph ceph-radosgw #实际上是安装这2个rpm
#初始化monitor和key
cd /etc/ceph/
ceph-deploy --overwrite-conf mon create-initial
#创建osd
ceph-deploy osd create --data /dev/sdb node1
ceph-deploy osd create --data /dev/sdc node1
ceph-deploy osd create --data /dev/sdd node1
ceph-deploy osd create --data /dev/sdb node2
ceph-deploy osd create --data /dev/sdc node2
ceph-deploy osd create --data /dev/sdd node2
ceph-deploy osd create --data /dev/sdb node3
ceph-deploy osd create --data /dev/sdc node3
ceph-deploy osd create --data /dev/sdd node3
#拷贝配置及密钥
ceph-deploy admin node1 node2 node3
chmod 644 /etc/ceph/ceph.client.admin.keyring
#创建mon
ceph-deploy --overwrite-conf mon create node1
ceph-deploy --overwrite-conf admin node1
#添加mgr
#ceph 12开始，monitor必须添加mgr
ceph-deploy mgr create node1:mon_mgr
#启用dashboard (在node1节点)
ceph mgr module enable dashboard
web访问http://10.0.0.11:7000 即可看到ceph的运行情况
#查看相关命令
ceph health
ceph -s
ceph osd tree
ceph df
ceph mon stat
ceph osd stat
ceph pg stat
ceph osd lspools
ceph auth list
Ceph L版与openstack Pike对接
环境opestack Pike的controller、computer、cinder三个节点已经安装完毕，并且能够正常的使用
node4为controller
node5为computer和cinder
一、环境准备，所有openstack主机都要操作
#所有openstack节点都添加主机名
#添加服务器主机名
echo '
10.0.0.11 node1
10.0.0.22 node2
10.0.0.33 node3
10.0.0.44 node4
10.0.0.55 node5
10.0.0.66 node6
'>>/etc/hosts
#创建ceph源
echo '#阿里ceph源
[ceph]
name=ceph
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/
gpgcheck=0
[ceph-noarch]
name=cephnoarch
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/
gpgcheck=0
[ceph-source]
name=ceph-source
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS/
gpgcheck=0
#'>/etc/yum.repos.d/ceph.repo
yum clean all && yum makecache #生成缓存
#关闭selinux、防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
firewall-cmd --state
sed -i '/^SELINUX=.*/c SELINUX=disabled' /etc/selinux/config
sed -i 's/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g' /etc/selinux/config
grep --color=auto '^SELINUX' /etc/selinux/config
setenforce 0
#将glance组件的节点安装ceph客户端
yum install python-rbd
#将nova和cinder组件的节点安装ceph客户端
yum install ceph-common
#创建POOL
ceph osd pool create volumes 128
ceph osd pool create images 128
ceph osd pool create vms 128
#初始化POOL
rbd pool init volumes
rbd pool init images
rbd pool init vms
#将ceph的配置文件导入到各个openstack节点上
ssh node4 tee /etc/ceph/ceph.conf </etc/ceph/ceph.conf
ssh node5 tee /etc/ceph/ceph.conf </etc/ceph/ceph.conf
#创建ceph用户和密钥
ceph auth get-or-create client.glance mon 'profile rbd' osd 'profile rbd pool=images'
ceph auth get-or-create client.cinder mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images'
ceph auth get-or-create client.cinder-backup mon 'profile rbd' osd 'profile rbd pool=backups'
#查询用户，写入文件
ceph auth get-or-create client.glance | ssh node4 tee /etc/ceph/ceph.client.glance.keyring
ssh node4 chown glance:glance /etc/ceph/ceph.client.glance.keyring
ceph auth get-or-create client.cinder | ssh node5 tee /etc/ceph/ceph.client.cinder.keyring
ssh node5 chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring
ceph auth get-or-create client.cinder-backup | ssh node5 tee /etc/ceph/ceph.client.cinder-backup.keyring
ssh node5 chown cinder:cinder /etc/ceph/ceph.client.cinder-backup.keyring
ceph auth get-key client.cinder | ssh node5 tee client.cinder.key
ceph与glance对接
#更改glance默认存储为ceph
#修改/etc/glance/glance-api.conf文件，添加一下内容
[DEFAULT]
default_store = rbd
show_image_direct_url = True
[glance_store]
#stores = file,http
#default_store = file
#filesystem_store_datadir = /var/lib/glance/images/
stores = rbd
default_store = rbd
rbd_store_pool = images
rbd_store_user = glance
rbd_store_ceph_conf = /etc/ceph/ceph.conf
rbd_store_chunk_size = 8
#重启服务
systemctl restart openstack-glance-api openstack-glance-registry
注意：在glance节点你需要查看/etc/ceph目录下是否有glance秘钥
ceph与nova对接，在computer节点上操作
#密钥加进libvirt
uuidgen
cat > secret.xml <<EOF
<secret ephemeral='no' private='no'>
<uuid>ff2e1190-30f5-4849-9c1c-886b1e1ee181</uuid>
<usage type='ceph'>
<name>client.cinder secret</name>
</usage>
</secret>
EOF
virsh secret-define --file secret.xml
virsh secret-set-value --secret ff2e1190-30f5-4849-9c1c-886b1e1ee181 --base64 $(cat client.cinder.key) && rm client.cinder.key secret.xml
virsh secret-list
UUID 用量
--------------------------------------------------------------------------------
ff2e1190-30f5-4849-9c1c-886b1e1ee181 ceph client.cinder secret
在computer节点的ceph配置文件里添加一下内容
ls -l /etc/ceph/
#ceph
echo '
[client]
rbd cache = true
rbd cache writethrough until flush = true
admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
log file = /var/log/qemu/qemu-guest-$pid.log
rbd concurrent management ops = 20
[client.cinder]
keyring = /etc/ceph/ceph.client.cinder.keyring
'>>/etc/ceph/ceph.conf
mkdir -p /var/run/ceph/guests/ /var/log/qemu/
chown qemu:libvirt /var/run/ceph/guests /var/log/qemu/
#使用ceph存储，在/etc/nova/nova.conf添加以下内容
[libvirt]
virt_type = qemu
images_type = rbd
images_rbd_pool = vms
images_rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_user = cinder
rbd_secret_uuid = 'ff2e1190-30f5-4849-9c1c-886b1e1ee181'
disk_cachemodes="network=writeback"
live_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
libvirt_inject_password = false
libvirt_inject_key = false
libvirt_inject_partition = -2
#重启服务
systemctl restart libvirtd.service openstack-nova-compute.service
ceph与cinder对接
#修改/etc/cinder/cinder.conf配置文件，注意：将之前的lvm的配置都删掉
[DEFAULT]
#enabled_backends = lvm
enabled_backends = ceph
[ceph]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
rbd_pool = volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_flatten_volume_from_snapshot = false
rbd_max_clone_depth = 5
rbd_store_chunk_size = 4
rados_connect_timeout = -1
glance_api_version = 2
rbd_user = cinder
rbd_secret_uuid = ff2e1190-30f5-4849-9c1c-886b1e1ee181
#重启服务
systemctl restart openstack-cinder-volume.service
