【十五】Openstack-Cinder
一、部署Cinder-Controller Node
#https://docs.openstack.org/cinder/train/install/index-rdo.html
#https://docs.openstack.org/cinder/train/install/cinder-controller-install-rdo.html
#在Mysql服务上操作 数据库 此环境：192.168.40.103
##【Install and configure controller node】
#1.[To create the database, complete these steps:]
#a.Use the database access client to connect to the database server as the root user:
mysql -u root -p
#b.Create the cinder database:
MariaDB [(none)]> CREATE DATABASE cinder;
#c.Grant proper access to the cinder database:
MariaDB [(none)]> GRANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'localhost' \
IDENTIFIED BY 'cinder123';
MariaDB [(none)]> GRANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'%' \
IDENTIFIED BY 'cinder123';
#controller2上操作
#2.[Source the admin credentials to gain access to admin-only CLI commands:]
bash admin-openrc.sh
#3.[To create the service credentials, complete these steps:]
#a.Create a cinder user:
[root@openstack-controller2 ~]# openstack user create --domain default --password-prompt cinder
User Password:
Repeat User Password:
+---------------------+----------------------------------+
| Field | Value |
+---------------------+----------------------------------+
| domain_id | default |
| enabled | True |
| id | 593ed1ffd8184a4f854ef0f8376d7f2f |
| name | cinder |
| options | {} |
| password_expires_at | None |
+---------------------+----------------------------------+
#b.Add the admin role to the cinder user:
openstack role add --project service --user cinder admin
#c.Create the cinderv2 and cinderv3 service entities:
[root@openstack-controller2 ~]# openstack service create --name cinderv2 \
> --description "OpenStack Block Storage" volumev2
+-------------+----------------------------------+
| Field | Value |
+-------------+----------------------------------+
| description | OpenStack Block Storage |
| enabled | True |
| id | 4b492d872ded472e966a14f8e1da7534 |
| name | cinderv2 |
| type | volumev2 |
+-------------+----------------------------------+
[root@openstack-controller2 ~]# openstack service create --name cinderv3 \
> --description "OpenStack Block Storage" volumev3
+-------------+----------------------------------+
| Field | Value |
+-------------+----------------------------------+
| description | OpenStack Block Storage |
| enabled | True |
| id | 86e215f2abaf49538cc1906cbaf0d50d |
| name | cinderv3 |
| type | volumev3 |
+-------------+----------------------------------+
#4.[Create the Block Storage service API endpoints:]
openstack endpoint create --region RegionOne \
volumev2 public http://openstack-vip.xks.local:8776/v2/%\(project_id\)s
openstack endpoint create --region RegionOne \
volumev2 internal http://openstack-vip.xks.local:8776/v2/%\(project_id\)s
openstack endpoint create --region RegionOne \
volumev2 admin http://openstack-vip.xks.local:8776/v2/%\(project_id\)s
openstack endpoint create --region RegionOne \
volumev3 public http://openstack-vip.xks.local:8776/v3/%\(project_id\)s
openstack endpoint create --region RegionOne \
volumev3 internal http://openstack-vip.xks.local:8776/v3/%\(project_id\)s
openstack endpoint create --region RegionOne \
volumev3 admin http://openstack-vip.xks.local:8776/v3/%\(project_id\)s
[root@openstack-controller2 ~]# openstack endpoint list | grep volume
| 0ef70c223f2e4ab385715f7b20fb01e8 | RegionOne | cinderv2 | volumev2 | True | internal | http://openstack-vip.xks.local:8776/v2/%(project_id)s |
| 1c7d39e7d7024cb9ba93d40f7ba934ec | RegionOne | cinderv3 | volumev3 | True | admin | http://openstack-vip.xks.local:8776/v3/%(project_id)s |
| 38324d0f6b0a4a6d8dbd30ecb7cb0c17 | RegionOne | cinderv3 | volumev3 | True | public | http://openstack-vip.xks.local:8776/v3/%(project_id)s |
| 7495c375c1b84f2dae1637a55b4f29c3 | RegionOne | cinderv2 | volumev2 | True | admin | http://openstack-vip.xks.local:8776/v2/%(project_id)s |
| 9ed4586354a64c0a90b2b60c0e2e9155 | RegionOne | cinderv3 | volumev3 | True | internal | http://openstack-vip.xks.local:8776/v3/%(project_id)s |
| bd97604edb4f48059712352c6e268f57 | RegionOne | cinderv2 | volumev2 | True | public | http://openstack-vip.xks.local:8776/v2/%(project_id)s |
#controller2操作
##【Install and configure components】
#1.[Install and configure components]
yum install -y openstack-cinder
#2.[Edit the /etc/cinder/cinder.conf file and complete the following actions:]
[database]
# ...
connection = mysql+pymysql://cinder:cinder123@openstack-vip.xks.local/cinder
[DEFAULT]
# ...
transport_url = rabbit://openstack:openstack123@openstack-vip.xks.local
[DEFAULT]
# ...
auth_strategy = keystone
[keystone_authtoken]
# ...
www_authenticate_uri = http://openstack-vip.xks.local:5000
auth_url = http://openstack-vip.xks.local:5000
memcached_servers = openstack-vip.xks.local:11211
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = service
username = cinder
password = cinder
[DEFAULT]
# ...
my_ip = 192.168.40.102
#3.[In the [oslo_concurrency] section, configure the lock path:]
[oslo_concurrency]
# ...
lock_path = /var/lib/cinder/tmp
#4.[Populate the Block Storage database:]
su -s /bin/sh -c "cinder-manage db sync" cinder
#查看数据库表
MariaDB [(none)]> use cinder;
MariaDB [cinder]> show tables;
+----------------------------+
| Tables_in_cinder |
+----------------------------+
| attachment_specs |
| backup_metadata |
| backups |
| cgsnapshots |
| clusters |
| consistencygroups |
| driver_initiator_data |
| encryption |
| group_snapshots |
| group_type_projects |
| group_type_specs |
| group_types |
| group_volume_type_mapping |
| groups |
| image_volume_cache_entries |
| messages |
| migrate_version |
| quality_of_service_specs |
| quota_classes |
| quota_usages |
| quotas |
| reservations |
| services |
| snapshot_metadata |
| snapshots |
| transfers |
| volume_admin_metadata |
| volume_attachment |
| volume_glance_metadata |
| volume_metadata |
| volume_type_extra_specs |
| volume_type_projects |
| volume_types |
| volumes |
| workers |
+----------------------------+
35 rows in set (0.000 sec)
##【Configure Compute to use Block Storage】
#Edit the /etc/nova/nova.conf file and add the following to it:
[cinder]
os_region_name = RegionOne
##【Finalize installation】
systemctl restart openstack-nova-api.service
systemctl enable openstack-cinder-api.service openstack-cinder-scheduler.service
systemctl start openstack-cinder-api.service openstack-cinder-scheduler.service
#配置Haproxy
[root@openstack-haproxy1 ~]# vim /etc/haproxy/haproxy.cfg
listen openstack-cinder-8776
bind 192.168.40.248:8776
mode tcp
#server 192.168.40.101 192.168.40.101:8776 check inter 3s fall 3 rise 5
server 192.168.40.102 192.168.40.102:8776 check inter 3s fall 3 rise 5
[root@openstack-haproxy1 ~]# systemctl restart haproxy
[root@openstack-haproxy1 ~]# netstat -ntlp | grep 8776
[root@openstack-controller2 ~]# cinder service-list
+------------------+---------------------------------+------+---------+-------+----------------------------+---------+-----------------+---------------+
| Binary | Host | Zone | Status | State | Updated_at | Cluster | Disabled Reason | Backend State |
+------------------+---------------------------------+------+---------+-------+----------------------------+---------+-----------------+---------------+
| cinder-scheduler | openstack-controller2.xks.local | nova | enabled | up | 2023-07-31T06:40:32.000000 | - | - | |
+------------------+---------------------------------+------+---------+-------+----------------------------+---------+-----------------+---------------+部署Cinder-storage node
#这里选择 openstack-mysql 服务器作为storage node
#在服务上进行添加磁盘
#为了不重启进行执行刷新 显示磁盘
[root@openstack-mysql scsi_host]# ls /sys/class/scsi_host/
host0 host1 host10 host11 host12 host13 host14 host15 host16 host17 host18 host19 host2 host20 host21 host22 host23 host24 host25 host26 host27 host28 host29 host3 host30 host31 host32 host4 host5 host6 host7 host8 host9
#host0~host32都要执行 可以刷出来
[root@openstack-mysql scsi_host]# echo "- - -" > /sys/class/scsi_host/host0/scan
[root@openstack-mysql scsi_host]# echo "- - -" > /sys/class/scsi_host/host1/scan
[root@openstack-mysql scsi_host]# echo "- - -" > /sys/class/scsi_host/host2/scan
....
[root@openstack-mysql scsi_host]# echo "- - -" > /sys/class/scsi_host/host32/scan
##Install and configure a storage node
#【Prerequisites】
#1.Install the supporting utility packages:
#Install the LVM packages:
yum install lvm2 device-mapper-persistent-data
#Start the LVM metadata service and configure it to start when the system boots:
systemctl enable lvm2-lvmetad.service
systemctl start lvm2-lvmetad.service
#2.Create the LVM physical volume /dev/sdb:
pvcreate /dev/sdb
#3.Create the LVM volume group cinder-volumes:
vgcreate cinder-volumes /dev/sdb
#4.Edit the /etc/lvm/lvm.conf file
devices {
...
filter = [ "a/sdb/", "r/.*/"]
##【Install and configure components】
#1.Install the packages:
#启用OpenStack库
yum install -y centos-release-openstack-train.noarch
yum install -y https://rdoproject.org/repos/rdo-release.rpm
yum install openstack-cinder targetcli python-keystone
#2.Edit the /etc/cinder/cinder.conf file and complete the following actions:
[database]
# ...
connection = mysql+pymysql://cinder:cinder123@openstack-vip.xks.local/cinder
[DEFAULT]
# ...
transport_url = rabbit://openstack:openstack123@openstack-vip.xks.local
[DEFAULT]
# ...
auth_strategy = keystone
[keystone_authtoken]
# ...
www_authenticate_uri = http://openstack-vip.xks.local:5000
auth_url = http://openstack-vip.xks.local:5000
memcached_servers = openstack-vip.xks.local:11211
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = service
username = cinder
password = cinder
[DEFAULT]
# ...
my_ip = 192.168.40.103
[lvm]
volume_driver = cinder.volume.drivers.lvm.LVMVolumeDriver
volume_group = cinder-volumes
target_protocol = iscsi
target_helper = lioadm
[DEFAULT]
# ...
enabled_backends = lvm
[DEFAULT]
# ...
glance_api_servers = http://openstack-vip.xks.local:9292
[oslo_concurrency]
# ...
lock_path = /var/lib/cinder/tmp
##【Finalize installation】
systemctl enable openstack-cinder-volume.service target.service
systemctl start openstack-cinder-volume.service target.service
[root@openstack-mysql scsi_host]# tail -f /var/log/cinder/*.log
2023-07-31 16:45:20.872 27136 INFO cinder.volume.manager [req-fc65380a-0ed7-47b7-813f-271fa050d2a8 - - - - -] Starting volume driver LVMVolumeDriver (3.0.0)
2023-07-31 16:45:21.291 27136 INFO cinder.brick.local_dev.lvm [req-fc65380a-0ed7-47b7-813f-271fa050d2a8 - - - - -] Logical Volume not found when querying LVM info. (vg_name=cinder-volumes, lv_name=cinder-volumes-pool
2023-07-31 16:45:23.126 27136 INFO cinder.volume.drivers.lvm [req-fc65380a-0ed7-47b7-813f-271fa050d2a8 - - - - -] Enabling LVM thin provisioning by default because a thin pool exists.
2023-07-31 16:45:23.946 27136 INFO cinder.volume.driver [req-fc65380a-0ed7-47b7-813f-271fa050d2a8 - - - - -] Driver hasn't implemented _init_vendor_properties()
2023-07-31 16:45:23.979 27136 INFO cinder.keymgr.migration [req-ca5ac3b2-d376-41c9-aa22-6ecf93d942ce - - - - -] Not migrating encryption keys because the ConfKeyManager's fixed_key is not in use.
2023-07-31 16:45:24.563 27136 INFO cinder.volume.manager [req-fc65380a-0ed7-47b7-813f-271fa050d2a8 - - - - -] Driver initialization completed successfully.
2023-07-31 16:45:24.568 27136 INFO cinder.manager [req-fc65380a-0ed7-47b7-813f-271fa050d2a8 - - - - -] Initiating service 2 cleanup
2023-07-31 16:45:24.573 27136 INFO cinder.manager [req-fc65380a-0ed7-47b7-813f-271fa050d2a8 - - - - -] Service 2 cleanup completed.
2023-07-31 16:45:24.616 27136 INFO cinder.volume.manager [req-fc65380a-0ed7-47b7-813f-271fa050d2a8 - - - - -] Initializing RPC dependent components of volume driver LVMVolumeDriver (3.0.0)
2023-07-31 16:45:25.191 27136 INFO cinder.volume.manager [req-fc65380a-0ed7-47b7-813f-271fa050d2a8 - - - - -] Driver post RPC initialization completed successfully.
##【Verify Cinder operation】
bash admin-openrc.sh
[root@openstack-controller2 ~]# openstack volume service list
+------------------+---------------------------------+------+---------+-------+----------------------------+
| Binary | Host | Zone | Status | State | Updated At |
+------------------+---------------------------------+------+---------+-------+----------------------------+
| cinder-scheduler | openstack-controller2.xks.local | nova | enabled | up | 2023-07-31T08:47:22.000000 |
| cinder-volume | openstack-mysql.xks.local@lvm | nova | enabled | up | 2023-07-31T08:47:25.000000 |
+------------------+---------------------------------+------+---------+-------+----------------------------+
[root@openstack-controller2 ~]# cinder service-list
+------------------+---------------------------------+------+---------+-------+----------------------------+---------+-----------------+---------------+
| Binary | Host | Zone | Status | State | Updated_at | Cluster | Disabled Reason | Backend State |
+------------------+---------------------------------+------+---------+-------+----------------------------+---------+-----------------+---------------+
| cinder-scheduler | openstack-controller2.xks.local | nova | enabled | up | 2023-07-31T08:47:42.000000 | - | - | |
| cinder-volume | openstack-mysql.xks.local@lvm | nova | enabled | up | 2023-07-31T08:47:45.000000 | - | - | up |
+------------------+---------------------------------+------+---------+-------+----------------------------+---------+-----------------+---------------+Dashboard创建卷
将分配的卷挂载到 虚拟机中
扩容卷
先格式化磁盘，挂载到/mnt中 写入一个xks.Txt文件
卸载磁盘进行扩容-先要进行分离卷
配置NFS后端存储
#Haproxy1 作为后端NFS Server
[root@openstack-haproxy1 ~]# yum install -y nfs-utils
Loaded plugins: fastestmirror
Determining fastest mirrors
* base: ftp.sjtu.edu.cn
* extras: ftp.sjtu.edu.cn
* updates: ftp.sjtu.edu.cn
base | 3.6 kB 00:00:00
extras | 2.9 kB 00:00:00
updates | 2.9 kB 00:00:00
Package 1:nfs-utils-1.3.0-0.68.el7.2.x86_64 already installed and latest version
Nothing to do
[root@openstack-haproxy1 ~]# mkdir /data/cinder
[root@openstack-haproxy1 ~]# vim /etc/exports
/data/glance *(rw,no_root_squash)
/data/cinder *(rw,no_root_squash)
[root@openstack-haproxy1 ~]# systemctl restart nfs
[root@openstack-haproxy1 ~]# cat /etc/exports
#配置cinder使用NFS
#【controller2】
[root@openstack-controller2 ~]# vim /etc/cinder/cinder.conf
[DEFAULT]
#......
enabled_backends = nfs
[nfs]
#......
volume_backend_name = openstack-NFS
volume_driver = cinder.volume.drivers.nfs.NfsDriver
nfs_shares_config= /etc/cinder/nfs_shares
nfs_mount_point_base = $state_path/mnt
[root@openstack-controller2 ~]# vim /etc/cinder/nfs_shares
[root@openstack-controller2 ~]# chown root.cinder /etc/cinder/nfs_shares
[root@openstack-controller2 ~]# systemctl restart openstack-cinder-api.service openstack-cinder-scheduler.service systemctl restart openstack-cinder-volume.service
[root@openstack-controller2 ~]# systemctl enable openstack-cinder-volume.service
[root@openstack-controller2 ~]# df -Th
Filesystem Type Size Used Avail Use% Mounted on
devtmpfs devtmpfs 2.5G 0 2.5G 0% /dev
tmpfs tmpfs 2.5G 0 2.5G 0% /dev/shm
tmpfs tmpfs 2.5G 20M 2.4G 1% /run
tmpfs tmpfs 2.5G 0 2.5G 0% /sys/fs/cgroup
/dev/mapper/centos-root xfs 50G 5.4G 45G 11% /
/dev/sda1 xfs 1014M 138M 877M 14% /boot
/dev/mapper/centos-home xfs 45G 33M 45G 1% /home
tmpfs tmpfs 494M 0 494M 0% /run/user/0
192.168.40.105:/data/glance nfs4 17G 6.4G 11G 38% /var/lib/glance/images
192.168.40.105:/data/cinder nfs4 17G 6.4G 11G 38% /var/lib/cinder/mnt/cf74ed63d6eb30cad6ca748fbb773278
[root@openstack-controller2 ~]# tail -n10 /var/log/cinder/*.log
#验证NFS和LVM
[root@openstack-controller2 ~]# cinder service-list
+------------------+-------------------------------------+------+---------+-------+----------------------------+---------+-----------------+---------------+
| Binary | Host | Zone | Status | State | Updated_at | Cluster | Disabled Reason | Backend State |
+------------------+-------------------------------------+------+---------+-------+----------------------------+---------+-----------------+---------------+
| cinder-scheduler | openstack-controller2.xks.local | nova | enabled | up | 2023-07-31T15:05:00.000000 | - | - | |
| cinder-volume | openstack-controller2.xks.local@nfs | nova | enabled | up | 2023-07-31T15:04:54.000000 | - | - | up |
| cinder-volume | openstack-mysql.xks.local@lvm | nova | enabled | up | 2023-07-31T15:04:54.000000 | - | - | up |
+------------------+-------------------------------------+------+---------+-------+----------------------------+---------+-----------------+---------------+
##创建磁盘类型并关联
否则在openstack管理界面创建磁盘的时候不能选择是NFS还是其他类型
#类型创建
[root@openstack-controller2 ~]# cinder type-create lvm
+--------------------------------------+------+-------------+-----------+
| ID | Name | Description | Is_Public |
+--------------------------------------+------+-------------+-----------+
| 1a251772-6dd0-494c-bc2a-f2eaf9d18666 | lvm | - | True |
+--------------------------------------+------+-------------+-----------+
[root@openstack-controller2 ~]# cinder type-create nfs
+--------------------------------------+------+-------------+-----------+
| ID | Name | Description | Is_Public |
+--------------------------------------+------+-------------+-----------+
| 092ded8e-be15-46d6-b0c2-0193bace88a8 | nfs | - | True |
+--------------------------------------+------+-------------+-----------+
#将磁盘类型与磁盘关联
#lvm 也就是在 openstack-mysql 服务器上配置后端别名
[root@openstack-mysql scsi_host]# vim /etc/cinder/cinder.conf
[lvm]
#...
volume_backend_name=openstack-lvm
[root@openstack-mysql scsi_host]# systemctl restart openstack-cinder-volume target
#nfs 也就是在 openstack-controller2 服务器上配置后端别名
[root@openstack-controller2 ~]# vim /etc/cinder/cinder.conf
[nfs]
#...
volume_backend_name = openstack-NFS
[root@openstack-mysql scsi_host]# systemctl restart openstack-cinder-volume target
cinder type-key lvm set volume_backend_name=openstack-lvm
cinder type-key nfs set volume_backend_name=openstack-NFS
#此时管理界面上就可以选择磁盘类型的卷了
ID和 nfs 存储上的id相对应
