02 Ceph 安装部署

合集 - Ceph(14)

1.01 Ceph 基础2022-10-21

2.02 Ceph 安装部署2022-10-18

3.03 块存储2022-10-214.04 对象存储网关2022-10-215.05 CephFS 文件系统2022-10-216.06 OSD 扩容和换盘2022-10-227.07 Ceph 集群运维2022-10-228.08 调整 CRUSH Map2022-10-289.09 RDB 高级功能2022-11-0210.10 RGW 高可用集群2022-11-0311.11 Ceph 集群测试2022-11-0312.12 Ceph 与 Kubernetes 集成2022-11-0813.13 Ceph 管理与监控2022-11-1914.14 SDK开发2022-11-20

收起

目录

• Ceph 安装手册
  • 部署架构图
  • Ceph 基础环境准备
    • 主机名和 hosts 设置
      • 设置主机名
      • 设置 hosts
    • 设置 ssh 无密码登陆
    • 安全设置
      • 关闭 selinux
      • 关闭 防火墙
    • ntp 时间同步
      • 安装 ntp 软件
      • 设置 时间同步
        • a | node0 节点
        • b | node[1|2] 节点
    • yum 源环境设置
      • 配置 Ceph 国内基础源、epel 源
  • Ceph 安装
    • 安装 ceph-deploy 软件
    • 安装 ceph 集群组件软件
  • Ceph 集群初始化
    • 创建目录，保存初始化集群信息
    • 设置 admin 认证 , ceph -s 正常显示
    • 集群不健康问题处理
    • 集群监控 mgr 部署
  • Ceph 集群添加 osd
    • 添加 osd
    • 查看 osd
    • 查看 集群状态
  • Ceph 集群扩容 monitor 服务
    • 查看仲裁情况
  • Ceph 集群扩容 mgr 服务

Ceph 安装手册

部署架构图

Ceph 基础环境准备

注意：以下步骤，除步骤 设置 ssh 无密码登陆，其他步骤所有节点均需要执行

主机名和 hosts 设置

设置主机名

$ hostanmectl set-hostname node[0|1|2]

设置 hosts

$ vim /etc/hosts

192.168.100.130 node0
192.168.100.131 node1
192.168.100.132 node2

设置 ssh 无密码登陆

此步骤只需要在 node0 上执行即可

# 生产密钥对
[root@node0 ~]$ ssh-keygen 

# 分发密钥
[root@node0 ~]$ ssh-copy-id node1

[root@node0 ~]$ ssh-copy-id node2

安全设置

关闭 selinux

$ vim /etc/selinux/config 

# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#     enforcing - SELinux security policy is enforced.
#     permissive - SELinux prints warnings instead of enforcing.
#     disabled - No SELinux policy is loaded.
SELINUX=disabled

$ setenforce 0

关闭 防火墙

$ systemctl disalbe firewalld && systemctl stop firewalld

ntp 时间同步

安装 ntp 软件

$ yum -y install ntp ntpdate ntp-doc

设置 时间同步

yum -y install ntp ntpdate ntp-doc

a | node0 节点

# 直接启动服务（生产环境建议配置为公司内网 ntp 服务器地址) 
$ systemctl enable ntpd --now
$ systemctl status ntpd

# 查看时间同步情况
$ ntpq -pn
     remote           refid      st t when poll reach   delay   offset  jitter
==============================================================================
-193.182.111.141 192.36.143.153   2 u 1644 1024   62  310.598   25.603  21.193
+162.159.200.123 10.115.8.6       3 u 1041 1024  377  270.740   -6.318   5.122
+185.209.85.222  194.190.168.1    2 u  595 1024  377  191.147  -29.251   6.177
*36.110.235.196  10.218.108.2     2 u 1017 1024  377   35.870    3.234   2.771

b | node[1|2] 节点

# 修改时间同步源地址为 node0 IP 地址
$ vim /etc/ntp.conf

......
# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
server 192.168.100.130 iburst

......

# 启动服务
$ systemctl enable ntpd --now
$ systemctl status ntpd

# 查看时间同步情况
$ ntpq -pn
     remote           refid      st t when poll reach   delay   offset  jitter
==============================================================================
*192.168.100.130 36.110.235.196   3 u  461 1024  377    0.334    1.744   1.644

yum 源环境设置

配置 Ceph 国内基础源、epel 源

# 设置源
$ cat << EOF > /etc/yum.repos.d/ceph.repo
[noarch]
name=ceph-noarch
baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch/
enabled=1
gpgchek=0

[x86_64]
name=ceph-x86_64
baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/x86_64/
enabled=1
gpgchek=0
EOF

# 更新仓库
$ yum clean all
$ yum makecache
$ yum repolist

Ceph 安装

以下步骤如无特殊说明，只需要在 node0 节点执行即可

安装 ceph-deploy 软件

# 安装 ceph-deploy

$ yum install python-pip
$ pip install ceph-deploy

# 查看 ceph-deploy 版本
$ ceph-deploy --version
2.0.1

安装 ceph 集群组件软件

此步骤需要在3个节点上运行

$ yum install ceph ceph-mds ceph-mgr ceph-mon ceph-radosgw -y
$ yum install -y --nogpgcheck librbd1
$ yum install -y --nogpgcheck libcephfs2
$ yum install -y --nogpgcheck ceph-mon
$ yum install -y --nogpgcheck ceph-mds
$ yum install -y --nogpgcheck ceph-mgr
$ yum install -y --nogpgcheck ceph-radosgw
$ yum install -y --nogpgcheck ceph
$ yum install ceph ceph-mds ceph-mgr ceph-mon ceph-radosgw -y

Ceph 集群初始化

创建目录，保存初始化集群信息

$ mkdir /data/ceph-deploy -pv
$ cd /data/ceph-deploy/

$ ceph-deploy new -h
$ ceph-deploy new --cluster-network 192.168.100.0/24 --public-network 192.168.100.0/24 node0
$ ls
ceph-deploy-ceph.log  ceph.conf  ceph.mon.keyring  

$ ceph-deploy mon create-initial
$ ls -l
total 108
-rw-------. 1 root root   113 Oct 13 14:03 ceph.bootstrap-mds.keyring
-rw-------. 1 root root   113 Oct 13 14:03 ceph.bootstrap-mgr.keyring
-rw-------. 1 root root   113 Oct 13 14:03 ceph.bootstrap-osd.keyring
-rw-------. 1 root root   113 Oct 13 14:03 ceph.bootstrap-rgw.keyring
-rw-------. 1 root root   151 Oct 13 14:03 ceph.client.admin.keyring
-rw-r--r--. 1 root root   267 Oct 13 13:45 ceph.conf
-rw-r--r--. 1 root root 78676 Oct 13 18:02 ceph-deploy-ceph.log
-rw-------. 1 root root    73 Oct 13 13:45 ceph.mon.keyring

$ ceph -s
[errno 2] error connecting to the cluster

设置 admin 认证 , ceph -s 正常显示

# 节点分发管理员配置文件到所有节点
$ ceph-deploy admin node0 node1 node2

$ ceph -s
  cluster:
    id:     97702c43-6cc2-4ef8-bdb5-855cfa90a260
    health: HEALTH_WARN
            mon is allowing insecure global_id reclaim
 
  services:
    mon: 1 daemons, quorum node0 (age 49s)
    mgr: no deameons active
    osd: 0 osds: 0 up, 0 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0.0 B used, 0 B / 0 B avail
    pgs: 

集群不健康问题处理

$ ceph config set mon auth_allow_insecure_global_id_reclaim false

$ ceph -s
  cluster:
    id:     97702c43-6cc2-4ef8-bdb5-855cfa90a260
    health: HEALTH_OK
 
  services:
    mon: 1 daemons, quorum node0 (age 49s)
    mgr: no deameons active
    osd: 0 osds: 0 up, 0 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0.0 B used, 0 B / 0 B avail
    pgs: 

## 参考文档：https://www.cnblogs.com/lvzhenjiang/p/14856572.html

集群监控 mgr 部署

$ ceph-deploy mgr create node0

$ ceph -s
  cluster:
    id:     97702c43-6cc2-4ef8-bdb5-855cfa90a260
    health: HEALTH_OK
 
  services:
    mon: 1 daemons, quorum node0 (age 2m)
    mgr: node0(active, since 5s)
    osd: 0 osds: 0 up, 0 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0.0 B used, 0 B / 0 B avail
    pgs: 

Ceph 集群添加 osd

添加 osd

$ ceph-deploy osd create node0 --data /dev/sdb
$ ceph-deploy osd create node1 --data /dev/sdb
$ ceph-deploy osd create node2 --data /dev/sdb

查看 osd

$ ceph osd tree
ID CLASS WEIGHT  TYPE NAME      STATUS REWEIGHT PRI-AFF 
-1       0.14639 root default                           
-3       0.04880     host node0                         
 0   hdd 0.04880         osd.0      up  1.00000 1.00000 
-5       0.04880     host node1                         
 1   hdd 0.04880         osd.1      up  1.00000 1.00000 
-7       0.04880     host node2                         
 2   hdd 0.04880         osd.2      up  1.00000 1.00000 

查看 集群状态

$ ceph -s
  cluster:
    id:     97702c43-6cc2-4ef8-bdb5-855cfa90a260
    health: HEALTH_OK
 
  services:
    mon: 1 daemons, quorum node0 (age 3h)
    mgr: node0(active, since 3h)
    osd: 3 osds: 3 up (since 3m), 3 in (since 3m)
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   3.0 GiB used, 147 GiB / 150 GiB avail
    pgs:

Ceph 集群扩容 monitor 服务

$ ceph-deploy mon add node1 --address 192.168.100.131
$ ceph-deploy mon add node2 --address 192.168.100.132

$ ceph -s
  cluster:
    id:     97702c43-6cc2-4ef8-bdb5-855cfa90a260
    health: HEALTH_OK
 
  services:
    mon: 3 daemons, quorum node0,node1,node2 (age 0.187697s)
    mgr: node0(active, since 3h)
    osd: 3 osds: 3 up (since 5m), 3 in (since 5m)
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   3.0 GiB used, 147 GiB / 150 GiB avail
    pgs:

查看仲裁情况

$ ceph quorum_status --format json-pretty

{
    "election_epoch": 12,
    "quorum": [
        0,
        1,
        2
    ],
    "quorum_names": [
        "node0",
        "node1",
        "node2"
    ],
    "quorum_leader_name": "node0",
    "quorum_age": 104,
    "monmap": {
        "epoch": 3,
        "fsid": "97702c43-6cc2-4ef8-bdb5-855cfa90a260",
        "modified": "2022-10-13 17:57:43.445773",
        "created": "2022-10-13 14:03:09.897152",
        "min_mon_release": 14,
        "min_mon_release_name": "nautilus",
        "features": {
            "persistent": [
                "kraken",
                "luminous",
                "mimic",
                "osdmap-prune",
                "nautilus"
            ],
            "optional": []
        },
        "mons": [
            {
                "rank": 0,
                "name": "node0",
                "public_addrs": {
                    "addrvec": [
                        {
                            "type": "v2",
                            "addr": "192.168.100.130:3300",
                            "nonce": 0
                        },
                        {
                            "type": "v1",
                            "addr": "192.168.100.130:6789",
                            "nonce": 0
                        }
                    ]
                },
                "addr": "192.168.100.130:6789/0",
                "public_addr": "192.168.100.130:6789/0"
            },
            {
                "rank": 1,
                "name": "node1",
                "public_addrs": {
                    "addrvec": [
                        {
                            "type": "v2",
                            "addr": "192.168.100.131:3300",
                            "nonce": 0
                        },
                        {
                            "type": "v1",
                            "addr": "192.168.100.131:6789",
                            "nonce": 0
                        }
                    ]
                },
                "addr": "192.168.100.131:6789/0",
                "public_addr": "192.168.100.131:6789/0"
            },
            {
                "rank": 2,
                "name": "node2",
                "public_addrs": {
                    "addrvec": [
                        {
                            "type": "v2",
                            "addr": "192.168.100.132:3300",
                            "nonce": 0
                        },
                        {
                            "type": "v1",
                            "addr": "192.168.100.132:6789",
                            "nonce": 0
                        }
                    ]
                },
                "addr": "192.168.100.132:6789/0",
                "public_addr": "192.168.100.132:6789/0"
            }
        ]
    }
}

$ ceph mon stat
e3: 3 mons at {node0=[v2:192.168.100.130:3300/0,v1:192.168.100.130:6789/0],node1=[v2:192.168.100.131:3300/0,v1:192.168.100.131:6789/0],node2=[v2:192.168.100.132:3300/0,v1:192.168.100.132:6789/0]}, election epoch 12, leader 0 node0, quorum 0,1,2 node0,node1,node2


$ ceph mon dump
epoch 3
fsid 97702c43-6cc2-4ef8-bdb5-855cfa90a260
last_changed 2022-10-13 17:57:43.445773
created 2022-10-13 14:03:09.897152
min_mon_release 14 (nautilus)
0: [v2:192.168.100.130:3300/0,v1:192.168.100.130:6789/0] mon.node0
1: [v2:192.168.100.131:3300/0,v1:192.168.100.131:6789/0] mon.node1
2: [v2:192.168.100.132:3300/0,v1:192.168.100.132:6789/0] mon.node2
dumped monmap epoch 3

Ceph 集群扩容 mgr 服务

$ ceph-deploy mgr create node1 node2

$ ceph -s
  cluster:
    id:     97702c43-6cc2-4ef8-bdb5-855cfa90a260
    health: HEALTH_OK
 
  services:
    mon: 3 daemons, quorum node0,node1,node2 (age 5m)
    mgr: node0(active, since 3h), standbys: node1, node2
    osd: 3 osds: 3 up (since 10m), 3 in (since 10m)
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   3.0 GiB used, 147 GiB / 150 GiB avail
    pgs: