k8s集群部署之环境介绍与etcd数据库集群部署

角色	IP	组件	配置
master-1	192.168.10.11	kube-apiserver kube-controller-manager kube-scheduler etcd	2c 2g
master-2	192.168.10.12	kube-apiserver kube-controller-manager kube-scheduler etcd	2c 2g
node-1	192.168.10.13	kubelet kube-proxy docker flannel etcd	2c 2g
node-2	192.168.10.14	kubelet kube-proxy docker flannel	2c 2g
nginx负载主	192.168.10.15 192.168.10.20（vip）	nginx	2c 2g
nginx负载备	192.168.10.16	nginx	2c 2g
Registry	192.168.10.17	harbor	2c 2g

多master集群架构图

环境准备，关闭SELinux

[root@mast-1 ~]# vi /etc/selinux/config 
 
 
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#     enforcing - SELinux security policy is enforced.
#     permissive - SELinux prints warnings instead of enforcing.
#     disabled - No SELinux policy is loaded.
SELINUX=disabled    永久关闭
# SELINUXTYPE= can take one of three two values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected.
#     mls - Multi Level Security protection.
SELINUXTYPE=targeted
[root@mast-1 ~]# setenforce 0
[root@mast-1 ~]# systemctl stop firewalld.service 
[root@mast-1 ~]# systemctl disable firewalld.service 

生成自签证书

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

[root@mast-1 ~]# mkdir k8s/{k8s-cert,etcd-cert} -p
[root@mast-1 ~]# cd k8s/
[root@mast-1 k8s]# ls
etcd-cert  k8s-cert
[root@mast-1 k8s]# cd etcd-cert/
[root@mast-1 etcd-cert]# cat etcd-cert.sh 
cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF
 
cat > ca-csr.json <<EOF
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF
 
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
 
#-----------------------
 
cat > server-csr.json <<EOF
{
    "CN": "etcd",
    "hosts": [
    "10.206.240.188",
    "10.206.240.189",
    "10.206.240.111"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}
EOF
 
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
 时间同步
[root@mast-1 etcd-cert]# ntpdate time.windows.com
19 Apr 18:08:01 ntpdate[18507]: adjust time server 52.175.49.4 offset -0.038617 sec
安装工具
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo
生成证书
[root@mast-1 etcd-cert]# bash etcd-cert.sh 
2019/04/19 18:12:51 [INFO] generating a new CA key and certificate from CSR
2019/04/19 18:12:51 [INFO] generate received request
2019/04/19 18:12:51 [INFO] received CSR
2019/04/19 18:12:51 [INFO] generating key: rsa-2048
2019/04/19 18:12:52 [INFO] encoded CSR
2019/04/19 18:12:52 [INFO] signed certificate with serial number 68673429772362340986397269642150956332201052001
2019/04/19 18:12:52 [INFO] generate received request
2019/04/19 18:12:52 [INFO] received CSR
2019/04/19 18:12:52 [INFO] generating key: rsa-2048
2019/04/19 18:12:53 [INFO] encoded CSR
2019/04/19 18:12:53 [INFO] signed certificate with serial number 481813885977621272201720696275880807248023743489
2019/04/19 18:12:53 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@mast-1 etcd-cert]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  etcd-cert.sh  server.csr  server-csr.json  server-key.pem  server.pem
生成CA证书
[root@mast-1 etcd-cert]# cat > ca-config.json <<EOF
> {
>   "signing": {
>     "default": {
>       "expiry": "87600h"
>     },
>     "profiles": {
>       "www": {
>          "expiry": "87600h",
>          "usages": [
>             "signing",
>             "key encipherment",
>             "server auth",
>             "client auth"
>         ]
>       }
>     }
>   }
> }
> EOF
[root@mast-1 etcd-cert]# 
[root@mast-1 etcd-cert]# cat > ca-csr.json <<EOF
> {
>     "CN": "etcd CA",
>     "key": {
>         "algo": "rsa",
>         "size": 2048
>     },
>     "names": [
>         {
>             "C": "CN",
>             "L": "Beijing",
>             "ST": "Beijing"
>         }
>     ]
> }
> EOF
[root@mast-1 etcd-cert]# 
[root@mast-1 etcd-cert]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2019/04/19 18:21:15 [INFO] generating a new CA key and certificate from CSR
2019/04/19 18:21:15 [INFO] generate received request
2019/04/19 18:21:15 [INFO] received CSR
2019/04/19 18:21:15 [INFO] generating key: rsa-2048
2019/04/19 18:21:15 [INFO] encoded CSR
2019/04/19 18:21:15 [INFO] signed certificate with serial number 601159451155869323088996388702263627368633639404
[root@mast-1 etcd-cert]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem 
生成etcd证书
[root@mast-1 etcd-cert]# cat > server-csr.json <<EOF
> {
>     "CN": "etcd",
>     "hosts": [
>     "192.168.10.11",
>     "192.168.10.12",
>     "192.168.10.13"
>     ],
>     "key": {
>         "algo": "rsa",
>         "size": 2048
>     },
>     "names": [
>         {
>             "C": "CN",
>             "L": "BeiJing",
>             "ST": "BeiJing"
>         }
>     ]
> }
> EOF
[root@mast-1 etcd-cert]# 
[root@mast-1 etcd-cert]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
2019/04/19 18:26:38 [INFO] generate received request
2019/04/19 18:26:38 [INFO] received CSR
2019/04/19 18:26:38 [INFO] generating key: rsa-2048
2019/04/19 18:26:38 [INFO] encoded CSR
2019/04/19 18:26:38 [INFO] signed certificate with serial number 20600129498404647940919904437550435044516988458
2019/04/19 18:26:38 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

　ETCD数据库集群部署　

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

[root@mast-1 ~]# mkdir APP 
[root@mast-1 ~]# mv etcd-v3.3.10-linux-amd64.tar.gz APP/
[root@mast-1 ~]# cd APP/
[root@mast-1 APP]# ls
etcd-v3.3.10-linux-amd64.tar.gz
[root@mast-1 APP]# ls
etcd-v3.3.10-linux-amd64.tar.gz
[root@mast-1 APP]# tar xf etcd-v3.3.10-linux-amd64.tar.gz 
[root@mast-1 APP]# ls
etcd-v3.3.10-linux-amd64  etcd-v3.3.10-linux-amd64.tar.gz
[root@mast-1 APP]# cd etcd-v3.3.10-linux-amd64/
[root@mast-1 etcd-v3.3.10-linux-amd64]# ls
Documentation  etcd  etcdctl  README-etcdctl.md  README.md  READMEv2-etcdctl.md
[root@mast-1 etcd-v3.3.10-linux-amd64]# mkdir -pv /opt/etcd/{cfg,bin,ssl}
mkdir: 已创建目录 "/opt/etcd"
mkdir: 已创建目录 "/opt/etcd/cfg"
mkdir: 已创建目录 "/opt/etcd/bin"
mkdir: 已创建目录 "/opt/etcd/ssl"
[root@mast-1 etcd-v3.3.10-linux-amd64]# mv etcd etcdctl /opt/etcd/bin/
[root@mast-1 etcd-v3.3.10-linux-amd64]# cd ~/k8s/
[root@mast-1 k8s]# ls
etcd-cert  etcd.sh  k8s-cert
生成etcd配置文件脚本
[root@mast-1 k8s]# cat etcd.sh 
#!/bin/bash
# example: ./etcd.sh etcd01 192.168.1.10 etcd02=https://192.168.1.11:2380,etcd03=https://192.168.1.12:2380
ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3
 
WORK_DIR=/opt/etcd
 
cat <<EOF >$WORK_DIR/cfg/etcd
#[Member]
ETCD_NAME="${ETCD_NAME}"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_LISTEN_CLIENT_URLS="https://${ETCD_IP}:2379"
 
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_IP}:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://${ETCD_IP}:2380,${ETCD_CLUSTER}"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
 
cat <<EOF >/usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
 
[Service]
Type=notify
EnvironmentFile=${WORK_DIR}/cfg/etcd
ExecStart=${WORK_DIR}/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=${WORK_DIR}/ssl/server.pem \
--key-file=${WORK_DIR}/ssl/server-key.pem \
--peer-cert-file=${WORK_DIR}/ssl/server.pem \
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
 
[Install]
WantedBy=multi-user.target
EOF
 
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd
 
启动有问题
[root@mast-1 k8s]# ./etcd.sh etcd01 192.168.10.11 etcd02=https://192.168.10.12:2380,etcd03=https://192.168.10.13:2380
Job for etcd.service failed because the control process exited with error code. See "systemctl status etcd.service" and "journalctl -xe" for details.
查看配置文件
[root@mast-1 k8s]# cat /opt/etcd/cfg/etcd 
#[Member]
ETCD_NAME="etcd01"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.10.11:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.10.11:2379"
 
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.11:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.11:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.10.11:2380,etcd02=https://192.168.10.12:2380,etcd03=https://192.168.10.13:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
 
 
 
复制证书
[root@mast-1 k8s]# cp etcd-cert/{ca,server-key,server}.pem /opt/etcd/ssl/
[root@mast-1 k8s]# systemctl start etcd
[root@mast-1 ~]# tail -10 /var/log/messages 
Apr 19 18:59:01 localhost etcd: 6571fb7574e87dba [logterm: 1, index: 3] sent MsgVote request to d1fbb74bc6a61e5c at term 55
Apr 19 18:59:02 localhost etcd: 6571fb7574e87dba is starting a new election at term 55
Apr 19 18:59:02 localhost etcd: 6571fb7574e87dba became candidate at term 56
Apr 19 18:59:02 localhost etcd: 6571fb7574e87dba received MsgVoteResp from 6571fb7574e87dba at term 56
Apr 19 18:59:02 localhost etcd: 6571fb7574e87dba [logterm: 1, index: 3] sent MsgVote request to d1fbb74bc6a61e5c at term 56
Apr 19 18:59:02 localhost etcd: 6571fb7574e87dba [logterm: 1, index: 3] sent MsgVote request to 9b449b0ff1d4c375 at term 56
Apr 19 18:59:03 localhost etcd: health check for peer 9b449b0ff1d4c375 could not connect: dial tcp 192.168.10.12:2380: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Apr 19 18:59:03 localhost etcd: health check for peer d1fbb74bc6a61e5c could not connect: dial tcp 192.168.10.13:2380: connect: connection refused (prober "ROUND_TRIPPER_SNAPSHOT")
Apr 19 18:59:03 localhost etcd: health check for peer 9b449b0ff1d4c375 could not connect: dial tcp 192.168.10.12:2380: connect: connection refused (prober "ROUND_TRIPPER_SNAPSHOT")
Apr 19 18:59:03 localhost etcd: health check for peer d1fbb74bc6a61e5c could not connect: dial tcp 192.168.10.13:2380: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE")
 
 
 
 
将etcd拷贝其他节点
[root@mast-1 k8s]# scp -r /opt/etcd 192.168.10.12:/opt/
The authenticity of host '192.168.10.12 (192.168.10.12)' can't be established.
ECDSA key fingerprint is SHA256:K3leIm01q6LfP3t9FEMEtJq0njLut9ZzCGxcHUEWSDk.
ECDSA key fingerprint is MD5:db:5c:1e:fd:4c:e4:fc:b2:bd:63:b9:81:92:5e:fd:f5.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.10.12' (ECDSA) to the list of known hosts.
root@192.168.10.12's password: 
etcd                                                                                                                                                         100%  509    54.0KB/s   00:00    
etcd                                                                                                                                                         100%   18MB  19.5MB/s   00:00    
etcdctl                                                                                                                                                      100%   15MB   8.9MB/s   00:01    
ca.pem                                                                                                                                                       100% 1265    43.5KB/s   00:00    
server-key.pem                                                                                                                                               100% 1675   242.4KB/s   00:00    
server.pem                                                                                                                                                   100% 1338   304.7KB/s   00:00    
[root@mast-1 k8s]# scp -r /opt/etcd 192.168.10.13:/opt/
The authenticity of host '192.168.10.13 (192.168.10.13)' can't be established.
ECDSA key fingerprint is SHA256:UB3BsWnL8Kd/kNjYziPzdMZFo8ucoJn49Ar7TkFaOUI.
ECDSA key fingerprint is MD5:31:d0:3c:eb:60:bd:0a:55:df:72:da:d3:4d:67:16:e6.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.10.13' (ECDSA) to the list of known hosts.
root@192.168.10.13's password: 
etcd                                                                                                                                                         100%  509   197.1KB/s   00:00    
etcd                                                                                                                                                         100%   18MB   6.8MB/s   00:02    
etcdctl                                                                                                                                                      100%   15MB  15.1MB/s   00:01    
ca.pem                                                                                                                                                       100% 1265    12.2KB/s   00:00    
server-key.pem                                                                                                                                               100% 1675    27.0KB/s   00:00    
server.pem                    
[root@mast-1 k8s]# scp /usr/lib/systemd/system/etcd.service 192.168.10.12:/usr/lib/systemd/system
root@192.168.10.12's password: 
etcd.service                                                                                                                                                 100%  923    28.2KB/s   00:00    
[root@mast-1 k8s]# scp /usr/lib/systemd/system/etcd.service 192.168.10.13:/usr/lib/systemd/system
root@192.168.10.13's password: 
etcd.service       
修改etcd2 的配置文件节点名字与IP地址
 [root@mast-2 cfg]# vi etcd 
 
#[Member]
ETCD_NAME="etcd02"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.10.12:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.10.12:2379"
 
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.12:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.12:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.10.11:2380,etcd02=https://192.168.10.12:2380,etcd03=https://192.168.10.13:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
修改etcd3 的配置文件节点名字与IP地址     
[root@node-1 cfg]# vi etcd 
 
#[Member]
ETCD_NAME="etcd03"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.10.13:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.10.13:2379"
 
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.13:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.13:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.10.11:2380,etcd02=https://192.168.10.12:2380,etcd03=https://192.168.10.13:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
启动节点1
[root@mast-1 k8s]# systemctl start etcd
启动节点2
[root@mast-2 cfg]# systemctl daemon-reload 
[root@mast-2 cfg]# systemctl start etcd
[root@mast-2 cfg]# 
启动节点3
[root@node-1 cfg]# systemctl daemon-reload 
[root@node-1 cfg]# systemctl start etcd
[root@node-1 cfg]# vi etcd 
 
[root@mast-1 ~]# tail -10 /var/log/messages 
Apr 19 19:16:44 localhost etcd: server is likely overloaded
Apr 19 19:17:02 localhost etcd: failed to send out heartbeat on time (exceeded the 100ms timeout for 1.126288546s)
Apr 19 19:17:02 localhost etcd: server is likely overloaded
Apr 19 19:17:02 localhost etcd: failed to send out heartbeat on time (exceeded the 100ms timeout for 1.12637708s)
Apr 19 19:17:02 localhost etcd: server is likely overloaded
Apr 19 19:17:02 localhost etcd: 6571fb7574e87dba [term: 634] received a MsgAppResp message with higher term from d1fbb74bc6a61e5c [term: 635]
Apr 19 19:17:02 localhost etcd: 6571fb7574e87dba became follower at term 635
Apr 19 19:17:02 localhost etcd: raft.node: 6571fb7574e87dba lost leader 6571fb7574e87dba at term 635
Apr 19 19:17:02 localhost etcd: 6571fb7574e87dba [logterm: 634, index: 9, vote: 0] cast MsgVote for 9b449b0ff1d4c375 [logterm: 634, index: 9] at term 635
Apr 19 19:17:02 localhost etcd: raft.node: 6571fb7574e87dba elected leader 9b449b0ff1d4c375 at term 635
 
检查集群监控状态，数据端口
[root@mast-1 ~]# /opt/etcd/bin/etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.10.11:2379,http
s://192.168.10.12:2379,https://192.168.10.13:2379" cluster-healthmember 6571fb7574e87dba is healthy: got healthy result from https://192.168.10.11:2379
member 9b449b0ff1d4c375 is healthy: got healthy result from https://192.168.10.12:2379
member d1fbb74bc6a61e5c is healthy: got healthy result from https://192.168.10.13:2379
cluster is healthy