一、Flannel网络部署
为Flannel生成证书
[root@linux-node1 ssl]# vim flanneld-csr.json
{
"CN": "flanneld",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
生成证书
[root@linux-node1 ssl]# cfssl gencert -ca=/opt/kubernetes/ssl/ca.pem \
-ca-key=/opt/kubernetes/ssl/ca-key.pem \
-config=/opt/kubernetes/ssl/ca-config.json \
-profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
2018/05/31 16:19:46 [INFO] generate received request
2018/05/31 16:19:46 [INFO] received CSR
2018/05/31 16:19:46 [INFO] generating key: rsa-2048
2018/05/31 16:19:46 [INFO] encoded CSR
2018/05/31 16:19:46 [INFO] signed certificate with serial number 655739903410343916627289078414999108781865584022
2018/05/31 16:19:46 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@linux-node1 ssl]# ll flanneld*
-rw-r--r-- 1 root root 997 May 31 16:19 flanneld.csr
-rw-r--r-- 1 root root 221 May 31 16:18 flanneld-csr.json
-rw------- 1 root root 1679 May 31 16:19 flanneld-key.pem
-rw-r--r-- 1 root root 1391 May 31 16:19 flanneld.pem
分发证书
[root@linux-node1 ssl]# scp flanneld*.pem 192.168.56.12:/opt/kubernetes/ssl/
[root@linux-node1 ssl]# scp flanneld*.pem 192.168.56.13:/opt/kubernetes/ssl/
下载Flannel软件包
# wget https://github.com/coreos/flannel/releases/download/v0.10.0/flannel-v0.10.0-linux-amd64.tar.gz
[root@linux-node1 ssl]# cd /usr/local/src/
[root@linux-node1 src]# pwd
/usr/local/src
[root@linux-node1 src]# tar zxf flannel-v0.10.0-linux-amd64.tar.gz
[root@linux-node1 src]# cp flanneld mk-docker-opts.sh /opt/kubernetes/bin/
#复制到linux-node2、linux-node3节点上
[root@linux-node1 src]# scp flanneld mk-docker-opts.sh 192.168.56.12:/opt/kubernetes/bin/
[root@linux-node1 src]# scp flanneld mk-docker-opts.sh 192.168.56.13:/opt/kubernetes/bin/
复制对应脚本到/opt/kubernetes/bin目录下。
[root@linux-node1 src]# cd /usr/local/src/kubernetes/cluster/centos/node/bin/
[root@linux-node1 bin]# ll
total 8
-rwxr-xr-x 1 root root 2590 Apr 12 23:24 mk-docker-opts.sh
-rwxr-xr-x 1 root root 850 Apr 12 23:24 remove-docker0.sh
[root@linux-node1 bin]# cp remove-docker0.sh /opt/kubernetes/bin/
[root@linux-node1 bin]# scp remove-docker0.sh 192.168.56.12:/opt/kubernetes/bin/
[root@linux-node1 bin]# scp remove-docker0.sh 192.168.56.13:/opt/kubernetes/bin/
配置Flannel
[root@linux-node1 bin]# vim /opt/kubernetes/cfg/flannel
FLANNEL_ETCD="-etcd-endpoints=https://192.168.56.11:2379,https://192.168.56.12:2379,https://192.168.56.13:2379"
FLANNEL_ETCD_KEY="-etcd-prefix=/kubernetes/network"
FLANNEL_ETCD_CAFILE="--etcd-cafile=/opt/kubernetes/ssl/ca.pem"
FLANNEL_ETCD_CERTFILE="--etcd-certfile=/opt/kubernetes/ssl/flanneld.pem"
FLANNEL_ETCD_KEYFILE="--etcd-keyfile=/opt/kubernetes/ssl/flanneld-key.pem"
复制配置到其它节点上
[root@linux-node1 bin]# scp /opt/kubernetes/cfg/flannel 192.168.56.12:/opt/kubernetes/cfg/
[root@linux-node1 bin]# scp /opt/kubernetes/cfg/flannel 192.168.56.13:/opt/kubernetes/cfg/
设置Flannel系统服务
[root@linux-node1 bin]# vim /usr/lib/systemd/system/flannel.service
[root@linux-node1 bin]# cat /usr/lib/systemd/system/flannel.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
Before=docker.service
[Service]
EnvironmentFile=-/opt/kubernetes/cfg/flannel
ExecStartPre=/opt/kubernetes/bin/remove-docker0.sh
ExecStart=/opt/kubernetes/bin/flanneld ${FLANNEL_ETCD} ${FLANNEL_ETCD_KEY} ${FLANNEL_ETCD_CAFILE} ${FLANNEL_ETCD_CERTFILE} ${FLANNEL_ETCD_KEYFILE}
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -d /run/flannel/docker
Type=notify
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
复制系统服务脚本到其它节点上
[root@linux-node1 bin]# scp /usr/lib/systemd/system/flannel.service 192.168.56.12:/usr/lib/systemd/system/
[root@linux-node1 bin]# scp /usr/lib/systemd/system/flannel.service 192.168.56.13:/usr/lib/systemd/system/
二、Flannel CNI集成
下载CNI插件
#下载CNI插件
https://github.com/containernetworking/plugins/releases
wget https://github.com/containernetworking/plugins/releases/download/v0.7.1/cni-plugins-amd64-v0.7.1.tgz
[root@linux-node1 bin]# mkdir /opt/kubernetes/bin/cni
[root@linux-node2 ssl]# mkdir /opt/kubernetes/bin/cni
[root@linux-node3 ssl]# mkdir /opt/kubernetes/bin/cni
[root@linux-node1 bin]# cd /usr/local/src/
[root@linux-node1 src]# tar zxf cni-plugins-amd64-v0.7.1.tgz -C /opt/kubernetes/bin/cni
[root@linux-node1 src]# scp -r /opt/kubernetes/bin/cni/* 192.168.56.12:/opt/kubernetes/bin/cni/
root@linux-node1 src]# scp -r /opt/kubernetes/bin/cni/* 192.168.56.13:/opt/kubernetes/bin/cni/
创建Etcd的key
root@linux-node1 src]# /opt/kubernetes/bin/etcdctl --ca-file /opt/kubernetes/ssl/ca.pem --cert-file /opt/kubernetes/ssl/flanneld.pem --key-file /opt/kubernetes/ssl/flanneld-key.pem \
--no-sync -C https://192.168.56.11:2379,https://192.168.56.12:2379,https://192.168.56.13:2379 \
mk /kubernetes/network/config '{ "Network": "10.2.0.0/16", "Backend": { "Type": "vxlan", "VNI": 1 }}' >/dev/null 2>&1
启动flannel
######################linux-node1启动#############################
[root@linux-node1 src]# systemctl daemon-reload
[root@linux-node1 src]# systemctl enable flannel
Created symlink from /etc/systemd/system/multi-user.target.wants/flannel.service to /usr/lib/systemd/system/flannel.service.
Created symlink from /etc/systemd/system/docker.service.requires/flannel.service to /usr/lib/systemd/system/flannel.service.
[root@linux-node1 src]# chmod +x /opt/kubernetes/bin/*
[root@linux-node1 src]# systemctl start flannel
######################linux-node2启动#############################
[root@linux-node2 src]# systemctl daemon-reload
[root@linux-node2 src]# systemctl enable flannel
Created symlink from /etc/systemd/system/multi-user.target.wants/flannel.service to /usr/lib/systemd/system/flannel.service.
Created symlink from /etc/systemd/system/docker.service.requires/flannel.service to /usr/lib/systemd/system/flannel.service.
[root@linux-node2 src]# chmod +x /opt/kubernetes/bin/*
[root@linux-node2 src]# systemctl start flannel
######################linux-node3启动#############################
[root@linux-node3 ssl]# systemctl daemon-reload
[root@linux-node3 ssl]# systemctl enable flannel
Created symlink from /etc/systemd/system/multi-user.target.wants/flannel.service to /usr/lib/systemd/system/flannel.service.
Created symlink from /etc/systemd/system/docker.service.requires/flannel.service to /usr/lib/systemd/system/flannel.service.
[root@linux-node3 ssl]# chmod +x /opt/kubernetes/bin/*
[root@linux-node3 ssl]# systemctl start flannel
查看服务状态
[root@linux-node1 src]# systemctl status flannel
[root@linux-node2 src]# systemctl status flannel
[root@linux-node3 ssl]# systemctl status flannel
[root@linux-node1 src]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.56.11 netmask 255.255.255.0 broadcast 192.168.56.255
inet6 fe80::20c:29ff:fea0:10ca prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:a0:10:ca txqueuelen 1000 (Ethernet)
RX packets 386026 bytes 62831393 (59.9 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 768980 bytes 1204891659 (1.1 GiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 10.2.9.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::2404:bbff:fee0:41a6 prefixlen 64 scopeid 0x20<link>
ether 26:04:bb:e0:41:a6 txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 8 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 232562 bytes 59255512 (56.5 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 232562 bytes 59255512 (56.5 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
linux-node1上查看网卡ifconfig
[root@linux-node2 src]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.56.12 netmask 255.255.255.0 broadcast 192.168.56.255
inet6 fe80::20c:29ff:fef9:ca25 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:f9:ca:25 txqueuelen 1000 (Ethernet)
RX packets 669708 bytes 372005370 (354.7 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 470811 bytes 63787801 (60.8 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 10.2.13.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::4031:7ff:fe2d:9c2f prefixlen 64 scopeid 0x20<link>
ether 42:31:07:2d:9c:2f txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 8 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 960 bytes 72576 (70.8 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 960 bytes 72576 (70.8 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
linux-node2上查看网卡ifconfig
[root@linux-node3 ssl]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.56.13 netmask 255.255.255.0 broadcast 192.168.56.255
inet6 fe80::20c:29ff:feca:800d prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:ca:80:0d txqueuelen 1000 (Ethernet)
RX packets 548452 bytes 359854149 (343.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 353542 bytes 51778767 (49.3 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 10.2.17.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::c86c:3ff:fe2e:d4ca prefixlen 64 scopeid 0x20<link>
ether ca:6c:03:2e:d4:ca txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 8 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 804 bytes 48598 (47.4 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 804 bytes 48598 (47.4 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
linux-node3上查看网卡ifconfig
[root@linux-node3 ssl]# cat /run/flannel/docker
DOCKER_OPT_BIP="--bip=10.2.17.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=true"
DOCKER_OPT_MTU="--mtu=1450"
DOCKER_OPTS=" --bip=10.2.17.1/24 --ip-masq=true --mtu=1450"
三、配置Docker使用Flannel
root@linux-node1 ~]# vim /usr/lib/systemd/system/docker.service
[Unit] #在Unit下面修改After和增加Requires
After=network-online.target firewalld.service flannel.service
Wants=network-online.target
Requires=flannel.service
[Service] #增加EnvironmentFile=-/run/flannel/docker
Type=notify
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/bin/dockerd $DOCKER_OPTS
将配置复制到另外两个节点
[root@linux-node1 src]# scp /usr/lib/systemd/system/docker.service 192.168.56.12:/usr/lib/systemd/system/
[root@linux-node1 src]# scp /usr/lib/systemd/system/docker.service 192.168.56.13:/usr/lib/systemd/system/
重启Docker
#####################linux-node1重启Docker服务########################
[root@linux-node1 src]# systemctl daemon-reload
[root@linux-node1 src]# systemctl restart docker
#####################linux-node3重启Docker服务########################
[root@linux-node2 src]# systemctl daemon-reload
[root@linux-node2 src]# systemctl restart docker
#####################linux-node3重启Docker服务########################
[root@linux-node3 ssl]# systemctl daemon-reload
[root@linux-node3 ssl]# systemctl restart docker
[root@linux-node1 src]# ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 10.2.9.1 netmask 255.255.255.0 broadcast 10.2.9.255
ether 02:42:18:d5:e0:d1 txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.56.11 netmask 255.255.255.0 broadcast 192.168.56.255
inet6 fe80::20c:29ff:fea0:10ca prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:a0:10:ca txqueuelen 1000 (Ethernet)
RX packets 426666 bytes 68740555 (65.5 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 809174 bytes 1211522272 (1.1 GiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 10.2.9.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::2404:bbff:fee0:41a6 prefixlen 64 scopeid 0x20<link>
ether 26:04:bb:e0:41:a6 txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 8 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 254544 bytes 65389446 (62.3 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 254544 bytes 65389446 (62.3 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
linux-node1上查看网卡ifconfig
[root@linux-node2 src]# ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 10.2.13.1 netmask 255.255.255.0 broadcast 10.2.13.255
ether 02:42:6a:cf:66:9f txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.56.12 netmask 255.255.255.0 broadcast 192.168.56.255
inet6 fe80::20c:29ff:fef9:ca25 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:f9:ca:25 txqueuelen 1000 (Ethernet)
RX packets 724138 bytes 379110250 (361.5 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 524129 bytes 71071850 (67.7 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 10.2.13.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::4031:7ff:fe2d:9c2f prefixlen 64 scopeid 0x20<link>
ether 42:31:07:2d:9c:2f txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 8 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 1264 bytes 88416 (86.3 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1264 bytes 88416 (86.3 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
linux-node2上查看网卡ifconfig
[root@linux-node3 ssl]# ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 10.2.17.1 netmask 255.255.255.0 broadcast 10.2.17.255
ether 02:42:e3:78:53:2d txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.56.13 netmask 255.255.255.0 broadcast 192.168.56.255
inet6 fe80::20c:29ff:feca:800d prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:ca:80:0d txqueuelen 1000 (Ethernet)
RX packets 584443 bytes 365047002 (348.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 390601 bytes 57109546 (54.4 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 10.2.17.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::c86c:3ff:fe2e:d4ca prefixlen 64 scopeid 0x20<link>
ether ca:6c:03:2e:d4:ca txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 8 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 880 bytes 52582 (51.3 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 880 bytes 52582 (51.3 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
linux-node3上查看网卡ifconfig
