062、如何使用flannel host-gw backend（2019-04-02 周二）
flannel 支持多种backend，前面学习的是 vxlan backend ，host-gw 是 flannel的另一个backend。
与vxlan不同，host-gw 不会封装数据包，而是在主机的路由表中创建到其他主机的subnet 路由条目，从而实现容器跨主机通信。要使用host-gw 首先要修改flannel的配置 flannel-config.json ： 将type中的vxlan改为host-gw
# 1、配置etcd数据库，更改type
[root@docker-machine ~]# cat flannel-config.json
{
"Network": "10.2.0.0/16",
"SubnetLen": 24,
"Backend": {
"Type": "host-gw"
}
}
[root@docker-machine ~]# etcdctl --endpoints=10.12.31.213:2379 set /docker-test/network/config < flannel-config.json
{
"Network": "10.2.0.0/16",
"SubnetLen": 24,
"Backend": {
"Type": "host-gw"
}
}
# 2、host1上重启flannel，修改mtu，重启docker
root@host1:~# ps -ef | grep flannel
root 7315 7226 0 17:36 pts/0 00:00:00 /usr/local/bin/flanneld-amd64 -etcd-endpoints=http://10.12.31.213:2379 -iface=ens160 -etcd-prefix=/docker-test/network
root 7437 7226 0 17:38 pts/0 00:00:00 grep --color=auto flannel
root@host1:~# kill -9 7315
root@host1:~# /usr/local/bin/flanneld-amd64 -etcd-endpoints=http://10.12.31.213:2379 -iface=ens160 -etcd-prefix=/docker-test/network &
[1] 7440
root@host1:~# I0402 17:38:43.723057 7440 main.go:529] Using interface with name ens160 and address 10.12.31.211
I0402 17:38:43.723121 7440 main.go:546] Defaulting external address to interface address (10.12.31.211)
I0402 17:38:43.723289 7440 main.go:244] Created subnet manager: Etcd Local Manager with Previous Subnet: 10.2.46.0/24
I0402 17:38:43.723307 7440 main.go:247] Installing signal handlers
I0402 17:38:43.725268 7440 main.go:388] Found network config - Backend type: host-gw
I0402 17:38:43.739204 7440 local_manager.go:147] Found lease (10.2.46.0/24) for current IP (10.12.31.211), reusing
I0402 17:38:43.751344 7440 main.go:311] Changing default FORWARD chain policy to ACCEPT
I0402 17:38:43.751523 7440 main.go:319] Wrote subnet file to /run/flannel/subnet.env
I0402 17:38:43.751546 7440 main.go:323] Running backend.
I0402 17:38:43.751616 7440 route_network.go:53] Watching for new subnet leases
I0402 17:38:43.756410 7440 main.go:431] Waiting for 22h59m59.974453402s to renew lease
I0402 17:38:43.756617 7440 route_network.go:85] Subnet added: 10.2.44.0/24 via 10.12.31.212
W0402 17:38:43.756637 7440 route_network.go:88] Ignoring non-host-gw subnet: type=vxlan
root@host1:~# cat /run/flannel/subnet.env
FLANNEL_NETWORK=10.2.0.0/16
FLANNEL_SUBNET=10.2.46.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false
root@host1:~# ip r
default via 10.12.28.6 dev ens160 onlink
10.2.44.0/24 via 10.2.44.0 dev flannel.1 onlink
10.2.46.0/24 dev docker0 proto kernel scope link src 10.2.46.1
10.12.28.0/22 dev ens160 proto kernel scope link src 10.12.31.211
172.22.0.0/16 via 10.12.28.1 dev ens160
root@host1:~# cat /etc/systemd/system/docker.service.d/10-machine.conf
[Service]
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --storage-driver overlay2 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=generic --bip=10.2.46.1/24 --mtu=1500
#--cluster-store=consul://10.12.31.213:8500 --cluster-advertise=ens160:2376
Environment=
root@host1:~# systemctl daemon-reload
root@host1:~# systemctl restart docker.service
# 3、host1上重启flannel，修改mtu，重启docker
root@host2:~# ps -ef | grep flannel
root 1572 1 0 Apr01 ? 00:00:33 /usr/local/bin/flanneld-amd64 -etcd-endpoints=http://10.12.31.213:2379 -iface=ens160 -etcd-prefix=/docker-test/network
root 18111 17898 0 17:41 pts/0 00:00:00 grep --color=auto flannel
root@host2:~# kill -9 1572
root@host2:~# /usr/local/bin/flanneld-amd64 -etcd-endpoints=http://10.12.31.213:2379 -iface=ens160 -etcd-prefix=/docker-test/network &
[1] 18120
root@host2:~# I0402 17:41:52.208836 18120 main.go:529] Using interface with name ens160 and address 10.12.31.212
I0402 17:41:52.208929 18120 main.go:546] Defaulting external address to interface address (10.12.31.212)
I0402 17:41:52.209142 18120 main.go:244] Created subnet manager: Etcd Local Manager with Previous Subnet: 10.2.44.0/24
I0402 17:41:52.209168 18120 main.go:247] Installing signal handlers
I0402 17:41:52.211324 18120 main.go:388] Found network config - Backend type: host-gw
I0402 17:41:52.237102 18120 local_manager.go:147] Found lease (10.2.44.0/24) for current IP (10.12.31.212), reusing
I0402 17:41:52.253167 18120 main.go:311] Changing default FORWARD chain policy to ACCEPT
I0402 17:41:52.253345 18120 main.go:319] Wrote subnet file to /run/flannel/subnet.env
I0402 17:41:52.253369 18120 main.go:323] Running backend.
I0402 17:41:52.253604 18120 route_network.go:53] Watching for new subnet leases
I0402 17:41:52.269068 18120 route_network.go:85] Subnet added: 10.2.46.0/24 via 10.12.31.211
W0402 17:41:52.271450 18120 route_network.go:102] Replacing existing route to 10.2.46.0/24 via 10.2.46.0 dev index 6 with 10.2.46.0/24 via 10.12.31.211 dev index 2.
I0402 17:41:52.272686 18120 main.go:431] Waiting for 22h59m59.965316418s to renew lease
root@host2:~# cat /run/flannel/subnet.env
FLANNEL_NETWORK=10.2.0.0/16
FLANNEL_SUBNET=10.2.44.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false
root@host2:~# ip r
default via 10.12.28.6 dev ens160 onlink
10.2.44.0/24 dev docker0 proto kernel scope link src 10.2.44.1
10.2.46.0/24 via 10.12.31.211 dev ens160
10.12.28.0/22 dev ens160 proto kernel scope link src 10.12.31.212
172.22.0.0/16 via 10.12.28.1 dev ens160
root@host2:~# cat /etc/systemd/system/docker.service.d/10-machine.conf
[Service]
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --storage-driver overlay2 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=generic --bip 10.2.44.1/24 --mtu=1500
# --cluster-store=consul://10.12.31.213:8500 --cluster-advertise=ens160:2376
Environment=
root@host2:~# systemctl daemon-reload
root@host2:~# systemctl restart docker.service
重新进行连通性测试
root@host1:~# docker exec bbox1 ip r
default via 10.2.46.1 dev eth0
10.2.46.0/24 dev eth0 scope link src 10.2.46.2
root@host1:~# docker exec bbox1 ping -c 2 10.2.44.2
PING 10.2.44.2 (10.2.44.2): 56 data bytes
64 bytes from 10.2.44.2: seq=0 ttl=62 time=0.641 ms
64 bytes from 10.2.44.2: seq=1 ttl=62 time=0.462 ms
--- 10.2.44.2 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max = 0.462/0.551/0.641 ms
root@host2:~# docker exec bbox2 ip r
default via 10.2.44.1 dev eth0
10.2.44.0/24 dev eth0 scope link src 10.2.44.2
host-gw 的MTU 为1500，所以需要修改docker启动参数--mtu值
下面对比 host-gw 和 vxlan 两种backend：
1、host-gw 把每个主机都配置成网关，主机知道其他主机的subnet和转发地址。vxlan则在主机间建立隧道，不同主机的容器都在一个大的网段内
2、虽然vxlan与host-gw使用不同的机制建立主机之间连接，但对于容器则无需任何改变，bbox1仍然可以与bbox2通信
3、由于vxlan需要对数据进行额外打包和拆包，性能稍逊于host-gw
