Docker swarm 容器流量追踪

南北向流量

 

 

node1 51.0.1.213   容器 busybox  10.0.5.174

node2 51.0.1.214   容器 busybox  10.0.5.173容器 busybox  10.0.5.174

 

 

 

busybox 测试容器
docker service create --replicas 3 --network es-network --publish 9830:90 --name busybox   51.0.1.213:5000/busybox   sleep 360000

[root@node1 ~]# docker exec -it 11b91701cff9 traceroute baidu.com
traceroute to baidu.com (220.181.38.148), 30 hops max, 46 byte packets
  1  bogon (172.18.0.1)  0.014 ms  0.011 ms  0.009 ms
  2  51.0.1.254 (51.0.1.254)  24.625 ms  2.312 ms  6.876 ms

流量经过172.18.0.1-------->51.0.1.254物理网关

默认路由直接扔给172.18.0.1
[root@node1 ~]# docker exec -it 11b91701cff9 ip route
default via 172.18.0.1 dev eth1
10.0.5.0/24 dev eth0 scope link  src 10.0.5.174
172.18.0.0/16 dev eth1 scope link  src 172.18.0.13

查看eth1@if397  接口
[root@node1 ~]# docexec -it 11b91701cff9 ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000
     link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
     inet 127.0.0.1/8 scope host lo
        valid_lft forever preferred_lft forever
394: eth0@if395: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1450 qdisc noqueue
     link/ether 02:42:0a:00:05:ae brd ff:ff:ff:ff:ff:ff
     inet 10.0.5.174/24 brd 10.0.5.255 scope global eth0
        valid_lft forever preferred_lft forever
396: eth1@if397: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
     link/ether 02:42:ac:12:00:0d brd ff:ff:ff:ff:ff:ff
     inet 172.18.0.13/16 brd 172.18.255.255 scope global eth1
        valid_lft forever preferred_lft forever

 
396: eth1@if397  这个397是在宿主机上面并没有单独在一个namespace里面
[root@node1 ~]# ip a|grep 397:
397: veth5a5ec90@if396: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master docker_gwbridge state UP group default

 
在查看下 bridge 397: veth5a5ec90@if396 接口插在  docker_gwbridge上面 ,docker_gwbridge接口
root@node1 ~]# brctl show  docker_gwbridge  veth51c05d7
bridge name     bridge id               STP enabled     interfaces
docker_gwbridge         8000.024265f8e0ff       no              veth51c05d7
                                                         veth5a5ec90
 
本机路由
[root@node1 ~]# ip route
default via 51.0.1.254 dev ens192 proto static metric 100
51.0.1.0/24 dev ens192 proto kernel scope link src 51.0.1.213 metric 100
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1
172.18.0.0/16 dev docker_gwbridge proto kernel scope link src 172.18.0.1
172.19.0.0/16 dev br-6d9ab3de6dee proto kernel scope link src 172.19.0.1

 

本机也有nat转换

 

[root@node1 ~]# iptables-save -t nat | grep -- '-A POSTROUTING'
-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE
-A POSTROUTING -o docker_gwbridge -m addrtype --src-type LOCAL -j MASQUERADE
-A POSTROUTING -s 172.19.0.0/16 ! -o br-6d9ab3de6dee -j MASQUERADE
-A POSTROUTING -s 172.18.0.0/16 ! -o docker_gwbridge -j MASQUERADE
-A POSTROUTING -s 172.17.0.2/32 -d 172.17.0.2/32 -p tcp -m tcp --dport 8081 -j MASQUERADE
-A POSTROUTING -s 172.17.0.2/32 -d 172.17.0.2/32 -p tcp -m tcp --dport 5001 -j MASQUERADE
-A POSTROUTING -s 172.17.0.2/32 -d 172.17.0.2/32 -p tcp -m tcp --dport 5000 -j MASQUERADE
-A POSTROUTING -s 172.19.0.2/32 -d 172.19.0.2/32 -p tcp -m tcp --dport 10514 -j MASQUERADE

 

 

 

东西向流量

[root@node1 ~]# docker exec -it 11b91701cff9 ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000
     link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
     inet 127.0.0.1/8 scope host lo
        valid_lft forever preferred_lft forever
394: eth0@if395: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1450 qdisc noqueue
     link/ether 02:42:0a:00:05:ae brd ff:ff:ff:ff:ff:ff
     inet 10.0.5.174/24 brd 10.0.5.255 scope global eth0
        valid_lft forever preferred_lft forever
396: eth1@if397: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
     link/ether 02:42:ac:12:00:0d brd ff:ff:ff:ff:ff:ff
     inet 172.18.0.13/16 brd 172.18.255.255 scope global eth1
        valid_lft forever preferred_lft forever
       
       
       
10.0.5.173 在节点node2上
[root@node1 ~]#  docker exec -it 11b91701cff9 ping 10.0.5.173
PING 10.0.5.173 (10.0.5.173): 56 data bytes
64 bytes from 10.0.5.173: seq=0 ttl=64 time=0.533 ms

394网卡对的另一侧是395
394: eth0@if395:

 

两个容器是通过vxlan通信
[root@node1 ~]# python nspy.py ip a re 395
1-arxqpf76ma
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
     link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
     inet 127.0.0.1/8 scope host lo
        valid_lft forever preferred_lft forever
2: br0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
     link/ether 46:58:9a:3c:78:61 brd ff:ff:ff:ff:ff:ff
     inet 10.0.5.1/24 brd 10.0.5.255 scope global br0
        valid_lft forever preferred_lft forever
...............................
395: veth31@if394: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master br0 state UP group default
     link/ether be:fd:96:c5:f5:55 brd ff:ff:ff:ff:ff:ff link-netnsid 6

查看arp表,由于刚才ping过10.0.5.174 这里会看到
[root@node1 ~]# ip netns exec 1-arxqpf76ma  ip neigh
10.0.5.149 dev vxlan0 lladdr 02:42:0a:00:05:95 PERMANENT
10.0.5.174 dev br0 lladdr 02:42:0a:00:05:ae STALE
10.0.5.153 dev vxlan0 lladdr 02:42:0a:00:05:99 PERMANENT
10.0.5.172 dev vxlan0 lladdr 02:42:0a:00:05:ac PERMANENT
10.0.5.168 dev vxlan0 lladdr 02:42:0a:00:05:a8 PERMANENT
10.0.5.170 dev vxlan0 lladdr 02:42:0a:00:05:aa PERMANENT
10.0.5.154 dev vxlan0 lladdr 02:42:0a:00:05:9a PERMANENT
10.0.5.163 dev vxlan0 lladdr 02:42:0a:00:05:a3 PERMANENT
10.0.5.173 dev vxlan0 lladdr 02:42:0a:00:05:ad PERMANENT

查看fdb表就可以看到51.0.1.214vxlan信息了
[root@node1 ~]# ip netns exec 1-arxqpf76ma bridge fdb
33:33:00:00:00:01 dev br0 self permanent
01:00:5e:00:00:01 dev br0 self permanent
46:58:9a:3c:78:61 dev veth10 master br0 permanent
33:33:00:00:00:01 dev veth10 self permanent
01:00:5e:00:00:01 dev veth10 self permanent
4e:7b:f0:26:29:be dev veth13 master br0 permanent
33:33:00:00:00:01 dev veth13 self permanent
01:00:5e:00:00:01 dev veth13 self permanent
d6:3d:92:20:1c:7f dev veth24 master br0 permanent
33:33:00:00:00:01 dev veth24 self permanent
01:00:5e:00:00:01 dev veth24 self permanent
7e:83:d7:13:d7:92 dev veth30 master br0 permanent
33:33:00:00:00:01 dev veth30 self permanent
01:00:5e:00:00:01 dev veth30 self permanent
be:fd:96:c5:f5:55 dev veth31 master br0 permanent
33:33:00:00:00:01 dev veth31 self permanent
01:00:5e:00:00:01 dev veth31 self permanent
be:3a:7e:3c:f3:e8 dev vxlan0 master br0 permanent
02:42:0a:00:05:95 dev vxlan0 dst 51.0.1.214 link-netnsid 0 self permanent
02:42:0a:00:05:99 dev vxlan0 dst 51.0.1.214 link-netnsid 0 self permanent
02:42:0a:00:05:9a dev vxlan0 dst 51.0.1.214 link-netnsid 0 self permanent
02:42:0a:00:05:a3 dev vxlan0 dst 51.0.1.215 link-netnsid 0 self permanent
02:42:0a:00:05:a8 dev vxlan0 dst 51.0.1.214 link-netnsid 0 self permanent
02:42:0a:00:05:aa dev vxlan0 dst 51.0.1.215 link-netnsid 0 self permanent
02:42:0a:00:05:ac dev vxlan0 dst 51.0.1.215 link-netnsid 0 self permanent
02:42:0a:00:05:ad dev vxlan0 dst 51.0.1.214 link-netnsid 0 self permanent
f6:81:d0:21:23:ab dev veth0 master br0 permanent
33:33:00:00:00:01 dev veth0 self permanent
01:00:5e:00:00:01 dev veth0 self permanent

 

from pyroute2 import netns, NSPopen
import subprocess
import sys
import re






end = sys.argv.index('re')
cmd = sys.argv[1:end]


pattern=sys.argv[end+1]

#print(cmd)
for ns in netns.listnetns():
     nsp = NSPopen(ns, cmd, stdout=subprocess.PIPE)
     result = nsp.communicate()[0]
     re_result = re.search(pattern, result)
     if re_result != None:
            print(ns)
            print(result)

nsp.wait()
nsp.release()