Flannel UDP 模式
Flannel UDP 模式
Flannel UDP 模式
一、环境信息
| 主机 | IP |
|---|---|
| ubuntu | 172.16.94.141 |
| 软件 | 版本 |
|---|---|
| docker | 26.1.4 |
| helm | v3.15.0-rc.2 |
| kind | 0.18.0 |
| clab | 0.54.2 |
| kubernetes | 1.23.4 |
| ubuntu os | Ubuntu 20.04.6 LTS |
| kernel | 5.11.5 内核升级文档 |
二、安装服务
kind 配置文件信息
$ cat install.sh
#!/bin/bash
date
set -v
# 1.prep noCNI env
cat <<EOF | kind create cluster --name=flannel-udp --image=kindest/node:v1.23.4 --config=-
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
# kind 默认使用 rancher cni，cni 我们需要自己创建
disableDefaultCNI: true
# 定义节点使用的 pod 网段
podSubnet: "10.244.0.0/16"
nodes:
- role: control-plane
- role: worker
- role: worker
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.evescn.com"]
endpoint = ["https://harbor.evescn.com"]
EOF
# 2.install cni
kubectl apply -f ./flannel.yaml
# 3.install necessary tools
# cd /opt/
# curl -o calicoctl -O -L "https://gh.api.99988866.xyz/https://github.com/containernetworking/plugins/releases/download/v0.9.0/cni-plugins-linux-amd64-v0.9.0.tgz"
# tar -zxvf cni-plugins-linux-amd64-v0.9.0.tgz
for i in $(docker ps -a --format "table {{.Names}}" | grep flannel)
do
echo $i
docker cp /opt/bridge $i:/opt/cni/bin/
docker cp /usr/bin/ping $i:/usr/bin/ping
docker exec -it $i bash -c "sed -i -e 's/jp.archive.ubuntu.com\|archive.ubuntu.com\|security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list"
docker exec -it $i bash -c "apt-get -y update >/dev/null && apt-get -y install net-tools tcpdump lrzsz bridge-utils >/dev/null 2>&1"
done
flannel.yaml
配置文件
# flannel.yaml
---
kind: Namespace
apiVersion: v1
metadata:
name: kube-flannel
labels:
pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
rules:
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-flannel-cfg
namespace: kube-system
labels:
tier: node
app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
"Network": "10.244.0.0/16",
"Backend": {
"Type": "udp"
}
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-flannel-ds
namespace: kube-system
labels:
tier: node
app: flannel
spec:
selector:
matchLabels:
app: flannel
template:
metadata:
labels:
tier: node
app: flannel
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
hostNetwork: true
priorityClassName: system-node-critical
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: flannel
initContainers:
- name: install-cni-plugin
image: harbor.dayuan1997.com/devops/rancher/mirrored-flannelcni-flannel-cni-plugin:v1.1.0
command:
- cp
args:
- -f
- /flannel
- /opt/cni/bin/flannel
volumeMounts:
- name: cni-plugin
mountPath: /opt/cni/bin
- name: install-cni
image: harbor.dayuan1997.com/devops/rancher/mirrored-flannelcni-flannel:v0.19.2
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
containers:
- name: kube-flannel
image: harbor.dayuan1997.com/devops/rancher/mirrored-flannelcni-flannel:v0.19.2
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: EVENT_QUEUE_DEPTH
value: "5000"
volumeMounts:
- name: run
mountPath: /run/flannel
- name: flannel-cfg
mountPath: /etc/kube-flannel/
- name: xtables-lock
mountPath: /run/xtables.lock
- name: tun
mountPath: /dev/net/tun
volumes:
- name: tun
hostPath:
path: /dev/net/tun
- name: run
hostPath:
path: /run/flannel
- name: cni-plugin
hostPath:
path: /opt/cni/bin
- name: cni
hostPath:
path: /etc/cni/net.d
- name: flannel-cfg
configMap:
name: kube-flannel-cfg
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
flannel.yaml 参数解释
Backend.Type- 含义: 用于指定 flannel 工作模式。
udp: flannel 工作在 upd 模式。
- 安装
k8s集群和flannel服务
# ./install.sh
Creating cluster "flannel-udp" ...
✓ Ensuring node image (kindest/node:v1.23.4) 🖼
✓ Preparing nodes 📦 📦 📦
✓ Writing configuration 📜
✓ Starting control-plane 🕹️
✓ Installing StorageClass 💾
✓ Joining worker nodes 🚜
Set kubectl context to "kind-flannel-udp"
You can now use your cluster with:
kubectl cluster-info --context kind-flannel-udp
Have a nice day! 👋
- 查看安装的服务
root@kind:~# kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-64897985d-h7kmt 1/1 Running 0 15m
kube-system coredns-64897985d-xccr7 1/1 Running 0 15m
kube-system etcd-flannel-udp-control-plane 1/1 Running 0 15m
kube-system kube-apiserver-flannel-udp-control-plane 1/1 Running 0 16m
kube-system kube-controller-manager-flannel-udp-control-plane 1/1 Running 0 15m
kube-system kube-flannel-ds-55rqz 1/1 Running 0 15m
kube-system kube-flannel-ds-92x7v 1/1 Running 0 15m
kube-system kube-flannel-ds-gkczb 1/1 Running 0 15m
kube-system kube-proxy-52nfm 1/1 Running 0 15m
kube-system kube-proxy-pff8g 1/1 Running 0 15m
kube-system kube-proxy-s55jd 1/1 Running 0 15m
kube-system kube-scheduler-flannel-udp-control-plane 1/1 Running 0 16m
local-path-storage local-path-provisioner-5ddd94ff66-s4q8r 1/1 Running 0 15m
k8s 集群安装 Pod 测试网络
root@kind:~# cat cni.yaml
apiVersion: apps/v1
kind: DaemonSet
#kind: Deployment
metadata:
labels:
app: cni
name: cni
spec:
#replicas: 1
selector:
matchLabels:
app: cni
template:
metadata:
labels:
app: cni
spec:
containers:
- image: harbor.dayuan1997.com/devops/nettool:0.9
name: nettoolbox
securityContext:
privileged: true
---
apiVersion: v1
kind: Service
metadata:
name: serversvc
spec:
type: NodePort
selector:
app: cni
ports:
- name: cni
port: 80
targetPort: 80
nodePort: 32000
root@kind:~# kubectl apply -f cni.yaml
daemonset.apps/cni created
service/serversvc created
root@kind:~# kubectl run net --image=harbor.dayuan1997.com/devops/nettool:0.9
pod/net created
- 查看安装服务信息
root@kind:~# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
cni-hs7kn 1/1 Running 0 16s 10.244.2.2 flannel-udp-worker2 <none> <none>
cni-qqj89 1/1 Running 0 16s 10.244.1.2 flannel-udp-worker <none> <none>
net 1/1 Running 0 5s 10.244.2.3 flannel-udp-worker2 <none> <none>
root@kind:~# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 23m
serversvc NodePort 10.96.45.119 <none> 80:32000/TCP 81s
三、测试网络
同节点 Pod 网络通讯
Pod节点信息
## ip 信息
root@kind:~# kubectl exec -it net -- ip a l
3: eth0@if5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1472 qdisc noqueue state UP group default
link/ether 9a:ad:9f:04:e2:ca brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.244.2.3/24 brd 10.244.2.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::98ad:9fff:fe04:e2ca/64 scope link
valid_lft forever preferred_lft forever
## 路由信息
root@kind:~# kubectl exec -it net -- ip r s
default via 10.244.2.1 dev eth0
10.244.0.0/16 via 10.244.2.1 dev eth0
10.244.2.0/24 dev eth0 proto kernel scope link src 10.244.2.3
查看 Pod 信息发现在主机的 IP 地址为 24 位掩码，在 fannel 中同一 Node 节点 Pod 均在同一个 2 层网络中，不同 Node 节点 Pod 在不同的 2 层网络中
Pod节点所在Node节点信息
root@kind:~# docker exec -it flannel-udp-worker2 bash
## ip 信息
root@flannel-udp-worker2:/# ip a l
2: flannel0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1472 qdisc fq_codel state UNKNOWN group default qlen 500
link/none
inet 10.244.2.0/32 scope global flannel0
valid_lft forever preferred_lft forever
inet6 fe80::f507:941d:f3b3:9a9a/64 scope link stable-privacy
valid_lft forever preferred_lft forever
3: cni0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1472 qdisc noqueue state UP group default qlen 1000
link/ether 3e:da:8b:79:9e:87 brd ff:ff:ff:ff:ff:ff
inet 10.244.2.1/24 brd 10.244.2.255 scope global cni0
valid_lft forever preferred_lft forever
inet6 fe80::3cda:8bff:fe79:9e87/64 scope link
valid_lft forever preferred_lft forever
4: veth605ed192@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1472 qdisc noqueue master cni0 state UP group default
link/ether 7a:d7:2a:be:19:e7 brd ff:ff:ff:ff:ff:ff link-netns cni-aef3599a-c10a-dbd7-3d34-e5527762dd36
inet6 fe80::78d7:2aff:febe:19e7/64 scope link
valid_lft forever preferred_lft forever
5: veth22fb4d65@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1472 qdisc noqueue master cni0 state UP group default
link/ether be:65:c8:d4:8a:fa brd ff:ff:ff:ff:ff:ff link-netns cni-26c7a19b-7af5-fdfc-a5af-4457f208276f
inet6 fe80::bc65:c8ff:fed4:8afa/64 scope link
valid_lft forever preferred_lft forever
7: eth0@if8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:ac:12:00:03 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 172.18.0.3/16 brd 172.18.255.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fc00:f853:ccd:e793::3/64 scope global nodad
valid_lft forever preferred_lft forever
inet6 fe80::42:acff:fe12:3/64 scope link
valid_lft forever preferred_lft forever
## 路由信息
root@flannel-udp-worker2:/# ip r s
default via 172.18.0.1 dev eth0
10.244.0.0/16 dev flannel0
10.244.2.0/24 dev cni0 proto kernel scope link src 10.244.2.1
172.18.0.0/16 dev eth0 proto kernel scope link src 172.18.0.3
Pod节点进行ping包测试，访问cni-hs7knPod节点
root@kind:~# kubectl exec -it net -- ping 10.244.2.2 -c 1
PING 10.244.2.2 (10.244.2.2): 56 data bytes
64 bytes from 10.244.2.2: seq=0 ttl=64 time=0.814 ms
--- 10.244.2.2 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 0.814/0.814/0.814 ms
Pod节点eth0网卡抓包
net~$ tcpdump -pne -i eth0
09:17:34.426532 9a:ad:9f:04:e2:ca > a2:bc:67:6e:85:77, ethertype IPv4 (0x0800), length 98: 10.244.2.3 > 10.244.2.2: ICMP echo request, id 52, seq 0, length 64
09:17:34.426615 a2:bc:67:6e:85:77 > 9a:ad:9f:04:e2:ca, ethertype IPv4 (0x0800), length 98: 10.244.2.2 > 10.244.2.3: ICMP echo reply, id 52, seq 0, length 64
通过抓包发现，数据包源 mac 地址: 9a:ad:9f:04:e2:ca 为 eth0 网卡 mac 地址，而目的 mac 地址: a2:bc:67:6e:85:77 ，通过对比 flannel-udp-worker2 节点 ip 信息中，没有 a2:bc:67:6e:85:77 mac 地址，而这个 mac 地址是目标 Pod eth0 网卡 mac 地址
这是因为在 fannel 中同 Node 节点 Pod 均在同一个 2 层网络中，通过广播 Pod 节点能获取到 cni-hs7kn Pod 节点 eth0 mac 地址，查看 Pod 节点 arp 信息验证
net~$ arp -n
Address HWtype HWaddress Flags Mask Iface
10.244.2.2 ether a2:bc:67:6e:85:77 C eth0
通过 veth pair 可以确定 Pod 节点 eth0 网卡对应的 veth pair 为 5: veth22fb4d65@if3 网卡
flannel-udp-worker2节点veth22fb4d65网卡抓包
root@flannel-udp-worker2:/# tcpdump -pne -i veth22fb4d65
09:20:39.747858 9a:ad:9f:04:e2:ca > a2:bc:67:6e:85:77, ethertype IPv4 (0x0800), length 98: 10.244.2.3 > 10.244.2.2: ICMP echo request, id 66, seq 0, length 64
09:20:39.747938 a2:bc:67:6e:85:77 > 9a:ad:9f:04:e2:ca, ethertype IPv4 (0x0800), length 98: 10.244.2.2 > 10.244.2.3: ICMP echo reply, id 66, seq 0, length 64
- 查看
flannel-udp-worker2主机路由信息，发现并在数据包会在通过10.244.2.0/24 dev cni0 proto kernel scope link src 10.244.2.1路由信息转发，但是因为在同一个2层网络，所以不回经过cni0网卡，也无法抓包到icmp信息
root@flannel-udp-worker2:/# tcpdump -pne -i cni0
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on cni0, link-type EN10MB (Ethernet), snapshot length 262144 bytes
^C
0 packets captured
0 packets received by filter
0 packets dropped by kernel
- cni0 网卡类似一个 l2 层交换机提供服务，查看 cni0 网卡的 mac 地址信息
root@flannel-udp-worker2:/# brctl showmacs cni0
port no mac addr is local? ageing timer
1 7a:d7:2a:be:19:e7 yes 0.00
1 7a:d7:2a:be:19:e7 yes 0.00
2 9a:ad:9f:04:e2:ca no 50.98
1 a2:bc:67:6e:85:77 no 50.98
2 be:65:c8:d4:8a:fa yes 0.00
2 be:65:c8:d4:8a:fa yes 0.00
flannel-udp-worker2节点veth605ed192网卡抓包
root@flannel-udp-worker2:/# tcpdump -pne -i cali6c08e41b212
09:24:06.390987 9a:ad:9f:04:e2:ca > a2:bc:67:6e:85:77, ethertype IPv4 (0x0800), length 98: 10.244.2.3 > 10.244.2.2: ICMP echo request, id 81, seq 36, length 64
09:24:06.391001 a2:bc:67:6e:85:77 > 9a:ad:9f:04:e2:ca, ethertype IPv4 (0x0800), length 98: 10.244.2.2 > 10.244.2.3: ICMP echo reply, id 81, seq 36, length 64
10.244.2.2Pod节点eth0网卡抓包
cni-4hvth~$tcpdump -pne -i eth0
09:58:13.026980 9a:ad:9f:04:e2:ca > a2:bc:67:6e:85:77, ethertype IPv4 (0x0800), length 98: 10.244.2.3 > 10.244.2.2: ICMP echo request, id 88, seq 0, length 64
09:58:13.027012 a2:bc:67:6e:85:77 > 9a:ad:9f:04:e2:ca, ethertype IPv4 (0x0800), length 98: 10.244.2.2 > 10.244.2.3: ICMP echo reply, id 88, seq 0, length 64
总结：同节点
Pod通讯，数据包通过veth pair送往Node节点后，cni0网卡作为l2层交换机发现目的mac为l2层网络插口mac地址信息，然后转发数据包送往目的Pod节点的veth pair网卡，最后通过veth pair网卡送往目的主机，整体流程中，源mac目标mac源ip目标ip均为发送变化
不同节点 Pod 网络通讯
-
Pod节点信息，查看前面: 同节点Pod网络通讯 -
Pod节点所在Node节点信息，查看前面: 同节点Pod网络通讯 -
Pod节点进行ping包测试，访问cni-qqj89Pod节点
root@kind:~# kubectl exec -it net -- ping 10.244.1.2 -c 1
PING 10.244.1.2 (10.244.1.2): 56 data bytes
64 bytes from 10.244.1.2: seq=0 ttl=60 time=9.777 ms
--- 10.244.1.2 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 9.777/9.777/9.777 ms
Node节点flannel-udp-worker2veth22fb4d65网卡抓包
root@flannel-udp-worker2:/# tcpdump -pne -i veth22fb4d65
10:39:51.048048 9a:ad:9f:04:e2:ca > 3e:da:8b:79:9e:87, ethertype IPv4 (0x0800), length 98: 10.244.2.3 > 10.244.1.2: ICMP echo request, id 94, seq 0, length 64
10:39:51.052527 3e:da:8b:79:9e:87 > 9a:ad:9f:04:e2:ca, ethertype IPv4 (0x0800), length 98: 10.244.1.2 > 10.244.2.3: ICMP echo reply, id 94, seq 0, length 64
数据包源 mac 地址: 9a:ad:9f:04:e2:ca 为 eth0 网卡 mac 地址，而目的 mac 地址: 3e:da:8b:79:9e:87 ，通过对比 flannel-udp-worker2 节点 ip 信息中，发现是 cni0 网卡接口 mac，而 cni0 ip: 10.244.2.1 则是源 Pod 路由的下一跳 ip 10.244.0.0/16 via 10.244.2.1 dev eth0，flannel 为 2 层网络模式通过路由送往数据到网关地址
flannel-vxlan-worker节点cni0网关网卡抓包
root@flannel-udp-worker2:/# tcpdump -pne -i cni0
10:39:51.048048 9a:ad:9f:04:e2:ca > 3e:da:8b:79:9e:87, ethertype IPv4 (0x0800), length 98: 10.244.2.3 > 10.244.1.2: ICMP echo request, id 94, seq 0, length 64
10:39:51.052527 3e:da:8b:79:9e:87 > 9a:ad:9f:04:e2:ca, ethertype IPv4 (0x0800), length 98: 10.244.1.2 > 10.244.2.3: ICMP echo reply, id 94, seq 0, length 64
数据包源 mac 地址: 9a:ad:9f:04:e2:ca 为 net Pod 节点 eth0 网卡 mac 地址，而目的 mac 地址: 3e:da:8b:79:9e:87 为 cni0 网卡 mac 地址
查看
flannel-vxlan-worker主机路由信息，发现并在数据包会在通过10.244.0.0/16 dev flannel0路由信息转发
Node节点flannel-udp-worker2flannel0网卡抓包
root@flannel-udp-worker2:/# tcpdump -pne -i flannel0
10:45:07.049216 ip: 10.244.2.3 > 10.244.1.2: ICMP echo request, id 116, seq 0, length 64
10:45:07.049693 ip: 10.244.1.2 > 10.244.2.3: ICMP echo reply, id 116, seq 0, length 64
icmp 包中，没有 mac 信息，只有源 ip 目的 ip 信息，查看 flannel0 网卡详细信息
root@flannel-udp-worker2:/# ip -d link show
2: flannel0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1472 qdisc fq_codel state UNKNOWN mode DEFAULT group default qlen 500
link/none promiscuity 0 minmtu 68 maxmtu 65535
tun type tun pi off vnet_hdr off persist off addrgenmode random numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
flannel0 是一个 TUN 模式设备
Node节点flannel-udp-worker2eth0网卡抓包
数据包中过滤 udp 数据包，发现 2 条数据信息，但是该数据加密，使用 decode 解密后，得到下面的数据信息
root@flannel-udp-worker2:/# ip a l
7: eth0@if8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:ac:12:00:03 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 172.18.0.3/16 brd 172.18.255.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fc00:f853:ccd:e793::3/64 scope global nodad
valid_lft forever preferred_lft forever
inet6 fe80::42:acff:fe12:3/64 scope link
valid_lft forever preferred_lft forever
root@flannel-udp-worker2:/# ip n s
172.18.0.4 dev eth0 lladdr 02:42:ac:12:00:04 REACHABLE
172.18.0.1 dev eth0 lladdr 02:42:24:45:33:28 STALE
172.18.0.2 dev eth0 lladdr 02:42:ac:12:00:02 STALE
10.244.2.3 dev cni0 lladdr 9a:ad:9f:04:e2:ca STALE
fe80::1 dev eth0 lladdr 02:42:24:45:33:28 router STALE
fc00:f853:ccd:e793::4 dev eth0 lladdr 02:42:ac:12:00:04 REACHABLE
fc00:f853:ccd:e793::1 dev eth0 lladdr 02:42:24:45:33:28 router STALE
icmp 包使用 udp 封装，外层数据包源 mac 地址：02:42:ac:12:00:03 目标 mac 地址：02:42:ac:12:00:04 ，分别为 flannel-udp-worker2 主机的 eth0 网卡的 mac 地址，和 flannel-udp-worker 主机 172.18.0.4 ip 的 mac 地址。内层数据包信息只有 ip 层往上数据层信息，没有 mac 层信息，数据包信息同 flannel0 网卡抓包信息
- 总结：查看上面抓包图片，可以分析在
udp模式中，数据包的转发流程中：- 数据包在送到
flannel0接口之前，mac地址，每经过一个节点，源mac和 目标mac均会发生改变。送到flannel0接口之后，会去掉mac地址信息 - 数据包在送到
flannel0接口之前，源ip和 目标ip不会发生改变。送到flannel0接口之后，内层源ip和 目标ip不会发生改变，会在原数据包网络层上进行udp封包 udp数据包内部的数据包，没有mac地址，并且flannel0网卡本身也没有mac地址信息。
- 数据包在送到
数据包流向
- 数据从
pod服务发出，通过查看本机路由表，送往cni0网关。路由:default via 169.254.1.1 dev eth0 cni0网关在flannel-udp-worker2主机上，到达cni0网卡后，flannel-udp-worker2主机查看自身路由后，会送往flannel0接口，因为目的地址为10.244.1.2。路由:10.244.0.0/16 dev flannel0flannel0接口为TUN/TAP接口，使用udp重新封装数据包，查询etcd中各节点注册的路由信息，发现目的地址10.244.1.0/24网段所在Node节点信息- 获取到目的 Node 节点信息信息，封装
udp数据包，数据封装完成后，会送往eth0 接口，并送往对端flannel-udp-worker主机。 - 对端
flannel-udp-worker主机接受到数据包后，发现这个是一个udp封装数据包，进行数据解密并送往flannel0接口 - 解封装后发现内部的数据包，目的地址为
10.244.1.2，通过查看本机路由表，送往cni0网卡。路由:10.244.1.0/24 dev cni0 proto kernel scope link src 10.244.1.1 - 通过
cni0网卡brctl showmacs cni0mac信息 ，最终会把数据包送到cni-qqj89主机
Service 网络通讯
- 查看
Service信息
root@kind:~# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 18m
serversvc NodePort 10.96.211.66 <none> 80:32000/TCP 45s
net服务上请求Pod所在Node节点32000端口
root@kind:~# kubectl exec -it net -- curl 172.18.0.3:32000
PodName: cni-qqj89 | PodIP: eth0 10.244.1.2/24
并在 net 服务 eth0 网卡 抓包查看
net~$ tcpdump -pne -i eth0
06:44:53.550795 ce:a0:79:38:8c:0d > 26:75:ac:05:67:04, ethertype IPv4 (0x0800), length 74: 10.244.2.3.38346 > 172.18.0.3.32000: Flags [S], seq 3848762339, win 64440, options [mss 1432,sackOK,TS val 1563697881 ecr 0,nop,wscale 7], length 0
06:44:53.550841 da:9e:b4:68:93:28 > ce:a0:79:38:8c:0d, ethertype IPv4 (0x0800), length 74: 172.18.0.3.32000 > 10.244.2.3.38346: Flags [S.], seq 922360601, ack 3848762340, win 65320, options [mss 1432,sackOK,TS val 3410413732 ecr 1563697881,nop,wscale 7], length 0
06:44:53.550850 ce:a0:79:38:8c:0d > 26:75:ac:05:67:04, ethertype IPv4 (0x0800), length 66: 10.244.2.3.38346 > 172.18.0.3.32000: Flags [.], ack 1, win 504, options [nop,nop,TS val 1563697881 ecr 3410413732], length 0
06:44:53.550898 ce:a0:79:38:8c:0d > 26:75:ac:05:67:04, ethertype IPv4 (0x0800), length 146: 10.244.2.3.38346 > 172.18.0.3.32000: Flags [P.], seq 1:81, ack 1, win 504, options [nop,nop,TS val 1563697881 ecr 3410413732], length 80
06:44:53.550910 da:9e:b4:68:93:28 > ce:a0:79:38:8c:0d, ethertype IPv4 (0x0800), length 66: 172.18.0.3.32000 > 10.244.2.3.38346: Flags [.], ack 81, win 510, options [nop,nop,TS val 3410413732 ecr 1563697881], length 0
06:44:53.551127 da:9e:b4:68:93:28 > ce:a0:79:38:8c:0d, ethertype IPv4 (0x0800), length 302: 172.18.0.3.32000 > 10.244.2.3.38346: Flags [P.], seq 1:237, ack 81, win 510, options [nop,nop,TS val 3410413732 ecr 1563697881], length 236
06:44:53.551133 ce:a0:79:38:8c:0d > 26:75:ac:05:67:04, ethertype IPv4 (0x0800), length 66: 10.244.2.3.38346 > 172.18.0.3.32000: Flags [.], ack 237, win 503, options [nop,nop,TS val 1563697881 ecr 3410413732], length 0
06:44:53.551179 da:9e:b4:68:93:28 > ce:a0:79:38:8c:0d, ethertype IPv4 (0x0800), length 113: 172.18.0.3.32000 > 10.244.2.3.38346: Flags [P.], seq 237:284, ack 81, win 510, options [nop,nop,TS val 3410413733 ecr 1563697881], length 47
06:44:53.551182 ce:a0:79:38:8c:0d > 26:75:ac:05:67:04, ethertype IPv4 (0x0800), length 66: 10.244.2.3.38346 > 172.18.0.3.32000: Flags [.], ack 284, win 503, options [nop,nop,TS val 1563697882 ecr 3410413733], length 0
06:44:53.551668 ce:a0:79:38:8c:0d > 26:75:ac:05:67:04, ethertype IPv4 (0x0800), length 66: 10.244.2.3.38346 > 172.18.0.3.32000: Flags [F.], seq 81, ack 284, win 503, options [nop,nop,TS val 1563697882 ecr 3410413733], length 0
06:44:53.551742 da:9e:b4:68:93:28 > ce:a0:79:38:8c:0d, ethertype IPv4 (0x0800), length 66: 172.18.0.3.32000 > 10.244.2.3.38346: Flags [F.], seq 284, ack 82, win 510, options [nop,nop,TS val 3410413733 ecr 1563697882], length 0
06:44:53.551748 ce:a0:79:38:8c:0d > 26:75:ac:05:67:04, ethertype IPv4 (0x0800), length 66: 10.244.2.3.38346 > 172.18.0.3.32000: Flags [.], ack 285, win 503, options [nop,nop,TS val 1563697882 ecr 3410413733], length 0
抓包数据显示 net 服务使用 56972 端口和 172.18.0.3 32000 端口进行 tcp 通讯。
conntrack信息
root@flannel-udp-worker2:/# conntrack -L | grep 32000
conntrack v1.4.6 (conntrack-tools): 11 flow entries have been shown.
tcp 6 118 TIME_WAIT src=10.244.2.3 dst=172.18.0.3 sport=38346 dport=32000 src=10.244.1.2 dst=10.244.2.0 sport=80 dport=54991 [ASSURED] mark=0 use=1
iptables信息
root@flannel-udp-worker2:/# iptables-save | grep 32000
-A KUBE-NODEPORTS -p tcp -m comment --comment "default/serversvc:cni" -m tcp --dport 32000 -j KUBE-SVC-CU7F3MNN62CF4ANP
-A KUBE-SVC-CU7F3MNN62CF4ANP -p tcp -m comment --comment "default/serversvc:cni" -m tcp --dport 32000 -j KUBE-MARK-MASQ
