flannel 关闭SNAT
默认情况下,flannel 访问集群外网络是通过 SNAT 成宿主机 ip 方式,在一些金融客户环境中为了能实现防火墙规则,需要直接针对 POD ip 进行进行规则配置,所以需要关闭 SNAT
- 关闭flannel 配置文件关于ip-masq 配置,删除配置文件 -ip-masq 参数
# cat /etc/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service
[Service]
Type=notify
ExecStart=/opt/k8s/bin/flanneld \
-etcd-cafile=/data/work/ca.pem \
-etcd-certfile=/data/work/flanneld.pem \
-etcd-keyfile=/data/work/flanneld-key.pem \
-etcd-endpoints=https://10.65.91.161:2379,https://10.65.91.162:2379,https://10.65.91.163:2379 \
-etcd-prefix=/kubernetes/network \
-iface=ens192
ExecStartPost=/opt/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=always
RestartSec=5
StartLimitInterval=0
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
#重启 flanneld
systemctl daemon-reload
systemctl restart flanneld
- 修改flannel 配置文件中 mk-docker-opts.sh,将 ipmasq=true 修改为 ipmasq=ipmasq=false
#!/bin/sh
usage() {
echo "$0 [-f FLANNEL-ENV-FILE] [-d DOCKER-ENV-FILE] [-i] [-c] [-m] [-k COMBINED-KEY]
Generate Docker daemon options based on flannel env file
OPTIONS:
-f Path to flannel env file. Defaults to /run/flannel/subnet.env
-d Path to Docker env file to write to. Defaults to /run/docker_opts.env
-i Output each Docker option as individual var. e.g. DOCKER_OPT_MTU=1500
-c Output combined Docker options into DOCKER_OPTS var
-k Set the combined options key to this value (default DOCKER_OPTS=)
-m Do not output --ip-masq (useful for older Docker version)
" >&2
exit 1
}
flannel_env="/run/flannel/subnet.env"
docker_env="/run/docker_opts.env"
combined_opts_key="DOCKER_OPTS"
indiv_opts=false
combined_opts=false
ipmasq=false
while getopts "f:d:icmk:?h" opt; do
case $opt in
f)
flannel_env=$OPTARG
;;
d)
docker_env=$OPTARG
;;
i)
indiv_opts=true
;;
c)
combined_opts=true
;;
m)
ipmasq=false
;;
k)
combined_opts_key=$OPTARG
;;
[\?h])
usage
;;
esac
done
if [ $indiv_opts = false ] && [ $combined_opts = false ]; then
indiv_opts=true
combined_opts=true
fi
if [ -f "$flannel_env" ]; then
. $flannel_env
fi
if [ -n "$FLANNEL_SUBNET" ]; then
DOCKER_OPT_BIP="--bip=$FLANNEL_SUBNET"
fi
if [ -n "$FLANNEL_MTU" ]; then
DOCKER_OPT_MTU="--mtu=$FLANNEL_MTU"
fi
if [ -n "$FLANNEL_IPMASQ" ] && [ $ipmasq = false ] ; then
if [ "$FLANNEL_IPMASQ" = true ] ; then
DOCKER_OPT_IPMASQ="--ip-masq=false"
elif [ "$FLANNEL_IPMASQ" = false ] ; then
DOCKER_OPT_IPMASQ="--ip-masq=false"
else
echo "Invalid value of FLANNEL_IPMASQ: $FLANNEL_IPMASQ" >&2
exit 1
fi
fi
eval docker_opts="\$${combined_opts_key}"
if [ "$docker_opts" ]; then
docker_opts="$docker_opts ";
fi
echo -n "" >$docker_env
for opt in $(set | grep "DOCKER_OPT_"); do
OPT_NAME=$(echo $opt | awk -F "=" '{print $1;}');
OPT_VALUE=$(eval echo "\$$OPT_NAME");
if [ "$indiv_opts" = true ]; then
echo "$OPT_NAME=\"$OPT_VALUE\"" >>$docker_env;
fi
docker_opts="$docker_opts $OPT_VALUE";
done
if [ "$combined_opts" = true ]; then
echo "${combined_opts_key}=\"${docker_opts}\"" >>$docker_env
fi
#重启 flanneld
systemctl daemon-reload
systemctl restart flanneld
# cat /usr/lib/systemd/system/docker.service
#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
Environment="PATH=/opt/k8s/bin:/bin:/sbin:/usr/bin:/usr/sbin"
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
systemctl daemon-reload
systemctl restart docker
# cat /run/flannel/docker
DOCKER_OPT_BIP="--bip=10.0.79.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=false"
DOCKER_OPT_MTU="--mtu=1500"
DOCKER_NETWORK_OPTIONS=" --bip=10.0.79.1/24 --ip-masq=false --mtu=1500"
[root@lgy-test-node01 10.65.91.164 ~ ]
# cat /run/flannel/subnet.env
FLANNEL_NETWORK=10.0.0.0/16
FLANNEL_SUBNET=10.0.79.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false
- 删除 node节点 POSTROUTING 规则,只剩下默认一条规则
iptables -t nat --line-numbers -vnL POSTROUTING
1 87 5856 KUBE-POSTROUTING all -- * * 0.0.0.0/0 0.0.0.0/0 /* kubernetes postrouting rules */
#删除方法
iptables -t nat -D POSTROUTING 3
iptables -t nat -D POSTROUTING 2
iptables -t nat -D POSTROUTING 1
tcpdump -i ens192 port 80 -vnn
10.0.79.2.59010 > 10.65.91.51.80
route add -host 10.0.79.2 gw 10.65.91.164