ovs nat + arp proxy
ovs说明的例子 The following flows provide an example of how toimplement a simple firewall that allows new connections from port 1 to port 2, and onlyallows established connections to send traffic from port 2 to port 1: table=0,priority=1,action=drop 低优先级丢弃 table=0,priority=10,arp,action=normal arp放行 table=0,priority=100,ip,ct_state=-trk,action=ct(table=1) 没有被追踪过的报文，进入connection tracker然后进入table 1 table=1,in_port=1,ip,ct_state=+trk+new,action=ct(commit),2 被追踪过且新的，1口的报文，进行commit操作，同时从2发出 table=1,in_port=1,ip,ct_state=+trk+est,action=2 被追踪过，已经建立的connection，直接从2发出 table=1,in_port=2,ip,ct_state=+trk+new,action=drop 被追踪过，2口进的，是新connection，丢弃 table=1,in_port=2,ip,ct_state=+trk+est,action=1 被追踪过，2口进，已经建立的connection，从1出
ovs-vsctl add-br arp_sw ip netns add ns2 ip link add tap0 type veth peer name tap0_br ip link set tap0 netns ns2 ip netns exec ns2 ip link set tap0 up ip netns exec ns2 ip link set lo up ip link set tap0_br up ip netns exec ns2 ip addr add 192.168.2.210/24 dev tap0 ip netns exec ns2 route add -net 192.168.2.0 netmask 255.255.255.0 dev tap0 ovs-vsctl add-port arp_sw tap0_br ovs-ofctl add-flow arp_sw table=0,in_port=1,arp,arp_tpa=192.168.1.81,arp_op=1,actions=move:"NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[]",mod_dl_src:"48:57:02:64:ea:1e",load:"0x02->NXM_OF_ARP_OP[]",move:"NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[]",load:"0x48570264ea1e->NXM_NX_ARP_SHA[]",move:"NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[]",load:"0xc0a80151->NXM_OF_ARP_SPA[]",in_port #ovs-ofctl add-flow arp_sw table=0,in_port=1,ip,nw_dst=192.168.1.81/24,action=ct\(commit,table=1,nat\(src=192.168.1.82\)\) #ovs-ofctl add-flow arp_sw table=1,ip,nw_src=192.168.1.82,nw_dst=192.168.1.81,actions=mod_dl_src:"48:57:02:64:e7:ae",host82 #ovs-ofctl add-flow arp_sw table=1,ip,nw_dst=192.168.2.210,actions=mod_dl_dst:"2e:a9:be:9e:4d:07"
[root@kunpeng82 devuser]# ovs-ofctl add-flow arp_sw table=0,in_port=1,arp,arp_tpa=192.168.1.81,arp_op=1,actions=move:"NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[]",mod_dl_src:"48:57:02:64:ea:1e",load:"0x02->NXM_OF_ARP_OP[]",move:"NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[]",load:"0x48570264ea1e->NXM_NX_ARP_SHA[]",move:"NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[]",load:"0xc0a80151->NXM_OF_ARP_SPA[]",in_port
[root@kunpeng82 devuser]# ovs-appctl ofproto/trace arp_sw in_port=1,arp,arp_spa=192.168.2.210,arp_sha=2e:a9:be:9e:4d:07,arp_tpa=192.168.1.81,arp_tha=ff:ff:ff:ff:ff:ff,arp_op=1 -generate Flow: arp,in_port=1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=00:00:00:00:00:00,arp_spa=192.168.2.210,arp_tpa=192.168.1.81,arp_op=1,arp_sha=2e:a9:be:9e:4d:07,arp_tha=ff:ff:ff:ff:ff:ff bridge("arp_sw") ---------------- 0. arp,in_port=1,arp_tpa=192.168.1.81,arp_op=1, priority 32768 move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[] -> NXM_OF_ETH_DST[] is now 00:00:00:00:00:00 mod_dl_src:48:57:02:64:ea:1e load:0x2->NXM_OF_ARP_OP[] move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[] -> NXM_NX_ARP_THA[] is now 2e:a9:be:9e:4d:07 load:0x48570264ea1e->NXM_NX_ARP_SHA[] move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[] -> NXM_OF_ARP_TPA[] is now 192.168.2.210 load:0xc0a80151->NXM_OF_ARP_SPA[] IN_PORT Final flow: arp,in_port=1,vlan_tci=0x0000,dl_src=48:57:02:64:ea:1e,dl_dst=00:00:00:00:00:00,arp_spa=192.168.1.81,arp_tpa=192.168.2.210,arp_op=2,arp_sha=48:57:02:64:ea:1e,arp_tha=2e:a9:be:9e:4d:07 Megaflow: recirc_id=0,eth,arp,in_port=1,dl_src=00:00:00:00:00:00,arp_spa=192.168.2.210,arp_tpa=192.168.1.81,arp_op=1,arp_sha=2e:a9:be:9e:4d:07,arp_tha=ff:ff:ff:ff:ff:ff Datapath actions: set(eth(src=48:57:02:64:ea:1e)),set(arp(sip=192.168.1.81,tip=192.168.2.210,op=2/0xff,sha=48:57:02:64:ea:1e,tha=2e:a9:be:9e:4d:07)),8 This flow is handled by the userspace slow path because it: - Uses action(s) not supported by datapath. [root@kunpeng82 devuser]#
[root@kunpeng82 devuser]# ovs-appctl ofproto/trace arp_sw in_port=1,ip,dl_src=02:ac:10:ff:01:01,dl_dst=48:57:02:64:ea:1e,nw_src=192.168.2.210,nw_dst=192.168.1.81,nw_proto=1 -generate Flow: icmp,in_port=1,vlan_tci=0x0000,dl_src=02:ac:10:ff:01:01,dl_dst=48:57:02:64:ea:1e,nw_src=192.168.2.210,nw_dst=192.168.1.81,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0 bridge("arp_sw") ---------------- 0. ip,in_port=1,nw_dst=192.168.1.0/24, priority 32768 ct(commit,table=1,nat(src=192.168.1.82)) nat(src=192.168.1.82) -> A clone of the packet is forked to recirculate. The forked pipeline will be resumed at table 1. -> Sets the packet to an untracked state, and clears all the conntrack fields. Final flow: unchanged Megaflow: recirc_id=0,eth,ip,in_port=1,nw_dst=192.168.1.0/24,nw_frag=no Datapath actions: ct(commit,nat(src=192.168.1.82)),recirc(0x2) =============================================================================== recirc(0x2) - resume conntrack with default ct_state=trk|new (use --ct-next to customize) =============================================================================== Flow: recirc_id=0x2,ct_state=new|trk,eth,icmp,in_port=1,vlan_tci=0x0000,dl_src=02:ac:10:ff:01:01,dl_dst=48:57:02:64:ea:1e,nw_src=192.168.2.210,nw_dst=192.168.1.81,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0 bridge("arp_sw") ---------------- thaw Resuming from table 1 1. No match. drop Final flow: unchanged Megaflow: recirc_id=0x2,eth,ip,in_port=1,nw_src=192.168.2.0/23,nw_dst=192.168.1.81,nw_frag=no Datapath actions: drop
[root@kunpeng82 devuser]# ovs-ofctl show arp_sw OFPT_FEATURES_REPLY (xid=0x2): dpid:000048570264e7ae n_tables:254, n_buffers:0 capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP actions: output enqueue set_vlan_vid set_vlan_pcp strip_vlan mod_dl_src mod_dl_dst mod_nw_src mod_nw_dst mod_nw_tos mod_tp_src mod_tp_dst 1(tap0_br): addr:f6:06:f4:df:35:df config: 0 state: 0 current: 10GB-FD COPPER speed: 10000 Mbps now, 0 Mbps max 2(enahisic2i3): addr:48:57:02:64:e7:ae config: 0 state: 0 current: 10GB-FD FIBER advertised: 10GB-FD supported: 10GB-FD FIBER AUTO_PAUSE speed: 10000 Mbps now, 10000 Mbps max LOCAL(arp_sw): addr:48:57:02:64:e7:ae config: 0 state: 0 speed: 0 Mbps now, 0 Mbps max OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0 [root@kunpeng82 devuser]#
[root@kunpeng82 devuser]# ovs-ofctl dump-flows arp_sw cookie=0x0, duration=4201.428s, table=0, n_packets=16, n_bytes=672, arp,in_port="tap0_br",arp_tpa=192.168.1.81,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:48:57:02:64:ea:1e,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x48570264ea1e->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xc0a80151->NXM_OF_ARP_SPA[],IN_PORT cookie=0x0, duration=769.446s, table=0, n_packets=106, n_bytes=10388, ip,in_port="tap0_br",nw_dst=192.168.1.0/24 actions=ct(commit,table=1,nat(src=192.168.1.82)) cookie=0x0, duration=16419.321s, table=0, n_packets=625511, n_bytes=75996564, priority=0 actions=NORMAL cookie=0x0, duration=761.900s, table=1, n_packets=106, n_bytes=10388, ip,nw_src=192.168.1.82,nw_dst=192.168.1.81 actions=mod_dl_src:48:57:02:64:e7:ae ---规则没有out cookie=0x0, duration=754.399s, table=1, n_packets=0, n_bytes=0, ip,nw_dst=192.168.2.210 actions=mod_dl_dst:2e:a9:be:9e:4d:07 ----规则没有out
[root@kunpeng82 devuser]# ovs-ofctl dump-flows arp_sw cookie=0x0, duration=4367.317s, table=0, n_packets=16, n_bytes=672, arp,in_port="tap0_br",arp_tpa=192.168.1.81,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:48:57:02:64:ea:1e,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x48570264ea1e->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xc0a80151->NXM_OF_ARP_SPA[],IN_PORT cookie=0x0, duration=935.335s, table=0, n_packets=106, n_bytes=10388, ip,in_port="tap0_br",nw_dst=192.168.1.0/24 actions=ct(commit,table=1,nat(src=192.168.1.82)) cookie=0x0, duration=16585.210s, table=0, n_packets=633130, n_bytes=76933068, priority=0 actions=NORMAL cookie=0x0, duration=9.796s, table=1, n_packets=0, n_bytes=0, ip,nw_src=192.168.1.82,nw_dst=192.168.1.81 actions=mod_dl_src:48:57:02:64:e7:ae,output:enahisic2i3 cookie=0x0, duration=5.264s, table=1, n_packets=0, n_bytes=0, ip,nw_dst=192.168.2.210 actions=mod_dl_dst:2e:a9:be:9e:4d:07,output:"tap0_br"
[root@kunpeng82 devuser]# ip netns exec ns2 ping 192.168.1.81 PING 192.168.1.81 (192.168.1.81) 56(84) bytes of data. 还是不通
[root@kunpeng82 devuser]# tcpdump -i arp_sw icmp -nnvv tcpdump: listening on arp_sw, link-type EN10MB (Ethernet), capture size 262144 bytes 07:38:57.347244 IP (tos 0x0, ttl 64, id 31892, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 4, length 64 07:38:58.387218 IP (tos 0x0, ttl 64, id 31984, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 5, length 64 07:38:59.427215 IP (tos 0x0, ttl 64, id 32049, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 6, length 64 07:39:00.467210 IP (tos 0x0, ttl 64, id 32110, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 7, length 64 07:39:01.507549 IP (tos 0x0, ttl 64, id 32206, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 8, length 64 07:39:02.547215 IP (tos 0x0, ttl 64, id 32226, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 9, length 64 07:39:03.587219 IP (tos 0x0, ttl 64, id 32240, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 10, length 64 07:39:04.627210 IP (tos 0x0, ttl 64, id 32304, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 11, length 64 07:39:05.667217 IP (tos 0x0, ttl 64, id 32404, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 12, length 64 07:39:06.707235 IP (tos 0x0, ttl 64, id 32444, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 13, length 64 07:39:07.747218 IP (tos 0x0, ttl 64, id 32464, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 14, length 64 ^C 11 packets captured 11 packets received by filter 0 packets dropped by kernel 10 packets dropped by interface [root@kunpeng82 devuser]# ^C [root@kunpeng82 devuser]# tcpdump -i enahisic2i3 icmp -nnvv tcpdump: listening on enahisic2i3, link-type EN10MB (Ethernet), capture size 262144 bytes 07:40:03.907167 IP (tos 0x0, ttl 64, id 33003, offset 0, flags [DF], proto ICMP (1), length 84) 192.168.1.82 > 192.168.1.81: ICMP echo request, id 29728, seq 68, length 64 07:40:03.907241 IP (tos 0x0, ttl 64, id 35457, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 68, length 64 07:40:04.947157 IP (tos 0x0, ttl 64, id 33086, offset 0, flags [DF], proto ICMP (1), length 84) 192.168.1.82 > 192.168.1.81: ICMP echo request, id 29728, seq 69, length 64 07:40:04.947214 IP (tos 0x0, ttl 64, id 35478, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 69, length 64 07:40:05.987163 IP (tos 0x0, ttl 64, id 33158, offset 0, flags [DF], proto ICMP (1), length 84) 192.168.1.82 > 192.168.1.81: ICMP echo request, id 29728, seq 70, length 64 07:40:05.987224 IP (tos 0x0, ttl 64, id 35579, offset 0, flags [none], proto ICMP (1), length 84) 192.168.1.81 > 192.168.1.82: ICMP echo reply, id 29728, seq 70, length 64 ^C 6 packets captured 6 packets received by filter 0 packets dropped by kernel
[root@localhost ~]# tcpdump -i enahisic2i3 icmp and dst host 192.168.1.81 -nnvv tcpdump: listening on enahisic2i3, link-type EN10MB (Ethernet), capture size 262144 bytes 14:27:10.535372 IP (tos 0x0, ttl 64, id 586, offset 0, flags [DF], proto ICMP (1), length 84) 192.168.2.210 > 192.168.1.81: ICMP echo request, id 26179, seq 142, length 64 14:27:11.575364 IP (tos 0x0, ttl 64, id 681, offset 0, flags [DF], proto ICMP (1), length 84) 192.168.2.210 > 192.168.1.81: ICMP echo request, id 26179, seq 143, length 64 ^C 2 packets captured 2 packets received by filter 0 packets dropped by kernel 2 packets dropped by interface
ovs-vsctl add-br arp_sw ip netns add ns2 ip link add tap0 type veth peer name tap0_br ip link set tap0 netns ns2 ip netns exec ns2 ip link set tap0 up ip netns exec ns2 ip link set lo up ip link set tap0_br up ip netns exec ns2 ip addr add 192.168.2.210/24 dev tap0 ip netns exec ns2 route add -net 192.168.2.0 netmask 255.255.255.0 dev tap0 ovs-vsctl add-port arp_sw tap0_br ovs-ofctl add-flow arp_sw table=0,in_port=1,arp,arp_tpa=10.0.0.1,arp_op=1,actions=move:"NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[]",mod_dl_src:"02:ac:10:ff:01:01",load:"0x02->NXM_OF_ARP_OP[]",move:"NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[]",load:"0x02ac10ff0101->NXM_NX_ARP_SHA[]",move:"NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[]",load:"0x0a000001->NXM_OF_ARP_SPA[]",in_port ovs-ofctl add-flow arp_sw table=0,in_port=1,icmp,nw_dst=10.0.0.1,icmp_type=8,icmp_code=0,actions=push:"NXM_OF_ETH_SRC[]",push:"NXM_OF_ETH_DST[]",pop:"NXM_OF_ETH_SRC[]",pop:"NXM_OF_ETH_DST[]",push:"NXM_OF_IP_SRC[]",push:"NXM_OF_IP_DST[]",pop:"NXM_OF_IP_SRC[]",pop:"NXM_OF_IP_DST[]",load:"0xff->NXM_NX_IP_TTL[]",load:"0->NXM_OF_ICMP_TYPE[]",in_port #### ovs-ofctl add-flow arp_sw table=0,in_port=1,arp,arp_tpa=192.168.1.81,arp_op=1,actions=move:"NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[]",mod_dl_src:"48:57:02:64:ea:1e",load:"0x02->NXM_OF_ARP_OP[]",move:"NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[]",load:"0x48570264ea1e->NXM_NX_ARP_SHA[]",move:"NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[]",load:"0xc0a80151->NXM_OF_ARP_SPA[]",in_port #ovs-ofctl add-flow arp_sw 'table=0,priority=1,in_port=2,ip,ct_state=+trk+est,action=ct(nat,table=1)' #ovs-ofctl add-flow arp_sw table=0,in_port=1,ip,nw_dst=192.168.1.81/24,action=ct\(commit,table=1,nat\(src=192.168.1.82\)\) #ovs-ofctl add-flow arp_sw table=1,ip,nw_src=192.168.1.82,nw_dst=192.168.1.81,actions=mod_dl_src:"48:57:02:64:e7:ae",2 #ovs-ofctl add-flow arp_sw table=1,ip,nw_dst=192.168.2.210,actions=mod_dl_dst:"2e:a9:be:9e:4d:07",1 #如果进入trk可以直接查找连接，如果没有执行相应action ovs-ofctl add-flow arp_sw 'table=0,priority=10,ip,ct_state=-trk,action=ct(nat,table=1)' #新建连接，执行ct commit；创建链接，执行nat规则 ovs-ofctl add-flow arp_sw 'table=1,in_port=1,ip,ct_state=+trk+new,action=ct(nat(src=192.168.1.82-192.168.1.82:5000-50000),commit),mod_dl_src:48:57:02:64:e7:ae,2' #到外网流量，已经建立号连接，直接发给2口 ovs-ofctl add-flow arp_sw 'table=1,in_port=1,ip,ct_state=+trk+est,action=mod_dl_src:48:57:02:64:e7:ae,2' #回来流量，已经建立连接，直接发给1口 ovs-ofctl add-flow arp_sw 'table=1,in_port=2,ip,nw_dst=192.168.2.210,ct_state=+trk+est,action=mod_dl_dst:2e:a9:be:9e:4d:07,1' --------------ovs是二层转发设备，需要更改目的mac
