部署OVN网络拓扑
OVN-安装软件包
/etc/yum.repos.d/CentOS-OpenStack-ocata.repo
# yum list installed | grep openvswitch
openvswitch.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-devel.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-ovn-central.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-ovn-common.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-ovn-docker.x86_64 1:2.6.1-10.1.git20161206.el7
openvswitch-ovn-host.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-ovn-vtep.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-test.noarch 1:2.9.0-3.el7 @centos-openstack-ocata
python2-openvswitch.noarch 1:2.9.0-3.el7 @centos-openstack-ocata
### 关闭防火墙和SELINUX
# systemctl stop firewalld
# systemctl disable firewalld
### Central节点
# yum install -y openvswitch-ovn-central.x86_64 openvswitch-ovn-host.x86_64
# systemctl enable ovn-northd openvswitch ovn-controller
# systemctl start ovn-northd ovn-controller
### Node节点
# yum install -y openvswitch.x86_64 openvswitch-ovn-host.x86_64
# systemctl enable openvswitch ovn-controller
# systemctl start ovn-controller
配置OVN
export Centralip=10.33.46.182
export Nodeip=10.33.46.68
### Central节点
# ovn-nbctl set-connection ptcp:6641:$Centralip
# ovn-sbctl set-connection ptcp:6642:$Centralip
# ovs-vsctl set open . external-ids:ovn-remote=tcp:$Centralip:6642
# ovs-vsctl set open . external-ids:ovn-encap-type=geneve
# ovs-vsctl set open . external-ids:ovn-encap-ip=$Centralip
### Node节点
# ovs-vsctl set open . external-ids:ovn-remote=tcp:$Centralip:6642
# ovs-vsctl set open . external-ids:ovn-encap-type=geneve
# ovs-vsctl set open . external-ids:ovn-encap-ip=$Nodeip
tcp 0 1 10.33.46.68:43132 10.33.46.68:6642 SYN_SENT 0 20089825 44936/ovn-controlle
#reboot后可以建立连接
system-id是ovn-sbctl中的Chassis的ID
OVN-L2网络
定义逻辑网络:创建一个逻辑交换机，然后添加两个交换机端口，并为端口设置物理地址
# $FQDN|md5sum|sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'
export vm1mac=02:d4:1d:8c:d9:8f
export vm2mac=02:d4:1d:8c:d9:8e
export vm1ip=172.16.255.11
export vm2ip=172.16.255.22
### Central节点
# ovn-nbctl ls-add ls1
# ovn-nbctl lsp-add ls1 ls1-vm1
# ovn-nbctl lsp-set-addresses ls1-vm1 $vm1mac
# ovn-nbctl lsp-set-port-security ls1-vm1 $vm1mac
# ovn-nbctl lsp-add ls1 ls1-vm2
# ovn-nbctl lsp-set-addresses ls1-vm2 $vm2mac
# ovn-nbctl lsp-set-port-security ls1-vm2 $vm2mac
伪造虚拟机:创建网络命名空间，并在br-int上添加端口，然后将端口添加到命名空间，最后通过设置端口的MAC地址和网卡名完成和交换机端口的映射
### Central节点
# ip netns add vm1
# ovs-vsctl add-port br-int vm1 -- set interface vm1 type=internal
# ip link set vm1 netns vm1
# ip netns exec vm1 ip link set vm1 address $vm1mac
# ip netns exec vm1 ip addr add $vm1ip/24 dev vm1
# ip netns exec vm1 ip link set vm1 up
# ovs-vsctl set Interface vm1 external_ids:iface-id=ls1-vm1
# ip netns exec vm1 ip addr show
### Node节点
# ip netns add vm2
# ovs-vsctl add-port br-int vm2 -- set interface vm2 type=internal
# ip link set vm2 netns vm2
# ip netns exec vm2 ip link set vm2 address $vm2mac
# ip netns exec vm2 ip addr add $vm2ip/24 dev vm2
# ip netns exec vm2 ip link set vm2 up
# ovs-vsctl set Interface vm2 external_ids:iface-id=ls1-vm2
# ip netns exec vm2 ip addr show
vm互ping，可ping通
OVN- L3网络
添加L3网关
### 创建逻辑路由
# ovn-nbctl lr-add edge1
### 创建逻辑交换机用于连接edge1和tenant1
# ovn-nbctl ls-add transit
### 连接edge1到逻辑交换机上
# ovn-nbctl lrp-add edge1 edge1-transit 02:d4:1d:8c:d9:ae 192.168.0.1/24
# ovn-nbctl lsp-add transit transit-edge1
# ovn-nbctl lsp-set-type transit-edge1 router
# ovn-nbctl lsp-set-addresses transit-edge1 02:d4:1d:8c:d9:ae
# ovn-nbctl lsp-set-options transit-edge1 router-port=edge1-transit
### 连接tenant1到逻辑交换机上
# ovn-nbctl lrp-add tenant1 tenant1-transit 02:d4:1d:8c:d9:af 192.168.0.2/24
# ovn-nbctl lsp-add transit transit-tenant1
# ovn-nbctl lsp-set-type transit-tenant1 router
# ovn-nbctl lsp-set-addresses transit-tenant1 02:d4:1d:8c:d9:af
# ovn-nbctl lsp-set-options transit-tenant1 router-port=tenant1-transit
### 添加静态路由
ovn-nbctl lr-route-add edge1 "20.0.0.0/24" 192.168.0.2
ovn-nbctl lr-route-add edge1 "10.0.0.0/24" 192.168.0.2
ovn-nbctl lr-route-add tenant1 "0.0.0.0/0" 192.168.0.1
ovn-nbctl lr-route-list edge1
ovn-nbctl lr-route-list tenant1
### 测试连通性
ip netns exec vm21 ping -c 2 192.168.0.1
网关与外网连接
### Central节点
### 创建外网逻辑交换机，并配置网关到叫交换机的连接
ovn-nbctl ls-add outside
ovn-nbctl lrp-add edge1 edge1-outside 02:d4:1d:8c:d9:be 192.168.233.177/24
ovn-nbctl lsp-add outside outside-edge1
ovn-nbctl lsp-set-type outside-edge1 router
ovn-nbctl lsp-set-addresses outside-edge1 02:d4:1d:8c:d9:be
ovn-nbctl lsp-set-options outside-edge1 router-port=edge1-outside
### 为外网网卡ens4创建网桥
ovs-vsctl add-br br-ex
### 为外网网卡ens4创建网桥到网络的映射
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=dataNet:br-ex
### 在逻辑交换机outside上添加本地网络端口，并且本地网络的名字为dataNet
ovn-nbctl lsp-add outside outside-localnet
ovn-nbctl lsp-set-addresses outside-localnet unknown
ovn-nbctl lsp-set-type outside-localnet localnet
ovn-nbctl lsp-set-options outside-localnet network_name=dataNet
### 关联外网网卡到网桥上
# ovs-vsctl add-port br-ex eth1
### 测试连通性（需要注意vm2的ip地址是不是没了，dhclient好像有些问题）
ip netns exec vm22 ping -c 2 192.168.233.177
### 设置网桥地址
ip addr add 192.168.233.7/24 dev br-ex
ip link set br-ex up
### 重置下路由
# ip route
# ip route del default via 192.168.233.1
# ip route del 192.168.233.0/24 dev eth1
设置SNAT
### Central节点
### 设置网关chassis
ovn-nbctl lrp-set-gateway-chassis edge1-outside 35a10447-0513-4f8f-a340-33220258b9d9 #为ovn-snctl show对应的Chassis ID
### 配置SNAT规则
ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=20.0.0.0/24 external_ip=192.168.233.177 -- add logical_router edge1 nat @nat
ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=10.0.0.0/24 external_ip=192.168.233.177 -- add logical_router edge1 nat @nat
/var/log/openvswitch/ovn-controller.log：
2019-07-09T00:37:14.205Z|00265|ofctrl|INFO|OpenFlow error: OFPT_ERROR (OF1.3) (xid=0x9f7): NXBAC_CT_DATAPATH_SUPPORT
OFPT_FLOW_MOD (OF1.3) (xid=0x9f7): ADD table:41 priority=25,ip,reg15=0x2,metadata=0x5,nw_src=20.0.0.0/24 cookie:0x71db37d actions=ct(commit,table=42,zone=NXM_NX_REG12[0..15],nat(src=192.168.233.177))
2019-07-09T00:37:14.205Z|00266|ofctrl|INFO|OpenFlow error: OFPT_ERROR (OF1.3) (xid=0x9fa): NXBAC_CT_DATAPATH_SUPPORT
OFPT_FLOW_MOD (OF1.3) (xid=0x9fa): ADD table:11 priority=100,ip,reg14=0x2,metadata=0x5,nw_dst=192.168.233.177 cookie:0x236f944b actions=ct(table=12,zone=NXM_NX_REG12[0..15],nat)
对应的命令：
ovs-ofctl add-flow br-int "table=41,priority=25,ip,reg15=0x2,metadata=0x5,nw_src=20.0.0.0/24 cookie:0x71db37d actions=ct(commit,table=42,zone=NXM_NX_REG12[0..15],nat(src=192.168.233.177))"
详细信息：
ovs-ofctl add-flow br-int "table=41,priority=25,ip,reg15=0x2,metadata=0x5,nw_src=20.0.0.0/24 cookie:0x71db37d actions=ct(commit,table=42,zone=NXM_NX_REG12[0..15],nat(src=192.168.233.177))" -v
失败原因：OFPT_FLOW_MOD
内核datapath不支持该特性，环境部署问题
查看发现：
# kmod list | grep openvs
openvswitch 84535 1 vport_geneve
libcrc32c 12644 2 xfs,openvswitch
正常情况下：（原因是我用的是虚机搭建的环境，centos7.2版本，7.4版本ok）
[root@Images ~]# kmod list | grep openvs
[root@Images ~]# modprobe openvswitch
[root@Images ~]# kmod list | grep openvs
openvswitch 106739 0
nf_nat_ipv6 14131 1 openvswitch
nf_defrag_ipv6 35104 2 openvswitch,nf_conntrack_ipv6
nf_nat_ipv4 14115 2 openvswitch,iptable_nat
nf_nat 26147 4 openvswitch,nf_nat_ipv4,nf_nat_ipv6,nf_nat_masquerade_ipv4
nf_conntrack 111302 8 openvswitch,nf_nat,nf_nat_ipv4,nf_nat_ipv6,xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_ipv4,nf_conntrack_ipv6
libcrc32c 12644 2 xfs,openvswitch
# ovs-dpctl show -v
02:d4:1d:8c:d9:be > c2:af:5a:9e:73:47, ethertype IPv4 (0x0800), length 98: 20.0.0.20 > 192.168.233.7: ICMP echo request, id 24614, seq 508, length 64
### 测试连通性
ip netns exec vm22 ping -c 2 192.168.233.7
简化环境后也可以：
流表分析
table 0 主要工作如下：
l 完成物理到逻辑的翻译，将逻辑信息，比如上面提到的信息记录到寄存器中。
l VM中的容器的报文用VLAN进行区分
l 别的chassis过来的报文，根据入端口和tunnel_id进行区分，然后获取出端口，这个在封装的时候已经有了
table 16-31 主要是将逻辑流表ingress pipeline 0-15 的操作部分转换为openflow流表，主要工作如下：
l 每个逻辑流表会映射一个或者多个openflow流表，通常报文只是匹配其中一条流表。
l ovn-controller使用逻辑流表的UUID的前32位作为openflow流表的cookie值。查看逻辑流表的UUID使用ovn-sbctl list Logical_Flow，对应上面cookie的逻辑流表的UUID的信息在这里。
l 一些逻辑流表可以映射到ovs的”conjunctive match”扩展名(参见这里)，这时候因为一条openflow流表对应了多条逻辑流表，所以cookie为0。这里的”conjunctive match”表示一个集合的匹配，比如tcp_src ∈ {80, 443, 8080} and tcp_dst ∈ {80, 443, 8080}。
l 一些逻辑流表可能不会转换成openflow流表，如果交换机上虚拟接口没有添加到ovs中，添加命令ovs-vsctl set Interface veth2_b external_ids:iface-id=ls2-vm4，那么相应的openflow流表将不会生成。
l 最后就是有一些逻辑流表和openflow流表很明显的对应操作关系，我们列一下
l next对应resubmit
l field = constant对应set_field
l output，将报文resubmit到表32，如果逻辑流表有多个output操作，那么每个都要resubmit到表32。
l get_arp(P, A)和get_nd(P, A)，通过讲参数存储在openflow字段中(上面例子中存储在NXM_NX_REG0，流表cookie=0x5dbc664)，然后resubmit到表66，然后ovn-controller从MAC_Binding表生成流填充，如果表66中有匹配项，其action将绑定的MAC存储在目的MAC地址字段中
l put_arp(P, A, E)和put_nd(P, A, E)讲参数存储到openflow的字段中(字段太多，查看上面流表cookie=0x92af5d1c)，然后更新MAC_Binding表中。
table 32-47 主要是将逻辑流表ingress pipeline的output action转换为openflow流表。以下详细介绍下：
表32主要是处理到其他宿主机中虚拟机的报文，讲VNI设置到metadata，然后resubmit到表33
表33主要是将报文resubmit到表34，对于多个逻辑output端口的时候，需要改为每个逻辑端口P，然后resubmit到表34
表34检查报文的逻辑ingress和egress的端口是否一致，一致则丢弃。剩下的resubmit到表48
table 48-63 主要是讲逻辑流表的egress pipeline部分转换成openflow流表，这块属于报文发送之前的最后验证，最终resubmit到表64，最终没有执行output的报文将被丢弃。
table 64 貌似和loopback有关，修改逻辑入端口。
table 65 逻辑到物理的转换，和表0相反，主要是将找到逻辑端口对应的物理端口，然后发送，如果虚拟机中还有容器的话，需要添加vlan头。
table 66 主要是对应MAC_Binding中的数据，来修改目的IP对应的目的MAC，功能类似arp。
# ovs-ofctl dump-flows br-int
//cookie没有值表示不是直接从逻辑流表转换而来的
//两个虚拟机进来的报文进行一些寄存器的操作，这个不是根据逻辑流表来的，但是和逻辑拓扑还是有关系的，具体这些寄存器的意义和获取我们下面介绍
cookie=0x0, table=0, priority=100,in_port=4 actions=load:0x1->NXM_NX_REG13[],load:0x6->NXM_NX_REG11[],load:0x8->NXM_NX_REG12[],load:0x3->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],resubmit(,16)
cookie=0x0, table=0, priority=100,in_port=3 actions=load:0x2->NXM_NX_REG13[],load:0x7->NXM_NX_REG11[],load:0x5->NXM_NX_REG12[],load:0x2->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],resubmit(,16)
//表示从其他宿主机发送过来的报文应该如何处理，这里的tun_id分别表示从两个逻辑交换中的哪一个发送过来的
cookie=0x0, table=0, priority=100,tun_id=0x3,in_port=7 actions=move:NXM_NX_TUN_ID[0..23]->OXM_OF_METADATA[0..23],load:0x3->NXM_NX_REG14[0..14],load:0x1->NXM_NX_REG10[1],resubmit(,16)
cookie=0x0, table=0, priority=100,tun_id=0x2,in_port=7 actions=move:NXM_NX_TUN_ID[0..23]->OXM_OF_METADATA[0..23],load:0x3->NXM_NX_REG14[0..14],load:0x1->NXM_NX_REG10[1],resubmit(,16)
//一些我们不关注的流表主要是一些错误报文的丢弃操作，相关流表已经删除了
//以下metadata不是1表示从逻辑交换发过来的报文怎么处理，前面的reg14表示从哪个逻辑端口发送过来的
cookie=0xa7c014e8, table=16, priority=50,reg14=0x2,metadata=0x3,dl_src=52:54:00:c1:68:71 actions=resubmit(,17)
cookie=0x3ed26758, table=16, priority=50,reg14=0x2,metadata=0x2,dl_src=52:54:00:c1:68:70 actions=resubmit(,17)
cookie=0x11dd5c04, table=16, priority=50,reg14=0x3,metadata=0x2,dl_src=52:54:00:c1:68:72 actions=resubmit(,17)
cookie=0x6126e3c1, table=16, priority=50,reg14=0x3,metadata=0x3,dl_src=52:54:00:c1:68:73 actions=resubmit(,17)
cookie=0x75e7ab7b, table=16, priority=50,reg14=0x1,metadata=0x2 actions=resubmit(,17)
cookie=0x8c78254f, table=16, priority=50,reg14=0x1,metadata=0x3 actions=resubmit(,17)
//以下metadata为1表示从逻辑路由过来的报文，需要进行怎样的操作
cookie=0xd9caf1fd, table=16, priority=50,reg14=0x1,metadata=0x1,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,17)
cookie=0xeac605df, table=16, priority=50,reg14=0x2,metadata=0x1,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,17)
cookie=0x819b5118, table=16, priority=50,reg14=0x1,metadata=0x1,dl_dst=52:54:00:c1:68:50 actions=resubmit(,17)
cookie=0xbe725a2b, table=16, priority=50,reg14=0x2,metadata=0x1,dl_dst=52:54:00:c1:68:60 actions=resubmit(,17)
//arp代答的流表
cookie=0xf4ca156, table=17, priority=90,arp,reg14=0x2,metadata=0x1,arp_tpa=192.168.2.1,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:52:54:00:c1:68:60,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x525400c16860->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xc0a80201->NXM_OF_ARP_SPA[],load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
cookie=0xb5d8c2e4, table=17, priority=90,arp,reg14=0x1,metadata=0x1,arp_tpa=192.168.1.1,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:52:54:00:c1:68:50,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x525400c16850->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xc0a80101->NXM_OF_ARP_SPA[],load:0x1->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
//arp回复报文的信息存入MAC_Binding
cookie=0x92af5d1c, table=17, priority=90,arp,metadata=0x1,arp_op=2 actions=push:NXM_NX_REG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ARP_SHA[],push:NXM_OF_ARP_SPA[],pop:NXM_NX_REG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.01.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_REG0[]
//icmp代答
cookie=0x815a3063, table=17, priority=90,icmp,metadata=0x1,nw_dst=192.168.1.1,icmp_type=8,icmp_code=0 actions=push:NXM_OF_IP_SRC[],push:NXM_OF_IP_DST[],pop:NXM_OF_IP_SRC[],pop:NXM_OF_IP_DST[],load:0xff->NXM_NX_IP_TTL[],load:0->NXM_OF_ICMP_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,18)
cookie=0xf3d609b1, table=17, priority=90,icmp,metadata=0x1,nw_dst=192.168.2.1,icmp_type=8,icmp_code=0 actions=push:NXM_OF_IP_SRC[],push:NXM_OF_IP_DST[],pop:NXM_OF_IP_SRC[],pop:NXM_OF_IP_DST[],load:0xff->NXM_NX_IP_TTL[],load:0->NXM_OF_ICMP_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,18)
//三个逻辑设备的流量继续往下走
cookie=0x56295f89, table=17, priority=0,metadata=0x1 actions=resubmit(,18)
cookie=0x791195e0, table=17, priority=0,metadata=0x3 actions=resubmit(,18)
cookie=0x4b1c93d4, table=17, priority=0,metadata=0x2 actions=resubmit(,18)
//arp通过
cookie=0x4a80a501, table=18, priority=90,arp,reg14=0x3,metadata=0x3,dl_src=52:54:00:c1:68:73,arp_sha=52:54:00:c1:68:73 actions=resubmit(,19)
cookie=0xc6c881ee, table=18, priority=90,arp,reg14=0x3,metadata=0x2,dl_src=52:54:00:c1:68:72,arp_sha=52:54:00:c1:68:72 actions=resubmit(,19)
cookie=0x9e2a7562, table=18, priority=90,arp,reg14=0x2,metadata=0x2,dl_src=52:54:00:c1:68:70,arp_sha=52:54:00:c1:68:70 actions=resubmit(,19)
cookie=0x686267fe, table=18, priority=90,arp,reg14=0x2,metadata=0x3,dl_src=52:54:00:c1:68:71,arp_sha=52:54:00:c1:68:71 actions=resubmit(,19)
//继续
cookie=0xb76a420f, table=18, priority=0,metadata=0x2 actions=resubmit(,19)
cookie=0x3ecbeeec, table=18, priority=0,metadata=0x1 actions=resubmit(,19)
cookie=0x78c16fb8, table=18, priority=0,metadata=0x3 actions=resubmit(,19)
//继续
cookie=0x76f9414c, table=19, priority=0,metadata=0x3 actions=resubmit(,20)
cookie=0xff75779d, table=19, priority=0,metadata=0x2 actions=resubmit(,20)
cookie=0xa4a71b19, table=19, priority=0,metadata=0x1 actions=resubmit(,20)
//继续
cookie=0x4c209f08, table=20, priority=0,metadata=0x3 actions=resubmit(,21)
cookie=0xc99c5154, table=20, priority=0,metadata=0x1 actions=resubmit(,21)
cookie=0xe187a6b4, table=20, priority=0,metadata=0x2 actions=resubmit(,21)
//conntrack记录
cookie=0x5c49d2d2, table=21, priority=100,ip,reg0=0x1/0x1,metadata=0x3 actions=ct(table=22,zone=NXM_NX_REG13[0..15])
cookie=0x596e0c95, table=21, priority=100,ip,reg0=0x1/0x1,metadata=0x2 actions=ct(table=22,zone=NXM_NX_REG13[0..15])
//模拟过网关时的操作
cookie=0xaea49216, table=21, priority=49,ip,metadata=0x1,nw_dst=192.168.1.0/24 actions=dec_ttl(),move:NXM_OF_IP_DST[]->NXM_NX_XXREG0[96..127],load:0xc0a80101->NXM_NX_XXREG0[64..95],mod_dl_src:52:54:00:c1:68:50,load:0x1->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,22)
cookie=0x3ebae949, table=21, priority=49,ip,metadata=0x1,nw_dst=192.168.2.0/24 actions=dec_ttl(),move:NXM_OF_IP_DST[]->NXM_NX_XXREG0[96..127],load:0xc0a80201->NXM_NX_XXREG0[64..95],mod_dl_src:52:54:00:c1:68:60,load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,22)
//继续
cookie=0xe3a08e2b, table=21, priority=0,metadata=0x3 actions=resubmit(,22)
cookie=0x80407476, table=21, priority=0,metadata=0x2 actions=resubmit(,22)
//获取MAC_Binding表里的数据，回复arp
cookie=0x5dbc664, table=22, priority=0,ip,metadata=0x1 actions=push:NXM_NX_REG0[],push:NXM_NX_XXREG0[96..127],pop:NXM_NX_REG0[],mod_dl_dst:00:00:00:00:00:00,resubmit(,66),pop:NXM_NX_REG0[],resubmit(,23)
//继续
cookie=0x66236a1, table=22, priority=0,metadata=0x2 actions=resubmit(,23)
cookie=0xefaed143, table=22, priority=0,metadata=0x3 actions=resubmit(,23)
//继续
cookie=0x3998ed82, table=23, priority=0,metadata=0x1 actions=resubmit(,24)
cookie=0xc475a7b3, table=23, priority=0,metadata=0x3 actions=resubmit(,24)
cookie=0xacda159d, table=23, priority=0,metadata=0x2 actions=resubmit(,24)
//????发送arp？
cookie=0xe51fffad, table=24, priority=100,ip,metadata=0x1,dl_dst=00:00:00:00:00:00 actions=controller(userdata=00.00.00.00.00.00.00.00.00.19.00.10.80.00.06.06.ff.ff.ff.ff.ff.ff.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.40.00.00.00.01.de.10.00.00.20.04.ff.ff.00.18.00.00.23.20.00.06.00.20.00.60.00.00.00.01.de.10.00.00.22.04.00.19.00.10.80.00.2a.02.00.01.00.00.00.00.00.00.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
//继续
cookie=0xd9c9912b, table=24, priority=0,metadata=0x1 actions=resubmit(,32)
cookie=0x9b703aff, table=24, priority=0,metadata=0x2 actions=resubmit(,25)
cookie=0xd44f4b41, table=24, priority=0,metadata=0x3 actions=resubmit(,25)
//conntrack lb
cookie=0xed10c525, table=25, priority=100,ip,reg0=0x4/0x4,metadata=0x3 actions=ct(table=26,zone=NXM_NX_REG13[0..15],nat)
cookie=0xb0869023, table=25, priority=100,ip,reg0=0x4/0x4,metadata=0x2 actions=ct(table=26,zone=NXM_NX_REG13[0..15],nat)
//conntrack
cookie=0xc8dfda6d, table=25, priority=100,ip,reg0=0x2/0x2,metadata=0x2 actions=ct(commit,zone=NXM_NX_REG13[0..15],exec(load:0->NXM_NX_CT_LABEL[0])),resubmit(,26)
cookie=0xf71a37ba, table=25, priority=100,ip,reg0=0x2/0x2,metadata=0x3 actions=ct(commit,zone=NXM_NX_REG13[0..15],exec(load:0->NXM_NX_CT_LABEL[0])),resubmit(,26)
//继续
cookie=0x3c4b37a7, table=25, priority=0,metadata=0x2 actions=resubmit(,26)
cookie=0x315f30b3, table=25, priority=0,metadata=0x3 actions=resubmit(,26)
//继续
cookie=0x4368d2e8, table=26, priority=0,metadata=0x3 actions=resubmit(,27)
cookie=0xf906a487, table=26, priority=0,metadata=0x2 actions=resubmit(,27)
cookie=0x1ab8df97, table=27, priority=0,metadata=0x3 actions=resubmit(,28)
cookie=0x8592b902, table=27, priority=0,metadata=0x2 actions=resubmit(,28)
cookie=0xe3f59b41, table=28, priority=0,metadata=0x3 actions=resubmit(,29)
cookie=0xba22fb48, table=28, priority=0,metadata=0x2 actions=resubmit(,29)
//泛洪
cookie=0x159f7998, table=29, priority=100,metadata=0x3,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=load:0xffff->NXM_NX_REG15[],resubmit(,32)
cookie=0xcbb8e72a, table=29, priority=100,metadata=0x2,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=load:0xffff->NXM_NX_REG15[],resubmit(,32)
//出口流量
cookie=0xc0e4e6a6, table=29, priority=50,metadata=0x2,dl_dst=52:54:00:c1:68:72 actions=load:0x3->NXM_NX_REG15[],resubmit(,32)
cookie=0x13381c84, table=29, priority=50,metadata=0x3,dl_dst=52:54:00:c1:68:73 actions=load:0x3->NXM_NX_REG15[],resubmit(,32)
cookie=0x23555b13, table=29, priority=50,metadata=0x2,dl_dst=52:54:00:c1:68:50 actions=load:0x1->NXM_NX_REG15[],resubmit(,32)
cookie=0x3f8b4ff9, table=29, priority=50,metadata=0x2,dl_dst=52:54:00:c1:68:70 actions=load:0x2->NXM_NX_REG15[],resubmit(,32)
cookie=0x615dbb2a, table=29, priority=50,metadata=0x3,dl_dst=52:54:00:c1:68:71 actions=load:0x2->NXM_NX_REG15[],resubmit(,32)
cookie=0xb88437bc, table=29, priority=50,metadata=0x3,dl_dst=52:54:00:c1:68:60 actions=load:0x1->NXM_NX_REG15[],resubmit(,32)
//????没有flags为2的标志
cookie=0x0, table=32, priority=150,reg10=0x2/0x2 actions=resubmit(,33)
//到逻辑路由的流量
cookie=0x0, table=32, priority=100,reg15=0xffff,metadata=0x3 actions=load:0x1->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[],load:0x3->NXM_NX_TUN_ID[0..23],output:7,resubmit(,33)
cookie=0x0, table=32, priority=100,reg15=0xffff,metadata=0x2 actions=load:0x1->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[],load:0x2->NXM_NX_TUN_ID[0..23],output:7,resubmit(,33)
//到逻辑交换的流量
cookie=0x0, table=32, priority=100,reg15=0x3,metadata=0x2 actions=load:0x2->NXM_NX_TUN_ID[0..23],output:7
cookie=0x0, table=32, priority=100,reg15=0x3,metadata=0x3 actions=load:0x3->NXM_NX_TUN_ID[0..23],output:7
//继续
cookie=0x0, table=32, priority=0 actions=resubmit(,33)
//????到网络节点需要NAT的流量，可是我们没有相应的配置
cookie=0x0, table=33, priority=100,reg15=0x1,metadata=0x3 actions=load:0x6->NXM_NX_REG11[],load:0x8->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, table=33, priority=100,reg15=0x2,metadata=0x1 actions=load:0x3->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, table=33, priority=100,reg15=0x2,metadata=0x2 actions=load:0x2->NXM_NX_REG13[],load:0x7->NXM_NX_REG11[],load:0x5->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, table=33, priority=100,reg15=0x1,metadata=0x2 actions=load:0x7->NXM_NX_REG11[],load:0x5->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, table=33, priority=100,reg15=0x1,metadata=0x1 actions=load:0x3->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, table=33, priority=100,reg15=0x2,metadata=0x3 actions=load:0x1->NXM_NX_REG13[],load:0x6->NXM_NX_REG11[],load:0x8->NXM_NX_REG12[],resubmit(,34)
//继续
cookie=0x0, table=33, priority=100,reg15=0xffff,metadata=0x2 actions=load:0x2->NXM_NX_REG13[],load:0x2->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[]
cookie=0x0, table=33, priority=100,reg15=0xffff,metadata=0x3 actions=load:0x1->NXM_NX_REG13[],load:0x2->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[]
//继续
cookie=0x0, table=34, priority=0 actions=load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],resubmit(,48)
//继续
cookie=0x38579acc, table=48, priority=0,metadata=0x1 actions=resubmit(,49)
cookie=0x402567e, table=48, priority=0,metadata=0x3 actions=resubmit(,49)
cookie=0x7e6e093d, table=48, priority=0,metadata=0x2 actions=resubmit(,49)
//继续
cookie=0xbce65dae, table=49, priority=0,metadata=0x2 actions=resubmit(,50)
cookie=0xf6e47c0e, table=49, priority=0,metadata=0x1 actions=resubmit(,50)
cookie=0xa630e910, table=49, priority=0,metadata=0x3 actions=resubmit(,50)
//conntrack
cookie=0xe6e35197, table=50, priority=100,ipv6,reg0=0x1/0x1,metadata=0x3 actions=ct(table=51,zone=NXM_NX_REG13[0..15])
cookie=0xa7a5e5f3, table=50, priority=100,ipv6,reg0=0x1/0x1,metadata=0x2 actions=ct(table=51,zone=NXM_NX_REG13[0..15])
cookie=0xa7a5e5f3, table=50, priority=100,ip,reg0=0x1/0x1,metadata=0x2 actions=ct(table=51,zone=NXM_NX_REG13[0..15])
cookie=0xe6e35197, table=50, priority=100,ip,reg0=0x1/0x1,metadata=0x3 actions=ct(table=51,zone=NXM_NX_REG13[0..15])
//继续
cookie=0x4e268323, table=50, priority=0,metadata=0x1 actions=resubmit(,51)
cookie=0x2e28bd0c, table=50, priority=0,metadata=0x2 actions=resubmit(,51)
cookie=0x7cca0b71, table=50, priority=0,metadata=0x3 actions=resubmit(,51)
//需要输出到逻辑路由的流量
cookie=0x1c84ef4, table=51, priority=100,reg15=0x2,metadata=0x1 actions=resubmit(,64)
cookie=0x83ce9e62, table=51, priority=100,reg15=0x1,metadata=0x1 actions=resubmit(,64)
//继续
cookie=0x51c9cccf, table=51, priority=0,metadata=0x2 actions=resubmit(,52)
cookie=0x7778d918, table=51, priority=0,metadata=0x3 actions=resubmit(,52)
//继续
cookie=0xa9ae4aaa, table=52, priority=0,metadata=0x2 actions=resubmit(,53)
cookie=0xe190604a, table=52, priority=0,metadata=0x3 actions=resubmit(,53)
cookie=0x934c95d9, table=53, priority=0,metadata=0x3 actions=resubmit(,54)
cookie=0x828e0c10, table=53, priority=0,metadata=0x2 actions=resubmit(,54)
//conntrack lb
cookie=0xb1d05c18, table=54, priority=100,ip,reg0=0x4/0x4,metadata=0x3 actions=ct(table=55,zone=NXM_NX_REG13[0..15],nat)
cookie=0x4b8234d9, table=54, priority=100,ip,reg0=0x4/0x4,metadata=0x2 actions=ct(table=55,zone=NXM_NX_REG13[0..15],nat)
//conntrack
cookie=0x6027420b, table=54, priority=100,ip,reg0=0x2/0x2,metadata=0x3 actions=ct(commit,zone=NXM_NX_REG13[0..15],exec(load:0->NXM_NX_CT_LABEL[0])),resubmit(,55)
cookie=0x76bd97bd, table=54, priority=100,ip,reg0=0x2/0x2,metadata=0x2 actions=ct(commit,zone=NXM_NX_REG13[0..15],exec(load:0->NXM_NX_CT_LABEL[0])),resubmit(,55)
//继续
cookie=0x390ebf5f, table=54, priority=0,metadata=0x2 actions=resubmit(,55)
cookie=0x6537ab93, table=54, priority=0,metadata=0x3 actions=resubmit(,55)
cookie=0x13159847, table=55, priority=0,metadata=0x3 actions=resubmit(,56)
cookie=0x439f6726, table=55, priority=0,metadata=0x2 actions=resubmit(,56)
//多播流量
cookie=0xb5641b45, table=56, priority=100,metadata=0x2,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,64)
cookie=0x7b1296c4, table=56, priority=100,metadata=0x3,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,64)
//到某个虚拟机的流量
cookie=0xcfbbf747, table=56, priority=50,reg15=0x3,metadata=0x2,dl_dst=52:54:00:c1:68:72 actions=resubmit(,64)
cookie=0xd39cd78f, table=56, priority=50,reg15=0x3,metadata=0x3,dl_dst=52:54:00:c1:68:73 actions=resubmit(,64)
cookie=0x46f7518d, table=56, priority=50,reg15=0x2,metadata=0x3,dl_dst=52:54:00:c1:68:71 actions=resubmit(,64)
cookie=0x10683faf, table=56, priority=50,reg15=0x2,metadata=0x2,dl_dst=52:54:00:c1:68:70 actions=resubmit(,64)
//继续
cookie=0xdf1a835, table=56, priority=50,reg15=0x1,metadata=0x3 actions=resubmit(,64)
cookie=0x69d25440, table=56, priority=50,reg15=0x1,metadata=0x2 actions=resubmit(,64)
//修改入端口，为重新循环做准备
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x1,metadata=0x1 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x3 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x2 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x1,metadata=0x3 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x1 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x1,metadata=0x2 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=0 actions=resubmit(,65)
//将报文重新resubmit到表16，表示过完一个逻辑网元，需要进入下一个逻辑网元了
cookie=0x0, table=65, priority=100,reg15=0x2,metadata=0x1 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x6->NXM_NX_REG11[],load:0x8->NXM_NX_REG12[],load:0x3->OXM_OF_METADATA[],load:0x1->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,16))
cookie=0x0, table=65, priority=100,reg15=0x1,metadata=0x2 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x3->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],load:0x1->OXM_OF_METADATA[],load:0x1->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,16))
cookie=0x0, table=65, priority=100,reg15=0x1,metadata=0x1 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x7->NXM_NX_REG11[],load:0x5->NXM_NX_REG12[],load:0x2->OXM_OF_METADATA[],load:0x1->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,16))
cookie=0x0, table=65, priority=100,reg15=0x1,metadata=0x3 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x3->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],load:0x1->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,16))
//到本地某个虚拟机的直接发送
cookie=0x0, table=65, priority=100,reg15=0x2,metadata=0x2 actions=output:3
cookie=0x0, table=65, priority=100,reg15=0x2,metadata=0x3 actions=output:4
//通过MAC_Binding修改IP对应的MAC
cookie=0x0, table=66, priority=100,reg0=0xc0a8025c,reg15=0x2,metadata=0x1 actions=mod_dl_dst:52:54:00:c1:68:73
cookie=0x0, table=66, priority=100,reg0=0xc0a8025b,reg15=0x2,metadata=0x1 actions=mod_dl_dst:52:54:00:c1:68:71
cookie=0x0, table=66, priority=100,reg0=0xc0a8015b,reg15=0x1,metadata=0x1 actions=mod_dl_dst:52:54:00:c1:68:70
cookie=0x0, table=66, priority=100,reg0=0,reg1=0,reg2=0,reg3=0,reg15=0x2,metadata=0x1 actions=mod_dl_dst:00:00:00:00:00:00
cookie=0x0, table=66, priority=100,reg0=0,reg1=0,reg2=0,reg3=0,reg15=0x1,metadata=0x1 actions=mod_dl_dst:00:00:00:00:00:00
寄存器意义
|
寄存器 |
功能 |
详解 |
|
metadata |
作为vni使用 |
是ovn的Logical Datapath Field，命令ovn-sbctl list Datapath_Binding查看tunnel_key，封装到geneve或者stt中 |
|
reg14 |
记录逻辑入端口 |
是ovn的Logical InputPort Field，命令ovn-sbctl list Port_Binding查看tunnel_key，封装到geneve或者stt中 |
|
reg15 |
记录逻辑出端口 |
是ovn的Logical OutputPort Field，命令ovn-sbctl list Port_Binding查看tunnel_key，封装到geneve或者stt中 |
|
reg13 |
逻辑端口的conntrack zone |
chassis内部有用，出了chassis无用 |
|
reg12 |
SNAT的conntrack zone |
也是chassis内部使用 |
|
reg11 |
DNAT的conntrack zone |
也是chassis内部使用 |
|
reg10 |
逻辑流表标志 |
可能是逻辑流表中的flags.loopback之类的标志 |
# ovs-ofctl dump-flows br-int
cookie=0x0, duration=7415.683s, table=0, n_packets=7599, n_bytes=735898, priority=100,in_port=vm21 actions=load:0x6->NXM_NX_REG13[],load:0x5->NXM_NX_REG11[],load:0x2->NXM_NX_REG12[],load:0x3->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],resubmit(,8)
cookie=0x0, duration=7415.684s, table=0, n_packets=156585, n_bytes=9893858, priority=100,in_port="patch-br-int-to",vlan_tci=0x0000/0x1000 actions=load:0x8->NXM_NX_REG13[],load:0x1->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],load:0x7->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],resubmit(,8)
cookie=0x0, duration=7415.683s, table=0, n_packets=0, n_bytes=0, priority=100,in_port="patch-br-int-to",dl_vlan=0 actions=strip_vlan,load:0x8->NXM_NX_REG13[],load:0x1->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],load:0x7->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],resubmit(,8)
cookie=0xded39c69, duration=7415.685s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x7,vlan_tci=0x1000/0x1000 actions=drop
cookie=0x1b7ef5c3, duration=7415.684s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x5,vlan_tci=0x1000/0x1000 actions=drop
cookie=0x4433dfa7, duration=7415.683s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x3,vlan_tci=0x1000/0x1000 actions=drop
cookie=0xc08e126a, duration=7415.685s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x3,dl_src=01:00:00:00:00:00/01:00:00:00:00:00 actions=drop
cookie=0x1b7ef5c3, duration=7415.684s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x5,dl_src=01:00:00:00:00:00/01:00:00:00:00:00 actions=drop
cookie=0x299365e2, duration=7415.683s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x7,dl_src=01:00:00:00:00:00/01:00:00:00:00:00 actions=drop
cookie=0x23f5facd, duration=7415.685s, table=8, n_packets=7584, n_bytes=733768, priority=50,reg14=0x5,metadata=0x5,dl_dst=02:d4:1d:8c:d9:af actions=resubmit(,9)
cookie=0x754b0851, duration=7415.683s, table=8, n_packets=0, n_bytes=0, priority=50,reg14=0x2,metadata=0x5,dl_dst=02:d4:1d:8c:d9:be actions=resubmit(,9)
cookie=0x36ac109e, duration=7415.685s, table=8, n_packets=7415, n_bytes=726670, priority=50,reg14=0x3,metadata=0x7 actions=resubmit(,9)
cookie=0x27036c09, duration=7415.683s, table=8, n_packets=156585, n_bytes=9893858, priority=50,reg14=0x2,metadata=0x7 actions=resubmit(,9)
cookie=0x737e698b, duration=7415.683s, table=8, n_packets=169, n_bytes=7098, priority=50,reg14=0x3,metadata=0x3 actions=resubmit(,9)
cookie=0xdc12a385, duration=7415.684s, table=8, n_packets=0, n_bytes=0, priority=50,reg14=0x5,metadata=0x5,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,9)
cookie=0x5cc8cc08, duration=7415.683s, table=8, n_packets=151968, n_bytes=9382386, priority=50,reg14=0x2,metadata=0x5,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,9)
cookie=0xa1f465ca, duration=7415.683s, table=8, n_packets=7599, n_bytes=735898, priority=50,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d actions=resubmit(,9)
cookie=0x5d8a6979, duration=7415.685s, table=9, n_packets=103, n_bytes=35226, priority=100,ip,metadata=0x5,nw_src=0.0.0.0/8 actions=drop
cookie=0x5d8a6979, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=100,ip,metadata=0x5,nw_src=127.0.0.0/8 actions=drop
cookie=0x5d8a6979, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=100,ip,metadata=0x5,nw_dst=0.0.0.0/8 actions=drop
cookie=0x5d8a6979, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=100,ip,metadata=0x5,nw_dst=127.0.0.0/8 actions=drop
cookie=0x4c80ad5c, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=100,ip,reg9=0/0x2,metadata=0x5,nw_src=192.168.233.177 actions=drop
cookie=0x863aa1e1, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=100,ip,reg9=0/0x2,metadata=0x5,nw_src=20.0.0.1 actions=drop
cookie=0x4c80ad5c, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=100,ip,reg9=0/0x2,metadata=0x5,nw_src=192.168.233.255 actions=drop
cookie=0x863aa1e1, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=100,ip,reg9=0/0x2,metadata=0x5,nw_src=20.0.0.255 actions=drop
cookie=0x5d8a6979, duration=7415.685s, table=9, n_packets=108412, n_bytes=6497220, priority=100,ip,metadata=0x5,nw_dst=224.0.0.0/4 actions=drop
cookie=0x5d8a6979, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=100,ip,metadata=0x5,nw_src=255.255.255.255 actions=drop
cookie=0x98e795ee, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=100,ipv6,metadata=0x5,ipv6_src=fe80::d4:1dff:fe8c:d9be actions=drop
cookie=0xc7e185f8, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=100,ipv6,metadata=0x5,ipv6_src=fe80::d4:1dff:fe8c:d9af actions=drop
cookie=0x944a74e0, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,icmp,metadata=0x5,nw_dst=20.0.0.1,icmp_type=8,icmp_code=0 actions=push:NXM_OF_IP_SRC[],push:NXM_OF_IP_DST[],pop:NXM_OF_IP_SRC[],pop:NXM_OF_IP_DST[],load:0xff->NXM_NX_IP_TTL[],load:0->NXM_OF_ICMP_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,10)
cookie=0xde91ac7f, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=90,icmp,metadata=0x5,nw_dst=192.168.233.177,icmp_type=8,icmp_code=0 actions=push:NXM_OF_IP_SRC[],push:NXM_OF_IP_DST[],pop:NXM_OF_IP_SRC[],pop:NXM_OF_IP_DST[],load:0xff->NXM_NX_IP_TTL[],load:0->NXM_OF_ICMP_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,10)
cookie=0x77f75a25, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,reg14=0x5,metadata=0x5,ipv6_dst=ff02::1:ff8c:d9af,nw_ttl=255,icmp_type=135,icmp_code=0,nd_target=fe80::d4:1dff:fe8c:d9af actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_SLL[],push:NXM_NX_IPV6_SRC[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[],controller(userdata=00.00.00.03.00.00.00.00.00.19.00.10.80.00.08.06.02.d4.1d.8c.d9.af.00.00.00.19.00.18.80.00.34.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.af.00.19.00.18.80.00.3e.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.af.00.19.00.10.80.00.42.06.02.d4.1d.8c.d9.af.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.00.00.00.00.01.1c.04.00.01.1e.04.ff.ff.00.18.00.00.23.20.00.07.00.00.00.01.14.04.00.00.00.00.00.00.00.01.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0x77f75a25, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,reg14=0x5,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9af,nw_ttl=255,icmp_type=135,icmp_code=0,nd_target=fe80::d4:1dff:fe8c:d9af actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_SLL[],push:NXM_NX_IPV6_SRC[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[],controller(userdata=00.00.00.03.00.00.00.00.00.19.00.10.80.00.08.06.02.d4.1d.8c.d9.af.00.00.00.19.00.18.80.00.34.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.af.00.19.00.18.80.00.3e.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.af.00.19.00.10.80.00.42.06.02.d4.1d.8c.d9.af.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.00.00.00.00.01.1c.04.00.01.1e.04.ff.ff.00.18.00.00.23.20.00.07.00.00.00.01.14.04.00.00.00.00.00.00.00.01.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0x7542fd0b, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,reg14=0x2,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9be,nw_ttl=255,icmp_type=135,icmp_code=0,nd_target=fe80::d4:1dff:fe8c:d9be actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_SLL[],push:NXM_NX_IPV6_SRC[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[],controller(userdata=00.00.00.03.00.00.00.00.00.19.00.10.80.00.08.06.02.d4.1d.8c.d9.be.00.00.00.19.00.18.80.00.34.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.be.00.19.00.18.80.00.3e.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.be.00.19.00.10.80.00.42.06.02.d4.1d.8c.d9.be.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.00.00.00.00.01.1c.04.00.01.1e.04.ff.ff.00.18.00.00.23.20.00.07.00.00.00.01.14.04.00.00.00.00.00.00.00.01.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0x7542fd0b, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,reg14=0x2,metadata=0x5,ipv6_dst=ff02::1:ff8c:d9be,nw_ttl=255,icmp_type=135,icmp_code=0,nd_target=fe80::d4:1dff:fe8c:d9be actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_SLL[],push:NXM_NX_IPV6_SRC[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[],controller(userdata=00.00.00.03.00.00.00.00.00.19.00.10.80.00.08.06.02.d4.1d.8c.d9.be.00.00.00.19.00.18.80.00.34.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.be.00.19.00.18.80.00.3e.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.be.00.19.00.10.80.00.42.06.02.d4.1d.8c.d9.be.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.00.00.00.00.01.1c.04.00.01.1e.04.ff.ff.00.18.00.00.23.20.00.07.00.00.00.01.14.04.00.00.00.00.00.00.00.01.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0xf0a998ea, duration=7415.686s, table=9, n_packets=0, n_bytes=0, priority=90,arp,reg14=0x2,metadata=0x5,arp_tpa=192.168.233.177,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:02:d4:1d:8c:d9:be,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x2d41d8cd9be->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xc0a8e9b1->NXM_OF_ARP_SPA[],load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
cookie=0xbc00a65b, duration=7415.685s, table=9, n_packets=169, n_bytes=7098, priority=90,arp,reg14=0x5,metadata=0x5,arp_tpa=20.0.0.1,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:02:d4:1d:8c:d9:af,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x2d41d8cd9af->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0x14000001->NXM_OF_ARP_SPA[],load:0x5->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
cookie=0x1f495ec9, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9af,icmp_type=128,icmp_code=0 actions=push:NXM_NX_IPV6_SRC[],push:NXM_NX_IPV6_DST[],pop:NXM_NX_IPV6_SRC[],pop:NXM_NX_IPV6_DST[],load:0xff->NXM_NX_IP_TTL[],load:0x81->NXM_NX_ICMPV6_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,10)
cookie=0x89f21ed8, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9be,icmp_type=128,icmp_code=0 actions=push:NXM_NX_IPV6_SRC[],push:NXM_NX_IPV6_DST[],pop:NXM_NX_IPV6_SRC[],pop:NXM_NX_IPV6_DST[],load:0xff->NXM_NX_IP_TTL[],load:0x81->NXM_NX_ICMPV6_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,10)
cookie=0x40c4377d, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,arp,metadata=0x5,arp_op=2 actions=push:NXM_NX_REG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ARP_SHA[],push:NXM_OF_ARP_SPA[],pop:NXM_NX_REG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.01.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_REG0[]
cookie=0x6636d7dd, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,udp,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,nw_src=0.0.0.0,nw_dst=255.255.255.255,tp_src=68,tp_dst=67 actions=resubmit(,10)
cookie=0x8095971d, duration=7415.684s, table=9, n_packets=7420, n_bytes=728380, priority=90,ip,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,nw_src=20.0.0.10 actions=resubmit(,10)
cookie=0xf90b1a37, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,metadata=0x5,nw_ttl=255,icmp_type=136,icmp_code=0 actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_TLL[],push:NXM_NX_ND_TARGET[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[]
cookie=0x89fa3891, duration=7415.684s, table=9, n_packets=194, n_bytes=16684, priority=80,icmp6,metadata=0x5,nw_ttl=255,icmp_type=135,icmp_code=0 actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_SLL[],push:NXM_NX_IPV6_SRC[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[]
cookie=0x612ff906, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=80,ip,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d actions=drop
cookie=0x612ff906, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=80,ipv6,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d actions=drop
cookie=0xb7bd9e1d, duration=7415.686s, table=9, n_packets=0, n_bytes=0, priority=60,ipv6,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9af actions=drop
cookie=0x52b8a0bd, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=60,ipv6,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9be actions=drop
cookie=0x28d382fa, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=60,ip,metadata=0x5,nw_dst=20.0.0.1 actions=drop
cookie=0xf16a2d3e, duration=7415.686s, table=9, n_packets=39348, n_bytes=2364630, priority=50,metadata=0x5,dl_dst=ff:ff:ff:ff:ff:ff actions=drop
cookie=0xedb0015b, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=30,ip,metadata=0x5,nw_ttl=1 actions=drop
cookie=0xedb0015b, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=30,ip,metadata=0x5,nw_ttl=0 actions=drop
cookie=0x5557ea52, duration=7415.686s, table=9, n_packets=348, n_bytes=14616, priority=0,metadata=0x3 actions=resubmit(,10)
cookie=0x2b40cc48, duration=7415.685s, table=9, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,10)
cookie=0x376d13ff, duration=7415.685s, table=9, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,10)
cookie=0xb57f57f7, duration=7415.686s, table=10, n_packets=169, n_bytes=7098, priority=90,arp,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,arp_spa=20.0.0.10,arp_sha=02:d4:1d:8c:d9:9d actions=resubmit(,11)
cookie=0xb34d78e8, duration=7415.685s, table=10, n_packets=10, n_bytes=420, priority=80,arp,reg14=0x2,metadata=0x3 actions=drop
cookie=0xb34d78e8, duration=7415.684s, table=10, n_packets=0, n_bytes=0, priority=80,icmp6,reg14=0x2,metadata=0x3,nw_ttl=255,icmp_type=135,icmp_code=0 actions=drop
cookie=0xb34d78e8, duration=7415.684s, table=10, n_packets=0, n_bytes=0, priority=80,icmp6,reg14=0x2,metadata=0x3,nw_ttl=255,icmp_type=136,icmp_code=0 actions=drop
cookie=0x5ab65a62, duration=7415.686s, table=10, n_packets=7589, n_bytes=735478, priority=0,metadata=0x3 actions=resubmit(,11)
cookie=0x30ab50cf, duration=7415.685s, table=10, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,11)
cookie=0x5efd761e, duration=7415.685s, table=10, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,11)
cookie=0xae6dae29, duration=7415.684s, table=11, n_packets=0, n_bytes=0, priority=50,ip,metadata=0x5,nw_dst=192.168.233.177 actions=load:0x1->OXM_OF_PKT_REG4[0],resubmit(,12)
cookie=0x93f6b9c, duration=7415.685s, table=11, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,12)
cookie=0xa99bdefc, duration=7415.685s, table=11, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,12)
cookie=0xd5587e18, duration=7415.684s, table=11, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,12)
cookie=0x6b45a8de, duration=7415.685s, table=12, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,13)
cookie=0x91478c72, duration=7415.685s, table=12, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,13)
cookie=0xa9e8bb31, duration=7415.685s, table=12, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,13)
cookie=0xaae4d01f, duration=7415.686s, table=13, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,14)
cookie=0x32f0ba0b, duration=7415.686s, table=13, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,14)
cookie=0xa02c3ecf, duration=7415.684s, table=13, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,14)
cookie=0x5b9d743a, duration=7415.686s, table=14, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,15)
cookie=0xcf2678fc, duration=7415.684s, table=14, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,15)
cookie=0x9a0b181d, duration=7415.684s, table=14, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,15)
cookie=0x8c7fbe9f, duration=7415.686s, table=15, n_packets=0, n_bytes=0, priority=300,ipv6,reg9=0x1/0x1,metadata=0x5 actions=dec_ttl(),resubmit(,16)
cookie=0x8c7fbe9f, duration=7415.685s, table=15, n_packets=0, n_bytes=0, priority=300,ip,reg9=0x1/0x1,metadata=0x5 actions=dec_ttl(),resubmit(,16)
cookie=0xa5be5a62, duration=7415.686s, table=15, n_packets=0, n_bytes=0, priority=129,ipv6,reg14=0x5,metadata=0x5,ipv6_dst=fe80::/64 actions=dec_ttl(),move:NXM_NX_IPV6_DST[]->NXM_NX_XXREG0[],load:0xd41dfffe8cd9af->NXM_NX_XXREG1[0..63],load:0xfe80000000000000->NXM_NX_XXREG1[64..127],mod_dl_src:02:d4:1d:8c:d9:af,load:0x5->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,16)
cookie=0x23104177, duration=7415.685s, table=15, n_packets=0, n_bytes=0, priority=129,ipv6,reg14=0x2,metadata=0x5,ipv6_dst=fe80::/64 actions=dec_ttl(),move:NXM_NX_IPV6_DST[]->NXM_NX_XXREG0[],load:0xd41dfffe8cd9be->NXM_NX_XXREG1[0..63],load:0xfe80000000000000->NXM_NX_XXREG1[64..127],mod_dl_src:02:d4:1d:8c:d9:be,load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,16)
cookie=0xf97c81e7, duration=7415.686s, table=15, n_packets=7415, n_bytes=726670, priority=49,ip,metadata=0x5,nw_dst=192.168.233.0/24 actions=dec_ttl(),move:NXM_OF_IP_DST[]->NXM_NX_XXREG0[96..127],load:0xc0a8e9b1->NXM_NX_XXREG0[64..95],mod_dl_src:02:d4:1d:8c:d9:be,load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,16)
cookie=0x1526dfe6, duration=7415.686s, table=15, n_packets=0, n_bytes=0, priority=49,ip,metadata=0x5,nw_dst=20.0.0.0/24 actions=dec_ttl(),move:NXM_OF_IP_DST[]->NXM_NX_XXREG0[96..127],load:0x14000001->NXM_NX_XXREG0[64..95],mod_dl_src:02:d4:1d:8c:d9:af,load:0x5->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,16)
cookie=0x1479186f, duration=7415.684s, table=15, n_packets=0, n_bytes=0, priority=1,ip,metadata=0x5 actions=dec_ttl(),load:0xc0a8e9b1->NXM_NX_XXREG0[96..127],load:0xc0a8e9b1->NXM_NX_XXREG0[64..95],mod_dl_src:02:d4:1d:8c:d9:be,load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,16)
cookie=0xff85f9d6, duration=7415.685s, table=15, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,16)
cookie=0x766070a, duration=7415.685s, table=15, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,16)
cookie=0xf2f4b9da, duration=7415.685s, table=16, n_packets=0, n_bytes=0, priority=200,reg9=0x1/0x1,metadata=0x5 actions=mod_dl_dst:02:d4:1d:8c:d9:be,resubmit(,17)
cookie=0xa12b693, duration=7415.684s, table=16, n_packets=0, n_bytes=0, priority=100,reg0=0x1400000a,reg15=0x5,metadata=0x5 actions=mod_dl_dst:02:d4:1d:8c:d9:9d,resubmit(,17)
cookie=0xdd6bdca0, duration=7415.685s, table=16, n_packets=0, n_bytes=0, priority=0,ipv6,metadata=0x5 actions=mod_dl_dst:00:00:00:00:00:00,resubmit(,66),resubmit(,17)
cookie=0x9e7a5a88, duration=7415.684s, table=16, n_packets=7415, n_bytes=726670, priority=0,ip,metadata=0x5 actions=push:NXM_NX_REG0[],push:NXM_NX_XXREG0[96..127],pop:NXM_NX_REG0[],mod_dl_dst:00:00:00:00:00:00,resubmit(,66),pop:NXM_NX_REG0[],resubmit(,17)
cookie=0xdf9fa20, duration=7415.685s, table=16, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,17)
cookie=0x853ab9e6, duration=7415.684s, table=16, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,17)
cookie=0x5d82efa, duration=7415.685s, table=17, n_packets=0, n_bytes=0, priority=200,reg9=0x1/0x1,metadata=0x5 actions=load:0x6->NXM_NX_REG15[],resubmit(,18)
cookie=0x8832e20e, duration=7415.686s, table=17, n_packets=0, n_bytes=0, priority=150,reg15=0x2,metadata=0x5,dl_dst=00:00:00:00:00:00 actions=load:0x6->NXM_NX_REG15[],resubmit(,18)
cookie=0x65331fc0, duration=7415.685s, table=17, n_packets=7415, n_bytes=726670, priority=50,reg15=0x2,metadata=0x5 actions=load:0x6->NXM_NX_REG15[],resubmit(,18)
cookie=0x617dd0d3, duration=7415.686s, table=17, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,18)
cookie=0xa8c020ff, duration=7415.685s, table=17, n_packets=0, n_bytes=0, priority=0,metadata=0x5 actions=resubmit(,18)
cookie=0xbe7a33b7, duration=7415.684s, table=17, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,18)
cookie=0xb0a974d1, duration=7415.685s, table=18, n_packets=0, n_bytes=0, priority=100,ipv6,metadata=0x5,dl_dst=00:00:00:00:00:00 actions=controller(userdata=00.00.00.09.00.00.00.00.ff.ff.00.18.00.00.23.20.00.06.00.80.00.00.00.00.00.01.de.10.00.01.2e.10.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0x947d2134, duration=7415.684s, table=18, n_packets=0, n_bytes=0, priority=100,ip,metadata=0x5,dl_dst=00:00:00:00:00:00 actions=controller(userdata=00.00.00.00.00.00.00.00.00.19.00.10.80.00.06.06.ff.ff.ff.ff.ff.ff.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.40.00.00.00.01.de.10.00.00.20.04.ff.ff.00.18.00.00.23.20.00.06.00.20.00.60.00.00.00.01.de.10.00.00.22.04.00.19.00.10.80.00.2a.02.00.01.00.00.00.00.00.00.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0xe7029d42, duration=7415.685s, table=18, n_packets=7415, n_bytes=726670, priority=0,metadata=0x5 actions=resubmit(,32)
cookie=0x354130d2, duration=7415.684s, table=18, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,19)
cookie=0xa0e9ffc, duration=7415.684s, table=18, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,19)
cookie=0x160901ae, duration=7415.684s, table=19, n_packets=156585, n_bytes=9893858, priority=100,reg14=0x2,metadata=0x7 actions=resubmit(,20)
cookie=0x6f016248, duration=7415.684s, table=19, n_packets=0, n_bytes=0, priority=100,arp,reg14=0x2,metadata=0x3,arp_tpa=20.0.0.10,arp_op=1 actions=resubmit(,20)
cookie=0x1c76bc17, duration=7415.685s, table=19, n_packets=0, n_bytes=0, priority=50,arp,metadata=0x3,arp_tpa=20.0.0.10,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:02:d4:1d:8c:d9:9d,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x2d41d8cd99d->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0x1400000a->NXM_OF_ARP_SPA[],move:NXM_NX_REG14[]->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
cookie=0x5a75b5c5, duration=7415.686s, table=19, n_packets=7415, n_bytes=726670, priority=0,metadata=0x7 actions=resubmit(,20)
cookie=0x25090509, duration=7415.684s, table=19, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,20)
cookie=0x43536dbb, duration=7415.685s, table=20, n_packets=0, n_bytes=0, priority=100,udp,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,nw_src=20.0.0.10,nw_dst=255.255.255.255,tp_src=68,tp_dst=67 actions=controller(userdata=00.00.00.02.00.00.00.00.00.01.de.10.00.00.00.63.14.00.00.0a.33.04.00.00.0e.10.01.04.ff.ff.ff.00.03.04.14.00.00.01.36.04.14.00.00.01,pause),resubmit(,21)
cookie=0x43536dbb, duration=7415.685s, table=20, n_packets=5, n_bytes=1710, priority=100,udp,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,nw_src=20.0.0.10,nw_dst=20.0.0.1,tp_src=68,tp_dst=67 actions=controller(userdata=00.00.00.02.00.00.00.00.00.01.de.10.00.00.00.63.14.00.00.0a.33.04.00.00.0e.10.01.04.ff.ff.ff.00.03.04.14.00.00.01.36.04.14.00.00.01,pause),resubmit(,21)
cookie=0x4f868dc8, duration=7415.684s, table=20, n_packets=0, n_bytes=0, priority=100,udp,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,nw_src=0.0.0.0,nw_dst=255.255.255.255,tp_src=68,tp_dst=67 actions=controller(userdata=00.00.00.02.00.00.00.00.00.01.de.10.00.00.00.63.14.00.00.0a.33.04.00.00.0e.10.01.04.ff.ff.ff.00.03.04.14.00.00.01.36.04.14.00.00.01,pause),resubmit(,21)
cookie=0x6829477e, duration=7415.684s, table=20, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,21)
cookie=0x9c47b2ee, duration=7415.684s, table=20, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,21)
cookie=0xfec4dff5, duration=7415.685s, table=21, n_packets=0, n_bytes=0, priority=100,udp,reg0=0x8/0x8,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,tp_src=68,tp_dst=67 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:02:d4:1d:8c:d9:9f,mod_nw_dst:20.0.0.10,mod_nw_src:20.0.0.1,mod_tp_src:67,mod_tp_dst:68,move:NXM_NX_REG14[]->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
cookie=0x883f0b8e, duration=7415.685s, table=21, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,22)
cookie=0xba05df68, duration=7415.684s, table=21, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,22)
cookie=0x2b1cd3f2, duration=7415.686s, table=22, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,23)
cookie=0x518bc984, duration=7415.685s, table=22, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,23)
cookie=0x1416a4cf, duration=7415.685s, table=23, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,24)
cookie=0x2545137, duration=7415.684s, table=23, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,24)
cookie=0x24578bfb, duration=7415.685s, table=24, n_packets=0, n_bytes=0, priority=100,metadata=0x3,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=load:0xffff->NXM_NX_REG15[],resubmit(,32)
cookie=0x40ea8458, duration=7415.684s, table=24, n_packets=151968, n_bytes=9382386, priority=100,metadata=0x7,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=load:0xffff->NXM_NX_REG15[],resubmit(,32)
cookie=0xf1103160, duration=7415.686s, table=24, n_packets=7584, n_bytes=733768, priority=50,metadata=0x3,dl_dst=02:d4:1d:8c:d9:af actions=load:0x3->NXM_NX_REG15[],resubmit(,32)
cookie=0x3efaae35, duration=7415.684s, table=24, n_packets=169, n_bytes=7098, priority=50,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d actions=load:0x2->NXM_NX_REG15[],resubmit(,32)
cookie=0xdf7b6600, duration=7415.684s, table=24, n_packets=0, n_bytes=0, priority=50,metadata=0x7,dl_dst=02:d4:1d:8c:d9:be actions=load:0x3->NXM_NX_REG15[],resubmit(,32)
cookie=0x711af6a2, duration=7415.685s, table=24, n_packets=12032, n_bytes=1238142, priority=0,metadata=0x7 actions=load:0xfffe->NXM_NX_REG15[],resubmit(,32)
cookie=0x0, duration=7415.685s, table=32, n_packets=0, n_bytes=0, priority=150,reg10=0x10/0x10,reg15=0xffff,metadata=0x7 actions=resubmit(,33)
cookie=0x0, duration=7415.685s, table=32, n_packets=0, n_bytes=0, priority=150,reg10=0x10/0x10,reg15=0xffff,metadata=0x3 actions=resubmit(,33)
cookie=0x0, duration=7415.684s, table=32, n_packets=0, n_bytes=0, priority=150,reg10=0x10/0x10,reg15=0xfffe,metadata=0x7 actions=resubmit(,33)
cookie=0x0, duration=7415.684s, table=32, n_packets=0, n_bytes=0, priority=150,reg10=0x2/0x2 actions=resubmit(,33)
cookie=0x0, duration=7415.685s, table=32, n_packets=151968, n_bytes=9382386, priority=100,reg15=0xffff,metadata=0x7 actions=load:0x3->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[],resubmit(,33)
cookie=0x0, duration=7415.684s, table=32, n_packets=0, n_bytes=0, priority=100,reg15=0xffff,metadata=0x3 actions=load:0x3->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[],resubmit(,33)
cookie=0x0, duration=7415.686s, table=32, n_packets=27369, n_bytes=2712776, priority=0 actions=resubmit(,33)
cookie=0x0, duration=7415.686s, table=33, n_packets=7584, n_bytes=733768, priority=100,reg15=0x3,metadata=0x3 actions=load:0x5->NXM_NX_REG11[],load:0x2->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.686s, table=33, n_packets=151968, n_bytes=9382386, priority=100,reg15=0xffff,metadata=0x7 actions=load:0x8->NXM_NX_REG13[],load:0x2->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[]
cookie=0x0, duration=7415.685s, table=33, n_packets=0, n_bytes=0, priority=100,reg15=0x2,metadata=0x7 actions=load:0x8->NXM_NX_REG13[],load:0x1->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.685s, table=33, n_packets=0, n_bytes=0, priority=100,reg15=0x2,metadata=0x5 actions=load:0x3->NXM_NX_REG11[],load:0x7->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.685s, table=33, n_packets=12032, n_bytes=1238142, priority=100,reg15=0xfffe,metadata=0x7 actions=load:0x8->NXM_NX_REG13[],load:0x2->NXM_NX_REG15[],resubmit(,34),load:0xfffe->NXM_NX_REG15[]
cookie=0x0, duration=7415.685s, table=33, n_packets=0, n_bytes=0, priority=100,reg15=0xffff,metadata=0x3 actions=load:0x6->NXM_NX_REG13[],load:0x2->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[]
cookie=0x0, duration=7415.685s, table=33, n_packets=7415, n_bytes=726670, priority=100,reg15=0x6,metadata=0x5 actions=load:0x2->NXM_NX_REG15[],load:0x3->NXM_NX_REG11[],load:0x7->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.684s, table=33, n_packets=169, n_bytes=7098, priority=100,reg15=0x2,metadata=0x3 actions=load:0x6->NXM_NX_REG13[],load:0x5->NXM_NX_REG11[],load:0x2->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.684s, table=33, n_packets=169, n_bytes=7098, priority=100,reg15=0x5,metadata=0x5 actions=load:0x3->NXM_NX_REG11[],load:0x7->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.684s, table=33, n_packets=0, n_bytes=0, priority=100,reg15=0x3,metadata=0x7 actions=load:0x1->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.686s, table=34, n_packets=0, n_bytes=0, priority=100,reg10=0/0x1,reg14=0x3,reg15=0x3,metadata=0x7 actions=drop
cookie=0x0, duration=7415.686s, table=34, n_packets=156585, n_bytes=9893858, priority=100,reg10=0/0x1,reg14=0x2,reg15=0x2,metadata=0x7 actions=drop
cookie=0x0, duration=7415.685s, table=34, n_packets=0, n_bytes=0, priority=100,reg10=0/0x1,reg14=0x2,reg15=0x2,metadata=0x5 actions=drop
cookie=0x0, duration=7415.685s, table=34, n_packets=0, n_bytes=0, priority=100,reg10=0/0x1,reg14=0x2,reg15=0x2,metadata=0x3 actions=drop
cookie=0x0, duration=7415.685s, table=34, n_packets=0, n_bytes=0, priority=100,reg10=0/0x1,reg14=0x3,reg15=0x3,metadata=0x3 actions=drop
cookie=0x0, duration=7415.684s, table=34, n_packets=0, n_bytes=0, priority=100,reg10=0/0x1,reg14=0x5,reg15=0x5,metadata=0x5 actions=drop
cookie=0x0, duration=7415.686s, table=34, n_packets=174720, n_bytes=11583690, priority=0 actions=load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],resubmit(,40)
cookie=0xab47d381, duration=7415.686s, table=40, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,41)
cookie=0xe30c8cbc, duration=7415.685s, table=40, n_packets=7584, n_bytes=733768, priority=0,metadata=0x5 actions=resubmit(,41)
cookie=0x64b884b4, duration=7415.685s, table=40, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,41)
cookie=0xf2d36f82, duration=7415.686s, table=41, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,42)
cookie=0xbb7b8f92, duration=7415.685s, table=41, n_packets=7584, n_bytes=733768, priority=0,metadata=0x5 actions=resubmit(,42)
cookie=0xafac9b88, duration=7415.684s, table=41, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,42)
cookie=0xd6d23b6d, duration=7415.684s, table=42, n_packets=0, n_bytes=0, priority=100,ip,reg15=0x2,metadata=0x5,nw_dst=192.168.233.177 actions=clone(ct_clear,move:NXM_NX_REG15[]->NXM_NX_REG14[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG10[],load:0x1->NXM_NX_REG10[0],load:0->NXM_NX_XXREG0[96..127],load:0->NXM_NX_XXREG0[64..95],load:0->NXM_NX_XXREG0[32..63],load:0->NXM_NX_XXREG0[0..31],load:0->NXM_NX_XXREG1[96..127],load:0->NXM_NX_XXREG1[64..95],load:0->NXM_NX_XXREG1[32..63],load:0->NXM_NX_XXREG1[0..31],load:0->OXM_OF_PKT_REG4[32..63],load:0->OXM_OF_PKT_REG4[0..31],load:0x1->OXM_OF_PKT_REG4[1],resubmit(,8))
cookie=0x3025b51b, duration=7415.686s, table=42, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,43)
cookie=0x8ef5f82b, duration=7415.685s, table=42, n_packets=7584, n_bytes=733768, priority=0,metadata=0x5 actions=resubmit(,43)
cookie=0xab15d779, duration=7415.684s, table=42, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,43)
cookie=0x4deb265c, duration=7415.685s, table=43, n_packets=169, n_bytes=7098, priority=100,reg15=0x5,metadata=0x5 actions=resubmit(,64)
cookie=0xd0d02e2e, duration=7415.684s, table=43, n_packets=7415, n_bytes=726670, priority=100,reg15=0x2,metadata=0x5 actions=resubmit(,64)
cookie=0x8a5f0c7c, duration=7415.686s, table=43, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,44)
cookie=0xabb769a7, duration=7415.684s, table=43, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,44)
cookie=0x7745fee3, duration=7415.684s, table=44, n_packets=0, n_bytes=0, priority=34000,udp,reg15=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9f,nw_src=20.0.0.1,tp_src=67,tp_dst=68 actions=resubmit(,45)
cookie=0xa677c8f, duration=7415.686s, table=44, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,45)
cookie=0xf9a88ae5, duration=7415.684s, table=44, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,45)
cookie=0x7cc31a30, duration=7415.686s, table=45, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,46)
cookie=0xe485076a, duration=7415.684s, table=45, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,46)
cookie=0xa2db634c, duration=7415.686s, table=46, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,47)
cookie=0x2eb88901, duration=7415.684s, table=46, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,47)
cookie=0xa47461af, duration=7415.685s, table=47, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,48)
cookie=0xf9838769, duration=7415.685s, table=47, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,48)
cookie=0x8b1f4a36, duration=7415.685s, table=48, n_packets=0, n_bytes=0, priority=90,ip,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d,nw_dst=255.255.255.255 actions=resubmit(,49)
cookie=0x8b1f4a36, duration=7415.684s, table=48, n_packets=0, n_bytes=0, priority=90,ip,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d,nw_dst=20.0.0.10 actions=resubmit(,49)
cookie=0x8b1f4a36, duration=7415.684s, table=48, n_packets=0, n_bytes=0, priority=90,ip,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d,nw_dst=224.0.0.0/4 actions=resubmit(,49)
cookie=0x6335fd40, duration=7415.685s, table=48, n_packets=0, n_bytes=0, priority=80,ipv6,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d actions=drop
cookie=0x6335fd40, duration=7415.684s, table=48, n_packets=0, n_bytes=0, priority=80,ip,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d actions=drop
cookie=0x1e836ae8, duration=7415.685s, table=48, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,49)
cookie=0x3106bcc, duration=7415.684s, table=48, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,49)
cookie=0xd0a2471c, duration=7415.684s, table=49, n_packets=0, n_bytes=0, priority=100,metadata=0x3,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,64)
cookie=0x22816064, duration=7415.684s, table=49, n_packets=151968, n_bytes=9382386, priority=100,metadata=0x7,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,64)
cookie=0x23119cfc, duration=7415.685s, table=49, n_packets=7415, n_bytes=726670, priority=50,reg15=0x2,metadata=0x7 actions=resubmit(,64)
cookie=0x87806516, duration=7415.684s, table=49, n_packets=7584, n_bytes=733768, priority=50,reg15=0x3,metadata=0x3 actions=resubmit(,64)
cookie=0xf9e7a385, duration=7415.684s, table=49, n_packets=0, n_bytes=0, priority=50,reg15=0x3,metadata=0x7 actions=resubmit(,64)
cookie=0x5555f23f, duration=7415.684s, table=49, n_packets=169, n_bytes=7098, priority=50,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d actions=resubmit(,64)
cookie=0x0, duration=7415.686s, table=64, n_packets=0, n_bytes=0, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x7 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.686s, table=64, n_packets=169, n_bytes=7098, priority=100,reg10=0x1/0x1,reg15=0x5,metadata=0x5 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.685s, table=64, n_packets=0, n_bytes=0, priority=100,reg10=0x1/0x1,reg15=0x3,metadata=0x3 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.685s, table=64, n_packets=0, n_bytes=0, priority=100,reg10=0x1/0x1,reg15=0x3,metadata=0x7 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.684s, table=64, n_packets=7415, n_bytes=726670, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x5 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.684s, table=64, n_packets=0, n_bytes=0, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x3 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.684s, table=64, n_packets=167136, n_bytes=10849922, priority=0 actions=resubmit(,65)
cookie=0x0, duration=7415.686s, table=65, n_packets=169, n_bytes=7098, priority=100,reg15=0x5,metadata=0x5 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x5->NXM_NX_REG11[],load:0x2->NXM_NX_REG12[],load:0x3->OXM_OF_METADATA[],load:0x3->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,8))
cookie=0x0, duration=7415.685s, table=65, n_packets=169, n_bytes=7098, priority=100,reg15=0x2,metadata=0x3 actions=output:vm21
cookie=0x0, duration=7415.684s, table=65, n_packets=7415, n_bytes=726670, priority=100,reg15=0x2,metadata=0x7 actions=output:"patch-br-int-to"
cookie=0x0, duration=7415.684s, table=65, n_packets=7415, n_bytes=726670, priority=100,reg15=0x2,metadata=0x5 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x1->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],load:0x7->OXM_OF_METADATA[],load:0x3->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,8))
cookie=0x0, duration=7415.684s, table=65, n_packets=7584, n_bytes=733768, priority=100,reg15=0x3,metadata=0x3 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x3->NXM_NX_REG11[],load:0x7->NXM_NX_REG12[],load:0x5->OXM_OF_METADATA[],load:0x5->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,8))
cookie=0x0, duration=7415.684s, table=65, n_packets=151968, n_bytes=9382386, priority=100,reg15=0x3,metadata=0x7 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x3->NXM_NX_REG11[],load:0x7->NXM_NX_REG12[],load:0x5->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,8))
cookie=0x0, duration=7415.686s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xc0a8e9b2,reg15=0x2,metadata=0x5 actions=mod_dl_dst:c2:af:5a:9e:73:47
cookie=0x0, duration=7415.685s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xc0a80fc8,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:f9:91:7a
cookie=0x0, duration=7415.685s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xc0a8e901,reg15=0x2,metadata=0x5 actions=mod_dl_dst:e8:61:1f:15:2b:94
cookie=0x0, duration=7415.685s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xac1e0a07,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:2a:b9:c8
cookie=0x0, duration=7415.685s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xac1e0c07,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:f7:00:67
cookie=0x0, duration=7415.685s, table=66, n_packets=7415, n_bytes=726670, priority=100,reg0=0xc0a8e907,reg15=0x2,metadata=0x5 actions=mod_dl_dst:c2:af:5a:9e:73:47
cookie=0x0, duration=7415.684s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xac1e0d07,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:18:1f:b9
cookie=0x0, duration=7415.684s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xac1e0b07,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:d8:1d:25
cookie=0x0, duration=7415.684s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xc0a8e903,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:92:59:af
cookie=0x0, duration=7415.686s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0,reg1=0,reg2=0,reg3=0,reg15=0x2,metadata=0x5 actions=mod_dl_dst:00:00:00:00:00:00
cookie=0x0, duration=7415.686s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xfe800000,reg1=0,reg2=0x25056ff,reg3=0xfe883878,reg15=0x2,metadata=0x5 actions=mod_dl_dst:00:50:56:88:38:78
cookie=0x0, duration=7415.684s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xfe800000,reg1=0,reg2=0xf8163eff,reg3=0xfef9917a,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:f9:91:7a
cookie=0x0, duration=7415.684s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xfe800000,reg1=0,reg2=0xf8163eff,reg3=0xfeb00567,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:b0:05:67
OVN-LB
lb-add LB VIP[:PORT] IP[:PORT]... [PROTOCOL]
lb-del LB [VIP] remove a load-balancer or just the VIP from
lb-list [LB] print load-balancers
lr-lb-add ROUTER LB add a load-balancer to ROUTER
lr-lb-del ROUTER [LB] remove load-balancers from ROUTER
lr-lb-list ROUTER print load-balancers
ls-lb-add SWITCH LB add a load-balancer to SWITCH
ls-lb-del SWITCH [LB] remove load-balancers from SWITCH
ls-lb-list SWITCH print load-balancers
配置负载均衡器规则
定义我们的负载均衡规则，即VIP和后端服务器IP池。 这里涉及的是在OVN北向数据库中创建一个条目，并捕获生成的UUID。 在的这次实验中，我们将使用位于实验室“数据”网络中的VIP 10.127.0.254。 我们将使用vm1/vm2的地址作为池IP。
uuid=`ovn-nbctl create load_balancer vips:10.127.0.254="172.16.255.130,172.16.255.131"`
echo $uuid
上述命令在北向数据库的load_balancer表中创建一个条目，并将生成的UUID存储到变量“uuid”。 我们将在后面的命令中引用这个变量。
在网关路由器上配置负载均衡
在OVN网关路由器“edge1”上开启负载均衡器功能。
ovn-nbctl set logical_router edge1 load_balancer=$uuid
ovn-nbctl set logical_router edge1 load_balancer=$uuid
您可以通过检查edge1的数据库条目来验证是否成功开启负载均衡器功能。
ovn-nbctl get logical_router edge1 load_balancer
ovn-nbctl get logical_router edge1 load_balancer
现在，我们可以从任何Ubuntu主机的全局命名空间连接到VIP。
# curl 10.127.0.254:8000
i am vm2
测试多次之后，可以确认负载平衡是相当随机的。
让我们看看禁用一个Web服务器会发生什么。 尝试停止在vm1命名空间中运行的python进程。 这是我得到的输出结果：
# curl 10.127.0.254:8000
curl: (7) Failed to connect to 10.127.0.254 port 8000: Connection refused
# curl 10.127.0.254:8000
i am vm2
# curl 10.127.0.254:8000
curl: (7) Failed to connect to 10.127.0.254 port 8000: Connection refused
# curl 10.127.0.254:8000
i am vm2
负载均衡器未执行任何类型的运行状态检查。 目前的计划是，运行状态检查将由协调解决方案（如Kubernetes）执行，该功能将在未来某个时间点被加入。
在进行下一个测试之前，在vm1上重新启动python Web服务器。
负载均衡器在虚拟机外部运行着，让我们来看看从内部虚拟机访问VIP时会发生什么。
# ip netns exec vm3 curl 10.127.0.254:8000
i am vm1
# ip netns exec vm3 curl 10.127.0.254:8000
i am vm2
# ip netns exec vm3 curl 10.127.0.254:8000
i am vm1
# ip netns exec vm3 curl 10.127.0.254:8000
i am vm2
注意日志中的客户端IP地址。第一个IP是上一轮测试的ubuntu1。第二个IP是edge1（来自vm3的请求）。为什么请求来自edge1而不是直接来自vm3？答案是，实现负载平衡的OVN开发人员使用了一种称为“代理模式”的方法，其中负载均衡器在某些情况下隐藏了客户端IP。为什么这是必要的？想想如果Web服务器看到vm3的真实IP会发生什么。来自服务器的响应将直接路由回到vm3，绕过edge1上的负载均衡器。从vm3的角度来看，它看起来像是向VIP发出请求，但收到了来自其中一个Web服务器的真实IP的回复。（如果不使用代理模式）负载均衡器就不工作了，这就是为什么代理模式功能很重要。
为了进行第二轮测试，先删除负载均衡器配置
ovn-nbctl clear logical_router edge1 load_balancer
ovn-nbctl destroy load_balancer $uuid
ovn-nbctl clear logical_router edge1 load_balancer
ovn-nbctl destroy load_balancer $uuid
在逻辑交换机上配置负载均衡
接下来的实验将负载均衡规则应用到逻辑交换机，会发生什么呢？ 由于我们将负载均衡从边缘移开，第一步需要创建一个带有内部VIP的新的负载均衡器。 我们将使用172.16.255.62作为VIP。
uuid=`ovn-nbctl create load_balancer vips:172.16.255.62="172.16.255.130,172.16.255.131"`
echo $uuid
uuid=`ovn-nbctl create load_balancer vips:172.16.255.62="172.16.255.130,172.16.255.131"`
echo $uuid
第一个测试:将负载均衡器应用于“内部”逻辑交换机。
# apply and verify
ovn-nbctl set logical_switch inside load_balancer=$uuid
ovn-nbctl get logical_switch inside load_balancer
# apply and verify
ovn-nbctl set logical_switch inside load_balancer=$uuid
ovn-nbctl get logical_switch inside load_balancer
然后从vm3测试（位于“inside”）:
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm1
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm1
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm2
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm1
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm1
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm2
ovn-nbctl clear logical_switch inside load_balancer
ovn-nbctl set logical_switch dmz load_balancer=$uuid
ovn-nbctl get logical_switch dmz load_balancer
ovn-nbctl clear logical_switch inside load_balancer
ovn-nbctl set logical_switch dmz load_balancer=$uuid
ovn-nbctl get logical_switch dmz load_balancer
然后再次从 vm3测试:
# ip netns exec vm3 curl 172.16.255.62:8000
# ip netns exec vm3 curl 172.16.255.62:8000
# ip netns exec vm1 curl 172.16.255.62:8000
# ip netns exec vm1 curl 172.16.255.62:8000
ovn-nbctl clear logical_switch dmz load_balancer
ovn-nbctl destroy load_balancer $uuid
ovn-nbctl clear logical_switch dmz load_balancer
ovn-nbctl destroy load_balancer $uuid
uuid=`ovn-nbctl create load_balancer vips:20.0.0.100="20.0.0.10,20.0.0.20"`
ovn-nbctl ls-lb-add inside $uuid
流表差异
table=12, priority=100,ip,metadata=0x1,nw_dst=20.0.0.100 actions=load:0x1->NXM_NX_XXREG0[96],resubmit(,13)
table=17, priority=65535,ct_state=-new+est-rel-inv+trk,metadata=0x1 actions=load:0x1->NXM_NX_XXREG0[98],resubmit(,18)
table=18, priority=110,ct_state=+new+trk,ip,metadata=0x1,nw_dst=20.0.0.100 actions=group:1
table=40, priority=100,ip,metadata=0x1 actions=load:0x1->NXM_NX_XXREG0[96],resubmit(,41)
table=40, priority=100,ipv6,metadata=0x1 actions=load:0x1->NXM_NX_XXREG0[96],resubmit(,41)
table=43, priority=65535,ct_state=-new+est-rel-inv+trk,metadata=0x1 actions=load:0x1->NXM_NX_XXREG0[98],resubmit(,44)
流量追踪
ping
[root@HikvisionOS ~]# ovs-appctl dpif/dump-flows br-int
recirc_id(0),in_port(3),ct_state(-new-est-rel-inv-trk),eth(src=02:d4:1d:8c:d9:9d,dst=02:d4:1d:8c:d9:9f),eth_type(0x0806),arp(sip=20.0.0.10,tip=20.0.0.1,op=1/0xff,sha=02:d4:1d:8c:d9:9d,tha=00:00:00:00:00:00), packets:0, bytes:0, used:never, actions:userspace(pid=4294963168,slow_path(action))
recirc_id(0x11d),in_port(5),ct_state(-new+est-rel-inv+trk),eth(src=02:d4:1d:8c:d9:9b,dst=02:d4:1d:8c:d9:9e),eth_type(0x0800),ipv4(src=10.0.0.10,dst=20.0.0.10,ttl=64,frag=no), packets:11, bytes:1078, used:0.296s, actions:ct_clear,ct_clear,set(eth(src=02:d4:1d:8c:d9:9f,dst=02:d4:1d:8c:d9:9d)),set(ipv4(src=10.0.0.10,dst=20.0.0.10,ttl=63)),3
recirc_id(0x11a),in_port(3),ct_state(-new+est-rel-inv+trk),eth(src=02:d4:1d:8c:d9:9e),eth_type(0x0800),ipv4(src=20.0.0.8/255.255.255.248,frag=no), packets:12, bytes:1176, used:0.296s, actions:ct(zone=9,nat),recirc(0x11b)
recirc_id(0),in_port(5),eth(src=02:d4:1d:8c:d9:9b),eth_type(0x0800),ipv4(src=10.0.0.10,dst=20.0.0.100,frag=no), packets:12, bytes:1176, used:0.296s, actions:ct(zone=9),recirc(0x117)
recirc_id(0x11b),in_port(3),eth(dst=02:d4:1d:8c:d9:9b),eth_type(0x0800),ipv4(dst=10.0.0.10,frag=no), packets:12, bytes:1176, used:0.296s, actions:5
recirc_id(0x11c),in_port(5),eth(src=02:d4:1d:8c:d9:9b,dst=02:d4:1d:8c:d9:9e),eth_type(0x0800),ipv4(dst=20.0.0.8/255.255.255.248,frag=no), packets:11, bytes:1078, used:0.296s, actions:ct(zone=9),recirc(0x119)
recirc_id(0),in_port(5),ct_state(-new-est-rel-inv-trk),eth(src=02:d4:1d:8c:d9:9b,dst=02:d4:1d:8c:d9:9e),eth_type(0x0806),arp(sip=10.0.0.10,tip=10.0.0.1,op=1/0xff,sha=02:d4:1d:8c:d9:9b,tha=00:00:00:00:00:00), packets:0, bytes:0, used:never, actions:userspace(pid=4294963166,slow_path(action))
recirc_id(0x119),in_port(5),ct_state(-new+est-rel-inv+trk),eth_type(0x0800),ipv4(frag=no), packets:11, bytes:1078, used:0.296s, actions:ct(zone=9,nat),recirc(0x11d)
recirc_id(0x117),in_port(5),ct_state(-new+est-rel-inv+trk),eth_type(0x0800),ipv4(frag=no), packets:11, bytes:1078, used:0.296s, actions:ct(zone=9,nat),recirc(0x11c)
recirc_id(0),in_port(3),ct_state(-new-est-rel-inv-trk),eth(src=02:d4:1d:8c:d9:9d,dst=02:d4:1d:8c:d9:9f),eth_type(0x0800),ipv4(src=20.0.0.10,dst=10.0.0.10,ttl=64,frag=no), packets:12, bytes:1176, used:0.296s, actions:ct_clear,ct_clear,set(eth(src=02:d4:1d:8c:d9:9e,dst=02:d4:1d:8c:d9:9b)),set(ipv4(src=20.0.0.10,dst=10.0.0.10,ttl=63)),ct(zone=9),recirc(0x11a)
ovs-appctl ofproto/trace br-int in_port=4,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,ipv4,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_proto=1,icmp_type=0,icmp_code=0 -generate
# ovs-appctl ofproto/trace br-int in_port=4,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,ipv4,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_proto=1,icmp_type=0,icmp_code=0 -generate
Flow: icmp,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
bridge("br-int")
----------------
0. in_port=4, priority 100
set_field:0x9->reg13
set_field:0x7->reg11
set_field:0xd->reg12
set_field:0x1->metadata
set_field:0x2->reg14
resubmit(,8)
8. reg14=0x2,metadata=0x1,dl_src=02:d4:1d:8c:d9:9b, priority 50, cookie 0x6047969c
resubmit(,9)
9. ip,reg14=0x2,metadata=0x1,dl_src=02:d4:1d:8c:d9:9b,nw_src=10.0.0.10, priority 90, cookie 0xb948ce75
resubmit(,10)
10. metadata=0x1, priority 0, cookie 0x5b23fa1b
resubmit(,11)
11. metadata=0x1, priority 0, cookie 0x85c5c31e
resubmit(,12)
12. ip,metadata=0x1,nw_dst=20.0.0.100, priority 100, cookie 0xdfbc9cba
load:0x1->NXM_NX_XXREG0[96]
resubmit(,13)
13. ip,reg0=0x1/0x1,metadata=0x1, priority 100, cookie 0xa5b7b054
ct(table=14,zone=NXM_NX_REG13[0..15])
drop
-> A clone of the packet is forked to recirculate. The forked pipeline will be resumed at table 14.
Final flow: icmp,reg0=0x1,reg11=0x7,reg12=0xd,reg13=0x9,reg14=0x2,metadata=0x1,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
Megaflow: recirc_id=0,eth,ip,in_port=4,vlan_tci=0x0000/0x1000,dl_src=02:d4:1d:8c:d9:9b,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_frag=no
Datapath actions: ct(zone=9),recirc(0x1cb)
===============================================================================
recirc(0x1cb) - resume conntrack with default ct_state=trk|new (use --ct-next to customize)
===============================================================================
Flow: recirc_id=0x1cb,ct_state=new|trk,ct_zone=9,eth,icmp,reg0=0x1,reg11=0x7,reg12=0xd,reg13=0x9,reg14=0x2,metadata=0x1,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
bridge("br-int")
----------------
thaw
Resuming from table 14
14. metadata=0x1, priority 0, cookie 0x50063cdd
resubmit(,15)
15. metadata=0x1, priority 0, cookie 0xf31c70df
resubmit(,16)
16. metadata=0x1, priority 0, cookie 0x13c4db5f
resubmit(,17)
17. metadata=0x1, priority 0, cookie 0x78c30bb9
resubmit(,18)
18. ct_state=+new+trk,ip,metadata=0x1,nw_dst=20.0.0.100, priority 110, cookie 0x752ce65e
group:1
ct(commit,table=19,zone=NXM_NX_REG13[0..15],nat(dst=20.0.0.10))
nat(dst=20.0.0.10)
-> A clone of the packet is forked to recirculate. The forked pipeline will be resumed at table 19.
Final flow: unchanged
Megaflow: recirc_id=0x1cb,ct_state=+new-est-rel-inv+trk,eth,icmp,in_port=4,vlan_tci=0x0000/0x1fff,vlan_tci1=0x0000/0x1fff,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_frag=no,icmp_type=0x0/0xff,icmp_code=0x0/0xff
Datapath actions: ct(commit,zone=9,nat(dst=20.0.0.10)),recirc(0x1cc)
===============================================================================
recirc(0x1cc) - resume conntrack with default ct_state=trk|new (use --ct-next to customize)
===============================================================================
Flow: recirc_id=0x1cc,ct_state=new|trk,ct_zone=9,eth,icmp,reg0=0x1,reg11=0x7,reg12=0xd,reg13=0x9,reg14=0x2,metadata=0x1,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
bridge("br-int")
----------------
thaw
Resuming from table 19
19. metadata=0x1, priority 0, cookie 0x5dc957e1
resubmit(,20)
20. metadata=0x1, priority 0, cookie 0x75f5bdfa
resubmit(,21)
21. metadata=0x1, priority 0, cookie 0xa21b1697
resubmit(,22)
22. metadata=0x1, priority 0, cookie 0x31cb2e34
resubmit(,23)
23. metadata=0x1, priority 0, cookie 0x3626ad6f
resubmit(,24)
24. metadata=0x1,dl_dst=02:d4:1d:8c:d9:9e, priority 50, cookie 0x502275b8
set_field:0x1->reg15
resubmit(,32)
32. priority 0
resubmit(,33)
33. reg15=0x1,metadata=0x1, priority 100
set_field:0x7->reg11
set_field:0xd->reg12
resubmit(,34)
34. priority 0
set_field:0->reg0
set_field:0->reg1
set_field:0->reg2
set_field:0->reg3
set_field:0->reg4
set_field:0->reg5
set_field:0->reg6
set_field:0->reg7
set_field:0->reg8
set_field:0->reg9
resubmit(,40)
40. ip,metadata=0x1, priority 100, cookie 0x14cc5da4
load:0x1->NXM_NX_XXREG0[96]
resubmit(,41)
41. metadata=0x1, priority 0, cookie 0x65381f07
resubmit(,42)
42. ip,reg0=0x1/0x1,metadata=0x1, priority 100, cookie 0x65dbb075
ct(table=43,zone=NXM_NX_REG13[0..15])
drop
-> A clone of the packet is forked to recirculate. The forked pipeline will be resumed at table 43.
Final flow: recirc_id=0x1cc,eth,icmp,reg0=0x1,reg11=0x7,reg12=0xd,reg13=0x9,reg14=0x2,reg15=0x1,metadata=0x1,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
Megaflow: recirc_id=0x1cc,eth,ip,in_port=4,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_dst=20.0.0.64/26,nw_frag=no
Datapath actions: ct(zone=9),recirc(0x1cd)
===============================================================================
recirc(0x1cd) - resume conntrack with default ct_state=trk|new (use --ct-next to customize)
===============================================================================
Flow: recirc_id=0x1cd,ct_state=new|trk,ct_zone=9,eth,icmp,reg0=0x1,reg11=0x7,reg12=0xd,reg13=0x9,reg14=0x2,reg15=0x1,metadata=0x1,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
bridge("br-int")
----------------
thaw
Resuming from table 43
43. metadata=0x1, priority 0, cookie 0x441f8496
resubmit(,44)
44. metadata=0x1, priority 0, cookie 0x10069659
resubmit(,45)
45. metadata=0x1, priority 0, cookie 0xe5a2272f
resubmit(,46)
46. metadata=0x1, priority 0, cookie 0xdfdd721e
resubmit(,47)
47. metadata=0x1, priority 0, cookie 0x103a342b
resubmit(,48)
48. metadata=0x1, priority 0, cookie 0x49deb0bb
resubmit(,49)
49. reg15=0x1,metadata=0x1, priority 50, cookie 0x74ad6dec
resubmit(,64)
64. priority 0
resubmit(,65)
65. reg15=0x1,metadata=0x1, priority 100
clone(ct_clear,set_field:0->reg11,set_field:0->reg12,set_field:0->reg13,set_field:0x5->reg11,set_field:0xb->reg12,set_field:0x3->metadata,set_field:0x2->reg14,set_field:0->reg10,set_field:0->reg15,set_field:0->reg0,set_field:0->reg1,set_field:0->reg2,set_field:0->reg3,set_field:0->reg4,set_field:0->reg5,set_field:0->reg6,set_field:0->reg7,set_field:0->reg8,set_field:0->reg9,set_field:0->in_port,resubmit(,8))
ct_clear
set_field:0->reg11
set_field:0->reg12
set_field:0->reg13
set_field:0x5->reg11
set_field:0xb->reg12
set_field:0x3->metadata
set_field:0x2->reg14
set_field:0->reg10
set_field:0->reg15
set_field:0->reg0
set_field:0->reg1
set_field:0->reg2
set_field:0->reg3
set_field:0->reg4
set_field:0->reg5
set_field:0->reg6
set_field:0->reg7
set_field:0->reg8
set_field:0->reg9
set_field:0->in_port
resubmit(,8)
8. reg14=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9e, priority 50, cookie 0x4a6a2617
resubmit(,9)
9. ip,metadata=0x3,nw_ttl=0, priority 30, cookie 0xcb4904dc
drop
Final flow: unchanged
Megaflow: recirc_id=0x1cd,ct_state=+new-est-rel-inv+trk,eth,ip,in_port=4,vlan_tci=0x0000/0x1000,dl_src=00:00:00:00:00:00/01:00:00:00:00:00,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.64/26,nw_ttl=0,nw_frag=no
Datapath actions: ct_clear
用的上的命令：
# ovs-ofctl -O OpenFlow13 dump-meters br-int
OFPST_METER_CONFIG reply (OF1.3) (xid=0x2):
ovn-nbctl list Logical_Router_Port
# ovn-sbctl list Logical_Flow
理论参考：
https://www.jianshu.com/p/40868a1428fc
https://docs.openstack.org/networking-ovn/latest/
https://www.ibm.com/developerworks/cn/cloud/library/1605-ovn-introduction/
https://www.sdnlab.com/19802.html
http://patchwork.ozlabs.org/patch/720449/
https://www.sdnlab.com/19216.html
https://www.sdnlab.com/19842.html
https://www.ibm.com/developerworks/cn/cloud/library/1603-ovn-ovs-openvswitch/index.html
https://www.cnblogs.com/silvermagic/p/7666089.html
https://www.cnblogs.com/silvermagic/p/7666072.html
https://www.cnblogs.com/silvermagic/p/7666111.html
https://www.cnblogs.com/silvermagic/p/7666117.html
https://www.cnblogs.com/silvermagic/p/7666124.html
https://www.itread01.com/content/1541340929.html
https://blog.csdn.net/ptmozhu/article/details/78644825?utm_source=blogxgwz3
OVN理论部分
OVN的架构和分析
OVN/CMS Plugin 是Neutron的一个插件，作为OVN 和 CMS 之间的接口 。它将CMS中的数据（存储在Neutron DB）翻译成一种“中间格式”。这种中间格式就是逻辑网络配置数据，这样CMS中的网络配置数据就能够被OVN的Northbound DB 所理解。
Northbound DB 里面的几乎所有的内容都是由 CMS 产生的，里面存的就是上面OVN/CMS Plugin翻译之后的逻辑网络的相关数据。比如 logical switch，logical router，logical port和ACL。OVN-northd 类似于一个集中的控制器，监听Northbound DB 数据库的内容变化，它把 Northbound DB 里面的逻辑网络的相关数据翻译成 Southbound DB 可理解的格式（logical datapath flows），并传递给 Southbound DB 进行存储，进而被所有的chassis 读取和应用
Southbound DB 处在 OVN 架构的核心，它是 OVN 中最重要的部分，它跟 OVN 的其他组件都有交互。 里面存的数据和 Northbound DB 语义完全不一样，主要包含 3 类数据：
一、物理网络数据，比如 hypervisor的 IP 地址，hypervisor的 tunnel 封装格式；
二、逻辑网络数据，比如报文如何在逻辑网络中转发；是OVN-northd 从Northbound DB 翻译过来的
三、物理网络和逻辑网络的绑定关系，比如逻辑端口关联到哪个 hypervisor上面。这类数据存储在binding表中，字段有uuid,chassis, logical_datapath, logical_port, mac, parent_port, tag, tunnel_key。
ovn-controller 是 OVN 里面的 agent，类似于 Neutron 里面的 ovs-agent，它也是运行在每个 hypervisor和软件网关之上。
它有下面2种功能：
（1）把物理网络的信息写到
Southbound DB 里面（这类信息就包括 Southbound DB中的第一类数据）；
（2）把 Southbound DB 里面存的一些数据转化成
Openflow flow 配到本地的 OVS table 里面，来实现报文的转发。
第2个功能的具体实现机制就是：
ovn-controller连接到到本地的ovsdb-server ，监控、读取、管理OpenvSwitch的配置信息；
ovn-controller作为ovs-vswitchd 的Openflow 控制器来控制流量的转发。另外，从架构图中就可看出ovn-controller是一种分布式SDN控制器。
ovs-vswitchd 和 ovsdb-server 是 OVS 的两个进程：
- ovs-vswitchd ：核心模块，实现交换功能，和Linux内核模块一起，实现基于流的交换；
- ovsdb-server ：是一个数据库。其保存了整个OVS的配置信息，包括接口，流表和VLAN等；ovs-vswitchd从其查询配置信息；
OVN Northbound DB
Northbound DB 是 OVN 和 CMS 之间的接口，Northbound DB 里面的几乎所有的内容都是由 CMS 产生的，ovn-northd 监听这个数据库的内容变化，然后翻译，保存到 Southbound DB 里面。
Northbound DB 里面主要有如下几张表：
Logical_Switch：每一行代表一个逻辑交换机，逻辑交换机有两种，一种是 overlay logical switches，对应于 neutron network，每创建一个 neutron network，networking-ovn 会在这张表里增加一行；另一种是 bridged logical switch，连接物理网络和逻辑网络，被 VTEP gateway 使用。Logical_Switch 里面保存了它包含的 logical port（指向 Logical_Port table）和应用在它上面的 ACL（指向 ACL table）。
Logical_Port：每一行代表一个逻辑端口，每创建一个 neutron port，networking-ovn 会在这张表里增加一行，每行保存的信息有端口的类型，比如 patch port，localnet port，端口的 IP 和 MAC 地址，端口的状态 UP/Down。
ACL：每一行代表一个应用到逻辑交换机上的 ACL 规则，如果逻辑交换机上面的所有端口都没有配置 security group，那么这个逻辑交换机上不应用 ACL。每条 ACL 规则包含匹配的内容，方向，还有动作。
Logical_Router：每一行代表一个逻辑路由器，每创建一个 neutron router，networking-ovn 会在这张表里增加一行，每行保存了它包含的逻辑的路由器端口。
Logical_Router_Port：每一行代表一个逻辑路由器端口，每创建一个 router interface，networking-ovn 会在这张表里加一行，它主要保存了路由器端口的 IP 和 MAC。
OVN Southbound DB
Southbound DB 里面有如下几张表：
Chassis：每一行表示一个 HV 或者 VTEP 网关，由 ovn-controller/ovn-controller-vtep 填写，包含 chassis 的名字和 chassis 支持的封装的配置（指向表 Encap），如果 chassis 是 VTEP 网关，VTEP 网关上和 OVN 关联的逻辑交换机也保存在这张表里。
Encap：保存着 tunnel 的类型和 tunnel endpoint IP 地址。
Logical_Flow：每一行表示一个逻辑的流表，这张表是 ovn-northd 根据 Nourthbound DB 里面二三层拓扑信息和 ACL 信息转换而来的，ovn-controller 把这个表里面的流表转换成 OVS 流表，配到 HV 上的 OVS table。流表主要包含匹配的规则，匹配的方向，优先级，table ID 和执行的动作。
Multicast_Group：每一行代表一个组播组，组播报文和广播报文的转发由这张表决定，它保存了组播组所属的 datapath，组播组包含的端口，还有代表 logical egress port 的 tunnel_key。
Datapath_Binding：每一行代表一个 datapath 和物理网络的绑定关系，每个 logical switch 和 logical router 对应一行。它主要保存了 OVN 给 datapath 分配的代表 logical datapath identifier 的 tunnel_key。
Port_Binding：这张表主要用来确定 logical port 处在哪个 chassis 上面。每一行包含的内容主要有 logical port 的 MAC 和 IP 地址，端口类型，端口属于哪个 datapath binding，代表 logical input/output port identifier 的 tunnel_key, 以及端口处在哪个 chassis。端口所处的 chassis 由 ovn-controller/ovn-controller 设置，其余的值由 ovn-northd 设置。
表 Chassis 和表 Encap 包含的是物理网络的数据，表 Logical_Flow 和表 Multicast_Group 包含的是逻辑网络的数据，表 Datapath_Binding 和表 Port_Binding 包含的是逻辑网络和物理网络绑定关系的数据。
OVN security group 对比 Neutron security group
OVN tunnel
OVN 支持的 tunnel 类型有三种，分别是 Geneve，STT 和 VXLAN。HV 与 HV 之间的流量，只能用 Geneve 和 STT 两种，HV 和 VTEP 网关之间的流量除了用Geneve和STT外，还能用 VXLAN，这是为了兼容硬件 VTEP 网关，因为大部分硬件VTEP网关只支持 VXLAN。
虽然 VXLAN 是数据中心常用的 tunnel 技术，但是 VXLAN header 是固定的，只能传递一个 VNID（VXLAN network identifier），如果想在 tunnel 里面传递更多的信息，VXLAN 实现不了。所以 OVN 选择了 Geneve 和 STT，Geneve 的头部有个 option 字段，支持 TLV 格式，用户可以根据自己的需要进行扩展，而 STT 的头部可以传递 64-bit 的数据，比 VXLAN 的 24-bit 大很多。
OVN tunnel 封装时使用了三种数据，
Logical datapath identifier（逻辑的数据通道标识符）：datapath 是 OVS 里面的概念，报文需要送到 datapath 进行处理，一个 datapath 对应一个 OVN 里面的逻辑交换机或者逻辑路由器，类似于 tunnel ID。这个标识符有 24-bit，由 ovn-northd 分配的，全局唯一，保存在 Southbound DB 里面的表 Datapath_Binding 的列 tunnel_key 里。
Logical input port identifier（逻辑的入端口标识符）：进入 logical datapath 的端口标识符，15-bit 长，由 ovn-northd 分配的，在每个 datapath 里面唯一。它可用范围是 1-32767，0 预留给内部使用。保存在 Southbound DB 里面的表 Port_Binding 的列 tunnel_key 里。
Logical output port identifier（逻辑的出端口标识符）：出 logical datapath 的端口标识符，16-bit 长，范围 0-32767 和 logical input port identifier 含义一样，范围 32768-65535 给组播组使用。对于每个 logical port，input port identifier 和 output port identifier 相同。
如果 tunnel 类型是 Geneve，Geneve header 里面的 VNI 字段填 logical datapath identifier，Option 字段填 logical input port identifier 和 logical output port identifier，TLV 的 class 为 0xffff，type 为 0，value 为 1-bit 0 + 15-bit logical input port identifier + 16-bit logical output port identifier。
如果 tunnel 类型是 STT，上面三个值填在 Context ID 字段，格式为 9-bit 0 + 15-bit logical input port identifier + 16-bit logical output port identifier + 24-bit logical datapath identifier。
OVS 的 tunnel 封装是由 Openflow 流表来做的，所以 ovn-controller 需要把这三个标识符写到本地 HV 的 Openflow flow table 里面，对于每个进入 br-int 的报文，都会有这三个属性，logical datapath identifier 和 logical input port identifier 在入口方向被赋值，分别存在 openflow metadata 字段和 Nicira 扩展寄存器 reg6 里面。报文经过 OVS 的 pipeline 处理后，如果需要从指定端口发出去，只需要把 Logical output port identifier 写在 Nicira 扩展寄存器 reg7 里面。
OVN tunnel 里面所携带的 logical input port identifier 和 logical output port identifier 可以提高流表的查找效率，OVS 流表可以通过这两个值来处理报文，不需要解析报文的字段。
从上一章节可以看到，OVN 里面的 tunnel 类型是由 HV 上面的 ovn-controller 来设置的，并不是由 CMS 指定的，并且 OVN 里面的 tunnel ID 又由 OVN 自己分配的，所以用 neutron 创建 network 时指定 tunnel 类型和 tunnel ID（比如 vnid）是无用的，OVN 不做处理。
OVN VTEP 网关
小结：
OVN存在的意义（目标）
- 可用于生产环境
- 简洁的设计
- 支持1000台以上的物理机环境（也支持相当数量的虚拟机/容器环境）
- 基于已有的OpenStack OVS 插件 来提升性能和稳定性
- 成为OpenStack+OVS集成场景下的首选方案
已经实现从OVS 平滑升级到 OVN
OVN 对于运行平台没有额外的要求，只要能够运行 OVS，就可以运行 OVN，可以和 Linux，Docker，DPDK 还有 Hyper-V 兼容，所以从 OVS 升级到 OVN 是非常简单快捷的。原有的网络、路由等数据不会丢失，也不需要对这些数据导入导出来进行数据迁移
另外 OVN 可以和很多 CMS（Cloud Management System）集成到一起，尤其是 OpenStack Neutron，这些 CMS 只需要添加一个 plugin 来配置 OVN 即可。
OVN对neutron的改变（以Ocata版本中的OVN和OVS 2.9版本来看OVN带来的变化）
OVN 里面数据的读写都是通过 OVSDB 协议来做的，取代了 neutron 里面的消息队列机制，neutron 变成了一个 API server 来处理用户的 REST 请求，其他的功能都交给 OVN 来做。
使得Neutron组件数量减少
OVN原生的ML2 driver替换掉 OVS ML2 driver 和 Neutron的OVS agent；
OVN原生支持L3和DHCP功能，这样就不再需要Neutron 的L3 agent、 DHCP agent 和DVR。
从 OVN 的架构可以看出，OVN 里面数据的读写都是通过 OVSDB来做的，取代了 Neutron 的消息队列机制，所以有了 OVN 之后，Neutron 里面所有的 agent 都不需要了，Neutron 变成了一个 API server 来处理用户的 REST 请求，其他的功能都交给 OVN 来做，只需要在 Neutron 里面加一个 plugin 来调用配置 OVN。
Neutron 里面的子项目 networking-ovn 就是实现 OVN 的 plugin。Plugin 使用 OVSDB 协议来把用户的配置写在 Northbound DB 里，ovn-northd 监听到 Northbound DB 配置发生改变，然后把配置翻译到 Southbound DB 里面。 ovn-controller 监控到 Southbound DB 数据的发生变化之后，进而更新本地的流表。
OVN 里面报文的处理都是通过 OVS OpenFlow 流表来实现的，而在 Neutron 里面二层报文处理是通过 OVS OpenFlow 流表来实现，三层报文处理是通过 Linux TCP/IP 协议栈来实现。
OVN L3 对比 Neutron L3
Neutron 的三层功能主要有路由，SNAT 和 Floating IP（也叫 DNAT），它是通 Linux kernel 的namespace 来实现的，每个路由器对应一个 namespace，利用 Linux TCP/IP 协议栈来做路由转发。
OVN 支持原生的三层功能，不需要借助 Linux TCP/IP stack，用OpenFlow 流表来实现路由查找，ARP 查找，TTL 和 MAC 地址的更改。OVN 的路由也是分布式的，路由器在每个计算节点上都有实例，有了 OVN 之后，不需要 Neutron L3 agent 了 和DVR了。
OVN和其它通用SDN控制器（比如OpenDayLight）的主要区别
OVN专注于实现云计算管理平台场景下的SDN控制器
OVN专注于实现二层和三层网络功能。除了在传输层实现了基于L4的ACL 外，基本上不在L4 ~ L7层实现某些功能。
OVN的实现了哪些功能？拥有哪些特性？
Logical switches：逻辑交换机，用来做二层转发。
L2/L3/L4 ACLs：二到四层的 ACL，可以根据报文的 MAC 地址，IP 地址，端口号来做访问控制。
Logical routers：逻辑路由器，分布式的，用来做三层转发。
Multiple tunnel overlays：支持多种隧道封装技术，有
Geneve，STT 和 VXLAN。
TOR switch or software logical switch gateways：支持使用硬件 TOR switch 或者软件逻辑 switch 当作网关来连接物理网络和虚拟网络。
ovs ovn 学习资料
0、A Primer on OVN
http://blog.spinhirne.com/2016/09/a-primer-on-ovn.html
1、Open Virtual Networking With Docker
http://docs.openvswitch.org/en/latest/howto/docker/
2、Multi-Host Docker network
https://wiredcraft.com/blog/multi-host-docker-network/
3、ovn-namespace
https://github.com/shettyg/ovn-namespace
4、OVN简介PPT
http://openvswitch.org/support/slides/OVN_Barcelona.pdf
5、What is Open Virtual Network (OVN)? How It Works （包含了各种关于网络虚拟化的介绍的连接）
https://www.sdxcentral.com/sdn/network-virtualization/definitions/what-is-open-virtual-network-ovn-how-it-works/
6、Open vSwitch 相关论文
http://openvswitch.org/support/papers/
７、OVN, Bringing Native Virtual Networking to OVS
https://networkheresy.com/category/open-vswitch/
8、基于Open vSwitch的OpenFlow实践
http://www.chenshake.com/based-on-openflow-practices-open-vswitch/
9、ovs源码分析
http://blog.csdn.net/column/details/openvswitch.html
10、ovs orbit
https://ovsorbit.org/
11、introduction to ovn
http://galsagie.github.io/2015/04/20/ovn-1/
12、Russell Bryant的博客
https://blog.russellbryant.net/category/ovs/
13、ovn architecture
http://openvswitch.org/support/dist-docs/ovn-architecture.7.html
14、OVN Logical Flows and ovn-trace
https://blog.russellbryant.net/2016/11/11/ovn-logical-flows-and-ovn-trace/
15、Justin Pettit的个人主页（其中包含了ovs, ovn相关的各种论文，博客和视频）
http://yuba.stanford.edu/~jpettit/
16、ovs 2.5.0源码分析
http://blog.csdn.net/one_clouder/article/category/6359278/1
17、netwoking-ovn - OpenStack Neutron integration with OVN
https://docs.openstack.org/networking-ovn/latest/
18、OVN路由功能详解
https://www.ibm.com/developerworks/cn/cloud/library/1605-ovn-introduction/index.html
19、OVS博客
http://www.cnblogs.com/popsuper1982/p/5848879.html
20、OVSDB RFC
https://datatracker.ietf.org/doc/rfc7047/
21、openstack底层技术-openflow在ovs中的应用
http://www.isjian.com/openstack/openstack-base-openflow-in-openvswitch/
命令行
yum install -y openvswitch.x86_64
yum install -y openvswitch-devel.x86_64
yum install -y openvswitch-ovn-central.x86_64
yum install -y openvswitch-ovn-common.x86_64
yum install -y openvswitch-ovn-docker.x86_64
yum install -y openvswitch-ovn-host.x86_64
yum install -y openvswitch-ovn-vtep.x86_64
yum install -y openvswitch-test.noarch
yum install -y python2-openvswitch.noarch
export Centralip=10.33.46.4
ovn-nbctl set-connection ptcp:6641:$Centralip
ovn-sbctl set-connection ptcp:6642:$Centralip
ovs-vsctl set open . external-ids:ovn-remote=tcp:$Centralip:6642
ovs-vsctl set open . external-ids:ovn-encap-type=geneve
ovs-vsctl set open . external-ids:ovn-encap-ip=$Centralip
ovs-vsctl set open . external-ids:ovn-remote=tcp:$Centralip:6642
ovs-vsctl set open . external-ids:ovn-encap-type=geneve
export Nodeip=10.33.46.4
ovs-vsctl set open . external-ids:ovn-encap-ip=$Nodeip
ovn-nbctl ls-add inside
ovn-nbctl ls-add dmz
ovn-nbctl lr-add tenant1
ovn-nbctl lrp-add tenant1 tenant1-dmz 02:d4:1d:8c:d9:9f 20.0.0.1/24
ovn-nbctl lsp-add dmz dmz-tenant1
ovn-nbctl lsp-set-type dmz-tenant1 router
ovn-nbctl lsp-set-addresses dmz-tenant1 02:d4:1d:8c:d9:9f
ovn-nbctl lsp-set-options dmz-tenant1 router-port=tenant1-dmz
ovn-nbctl lrp-add tenant1 tenant1-inside 02:d4:1d:8c:d9:9e 10.0.0.1/24
ovn-nbctl lsp-add inside inside-tenant1
ovn-nbctl lsp-set-type inside-tenant1 router
ovn-nbctl lsp-set-addresses inside-tenant1 02:d4:1d:8c:d9:9e
ovn-nbctl lsp-set-options inside-tenant1 router-port=tenant1-inside
ovn-nbctl show
ovn-nbctl lsp-add dmz dmz-vm1
ovn-nbctl lsp-set-addresses dmz-vm1 "02:d4:1d:8c:d9:9d 20.0.0.10"
ovn-nbctl lsp-set-port-security dmz-vm1 "02:d4:1d:8c:d9:9d 20.0.0.10"
ovn-nbctl lsp-add dmz dmz-vm2
ovn-nbctl lsp-set-addresses dmz-vm2 "02:d4:1d:8c:d9:9c 20.0.0.20"
ovn-nbctl lsp-set-port-security dmz-vm2 "02:d4:1d:8c:d9:9c 20.0.0.20"
ovn-nbctl lsp-add inside inside-vm3
ovn-nbctl lsp-set-addresses inside-vm3 "02:d4:1d:8c:d9:9b 10.0.0.10"
ovn-nbctl lsp-set-port-security inside-vm3 "02:d4:1d:8c:d9:9b 10.0.0.10"
ovn-nbctl lsp-add inside inside-vm4
ovn-nbctl lsp-set-addresses inside-vm4 "02:d4:1d:8c:d9:9a 10.0.0.20"
ovn-nbctl lsp-set-port-security inside-vm4 "02:d4:1d:8c:d9:9a 10.0.0.20"
ovn-nbctl show
ovn-nbctl create DHCP_Options cidr=20.0.0.0/24 options="\"server_id\"=\"20.0.0.1\" \"server_mac\"=\"02:d4:1d:8c:d9:9f\" \"lease_time\"=\"36000\" \"router\"=\"20.0.0.1\""
ovn-nbctl create DHCP_Options cidr=10.0.0.0/24 options="\"server_id\"=\"10.0.0.1\" \"server_mac\"=\"02:d4:1d:8c:d9:9e\" \"lease_time\"=\"360000\" \"router\"=\"10.0.0.1\""
ovn-nbctl dhcp-options-list
ovn-nbctl lsp-set-dhcpv4-options dmz-vm1 a5c06e37-496f-4b9c-ad79-178bd266d128
ovn-nbctl lsp-set-dhcpv4-options dmz-vm2 a5c06e37-496f-4b9c-ad79-178bd266d128
ovn-nbctl lsp-set-dhcpv4-options inside-vm3 38975d0a-658c-4064-a203-361a708045b1
ovn-nbctl lsp-set-dhcpv4-options inside-vm4 38975d0a-658c-4064-a203-361a708045b1
ovn-nbctl dhcp-options-list
ovn-nbctl dhcp-options-show 38975d0a-658c-4064-a203-361a708045b1
ovn-nbctl dhcp-options-get-options 38975d0a-658c-4064-a203-361a708045b1
ovn-nbctl lsp-get-dhcpv4-options dmz-vm1
ovn-nbctl lsp-get-dhcpv4-options dmz-vm2
ovn-nbctl lsp-get-dhcpv4-options inside-vm3
ovn-nbctl lsp-get-dhcpv4-options inside-vm4
ovs-vsctl show
ovs-vsctl del-port br-int vm1
ip netns add vm1
ovs-vsctl add-port br-int vm1 -- set interface vm1 type=internal
ip link set vm1 address 02:d4:1d:8c:d9:9d
ip link set vm1 netns vm1
ovs-vsctl set Interface vm1 external_ids:iface-id=dmz-vm1
ip netns exec vm1 dhclient vm1
ip netns exec vm1 ip addr show vm1
ip netns add vm2
ovs-vsctl add-port br-int vm2 -- set interface vm2 type=internal
ip link set vm2 address 02:d4:1d:8c:d9:9c
ip link set vm2 netns vm2
ovs-vsctl set Interface vm2 external_ids:iface-id=dmz-vm2
ip netns exec vm2 killall dhclient
ip netns exec vm2 dhclient vm2
ip netns exec vm2 ip addr show vm2
ip netns add vm3
ovs-vsctl add-port br-int vm3 -- set interface vm3 type=internal
ip link set vm3 address 02:d4:1d:8c:d9:9b
ip link set vm3 netns vm3
ovs-vsctl set Interface vm3 external_ids:iface-id=inside-vm3
ip netns exec vm3 killall dhclient
ip netns exec vm3 dhclient vm3
ip netns exec vm3 ip addr show vm3
ip netns add vm4
ovs-vsctl add-port br-int vm4 -- set interface vm4 type=internal
ip link set vm4 address 02:d4:1d:8c:d9:9a
ip link set vm4 netns vm4
ovs-vsctl set Interface vm4 external_ids:iface-id=inside-vm4
ip netns exec vm4 killall dhclient
ip netns exec vm4 dhclient vm4
ip netns exec vm4 ip addr show vm4
ip netns exec vm1 ping -c 2 20.0.0.1
ip netns exec vm1 ping -c 2 20.0.0.20
ip netns exec vm1 ping -c 2 10.0.0.10
ovn-sbctl show
ovn-nbctl lr-add edge1
ovn-nbctl ls-add transit
ovn-nbctl lrp-add edge1 edge1-transit 02:d4:1d:8c:d9:ae 192.168.0.1/24
ovn-nbctl lsp-add transit transit-edge1
ovn-nbctl lsp-set-type transit-edge1 router
ovn-nbctl lsp-set-addresses transit-edge1 02:d4:1d:8c:d9:ae
ovn-nbctl lsp-set-options transit-edge1 router-port=edge1-transit
ovn-nbctl lrp-add tenant1 tenant1-transit 02:d4:1d:8c:d9:af 192.168.0.2/24
ovn-nbctl lsp-add transit transit-tenant1
ovn-nbctl lsp-set-type transit-tenant1 router
ovn-nbctl lsp-set-addresses transit-tenant1 02:d4:1d:8c:d9:af
ovn-nbctl lsp-set-options transit-tenant1 router-port=tenant1-transit
ovn-nbctl lr-route-list edge1
ip netns exec vm1 ping -c 2 192.168.0.1
ovn-nbctl lr-route-add tenant1 "0.0.0.0/0" 192.168.0.1
ip netns exec vm1 ping -c 2 192.168.0.1
ovn-nbctl lr-route-add edge1 "10.0.0.0/24" 192.168.0.2
ovn-nbctl lr-route-add edge1 "20.0.0.0/24" 192.168.0.2
ip netns exec vm1 ping -c 2 192.168.0.1
ovn-nbctl ls-add outside
ovn-nbctl lrp-add edge1 edge1-outside 02:d4:1d:8c:d9:be 192.168.200.16/24
ovn-nbctl lsp-add outside outside-edge1
ovn-nbctl lsp-set-type outside-edge1 router
ovn-nbctl lsp-set-addresses outside-edge1 02:d4:1d:8c:d9:be
ovn-nbctl lsp-set-options outside-edge1 router-port=edge1-outside
# Central节点
# 创建外网逻辑交换机，并配置网关到叫交换机的连接
ovn-nbctl ls-add outside
ovn-nbctl lrp-add edge1 edge1-outside 02:d4:1d:8c:d9:be 192.168.200.16/24
ovn-nbctl lsp-add outside outside-edge1
ovn-nbctl lsp-set-type outside-edge1 router
ovn-nbctl lsp-set-addresses outside-edge1 02:d4:1d:8c:d9:be
ovn-nbctl lsp-set-options outside-edge1 router-port=edge1-outside
# 为外网网卡ens4创建网桥
ovs-vsctl add-br br-ex
# 为外网网卡ens4创建网桥到网络的映射
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=dataNet:br-ex
# 在逻辑交换机outside上添加本地网络端口，并且本地网络的名字为dataNet
ovn-nbctl lsp-add outside outside-localnet
ovn-nbctl lsp-set-addresses outside-localnet unknown
ovn-nbctl lsp-set-type outside-localnet localnet
ovn-nbctl lsp-set-options outside-localnet network_name=dataNet
# 关联外网网卡ens4到网桥上
ovs-vsctl add-port br-ex ens3
# 测试连通性（需要注意vm2的ip地址是不是没了，dhclient好像有些问题）
ip netns exec vm2 ping -c 2 192.168.200.16
# 设置网桥地址
ip addr add 192.168.200.17/24 dev br-ex
ip link set br-ex up
ip route del 192.168.200.0/24 dev ens3
#设置SNAT
# Central节点
# 设置网关chassis
ovn-nbctl lrp-set-gateway-chassis edge1-outside ddeaaec0-eb6e-4ae2-a4cd-2d97e6696e6d
# 配置SNAT规则
ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=20.0.0.0/24 external_ip=192.168.200.16 -- add logical_router edge1 nat @nat
ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=10.0.0.0/24 external_ip=192.168.200.16 -- add logical_router edge1 nat @nat
# 测试连通性
ip netns exec vm2 ping -c 2 192.168.200.17
# Node节点
ip netns exec vm4 ping -c 2 192.168.200.1
命令：
ovn-sbctl list SB_Globalovn-nbctl list NB_Global
ovn-sbctl list Port_Binding
ovn-nbctl find Logical_Switch_Port name="f6c17ce0-083b-4def-a6d8-9ebb7a69da04"ovn-sbctl list Connection
ovn-sbctl list SSL
ovn-sbctl list DHCP_Options
ovn-sbctl list DHCPv6_Options
ovn-sbctl lflow-list (逻辑流表项154个)
ovn-sbctl list Logical_Flow (逻辑流154个)
https://blog.csdn.net/zhengmx100/article/details/75426710
https://blog.csdn.net/zhengmx100/article/details/75426014
ovn-nbctl list NB_Global
ovn-nbctl list Logical_Switch_Port
ovn-nbctl list ACL
ovn-nbctl list Address_Set
ovn-nbctl list Connection 配置数据库连接到ovsdb客户端
ovn-nbctl list DHCP_Options
对IPv6的支持
DHCPv6 stateful
ipv6 neutron应用（一）
一、neutron支持ipv6，有2个重要的属性
1、ipv6_ra_mode
2、ipv6_address_mode
这2个属性都可以设置下面三个值
1、slaac
2、dhcpv6-stateful
3、dhcpv6-stateless
ipv6_address_mode用于客户端如何获取ipv6地址
ipv6_ra_mode用于在子网内路由器的通告，用icmpv6报文来实现，type为134的报文
slaac，无状态地址自动配置，客户端通过路由器返回的RA消息配置本地地址
dhcpv6-stateful，ip地址和dns等信息从dhcp服务器获取
dhcpv6-stateless，ip地址从路由器RA消息获取，dns等信息从dhcp服务器获取
我们目前支持的是dhcpv6-stateful+dhcpv6-stateful
实例通过 DHCPv6服务器接收一个 IPv6 地址，使用 DHCPv6 服务器获得其它可选信息，该DHCPv6服务来自openstack
二、neutron应用
1、创建ipv6网络，子网
neutron subnet-create --name subnet_v6 --enable-dhcp --ip-version 6 --ipv6-ra-mode dhcpv6-stateful --ipv6-address-mode dhcpv6-stateful --dns-nameserver 2001:4860:4860::8844 ipv6_test 2001:1001::/64
2、网络绑定路由器，开通该网络虚机
neutron router-interface-add ipv6_test subnet=2e3f9776-3545-4271-8a96-2e6e285d9500
nova boot wx_ipv6_test1 --flavor 2 --image 78544ead-fc8b-4b0e-9ff4-f21cfae42a55 --nic net-id=9c7a3663-a779-444a-9f97-07b60a592a03
OpenStack 中的5种分配IPv6地址的方式
No options specified（Default），
SLAAC: Address discovered from an OpenStack router
SLAAC: Address discovered from an external router
DHCPv6 stateless : Address discovered from OpenStack Router and additional information from OpenStack DHCP
DHCPv6 stateful : Address discovered from OpenStack DHCP
接下来对上述五种方式进行说明：
手工配置，即需要管理员对虚拟机逐台手工配置IPv6地址。
SLAAC (StateLess Autoconfiguration)其实就是利用路由宣告消息（RA）来确定前缀和长度，利用EUI-64算法计算出接口ID。
OpenStack 中又将SLAAC细分为2种，第一种就是利用OpenStack router 路由宣告消息（RA）报文来生成IPv6地址、
第二种SLAAC:利用外部 router的路由宣告消息（RA）报文来生成IPv6地址
无状态DHCPv6,利用OpenStack router 路由宣告消息（RA）报文来生成IPv6地址 ,从OpenStack DHCPv6服务器获取其它信息，比如DNS服务器地址、NTP服务器地址、WINS服务器地址、TFTP服务器地址、IP电话服务器地址、证书服务器地址等。
有状态DHCPv6，从从OpenStack DHCPv6服务器获取IPv6地址及其它信息，比如DNS服务器地址、NTP服务器地址、WINS服务器地址、TFTP服务器地址、IP电话服务器地址、证书服务器地址等。
手动配置测试ipv6-dhcp
ovn-nbctl lsp-add 98e1793b-b6a7-4d0b-a9c8-336c467795bd ljx-vm1
ovn-nbctl lsp-set-addresses ljx-vm1 "02:d4:1d:8c:d9:10 2003::2"
ovn-nbctl create DHCP_Options cidr=2003::/64 options="\"server_id\"=\"02:d4:1d:8c:99:99\" \"ia_addr\"=\"2003::1\" \"dns_server\"=\"2003::1\" \"domain_search\"=\"openstacklocal\" "
ovn-nbctl set DHCP_Options deb8260c-241e-46f3-a275-24bf1eb8a590 options="\"server_id\"=\"02:d4:1d:8c:99:99\" \"ia_addr\"=\"2003::1\" \"dns_server\"=\"2003::1\" \"domain_search\"=\"openstacklocal\" "
ovn-nbctl list DHCP_OPTIONS
ovn-nbctl lsp-set-dhcpv6-options ljx-vm1 deb8260c-241e-46f3-a275-24bf1eb8a590
ovn-nbctl lsp-get-dhcpv6-options ljx-vm1
ip netns add ljx
ovs-vsctl add-port br-int ljx-vm1 -- set interface ljx-vm1 type=internal
ovs-vsctl set Interface ljx-vm1 external_ids:iface-id=ljx-vm1
ip link set ljx-vm1 netns ljx
ovs-vsctl set Interface ljx-vm1 external_ids:iface-id=ljx-vm1
ip netns exec ljx dhclient ljx-vm1
ip netns exec ljx ip addr show ljx-vm1
ovn-nbctl find Logical_Switch_Port name="ljx-vm1"
https://blog.csdn.net/zhengmx100/article/details/78854478
