Linux 虚拟网络 IPSec

IPSec

img

IPsec(Internet Protocol Security)是为IP网络提供安全性的协议和服务的集合,它是VPN(Virtual Private Network,虚拟专用网)中常用的一种技术。 由于IP报文本身没有集成任何安全特性,IP数据包在公用网络如Internet中传输可能会面临被伪造、窃取或篡改的风险。通信双方通过IPsec建立一条IPsec隧道,IP数据包通过IPsec隧道进行加密传输,有效保证了数据在不安全的网络环境如Internet中传输的安全性。

一、手动搭建 ipsec 网络

a | 拓扑

拓扑

-----------------            -----------------
|      ns1      |            |      ns1      |
|   1.1.1.2/24  |            |   1.1.2.2/24  |
|       |       |            |       |       |
|       |       |            |       |       |
|       |       |            |       |       |
|-------|-------|            |-------|-------|      
      ens33                        ens33
        |----------------------------|
172.16.94.141     ipsec tunnel     172.16.94.142

b | 配置命令

  • host 141 主机配置命令
# host 141
## 创建名称空间
ip netns a ns1

## 创建 1对 veth pair 网卡
ip l a int0 type veth peer name br0
ip l s br0 up
ip a a 1.1.1.1/24 dev br0

## ns1 配置信息,网卡 int0 分配到 ns1
ip l s int0 netns ns1
ip netns exec ns1 ip l s int0 up
ip netns exec ns1 ip a a 1.1.1.2/24  dev int0
ip netns exec ns1 ip r a default via 1.1.1.1 dev int0

## create the ip xfrm state and ip xfrm policy:
# ID=0x`dd if=/dev/urandom count=4 bs=1 2> /dev/null| xxd -p -c 8``
# KEY=0x`dd if=/dev/urandom count=20 bs=1 2> /dev/null| xxd -p -c 40`

ID=0xfe51d978
KEY=0xfa42aa6bc685beb4d967057134dd8e327ca179a6
ip xfrm state flush
ip xfrm policy flush
ip xfrm state add src 172.16.94.141 dst 172.16.94.142 proto esp spi $ID reqid $ID mode tunnel aead 'rfc4106(gcm(aes))' $KEY 128
ip xfrm state add src 172.16.94.142 dst 172.16.94.141 proto esp spi $ID reqid $ID mode tunnel aead 'rfc4106(gcm(aes))' $KEY 128

ip xfrm policy add src 1.1.1.2/24 dst 1.1.2.2/24 dir out tmpl src 172.16.94.141 dst 172.16.94.142 proto esp reqid $ID mode tunnel
ip xfrm policy add src 1.1.2.2/24 dst 1.1.1.2/24 dir fwd tmpl src 172.16.94.142 dst 172.16.94.141 proto esp reqid $ID mode tunnel
ip xfrm policy add src 1.1.2.2/24 dst 1.1.1.2/24 dir in  tmpl src 172.16.94.142 dst 172.16.94.141 proto esp reqid $ID mode tunnel

  • host 142 主机配置命令
# host 142
## 创建名称空间
ip netns add ns1

## 创建 1对 veth pair 网卡
ip l a int0 type veth peer name br0
ip l s br0 up
ip a a 1.1.2.1/24 dev br0

## ns1 配置信息,网卡 int0 分配到 ns1
ip l s int0 netns ns1
ip netns exec ns1 ip l s int0 up
ip netns exec ns1 ip a a 1.1.2.2/24 dev int0
ip netns exec ns1 ip r a default via 1.1.2.1 dev int0


## create the ip xfrm state and ip xfrm policy:
# ID=0x`dd if=/dev/urandom count=4 bs=1 2> /dev/null| xxd -p -c 8``
# KEY=0x`dd if=/dev/urandom count=20 bs=1 2> /dev/null| xxd -p -c 40`

ID=0xfe51d978
KEY=0xfa42aa6bc685beb4d967057134dd8e327ca179a6
ip xfrm state flush
ip xfrm policy flush
ip xfrm state add src 172.16.94.142 dst 172.16.94.141 proto esp spi $ID reqid $ID mode tunnel aead 'rfc4106(gcm(aes))' $KEY 128
ip xfrm state add src 172.16.94.141 dst 172.16.94.142 proto esp spi $ID reqid $ID mode tunnel aead 'rfc4106(gcm(aes))' $KEY 128

ip xfrm policy add src 1.1.2.2/24 dst 1.1.1.2/24 dir out tmpl src 172.16.94.142 dst 172.16.94.141 proto esp reqid $ID mode tunnel
ip xfrm policy add src 1.1.1.2/24 dst 1.1.2.2/24 dir fwd tmpl src 172.16.94.141 dst 172.16.94.142 proto esp reqid $ID mode tunnel
ip xfrm policy add src 1.1.1.2/24 dst 1.1.2.2/24 dir in  tmpl src 172.16.94.141 dst 172.16.94.142 proto esp reqid $ID mode tunnel
  • host 141 主机 ipsec 信息
root@kind:~# ip x s
src 172.16.94.142 dst 172.16.94.141
        proto esp spi 0xfe51d978 reqid 4266776952 mode tunnel
        replay-window 0 
        aead rfc4106(gcm(aes)) 0xfa42aa6bc685beb4d967057134dd8e327ca179a6 128
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
        sel src 0.0.0.0/0 dst 0.0.0.0/0 
src 172.16.94.141 dst 172.16.94.142
        proto esp spi 0xfe51d978 reqid 4266776952 mode tunnel
        replay-window 0 
        aead rfc4106(gcm(aes)) 0xfa42aa6bc685beb4d967057134dd8e327ca179a6 128
        anti-replay context: seq 0x0, oseq 0x2, bitmap 0x00000000
        sel src 0.0.0.0/0 dst 0.0.0.0/0 

root@kind:~# ip x p
src 1.1.2.2/24 dst 1.1.1.2/24 
        dir in priority 0 
        tmpl src 172.16.94.142 dst 172.16.94.141
                proto esp reqid 4266776952 mode tunnel
src 1.1.2.2/24 dst 1.1.1.2/24 
        dir fwd priority 0 
        tmpl src 172.16.94.142 dst 172.16.94.141
                proto esp reqid 4266776952 mode tunnel
src 1.1.1.2/24 dst 1.1.2.2/24 
        dir out priority 0 
        tmpl src 172.16.94.141 dst 172.16.94.142
                proto esp reqid 4266776952 mode tunnel
  • host 142 主机 ipsec 信息
root@clab:~# ip x s
src 172.16.94.141 dst 172.16.94.142
        proto esp spi 0xfe51d978 reqid 4266776952 mode tunnel
        replay-window 0 
        aead rfc4106(gcm(aes)) 0xfa42aa6bc685beb4d967057134dd8e327ca179a6 128
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
        sel src 0.0.0.0/0 dst 0.0.0.0/0 
src 172.16.94.142 dst 172.16.94.141
        proto esp spi 0xfe51d978 reqid 4266776952 mode tunnel
        replay-window 0 
        aead rfc4106(gcm(aes)) 0xfa42aa6bc685beb4d967057134dd8e327ca179a6 128
        anti-replay context: seq 0x0, oseq 0x1, bitmap 0x00000000
        sel src 0.0.0.0/0 dst 0.0.0.0/0 

root@clab:~# ip x p
src 1.1.1.2/24 dst 1.1.2.2/24 
        dir in priority 0 
        tmpl src 172.16.94.141 dst 172.16.94.142
                proto esp reqid 4266776952 mode tunnel
src 1.1.1.2/24 dst 1.1.2.2/24 
        dir fwd priority 0 
        tmpl src 172.16.94.141 dst 172.16.94.142
                proto esp reqid 4266776952 mode tunnel
src 1.1.2.2/24 dst 1.1.1.2/24 
        dir out priority 0 
        tmpl src 172.16.94.142 dst 172.16.94.141
                proto esp reqid 4266776952 mode tunnel

c | 测试网络

root@kind:~# ip netns exec ns1 ping 1.1.2.2 -c 1
PING 1.1.2.2 (1.1.2.2) 56(84) bytes of data.
64 bytes from 1.1.2.2: icmp_seq=1 ttl=62 time=0.563 ms

--- 1.1.2.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.563/0.563/0.563/0.000 ms

d | 抓包分析

  • 141 主机 ns 名称空间进行 ping 包测试
root@kind:~# ip netns exec ns1 ping 1.1.2.2 -c 1
PING 1.1.2.2 (1.1.2.2) 56(84) bytes of data.
64 bytes from 1.1.2.2: icmp_seq=1 ttl=62 time=9.43 ms

--- 1.1.2.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 9.431/9.431/9.431/0.000 ms
  • 141 主机 ns1 名称空间 int0 网卡抓包
root@kind:~# ip netns exec ns1 ip a l
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
6: int0@if5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether c6:d8:82:be:42:55 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 1.1.1.2/24 scope global int0
       valid_lft forever preferred_lft forever
    inet6 fe80::c4d8:82ff:febe:4255/64 scope link 
       valid_lft forever preferred_lft forever

root@kind:~# ip netns exec ns1 tcpdump -pne -i int0
08:24:12.078337 c6:d8:82:be:42:55 > 16:48:39:b5:42:19, ethertype IPv4 (0x0800), length 98: 1.1.1.2 > 1.1.2.2: ICMP echo request, id 7429, seq 1, length 64
08:24:12.079530 16:48:39:b5:42:19 > c6:d8:82:be:42:55, ethertype IPv4 (0x0800), length 98: 1.1.2.2 > 1.1.1.2: ICMP echo reply, id 7429, seq 1, length 64

mac 地址为 int0 网卡 mac 地址,目的 mac 地址为 veth pair 网卡 br0 mac 地址

  • 141 主机网卡,路由信息
# 网卡信息
root@kind:~# ip a l
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:18:be:6c brd ff:ff:ff:ff:ff:ff
    altname enp2s1
    inet 172.16.94.141/24 brd 172.16.94.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe18:be6c/64 scope link 
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:f1:91:d1:46 brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
       valid_lft forever preferred_lft forever
4: br-e9fae772bb31: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:7e:b1:2f:69 brd ff:ff:ff:ff:ff:ff
    inet 172.18.0.1/16 brd 172.18.255.255 scope global br-e9fae772bb31
       valid_lft forever preferred_lft forever
    inet6 fc00:f853:ccd:e793::1/64 scope global tentative 
       valid_lft forever preferred_lft forever
    inet6 fe80::1/64 scope link tentative 
       valid_lft forever preferred_lft forever
5: br0@if6: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 16:48:39:b5:42:19 brd ff:ff:ff:ff:ff:ff link-netns ns1
    inet 1.1.1.1/24 scope global br0
       valid_lft forever preferred_lft forever
    inet6 fe80::1448:39ff:feb5:4219/64 scope link 
       valid_lft forever preferred_lft forever

# 路由信息
root@kind:~# ip r s
default via 172.16.94.2 dev ens33 proto static 
1.1.1.0/24 dev br0 proto kernel scope link src 1.1.1.1 
172.16.94.0/24 dev ens33 proto kernel scope link src 172.16.94.141 
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 
172.18.0.0/16 dev br-e9fae772bb31 proto kernel scope link src 172.18.0.1 linkdown 
  • 141 主机 br0 网卡抓包
root@kind:~# tcpdump -pne -i br0
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on br0, link-type EN10MB (Ethernet), capture size 262144 bytes
08:26:05.743927 c6:d8:82:be:42:55 > 16:48:39:b5:42:19, ethertype IPv4 (0x0800), length 98: 1.1.1.2 > 1.1.2.2: ICMP echo request, id 7554, seq 1, length 64
08:26:05.744621 16:48:39:b5:42:19 > c6:d8:82:be:42:55, ethertype IPv4 (0x0800), length 98: 1.1.2.2 > 1.1.1.2: ICMP echo reply, id 7554, seq 1, length 64

抓包信息同 ns1 名称空间 int0 网卡,因为他们互为 veth pair 网卡

  • 141 主机 eth0 网卡抓包,并使用 wireshare 分析

img

搜索 esp 数据包, ipsec 模式下,数据包是密文 esp 数据包,需要进行解密后,才能查看。解密 esp 数据包,获取到如下图数据信息

img

esp 数据层,下面有多了新的数据层,包括 ip 层信息和 icmp 信息

img

ipsec 模式下,内部数据包信息没有 mac 信息,只到了 ip 层后,即进行了 ipsec 封装,比较类似 ipip 模式, ipip 数据包内层包也没有 mac 信息,但是 ipip 数据包非加密,不需要使用解密即可查看到包信息

  • 142 主机 eth0 网卡抓包,并使用 wireshare 分析

img

数据包信息同 141 主机 eth0 网卡信息

  • 142 主机 br0 网卡抓包
root@clab:~# tcpdump -pne -i br0
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on br0, link-type EN10MB (Ethernet), capture size 262144 bytes
08:33:16.508361 1a:1d:1f:7c:5f:ec > 9e:1e:fc:13:e9:4d, ethertype IPv4 (0x0800), length 98: 1.1.1.2 > 1.1.2.2: ICMP echo request, id 8004, seq 1, length 64
08:33:16.508397 9e:1e:fc:13:e9:4d > 1a:1d:1f:7c:5f:ec, ethertype IPv4 (0x0800), length 98: 1.1.2.2 > 1.1.1.2: ICMP echo reply, id 8004, seq 1, length 64

mac 地址为 br0 网卡 mac 地址,目的 mac 地址为 ns1 名称空间 int0 网卡 mac 地址

root@clab:~# ip netns exec ns1 ip a l
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
6: int0@if5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 9e:1e:fc:13:e9:4d brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 1.1.2.2/24 scope global int0
       valid_lft forever preferred_lft forever
    inet6 fe80::9c1e:fcff:fe13:e94d/64 scope link 
       valid_lft forever preferred_lft forever

e | 资源回收

## 141
# ip netns del ns1 && reboot

## 142
# ip netns del ns1 && reboot

二、使用 Containerlab 模拟网络

a | 拓扑

拓扑

b | 网络拓扑文件

# ipsec.clab.yml
name: ipsec
topology:
  nodes:
    gw1:
      kind: linux
      image: vyos/vyos:1.2.8
      cmd: /sbin/init
      binds:
        - /lib/modules:/lib/modules
        - ./startup-conf/gw1.cfg:/opt/vyatta/etc/config/config.boot

    gw2:
      kind: linux
      image: vyos/vyos:1.2.8
      cmd: /sbin/init
      binds:
        - /lib/modules:/lib/modules
        - ./startup-conf/gw2.cfg:/opt/vyatta/etc/config/config.boot

    server1:
      kind: linux
      image: harbor.dayuan1997.com/devops/nettool:0.9
      exec:
      - ip addr add 10.1.5.10/24 dev net0
      - ip route replace default via 10.1.5.1

    server2:
      kind: linux
      image: harbor.dayuan1997.com/devops/nettool:0.9
      exec:
      - ip addr add 10.1.8.10/24 dev net0
      - ip route replace default via 10.1.8.1


  links:
    - endpoints: ["gw1:eth1", "server1:net0"]
    - endpoints: ["gw2:eth1", "server2:net0"]
    - endpoints: ["gw1:eth2", "gw2:eth2"]

c | VyOS 配置文件

  • gw1.cfg
配置文件
# ./startup-conf/gw1.cfg

interfaces {
    ethernet eth1 {
        address 10.1.5.1/24
        duplex auto
        smp-affinity auto
        speed auto
    }
    ethernet eth2 {
        address 172.12.1.10/24
        duplex auto
        smp-affinity auto
        speed auto
    }
    loopback lo {
    }
}
protocols {
    static {
        # 静态路由,指定到达 10.1.8.0/24 网络的下一条为 172.12.1.11
        route 10.1.8.0/24 {
            next-hop 172.12.1.11 {
            }
        }
    }
}
system {
    config-management {
        commit-revisions 100
    }
    console {
        device ttyS0 {
            speed 9600
        }
    }
    host-name vyos
    login {
        user vyos {
            authentication {
                encrypted-password $6$QxPS.uk6mfo$9QBSo8u1FkH16gMyAVhus6fU3LOzvLR9Z9.82m3tiHFAxTtIkhaZSWssSgzt4v4dGAL8rhVQxTg0oAG9/q11h/
                plaintext-password ""
            }
            level admin
        }
    }
    ntp {
        server 0.pool.ntp.org {
        }
        server 1.pool.ntp.org {
        }
        server 2.pool.ntp.org {
        }
    }
    syslog {
        global {
            facility all {
                level info
            }
            facility protocols {
                level debug
            }
        }
    }
    time-zone UTC
}
vpn {
    ipsec {
        esp-group office-srv-esp {
            compression disable
            lifetime 1800
            mode tunnel
            pfs enable
            proposal 1 {
                encryption aes256
                hash sha1
            }
        }
        ike-group office-srv-ike {
            close-action none
            ikev2-reauth no
            key-exchange ikev1
            lifetime 3600
            proposal 1 {
                dh-group 2
                encryption aes256
                hash sha1
            }
        }
        # 设置 eth2 网卡为 ipsec 模式
        ipsec-interfaces {
            interface eth2
        }
        site-to-site {
            peer 172.12.1.11 {
                authentication {
                    mode pre-shared-secret
                    pre-shared-secret 0xfa42aa6bc685beb4d967057134dd8e327ca179a6
                }
                connection-type initiate
                ike-group office-srv-ike
                ikev2-reauth inherit
                local-address 172.12.1.10
                tunnel 0 {
                    allow-nat-networks disable
                    allow-public-networks disable
                    esp-group office-srv-esp
                    local {
                        prefix 10.1.5.0/24
                    }
                    remote {
                        prefix 10.1.8.0/24
                    }
                }
            }
        }
    }
}


/* Warning: Do not remove the following line. */
/* === vyatta-config-version: "nat@4:snmp@1:pptp@1:ntp@1:dns-forwarding@1:ipsec@5:broadcast-relay@1:conntrack-sync@1:pppoe-server@2:config-management@1:vrrp@2:dhcp-server@5:quagga@7:qos@1:wanloadbalance@3:mdns@1:ssh@1:zone-policy@1:dhcp-relay@2:webgui@1:webproxy@1:firewall@5:conntrack@1:cluster@1:webproxy@2:l2tp@1:system@10" === */
/* Release version: 1.2.8 */
  • gw2.cfg
配置文件
# ./startup-conf/gw2.cfg

interfaces {
    ethernet eth1 {
        address 10.1.8.1/24
        duplex auto
        smp-affinity auto
        speed auto
    }
    ethernet eth2 {
        address 172.12.1.11/24
        duplex auto
        smp-affinity auto
        speed auto
    }
    loopback lo {
    }
}
protocols {
    static {
        # 静态路由,指定到达 10.1.8.0/24 网络的下一条为 172.12.1.10
        route 10.1.5.0/24 {
            next-hop 172.12.1.10 {
            }
        }
    }
}
system {
    config-management {
        commit-revisions 100
    }
    console {
        device ttyS0 {
            speed 9600
        }
    }
    host-name vyos
    login {
        user vyos {
            authentication {
                encrypted-password $6$QxPS.uk6mfo$9QBSo8u1FkH16gMyAVhus6fU3LOzvLR9Z9.82m3tiHFAxTtIkhaZSWssSgzt4v4dGAL8rhVQxTg0oAG9/q11h/
                plaintext-password ""
            }
            level admin
        }
    }
    ntp {
        server 0.pool.ntp.org {
        }
        server 1.pool.ntp.org {
        }
        server 2.pool.ntp.org {
        }
    }
    syslog {
        global {
            facility all {
                level info
            }
            facility protocols {
                level debug
            }
        }
    }
    time-zone UTC
}
vpn {
    ipsec {
        esp-group office-srv-esp {
            compression disable
            lifetime 1800
            mode tunnel
            pfs enable
            proposal 1 {
                encryption aes256
                hash sha1
            }
        }
        ike-group office-srv-ike {
            close-action none
            ikev2-reauth no
            key-exchange ikev1
            lifetime 3600
            proposal 1 {
                dh-group 2
                encryption aes256
                hash sha1
            }
        }
        # 设置 eth2 网卡为 ipsec 模式
        ipsec-interfaces {
            interface eth2
        }
        site-to-site {
            peer 172.12.1.10 {
                authentication {
                    mode pre-shared-secret
                    pre-shared-secret 0xfa42aa6bc685beb4d967057134dd8e327ca179a6
                }
                connection-type initiate
                ike-group office-srv-ike
                ikev2-reauth inherit
                local-address 172.12.1.11
                tunnel 0 {
                    allow-nat-networks disable
                    allow-public-networks disable
                    esp-group office-srv-esp
                    local {
                        prefix 10.1.8.0/24
                    }
                    remote {
                        prefix 10.1.5.0/24
                    }
                }
            }
        }
    }
}


/* Warning: Do not remove the following line. */
/* === vyatta-config-version: "nat@4:snmp@1:pptp@1:ntp@1:dns-forwarding@1:ipsec@5:broadcast-relay@1:conntrack-sync@1:pppoe-server@2:config-management@1:vrrp@2:dhcp-server@5:quagga@7:qos@1:wanloadbalance@3:mdns@1:ssh@1:zone-policy@1:dhcp-relay@2:webgui@1:webproxy@1:firewall@5:conntrack@1:cluster@1:webproxy@2:l2tp@1:system@10" === */
/* Release version: 1.2.8 */

d | 部署服务

# tree -L 2 ./
./
└── ipsec.clab.yml

# clab deploy -t ipsec.clab.yml
INFO[0000] Containerlab v0.54.2 started                 
INFO[0000] Parsing & checking topology file: clab.yaml  
INFO[0000] Creating docker network: Name="clab", IPv4Subnet="172.20.20.0/24", IPv6Subnet="2001:172:20:20::/64", MTU=1500 
INFO[0000] Creating lab directory: /root/wcni-kind/network/5-demo-cni/9-ipsec/1-clab-ipsec/clab-ipsec 
INFO[0000] Creating container: "gw1"                    
INFO[0000] Creating container: "server2"                
INFO[0001] Creating container: "gw2"                    
INFO[0001] Creating container: "server1"                
INFO[0002] Created link: gw2:eth1 <--> server2:net0     
INFO[0002] Created link: gw1:eth2 <--> gw2:eth2         
INFO[0002] Created link: gw1:eth1 <--> server1:net0     
INFO[0002] Executed command "ip addr add 10.1.8.10/24 dev net0" on the node "server2". stdout: 
INFO[0002] Executed command "ip route replace default via 10.1.8.1" on the node "server2". stdout: 
INFO[0002] Executed command "ip addr add 10.1.5.10/24 dev net0" on the node "server1". stdout: 
INFO[0002] Executed command "ip route replace default via 10.1.5.1" on the node "server1". stdout: 
INFO[0002] Adding containerlab host entries to /etc/hosts file 
INFO[0002] Adding ssh config for containerlab nodes     
INFO[0002] 🎉 New containerlab version 0.55.1 is available! Release notes: https://containerlab.dev/rn/0.55/#0551
Run 'containerlab version upgrade' to upgrade or go check other installation options at https://containerlab.dev/install/ 
+---+--------------------+--------------+------------------------------------------+-------+---------+----------------+----------------------+
| # |        Name        | Container ID |                  Image                   | Kind  |  State  |  IPv4 Address  |     IPv6 Address     |
+---+--------------------+--------------+------------------------------------------+-------+---------+----------------+----------------------+
| 1 | clab-ipsec-gw1     | 1536c5495e4c | vyos/vyos:1.2.8                          | linux | running | 172.20.20.3/24 | 2001:172:20:20::3/64 |
| 2 | clab-ipsec-gw2     | b1d45b33209e | vyos/vyos:1.2.8                          | linux | running | 172.20.20.4/24 | 2001:172:20:20::4/64 |
| 3 | clab-ipsec-server1 | 44dff0da2f32 | harbor.dayuan1997.com/devops/nettool:0.9 | linux | running | 172.20.20.5/24 | 2001:172:20:20::5/64 |
| 4 | clab-ipsec-server2 | 18b12c6c35ca | harbor.dayuan1997.com/devops/nettool:0.9 | linux | running | 172.20.20.2/24 | 2001:172:20:20::2/64 |
+---+--------------------+--------------+------------------------------------------+-------+---------+----------------+----------------------+

e | 查看 4 个容器路由表信息

## clab-ipsec-server1 主机路由
root@kind:~# lo clab-ipsec-server1 ip r s
default via 10.1.5.1 dev net0 
10.1.5.0/24 dev net0 proto kernel scope link src 10.1.5.10 
172.20.20.0/24 dev eth0 proto kernel scope link src 172.20.20.5 

## clab-ipsec-server2 主机路由
root@kind:~# lo clab-ipsec-server2 ip r s
default via 10.1.8.1 dev net0 
10.1.8.0/24 dev net0 proto kernel scope link src 10.1.8.10 
172.20.20.0/24 dev eth0 proto kernel scope link src 172.20.20.3 

## clab-ipsec-gw1 主机路由
root@kind:~# lo clab-ipsec-gw1 ip r s
default via 172.20.20.1 dev eth0 
10.1.5.0/24 dev eth1 proto kernel scope link src 10.1.5.1 
10.1.8.0/24 via 172.12.1.11 dev eth2 proto static metric 20 
172.12.1.0/24 dev eth2 proto kernel scope link src 172.12.1.10 
172.20.20.0/24 dev eth0 proto kernel scope link src 172.20.20.3 

## clab-ipsec-gw2 主机路由
root@kind:~# lo clab-ipsec-gw2 ip r s
default via 172.20.20.1 dev eth0 
10.1.5.0/24 via 172.12.1.10 dev eth2 proto static metric 20 
10.1.8.0/24 dev eth1 proto kernel scope link src 10.1.8.1 
172.12.1.0/24 dev eth2 proto kernel scope link src 172.12.1.11 
172.20.20.0/24 dev eth0 proto kernel scope link src 172.20.20.4 

f | 查看 gw1 gw2 容器 ipsec 信息

  • gw1
root@gw1:/# ip x s
src 172.12.1.10 dst 172.12.1.11
        proto esp spi 0xc02f0625 reqid 1 mode tunnel
        replay-window 0 flag af-unspec
        auth-trunc hmac(sha1) 0x777f6ab1a07c5f8f7f826c8c0f8f5612365e817f 96
        enc cbc(aes) 0xd1489ba1e8aa40afd37a7d0d22421ad6fb0ada708d9b38d7b23a27fa07b6681b
        anti-replay context: seq 0x0, oseq 0x1, bitmap 0x00000000
src 172.12.1.11 dst 172.12.1.10
        proto esp spi 0xce02f639 reqid 1 mode tunnel
        replay-window 32 flag af-unspec
        auth-trunc hmac(sha1) 0x976fafcd9b08a386752164ac125f707d37687c06 96
        enc cbc(aes) 0xdc5425b5217d32429fa36ef54c31596d85e7c3131e1644b592d806e9f1d240a0
        anti-replay context: seq 0x1, oseq 0x0, bitmap 0x00000001
src 172.12.1.10 dst 172.12.1.11
        proto esp spi 0xc314ee5d reqid 1 mode tunnel
        replay-window 0 flag af-unspec
        auth-trunc hmac(sha1) 0xeb0e331f9cd39ee36136f343622e9a9743fa6d1d 96
        enc cbc(aes) 0xecdfd09c35246b373f079772f1aaaec41f3727e77cb868926f85a9dcb2fd8567
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.12.1.11 dst 172.12.1.10
        proto esp spi 0xc509de2e reqid 1 mode tunnel
        replay-window 32 flag af-unspec
        auth-trunc hmac(sha1) 0xc974bb749d7fef047b59909ab00aca5a1b911331 96
        enc cbc(aes) 0x37ff49c439db9478b2e8db84c930b85b63713fe83eefd03f45d2fd21eaedfc23
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.12.1.10 dst 172.12.1.11
        proto esp spi 0xcf4843b8 reqid 1 mode tunnel
        replay-window 0 flag af-unspec
        auth-trunc hmac(sha1) 0x7965eefa6bd78f0f9c7fffef3cff1fcc08f43143 96
        enc cbc(aes) 0x3ad728a2de6193fa8905156700682b375c16e1e0fa7289ea3955fb68db7bb1ec
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.12.1.11 dst 172.12.1.10
        proto esp spi 0xc56b829e reqid 1 mode tunnel
        replay-window 32 flag af-unspec
        auth-trunc hmac(sha1) 0xff14b902b38dff70404f24a1db17045d9cf39e18 96
        enc cbc(aes) 0x86cc1d0277dc82b8d68876844519512cf92e2b839ec92769cd939616728c5259
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.12.1.10 dst 172.12.1.11
        proto esp spi 0xcf36c7b0 reqid 1 mode tunnel
        replay-window 0 flag af-unspec
        auth-trunc hmac(sha1) 0x2aaee791be690e2bbf80545901ad34e11081c805 96
        enc cbc(aes) 0x02a852201075f460b20a000b43c8a5772da5da82dd22b08016ac6904ae5567ad
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.12.1.11 dst 172.12.1.10
        proto esp spi 0xc7bb938b reqid 1 mode tunnel
        replay-window 32 flag af-unspec
        auth-trunc hmac(sha1) 0xb6bc417e23a2b13f6d0ebb10d80aefd376e2118c 96
        enc cbc(aes) 0xfdb2ea4398ce72d432f77bcc571d8f7688e35b11aaf4ef2a474268f79ba05d81
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000

root@gw1:/# show vpn ike sa
Peer ID / IP                            Local ID / IP               
------------                            -------------
172.12.1.11                             172.12.1.10                            

    State  IKEVer  Encrypt  Hash    D-H Group      NAT-T  A-Time  L-Time
    -----  ------  -------  ----    ---------      -----  ------  ------
    up     IKEv1   aes256   sha1_96 2(MODP_1024)   no     1440    3600   

 
root@gw1:/# show vpn ipsec sa
Connection                 State    Up         Bytes In/Out    Remote address    Remote ID    Proposal
-------------------------  -------  ---------  --------------  ----------------  -----------  ------------------------------------------------
peer-172.12.1.11-tunnel-0  up       8 minutes  N/A             172.12.1.11       N/A          AES_CBC_256/HMAC_SHA1_96/PRF_HMAC_SHA1/MODP_1024
  • gw2
root@gw2:/# ip x s
src 172.12.1.11 dst 172.12.1.10
        proto esp spi 0xce02f639 reqid 1 mode tunnel
        replay-window 0 flag af-unspec
        auth-trunc hmac(sha1) 0x976fafcd9b08a386752164ac125f707d37687c06 96
        enc cbc(aes) 0xdc5425b5217d32429fa36ef54c31596d85e7c3131e1644b592d806e9f1d240a0
        anti-replay context: seq 0x0, oseq 0x1, bitmap 0x00000000
src 172.12.1.10 dst 172.12.1.11
        proto esp spi 0xc02f0625 reqid 1 mode tunnel
        replay-window 32 flag af-unspec
        auth-trunc hmac(sha1) 0x777f6ab1a07c5f8f7f826c8c0f8f5612365e817f 96
        enc cbc(aes) 0xd1489ba1e8aa40afd37a7d0d22421ad6fb0ada708d9b38d7b23a27fa07b6681b
        anti-replay context: seq 0x1, oseq 0x0, bitmap 0x00000001
src 172.12.1.11 dst 172.12.1.10
        proto esp spi 0xc509de2e reqid 1 mode tunnel
        replay-window 0 flag af-unspec
        auth-trunc hmac(sha1) 0xc974bb749d7fef047b59909ab00aca5a1b911331 96
        enc cbc(aes) 0x37ff49c439db9478b2e8db84c930b85b63713fe83eefd03f45d2fd21eaedfc23
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.12.1.10 dst 172.12.1.11
        proto esp spi 0xc314ee5d reqid 1 mode tunnel
        replay-window 32 flag af-unspec
        auth-trunc hmac(sha1) 0xeb0e331f9cd39ee36136f343622e9a9743fa6d1d 96
        enc cbc(aes) 0xecdfd09c35246b373f079772f1aaaec41f3727e77cb868926f85a9dcb2fd8567
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.12.1.11 dst 172.12.1.10
        proto esp spi 0xc56b829e reqid 1 mode tunnel
        replay-window 0 flag af-unspec
        auth-trunc hmac(sha1) 0xff14b902b38dff70404f24a1db17045d9cf39e18 96
        enc cbc(aes) 0x86cc1d0277dc82b8d68876844519512cf92e2b839ec92769cd939616728c5259
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.12.1.10 dst 172.12.1.11
        proto esp spi 0xcf4843b8 reqid 1 mode tunnel
        replay-window 32 flag af-unspec
        auth-trunc hmac(sha1) 0x7965eefa6bd78f0f9c7fffef3cff1fcc08f43143 96
        enc cbc(aes) 0x3ad728a2de6193fa8905156700682b375c16e1e0fa7289ea3955fb68db7bb1ec
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.12.1.11 dst 172.12.1.10
        proto esp spi 0xc7bb938b reqid 1 mode tunnel
        replay-window 0 flag af-unspec
        auth-trunc hmac(sha1) 0xb6bc417e23a2b13f6d0ebb10d80aefd376e2118c 96
        enc cbc(aes) 0xfdb2ea4398ce72d432f77bcc571d8f7688e35b11aaf4ef2a474268f79ba05d81
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.12.1.10 dst 172.12.1.11
        proto esp spi 0xcf36c7b0 reqid 1 mode tunnel
        replay-window 32 flag af-unspec
        auth-trunc hmac(sha1) 0x2aaee791be690e2bbf80545901ad34e11081c805 96
        enc cbc(aes) 0x02a852201075f460b20a000b43c8a5772da5da82dd22b08016ac6904ae5567ad
        anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000

root@gw2:/# show vpn ike sa
Peer ID / IP                            Local ID / IP               
------------                            -------------
172.12.1.10                             172.12.1.11                            

    State  IKEVer  Encrypt  Hash    D-H Group      NAT-T  A-Time  L-Time
    -----  ------  -------  ----    ---------      -----  ------  ------
    up     IKEv1   aes256   sha1_96 2(MODP_1024)   no     1620    3600   

 
root@gw2:/# show vpn ipsec sa
Connection                 State    Up          Bytes In/Out    Remote address    Remote ID    Proposal
-------------------------  -------  ----------  --------------  ----------------  -----------  ------------------------------------------------
peer-172.12.1.10-tunnel-0  up       10 minutes  N/A             172.12.1.10       N/A          AES_CBC_256/HMAC_SHA1_96/PRF_HMAC_SHA1/MODP_1024

g| 抓包分析

root@kind:~# lo clab-ipsec-server1 ping 10.1.8.10 -c 10
PING 10.1.8.10 (10.1.8.10): 56 data bytes
64 bytes from 10.1.8.10: seq=0 ttl=62 time=0.596 ms
64 bytes from 10.1.8.10: seq=1 ttl=62 time=0.183 ms
64 bytes from 10.1.8.10: seq=2 ttl=62 time=0.178 ms
64 bytes from 10.1.8.10: seq=3 ttl=62 time=0.167 ms
64 bytes from 10.1.8.10: seq=4 ttl=62 time=0.162 ms
64 bytes from 10.1.8.10: seq=5 ttl=62 time=0.170 ms
64 bytes from 10.1.8.10: seq=6 ttl=62 time=0.180 ms
64 bytes from 10.1.8.10: seq=7 ttl=62 time=0.173 ms
64 bytes from 10.1.8.10: seq=8 ttl=62 time=0.174 ms
64 bytes from 10.1.8.10: seq=9 ttl=62 time=0.178 ms

--- 10.1.8.10 ping statistics ---
10 packets transmitted, 10 packets received, 0% packet loss
round-trip min/avg/max = 0.162/0.216/0.596 ms
  • s1 包分析

server1 net0 网卡

root@kind:~# lo clab-ipsec-server1 tcpdump -pne -i net0

09:12:22.004636 aa:c1:ab:c8:ef:52 > aa:c1:ab:c2:d9:8b, ethertype IPv4 (0x0800), length 98: 10.1.5.10 > 10.1.8.10: ICMP echo request, id 40, seq 0, length 64
09:12:22.004734 aa:c1:ab:c2:d9:8b > aa:c1:ab:c8:ef:52, ethertype IPv4 (0x0800), length 98: 10.1.8.10 > 10.1.5.10: ICMP echo reply, id 40, seq 0, length 64
root@kind:~# lo clab-ipsec-server1 ip a l
43: net0@if44: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9500 qdisc noqueue state UP group default 
    link/ether aa:c1:ab:c8:ef:52 brd ff:ff:ff:ff:ff:ff link-netnsid 1
    inet 10.1.5.10/24 scope global net0
       valid_lft forever preferred_lft forever
    inet6 fe80::a8c1:abff:fec8:ef52/64 scope link 
       valid_lft forever preferred_lft forever

root@kind:~# lo clab-ipsec-server1 ip n s
10.1.5.1 dev net0 lladdr aa:c1:ab:c2:d9:8b REACHABLE

icmp 包中,源 mac 地址:aa:c1:ab:c8:ef:52 目标 mac 地址:aa:c1:ab:c2:d9:8b , 分别为 s1 主机的 net0 网卡的 mac 地址,和 gw1 路由器 10.1.5.1 ipmac 地址

  • gw1.cap 包分析

数据包到达 gw1 路由器 eth1 网卡后,解包后发现数据包目的地址为 10.1.8.10,查询路由表信息后会送往 eth2 网卡 10.1.8.0/24 via 172.12.1.11 dev eth2 proto static metric 20,而 eth2 网卡被设置为 ipsec 模式 set vpn ipsec ipsec-interfaces interface eth2 ,会对数据包进行加密,加密后为 esp 数据包

img

对数据包进行解密,加密模式为: AES_CBC_256 认证模式: HMAC_SHA1_96,解密后获取到源数据包信息

img

  • 内层数据包没有 mac 层信息,比较类似 ipip 模式。
  • 外层数据包 ip 层中,源 ip 目的 ip 分别为 gw1 路由器和 gw2 路由器 eth2 网卡地址。
  • 外层数据包 icmp 包中,源 mac 地址:aa:c1:ab:24:8c:58 目标 mac 地址:aa:c1:ab:85:59:c9 , 分别为 gw1 主机的 eth2 网卡的 mac 地址,和 gw2 路由器 eth2 网卡的 mac 地址

h | 数据包流向

拓扑

  • 数据从 server1 服务器发出,通过查看本机路由表,送往 gw1 网关。路由: default via 10.1.5.1 dev net0
  • gw1 网关查看自身路由后,会送往 eth0 接口,因为目的地址为 10.1.8.10。路由: 10.1.8.0/24 via 172.12.1.11 dev eth2 proto static metric 20
  • eth2 接口为 ipsec 模式,会重新封装数据包,封装信息查看 ip x s 配置信息 src 172.12.1.10 dst 172.12.1.11 proto esp spi 0xc9d77d1d reqid 1 mode tunnel ... auth-trunc hmac(sha1) 0xeb2ef38fcbb796217cc58bed14ab0c12a09deb32 96 ... enc cbc(aes) 0x2c59ec6be9ddcbfc9e1547f206d6892ac257e8e0d6285cd97c768c7bc5669124
  • 数据封装完成后,会送往 eth2 接口,并送往对端 gw2 主机。
  • 对端 gw2 主机接受到数据包后,发现这个是一个 ipsec 数据包,使用 ipsec 信息对包解密。
  • 解密数据包后发现内部的数据包,目的地址为 10.1.8.10 ,通过查看本机路由表,送往 eth1 网卡。路由: 10.1.8.0/24 dev eth1 proto kernel scope link src 10.1.8.1
  • 通过 eth1 网卡,最终会把数据包送到 server2 主机

i | VyOS cli 配置

  • gw1 cli 接口配置
root@gw1:/# show configuration commands

set interfaces ethernet eth1 address '10.1.5.1/24'
set interfaces ethernet eth1 duplex 'auto'
set interfaces ethernet eth1 smp-affinity 'auto'
set interfaces ethernet eth1 speed 'auto'
set interfaces ethernet eth2 address '172.12.1.10/24'
set interfaces ethernet eth2 duplex 'auto'
set interfaces ethernet eth2 smp-affinity 'auto'
set interfaces ethernet eth2 speed 'auto'
set interfaces loopback lo

set protocols static route 10.1.8.0/24 next-hop 172.12.1.11

set vpn ipsec esp-group office-srv-esp compression 'disable'
set vpn ipsec esp-group office-srv-esp lifetime '1800'
set vpn ipsec esp-group office-srv-esp mode 'tunnel'
set vpn ipsec esp-group office-srv-esp pfs 'enable'
set vpn ipsec esp-group office-srv-esp proposal 1 encryption 'aes256'
set vpn ipsec esp-group office-srv-esp proposal 1 hash 'sha1'
set vpn ipsec ike-group office-srv-ike ikev2-reauth 'no'
set vpn ipsec ike-group office-srv-ike key-exchange 'ikev1'
set vpn ipsec ike-group office-srv-ike lifetime '3600'
set vpn ipsec ike-group office-srv-ike proposal 1 encryption 'aes256'
set vpn ipsec ike-group office-srv-ike proposal 1 hash 'sha1'
## IPsec
set vpn ipsec ipsec-interfaces interface eth2
set vpn ipsec site-to-site peer 172.12.1.11 authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer 172.12.1.11 authentication pre-shared-secret '0xfa42aa6bc685beb4d967057134dd8e327ca179a6'
set vpn ipsec site-to-site peer 172.12.1.11 ike-group 'office-srv-ike'
set vpn ipsec site-to-site peer 172.12.1.11 local-address '172.12.1.10'
set vpn ipsec site-to-site peer 172.12.1.11 tunnel 0 allow-nat-networks 'disable'
set vpn ipsec site-to-site peer 172.12.1.11 tunnel 0 allow-public-networks 'disable'
set vpn ipsec site-to-site peer 172.12.1.11 tunnel 0 esp-group 'office-srv-esp'
set vpn ipsec site-to-site peer 172.12.1.11 tunnel 0 local prefix '10.1.5.0/24'
set vpn ipsec site-to-site peer 172.12.1.11 tunnel 0 remote prefix '10.1.8.0/24'

set system config-management commit-revisions '100'
set system console device ttyS0 speed '9600'
set system host-name 'vyos'
set system login user vyos authentication encrypted-password '$6$QxPS.uk6mfo$9QBSo8u1FkH16gMyAVhus6fU3LOzvLR9Z9.82m3tiHFAxTtIkhaZSWssSgzt4v4dGAL8rhVQxTg0oAG9/q11h/'
set system login user vyos authentication plaintext-password ''
set system login user vyos level 'admin'
set system ntp server 0.pool.ntp.org
set system ntp server 1.pool.ntp.org
set system ntp server 2.pool.ntp.org
set system syslog global facility all level 'info'
set system syslog global facility protocols level 'debug'
set system time-zone 'UTC'
  • gw2 cli 接口配置
root@gw2:/# show configuration commands
set interfaces ethernet eth1 address '10.1.8.1/24'
set interfaces ethernet eth1 duplex 'auto'
set interfaces ethernet eth1 smp-affinity 'auto'
set interfaces ethernet eth1 speed 'auto'
set interfaces ethernet eth2 address '172.12.1.11/24'
set interfaces ethernet eth2 duplex 'auto'
set interfaces ethernet eth2 smp-affinity 'auto'
set interfaces ethernet eth2 speed 'auto'
set interfaces loopback lo

set protocols static route 10.1.5.0/24 next-hop 172.12.1.10

set vpn ipsec esp-group office-srv-esp compression 'disable'
set vpn ipsec esp-group office-srv-esp lifetime '1800'
set vpn ipsec esp-group office-srv-esp mode 'tunnel'
set vpn ipsec esp-group office-srv-esp pfs 'enable'
set vpn ipsec esp-group office-srv-esp proposal 1 encryption 'aes256'
set vpn ipsec esp-group office-srv-esp proposal 1 hash 'sha1'
set vpn ipsec ike-group office-srv-ike ikev2-reauth 'no'
set vpn ipsec ike-group office-srv-ike key-exchange 'ikev1'
set vpn ipsec ike-group office-srv-ike lifetime '3600'
set vpn ipsec ike-group office-srv-ike proposal 1 encryption 'aes256'
set vpn ipsec ike-group office-srv-ike proposal 1 hash 'sha1'
## IPsec
set vpn ipsec ipsec-interfaces interface eth2

set vpn ipsec site-to-site peer 172.12.1.10 authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer 172.12.1.10 authentication pre-shared-secret '0xfa42aa6bc685beb4d967057134dd8e327ca179a6'
set vpn ipsec site-to-site peer 172.12.1.10 ike-group 'office-srv-ike'
set vpn ipsec site-to-site peer 172.12.1.10 local-address '172.12.1.11'
set vpn ipsec site-to-site peer 172.12.1.10 tunnel 0 allow-nat-networks 'disable'
set vpn ipsec site-to-site peer 172.12.1.10 tunnel 0 allow-public-networks 'disable'
set vpn ipsec site-to-site peer 172.12.1.10 tunnel 0 esp-group 'office-srv-esp'
set vpn ipsec site-to-site peer 172.12.1.10 tunnel 0 local prefix '10.1.8.0/24'
set vpn ipsec site-to-site peer 172.12.1.10 tunnel 0 remote prefix '10.1.5.0/24'

set system config-management commit-revisions '100'
set system console device ttyS0 speed '9600'
set system host-name 'vyos'
set system login user vyos authentication encrypted-password '$6$QxPS.uk6mfo$9QBSo8u1FkH16gMyAVhus6fU3LOzvLR9Z9.82m3tiHFAxTtIkhaZSWssSgzt4v4dGAL8rhVQxTg0oAG9/q11h/'
set system login user vyos authentication plaintext-password ''
set system login user vyos level 'admin'
set system ntp server 0.pool.ntp.org
set system ntp server 1.pool.ntp.org
set system ntp server 2.pool.ntp.org
set system syslog global facility all level 'info'
set system syslog global facility protocols level 'debug'
set system time-zone 'UTC'

j | 销毁服务

# clab destroy -t ipsec.clab.yml
posted @   evescn  阅读(225)  评论(0编辑  收藏  举报
相关博文:
阅读排行:
· 震惊!C++程序真的从main开始吗?99%的程序员都答错了
· 别再用vector<bool>了!Google高级工程师:这可能是STL最大的设计失误
· 单元测试从入门到精通
· 【硬核科普】Trae如何「偷看」你的代码?零基础破解AI编程运行原理
· 上周热点回顾(3.3-3.9)
  1. 1 毛不易
  2. 2 青丝 等什么君(邓寓君)
  3. 3 最爱 周慧敏
  4. 4 青花 (Live) 摩登兄弟刘宇宁/周传雄
  5. 5 怨苍天变了心 葱香科学家(王悠然)
  6. 6 吹梦到西洲 恋恋故人难/黄诗扶/王敬轩(妖扬)
  7. 7 姑娘别哭泣 柯柯柯啊
  8. 8 我会好好的 王心凌
  9. 9 半生雪 七叔-叶泽浩
  10. 10 用力活着 张茜
  11. 11 山茶花读不懂白玫瑰 梨笑笑
  12. 12 赴春寰 张壹ZHANG/Mukyo木西/鹿予/弦上春秋Official
  13. 13 故事终章 程响
  14. 14 沿海独白 王唯一(九姨太)
  15. 15 若把你 越南电音 云音乐AI/网易天音
  16. 16 世间美好与你环环相扣 柏松
  17. 17 愿你如愿 陆七言
  18. 18 多情种 胡杨林
  19. 19 和你一样 李宇春
  20. 20 晚风心里吹 李克勤
  21. 21 世面 黄梓溪
  22. 22 等的太久 杨大六
  23. 23 微醺状态 张一
  24. 24 醉今朝 安小茜
  25. 25 阿衣莫 阿吉太组合
  26. 26 折风渡夜 沉默书生
  27. 27 星河万里 王大毛
  28. 28 满目星辰皆是你 留小雨
  29. 29 老人与海 海鸣威/吴琼
  30. 30 海底 一支榴莲
  31. 31 只要有你 曹芙嘉
  32. 32 兰花指 阿里郎
  33. 33 口是心非 张大帅
  34. 34 爱不得忘不舍 白小白
  35. 35 惊鸿醉 指尖笑
  36. 36 如愿 葱香科学家(王悠然)
  37. 37 晚风心里吹 阿梨粤
  38. 38 惊蛰·归云 陈拾月(只有影子)/KasaYAYA
  39. 39 风飞沙 迪克牛仔
  40. 40 把孤独当做晚餐 井胧
  41. 41 星星点灯 郑智化
  42. 42 客子光阴 七叔-叶泽浩
  43. 43 走马观花 王若熙
  44. 44 沈园外 阿YueYue/戾格/小田音乐社
  45. 45 盗将行 花粥/马雨阳
  46. 46 她的眼睛会唱歌 张宇佳
  47. 47 一笑江湖 姜姜
  48. 48 虎二
  49. 49 人间烟火 程响
  50. 50 不仅仅是喜欢 萧全/孙语赛
  51. 51 你的眼神(粤语版) Ecrolyn
  52. 52 剑魂 李炜
  53. 53 虞兮叹 闻人听書_
  54. 54 时光洪流 程响
  55. 55 桃花诺 G.E.M.邓紫棋
  56. 56 行星(PLANET) 谭联耀
  57. 57 别怕我伤心 悦开心i/张家旺
  58. 58 上古山海经 小少焱
  59. 59 你的眼神 七元
  60. 60 怨苍天变了心 米雅
  61. 61 绝不会放过 王亚东
  62. 62 可笑的孤独 黄静美
  63. 63 错位时空 艾辰
  64. 64 像个孩子 仙屁孩
  65. 65 完美世界 [主题版] 水木年华
  66. 66 我们的时光 赵雷
  67. 67 万字情诗 椒椒JMJ
  68. 68 妖王 浮生
  69. 69 天地无霜 (合唱版) 杨紫/邓伦
  70. 70 塞北殇 王若熙
  71. 71 花亦山 祖娅纳惜
  72. 72 醉今朝 是可乐鸭
  73. 73 欠我个未来 艾岩
  74. 74 缘分一道桥 容云/青峰AomineDaiky
  75. 75 不知死活 子无余/严书
  76. 76 不可说 霍建华/赵丽颖
  77. 77 孤勇者 陈奕迅
  78. 78 让酒 摩登兄弟刘宇宁
  79. 79 红尘悠悠DJ沈念版 颜一彦
  80. 80 折风渡夜 (DJ名龙版) 泽国同学
  81. 81 吹灭小山河 国风堂/司南
  82. 82 等什么君 - 辞九门回忆 张大帅
  83. 83 绝世舞姬 张曦匀/戚琦
  84. 84 阿刁(无修音版|live) 张韶涵网易云资讯台
  85. 85 往事如烟 蓝波
  86. 86 清明上河图 李玉刚
  87. 87 望穿秋水 坤坤阿
  88. 88 太多 杜宣达
  89. 89 小阿七
  90. 90 霞光-《精灵世纪》片尾曲 小时姑娘
  91. 91 放开 爱乐团王超
  92. 92 醉仙美 娜美
  93. 93 虞兮叹(完整版) 黎林添娇kiki
  94. 94 单恋一枝花 夏了个天呐(朴昱美)/七夕
  95. 95 一个人挺好 (DJ版) 69/肖涵/沈子凡
  96. 96 一笑江湖 闻人听書_
  97. 97 赤伶 李玉刚
  98. 98 达拉崩吧 (Live) 周深
  99. 99 等你归来 程响
  100. 100 责无旁贷 阿悠悠
  101. 101 你是人间四月天(钢琴弹唱版) 邵帅
  102. 102 虐心 徐良/孙羽幽
  103. 103 大天蓬 (女生版) 清水er
  104. 104 赤伶 是二智呀
  105. 105 有种关系叫知己 刘大壮
  106. 106 怎随天下 王若熙
  107. 107 有人 赵钶
  108. 108 海底 三块木头
  109. 109 有何不可 许嵩
  110. 110 大天蓬 (抖音版) 璐爷
  111. 111 我吹过你吹过的晚风(翻自 ac) 辛辛
  112. 112 只爱西经 林一
  113. 113 关山酒 等什么君(邓寓君)
  114. 114 曾经的你 年少不川
  115. 115 倔强 五月天
  116. 116 Lydia F.I.R.
  117. 117 爱你 王心凌
  118. 118 杀破狼 哥哥妹妹
  119. 119 踏山河 七叔-叶泽浩
  120. 120 错过的情人 雷婷
  121. 121 你看到的我 黄勇/任书怀
  122. 122 新欢渡旧爱 黄静美
  123. 123 慕容晓晓-黄梅戏(南柯一梦 / 明洋 remix) 南柯一梦/MINGYANG
  124. 124 浮白 花粥/王胜娚
  125. 125 叹郁孤 霄磊
  126. 126 贝加尔湖畔 (Live) 李健
  127. 127 不虞 王玖
  128. 128 麻雀 李荣浩
  129. 129 一场雨落下来要用多久 鹿先森乐队
  130. 130 野狼disco 宝石Gem
  131. 131 我们不该这样的 张赫煊
  132. 132 海底 一支榴莲
  133. 133 爱情错觉 王娅
  134. 134 你一定要幸福 何洁
  135. 135 往后余生 马良
  136. 136 放你走 正点
  137. 137 只要平凡 张杰/张碧晨
  138. 138 只要平凡-小石头和孩子们 小石头和孩子们
  139. 139 红色高跟鞋 (Live) 韩雪/刘敏涛/万茜
  140. 140 明月天涯 五音Jw
  141. 141 华年 鹿先森乐队
  142. 142 分飞 徐怀钰
  143. 143 你是我撞的南墙 刘楚阳
  144. 144 同簪 小时姑娘/HITA
  145. 145 我的将军啊-唯美独特女版 熙宝(陆迦卉)
  146. 146 我的将军啊(女版戏腔) Mukyo木西
  147. 147 口是心非 南柯nanklo/乐小桃
  148. 148 DAY BY DAY (Japanese Ver.) T-ara
  149. 149 我承认我怕黑 雅楠
  150. 150 我要找到你 冯子晨
  151. 151 你的答案 子尧
  152. 152 一剪梅 费玉清
  153. 153 纸船 薛之谦/郁可唯
  154. 154 那女孩对我说 (完整版) Uu
  155. 155 我好像在哪见过你 薛之谦
  156. 156 林中鸟 葛林
  157. 157 渡我不渡她 (正式版) 苏谭谭
  158. 158 红尘来去梦一场 大壮
  159. 159 都说 龙梅子/老猫
  160. 160 산다는 건 (Cheer Up) 洪真英
  161. 161 听说 丛铭君
  162. 162 那个女孩 张泽熙
  163. 163 最近 (正式版) 王小帅
  164. 164 不谓侠 萧忆情Alex
  165. 165 芒种 音阙诗听/赵方婧
  166. 166 恋人心 魏新雨
  167. 167 Trouble Is A Friend Lenka
  168. 168 风筝误 刘珂矣
  169. 169 米津玄師-lemon(Ayasa绚沙 Remix) Ayasa
  170. 170 可不可以 张紫豪
  171. 171 告白の夜 Ayasa
  172. 172 知否知否(翻自 胡夏) 凌之轩/rainbow苒
  173. 173 琵琶行 奇然/沈谧仁
  174. 174 一曲相思 半阳
  175. 175 起风了 吴青峰
  176. 176 胡广生 任素汐
  177. 177 左手指月 古琴版 古琴唐彬/古琴白无瑕
  178. 178 清明上河图 排骨教主
  179. 179 左手指月 萨顶顶
  180. 180 刚刚好 薛之谦
  181. 181 悟空 戴荃
  182. 182 易燃易爆炸 陈粒
  183. 183 漫步人生路 邓丽君
  184. 184 不染 萨顶顶
  185. 185 不染 毛不易
  186. 186 追梦人 凤飞飞
  187. 187 笑傲江湖 刘欢/王菲
  188. 188 沙漠骆驼 展展与罗罗
  189. 189 外滩十八号 男才女貌
  190. 190 你懂得 小沈阳/沈春阳
  191. 191 铁血丹心 罗文/甄妮
  192. 192 温柔乡 陈雅森
  193. 193 似水柔情 王备
  194. 194 我只能爱你 彭青
  195. 195 年轻的战场 张杰
  196. 196 七月七日晴 许慧欣
  197. 197 心爱 金学峰
  198. 198 Something Just Like This (feat. Romy Wave) Anthony Keyrouz/Romy Wave
  199. 199 ブルーバード いきものがかり
  200. 200 舞飞扬 含笑
  201. 201 时间煮雨 郁可唯
  202. 202 英雄一怒为红颜 小壮
  203. 203 天下有情人 周华健/齐豫
  204. 204 白狐 陈瑞
  205. 205 River Flows In You Martin Ermen
  206. 206 相思 毛阿敏
  207. 207 只要有你 那英/孙楠
  208. 208 Croatian Rhapsody Maksim Mrvica
  209. 209 来生缘 刘德华
  210. 210 莫失莫忘 麦振鸿
  211. 211 往后余生 王贰浪
  212. 212 雪见—仙凡之旅 麦振鸿
  213. 213 让泪化作相思雨 南合文斗
  214. 214 追梦人 阿木
  215. 215 真英雄 张卫健
  216. 216 天使的翅膀 安琥
  217. 217 生生世世爱 吴雨霏
  218. 218 爱我就跟我走 王鹤铮
  219. 219 特别的爱给特别的你 伍思凯
  220. 220 杜婧荧/王艺翔
  221. 221 I Am You Kim Taylor
  222. 222 起风了 买辣椒也用券
  223. 223 江湖笑 周华健
  224. 224 半壶纱 刘珂矣
  225. 225 Jar Of Love 曲婉婷
  226. 226 野百合也有春天 孟庭苇
  227. 227 后来 刘若英
  228. 228 不仅仅是喜欢 萧全/孙语赛
  229. 229 Time (Official) MKJ
  230. 230 纸短情长 (完整版) 烟把儿
  231. 231 离人愁 曲肖冰
  232. 232 难念的经 周华健
  233. 233 佛系少女 冯提莫
  234. 234 红昭愿 音阙诗听
  235. 235 BINGBIAN病变 Cubi/多多Aydos
  236. 236 说散就散 袁娅维TIA RAY
  237. 237 慢慢喜欢你 莫文蔚
  238. 238 最美的期待 周笔畅
  239. 239 牵丝戏 银临/Aki阿杰
  240. 240 夜的钢琴曲 K. Williams
兰花指 - 阿里郎
00:00 / 00:00
An audio error has occurred, player will skip forward in 2 seconds.

Loading

点击右上角即可分享
微信分享提示