gvisor netstack
https://github.com/google/gvisor/issues/1397
pkg/sentry/syscalls/linux/sys_socket.go
root@cloud:~/onlyGvisor/gvisor# docker exec -it test ping 8.8.8.8 PING 8.8.8.8 (8.8.8.8): 56 data bytes 64 bytes from 8.8.8.8: seq=0 ttl=42 time=29.027 ms 64 bytes from 8.8.8.8: seq=1 ttl=42 time=23.938 ms 64 bytes from 8.8.8.8: seq=2 ttl=42 time=11.870 ms 64 bytes from 8.8.8.8: seq=3 ttl=42 time=11.563 ms
root@cloud:~# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 13802c34815d debian "/bin/bash" 7 seconds ago Up 6 seconds eloquent_raman root@cloud:~# docker inspect 13802c34815d | grep Pid | head -n 1 "Pid": 943010,
root@cloud:~# dlv attach 943010 Type 'help' for list of commands. (dlv) b pkg/sentry/socket/netstack/netstack.go:2884 Command failed: could not find statement at pkg/sentry/socket/netstack/netstack.go:2884, please use a line with a statement (dlv) b pkg/sentry/socket/netstack/netstack.go:2884 Command failed: could not find statement at pkg/sentry/socket/netstack/netstack.go:2884, please use a line with a statement (dlv) b pkg/sentry/socket/netstack/netstack.go:2719 Breakpoint 1 set at 0x642930 for gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*socketOpsCommon).RecvMsg() pkg/sentry/socket/netstack/netstack.go:2719 (dlv) c > gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*socketOpsCommon).RecvMsg() pkg/sentry/socket/netstack/netstack.go:2719 (hits goroutine(12):1 total:1) (PC: 0x642930) Warning: debugging optimized function (dlv) bt
(dlv) b sys_socket.go:172 Breakpoint 2 set at 0x587f30 for gvisor.dev/gvisor/pkg/sentry/syscalls/linux.Socket() pkg/sentry/syscalls/linux/sys_socket.go:172 (dlv) c > gvisor.dev/gvisor/pkg/sentry/syscalls/linux.Socket() pkg/sentry/syscalls/linux/sys_socket.go:172 (hits goroutine(269):1 total:1) (PC: 0x587f30) Warning: debugging optimized function (dlv) bt 0 0x0000000000587f30 in gvisor.dev/gvisor/pkg/sentry/syscalls/linux.Socket at pkg/sentry/syscalls/linux/sys_socket.go:172 1 0x0000000000522ea4 in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).executeSyscall at pkg/sentry/kernel/task_syscall.go:104 2 0x0000000000523c5c in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscallInvoke at pkg/sentry/kernel/task_syscall.go:239 3 0x00000000005238dc in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscallEnter at pkg/sentry/kernel/task_syscall.go:199 4 0x00000000005233e0 in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscall at pkg/sentry/kernel/task_syscall.go:174 5 0x0000000000518e00 in gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute at pkg/sentry/kernel/task_run.go:282 6 0x0000000000517d9c in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).run at pkg/sentry/kernel/task_run.go:97 7 0x0000000000077c84 in runtime.goexit at src/runtime/asm_arm64.s:1136
gdb func (s *socketOpsCommon) RecvMsg(t
root@f2b9fb2551cd:/# root@cloud:~# docker run -it --runtime=runsc-kvm --rm debian /bin/bash root@13802c34815d:/# ping 8.8.8.8 PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data. 64 bytes from 8.8.8.8: icmp_seq=1 ttl=0 time=40718 ms 64 bytes from 8.8.8.8: icmp_seq=2 ttl=0 time=41928 ms 64 bytes from 8.8.8.8: icmp_seq=3 ttl=0 time=11.9 ms 64 bytes from 8.8.8.8: icmp_seq=4 ttl=0 time=11.5 ms 64 bytes from 8.8.8.8: icmp_seq=5 ttl=0 time=12.0 ms ^C --- 8.8.8.8 ping statistics --- 5 packets transmitted, 5 received, 0% packet loss, time 760ms rtt min/avg/max/mdev = 11.508/16536.283/41928.166/20241.902 ms, pipe 2
root@cloud:~# dlv attach 943010 Type 'help' for list of commands. (dlv) b pkg/sentry/socket/netstack/netstack.go:2884 Command failed: could not find statement at pkg/sentry/socket/netstack/netstack.go:2884, please use a line with a statement (dlv) b pkg/sentry/socket/netstack/netstack.go:2884 Command failed: could not find statement at pkg/sentry/socket/netstack/netstack.go:2884, please use a line with a statement (dlv) b pkg/sentry/socket/netstack/netstack.go:2719 Breakpoint 1 set at 0x642930 for gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*socketOpsCommon).RecvMsg() pkg/sentry/socket/netstack/netstack.go:2719 (dlv) c > gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*socketOpsCommon).RecvMsg() pkg/sentry/socket/netstack/netstack.go:2719 (hits goroutine(12):1 total:1) (PC: 0x642930) Warning: debugging optimized function (dlv) bt 0 0x0000000000642930 in gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*socketOpsCommon).RecvMsg at pkg/sentry/socket/netstack/netstack.go:2719 1 0x000000000058b024 in gvisor.dev/gvisor/pkg/sentry/syscalls/linux.recvSingleMsg at pkg/sentry/syscalls/linux/sys_socket.go:776 2 0x000000000058a4c0 in gvisor.dev/gvisor/pkg/sentry/syscalls/linux.RecvMsg at pkg/sentry/syscalls/linux/sys_socket.go:644 3 0x0000000000522ea4 in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).executeSyscall at pkg/sentry/kernel/task_syscall.go:104 4 0x0000000000523c5c in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscallInvoke at pkg/sentry/kernel/task_syscall.go:239 5 0x00000000005238dc in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscallEnter at pkg/sentry/kernel/task_syscall.go:199 6 0x00000000005233e0 in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscall at pkg/sentry/kernel/task_syscall.go:174 7 0x0000000000518e00 in gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute at pkg/sentry/kernel/task_run.go:282 8 0x0000000000517d9c in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).run at pkg/sentry/kernel/task_run.go:97 9 0x0000000000077c84 in runtime.goexit at src/runtime/asm_arm64.s:1136
root@13802c34815d:/# telnet 10.10.16.48 22 Trying 10.10.16.48... Connected to 10.10.16.48. Escape character is '^]'.
tcp 也call (*socketOpsCommon).RecvMsg
(dlv) b fillCmsgInq Breakpoint 4 set at 0x641920 for gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*socketOpsCommon).fillCmsgInq() pkg/sentry/socket/netstack/netstack.go:2517 (dlv) c > gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*socketOpsCommon).fillCmsgInq() pkg/sentry/socket/netstack/netstack.go:2517 (hits goroutine(7075):1 total:1) (PC: 0x641920) Warning: debugging optimized function (dlv) bt 0 0x0000000000641920 in gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*socketOpsCommon).fillCmsgInq at pkg/sentry/socket/netstack/netstack.go:2517 1 0x0000000000641f10 in gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*socketOpsCommon).nonBlockingRead at pkg/sentry/socket/netstack/netstack.go:2628 2 0x0000000000642a2c in gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*socketOpsCommon).RecvMsg at pkg/sentry/socket/netstack/netstack.go:2732 3 0x000000000058ba1c in gvisor.dev/gvisor/pkg/sentry/syscalls/linux.recvFrom at pkg/sentry/syscalls/linux/sys_socket.go:864 4 0x000000000058be68 in gvisor.dev/gvisor/pkg/sentry/syscalls/linux.RecvFrom at pkg/sentry/syscalls/linux/sys_socket.go:889 5 0x0000000000522ea4 in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).executeSyscall at pkg/sentry/kernel/task_syscall.go:104 6 0x0000000000523c5c in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscallInvoke at pkg/sentry/kernel/task_syscall.go:239 7 0x00000000005238dc in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscallEnter at pkg/sentry/kernel/task_syscall.go:199 8 0x00000000005233e0 in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscall at pkg/sentry/kernel/task_syscall.go:174 9 0x0000000000518e00 in gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute at pkg/sentry/kernel/task_run.go:282 10 0x0000000000517d9c in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).run at pkg/sentry/kernel/task_run.go:97 11 0x0000000000077c84 in runtime.goexit at src/runtime/asm_arm64.s:1136 (dlv)
tcpdump
root@cloud:~# docker run -it --runtime=runsc-kvm -v share:/share --name test --rm debian /bin/bash root@dc6ca0fab5ce:/# ip a 2: eth0: <UP,LOWER_UP> mtu 1500 link/ether 02:42:ac:11:00:02 brd ff:ff:ff:ff:ff:ff inet 172.17.0.2/16 scope global dynamic 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope global dynamic root@dc6ca0fab5ce:/# uname -a Linux dc6ca0fab5ce 4.4.0 #1 SMP Sun Jan 10 15:06:54 PST 2016 aarch64 GNU/Linux root@dc6ca0fab5ce:/# ping 8.8.8.8 PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data. 64 bytes from 8.8.8.8: icmp_seq=1 ttl=0 time=12.5 ms 64 bytes from 8.8.8.8: icmp_seq=2 ttl=0 time=11.5 ms 64 bytes from 8.8.8.8: icmp_seq=3 ttl=0 time=11.4 ms ^C --- 8.8.8.8 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 4ms rtt min/avg/max/mdev = 11.361/11.794/12.509/0.524 ms root@dc6ca0fab5ce:/#
414: vethaa35e02@if413: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master docker0 state UP group default link/ether de:c9:ea:b0:24:63 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet6 fe80::dcc9:eaff:feb0:2463/64 scope link valid_lft forever preferred_lft forever root@cloud:~# brctl show bridge name bridge id STP enabled interfaces docker0 8000.02429967f5bc no vethaa35e02 virbr0 8000.000000000000 no root@cloud:~# tcpdump -i vethaa35e02 icmp tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on vethaa35e02, link-type EN10MB (Ethernet), capture size 262144 bytes 17:55:10.268057 IP 172.17.0.2 > dns.google: ICMP echo request, id 18038, seq 1, length 64 17:55:10.279434 IP dns.google > 172.17.0.2: ICMP echo reply, id 18038, seq 1, length 64 17:55:11.268139 IP 172.17.0.2 > dns.google: ICMP echo request, id 18038, seq 2, length 64 17:55:11.279335 IP dns.google > 172.17.0.2: ICMP echo reply, id 18038, seq 2, length 64 17:55:12.268932 IP 172.17.0.2 > dns.google: ICMP echo request, id 18038, seq 3, length 64 17:55:12.280037 IP dns.google > 172.17.0.2: ICMP echo reply, id 18038, seq 3, length 64
goroutine 1127131 [running]: panic(0x1534460, 0x21ca320) GOROOT/src/runtime/panic.go:1064 +0x46d fp=0xc0006babe0 sp=0xc0006bab28 pc=0x43611d runtime.panicdivide() GOROOT/src/runtime/panic.go:191 +0x5b fp=0xc0006bac00 sp=0xc0006babe0 pc=0x43488b gvisor.dev/gvisor/pkg/tcpip/stack.(*ConnTrack).bucket(0xc00034cf50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x600000000, 0x800, 0x0) pkg/tcpip/stack/conntrack.go:508 +0x3df fp=0xc0006bacf8 sp=0xc0006bac00 pc=0x9fb3bf gvisor.dev/gvisor/pkg/tcpip/stack.(*ConnTrack).connForTID(0xc00034cf50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x600000000, 0x800, 0x0, 0x0) pkg/tcpip/stack/conntrack.go:247 +0xe0 fp=0xc0006baea8 sp=0xc0006bacf8 pc=0x9f8b10 gvisor.dev/gvisor/pkg/tcpip/stack.(*ConnTrack).originalDst(0xc00034cf50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x14dfac0, 0xc00118e72f, 0x203000, ...) pkg/tcpip/stack/conntrack.go:623 +0x158 fp=0xc0006bafa0 sp=0xc0006baea8 pc=0x9fbe78 gvisor.dev/gvisor/pkg/tcpip/stack.(*IPTables).OriginalDst(...) pkg/tcpip/stack/iptables.go:422 gvisor.dev/gvisor/pkg/tcpip/transport/tcp.(*endpoint).GetSockOpt(0xc001543800, 0x14b57e0, 0xc001739a20, 0xc00118e720) pkg/tcpip/transport/tcp/endpoint.go:2017 +0x1eb fp=0xc0006bb118 sp=0xc0006bafa0 pc=0xcde1eb gvisor.dev/gvisor/pkg/sentry/socket/netstack.getSockOptIP(0xc0027daa80, 0x7feaa07c2758, 0xc001543800, 0x50, 0x1000, 0x2, 0x7fe7b621fc80, 0x0, 0x447a17) pkg/sentry/socket/netstack/netstack.go:1632 +0xfe7 fp=0xc0006bb2c8 sp=0xc0006bb118 pc=0xd4f0e7 gvisor.dev/gvisor/pkg/sentry/socket/netstack.GetSockOpt(0xc0027daa80, 0x18e2960, 0xc00154be00, 0x7feaa07c2758, 0xc001543800, 0x2, 0x1, 0x0, 0x50, 0x1000, ...) pkg/sentry/socket/netstack/netstack.go:1017 +0x1bf fp=0xc0006bb350 sp=0xc0006bb2c8 pc=0xd489ff gvisor.dev/gvisor/pkg/sentry/socket/netstack.(*SocketOperations).GetSockOpt(0xc00154be00, 0xc0027daa80, 0x0, 0x50, 0x20001280, 0x1000, 0x0, 0x0, 0x0) pkg/sentry/socket/netstack/netstack.go:1000 +0xba3 fp=0xc0006bb588 sp=0xc0006bb350 pc=0xd48793 gvisor.dev/gvisor/pkg/sentry/syscalls/linux.getSockOpt(0xc0027daa80, 0x7feaa07c2640, 0xc00154be00, 0x0, 0x50, 0x20001280, 0x1000, 0x10, 0x90, 0x10) pkg/sentry/syscalls/linux/sys_socket.go:514 +0xd7 fp=0xc0006bb628 sp=0xc0006bb588 pc=0xc458a7 gvisor.dev/gvisor/pkg/sentry/syscalls/linux.GetSockOpt(0xc0027daa80, 0x3, 0x0, 0x50, 0x20001280, 0x20000040, 0x0, 0x0, 0x0, 0x0, ...) pkg/sentry/syscalls/linux/sys_socket.go:468 +0x2b7 fp=0xc0006bb740 sp=0xc0006bb628 pc=0xc45427 gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).executeSyscall(0xc0027daa80, 0x37, 0x3, 0x0, 0x50, 0x20001280, 0x20000040, 0x0, 0x340, 0x0, ...) pkg/sentry/kernel/task_syscall.go:103 +0x44e fp=0xc0006bb9e0 sp=0xc0006bb740 pc=0xba224e gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscallInvoke(0xc0027daa80, 0x37, 0x3, 0x0, 0x50, 0x20001280, 0x20000040, 0x0, 0x407baa, 0xc0027db357) pkg/sentry/kernel/task_syscall.go:238 +0xb5 fp=0xc0006bbac0 sp=0xc0006bb9e0 pc=0xba3905 gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscallEnter(0xc0027daa80, 0x37, 0x3, 0x0, 0x50, 0x20001280, 0x20000040, 0x0, 0x1132c62, 0x2bb88b2) pkg/sentry/kernel/task_syscall.go:198 +0x10a fp=0xc0006bbb70 sp=0xc0006bbac0 pc=0xba326a gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).doSyscall(0xc0027daa80, 0x2, 0xc0027daa80) pkg/sentry/kernel/task_syscall.go:173 +0x1e8 fp=0xc0006bbcb0 sp=0xc0006bbb70 pc=0xba2a58 gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute(0x0, 0xc0027daa80, 0x18aee00, 0x0) pkg/sentry/kernel/task_run.go:275 +0x11d9 fp=0xc0006bbee8 sp=0xc0006bbcb0 pc=0xb8f399 gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).run(0xc0027daa80, 0xab29) pkg/sentry/kernel/task_run.go:93 +0x31d fp=0xc0006bbfd0 sp=0xc0006bbee8 pc=0xb8d56d runtime.goexit() src/runtime/asm_amd64.s:1373 +0x1 fp=0xc0006bbfd8 sp=0xc0006bbfd0 pc=0x46b961 created by gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).Start pkg/sentry/kernel/task_start.go:318 +0x19e
