openstack--9--深入理解虚拟机
登录计算节点查看进程
[root@linux-node2 ~]# ps aux | grep kvm root 824 0.0 0.0 0 0 ? S< 10:19 0:00 [kvm-irqfd-clean] qemu 9762 18.2 3.2 538924 131596 ? Sl 22:11 0:59 /usr/libexec/qemu-kvm -name instance-00000002 -S -machine pc-i440fx-rhel7.0.0,accel=kvm,usb=off -cpu SandyBridge,+vme,+ds,+ss,+ht,+vmx,+pcid,+osxsave,+hypervisor,+arat,+tsc_adjust -m 64 -realtime mlock=off -smp 1,sockets=1,cores=1,threads=1 -uuid 4e810c6c-5c56-4160-a29c-b240a4550679 -smbios type=1,manufacturer=Fedora Project,product=OpenStack Nova,version=13.1.2-1.el7,serial=df6d7bf8-67af-42d3-b535-d710f13b0201,uuid=4e810c6c-5c56-4160-a29c-b240a4550679,family=Virtual Machine -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-2-instance-00000002/monitor.sock,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kv-pit.lost_tick_policy=discard -no-hpet -no-shutdown -boot strict=on -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -drive file=/var/lib/nova/instances/4e810c6c-5c56-4160-a29c-b240a4550679/disk,format=qcow2,if=none,id=drive-virtio-disk0,cache=none -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=26,id=hostnet0,vhost=on,vhostfd=28 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=fa:16:3e:ff:a2:6e,bus=pci.0,addr=0x3 -chardev file,id=charserial0,path=/var/lib/nova/instances/4e810c6c-5c56-4160-a29c-b240a4550679/console.log -device isa-serial,chardev=charserial0,id=serial0 -chardev pty,id=charserial1 -device isa-serial,chardev=charserial1,id=serial1 -device usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -k en-us -vga cirrus -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -msg timestamp=on root 9781 0.0 0.0 0 0 ? S 22:11 0:00 [kvm-pit/9762] root 9903 0.0 0.0 112648 964 pts/0 S+ 22:16 0:00 grep --colour=auto kvm [root@linux-node2 ~]#
查看端口监听
[root@linux-node2 ~]# netstat -lntp Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 0.0.0.0:5900 0.0.0.0:* LISTEN 9762/qemu-kvm tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 1/systemd tcp 0 0 192.168.122.1:53 0.0.0.0:* LISTEN 3916/dnsmasq tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 1155/sshd tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 1250/master tcp6 0 0 :::111 :::* LISTEN 1/systemd tcp6 0 0 :::22 :::* LISTEN 1155/sshd tcp6 0 0 ::1:25 :::* LISTEN 1250/master
查看桥接
[root@linux-node2 ~]# brctl show bridge name bridge id STP enabled interfaces brqac1b0655-93 8000.000c29f17278 no eth0 tap04087c6c-47 virbr0 8000.525400747276 yes virbr0-nic [root@linux-node2 ~]#
查看网络设备
[root@linux-node2 ~]# ifconfig
brqac1b0655-93: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.56.12 netmask 255.255.255.0 broadcast 192.168.56.255
inet6 fe80::a846:99ff:fedb:caf3 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:f1:72:78 txqueuelen 0 (Ethernet)
RX packets 23781 bytes 5440813 (5.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 34638 bytes 8215484 (7.8 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet6 fe80::20c:29ff:fef1:7278 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:f1:72:78 txqueuelen 1000 (Ethernet)
RX packets 169659 bytes 109732624 (104.6 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 96397 bytes 29661554 (28.2 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
登录控制节点查看
[root@linux-node1 ~]# openstack server list +--------------------------------------+------+--------+---------------------------+ | ID | Name | Status | Networks | +--------------------------------------+------+--------+---------------------------+ | 4e810c6c-5c56-4160-a29c-b240a4550679 | demo | ACTIVE | public-net=192.168.56.102 | +--------------------------------------+------+--------+---------------------------+ [root@linux-node1 ~]#
计算节点查看虚拟机存放路径
虚拟机在下面路径下，这里的名字和上面看到的是一致的
[root@linux-node2 ~]# cd /var/lib/nova/instances/ [root@linux-node2 instances]# ls 4e810c6c-5c56-4160-a29c-b240a4550679 _base compute_nodes locks [root@linux-node2 instances]# cd 4e810c6c-5c56-4160-a29c-b240a4550679/ [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls console.log disk disk.info libvirt.xml [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls -alh total 2.3M drwxr-xr-x 2 nova nova 69 Feb 18 22:11 . drwxr-xr-x 5 nova nova 93 Feb 18 22:11 .. -rw-rw---- 1 qemu qemu 19K Feb 18 22:11 console.log -rw-r--r-- 1 qemu qemu 2.3M Feb 18 22:12 disk -rw-r--r-- 1 nova nova 79 Feb 18 22:11 disk.info -rw-r--r-- 1 nova nova 2.6K Feb 18 22:11 libvirt.xml [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]#
下面这个就是镜像
[root@linux-node2 instances]# pwd /var/lib/nova/instances [root@linux-node2 instances]# cd _base/ [root@linux-node2 _base]# ls 8c31d8bd210f46f11ae77d077c33cff34e274fcb [root@linux-node2 _base]# file 8c31d8bd210f46f11ae77d077c33cff34e274fcb 8c31d8bd210f46f11ae77d077c33cff34e274fcb: x86 boot sector; GRand Unified Bootloader, stage1 version 0x3, stage2 address 0x2000, stage2 segment 0x200; partition 1: ID=0x83, active, starthead 0, startsector 16065, 64260 sectors, code offset 0x48 [root@linux-node2 _base]#
控制节点查看镜像
[root@linux-node1 ~]# openstack image list +--------------------------------------+--------+--------+ | ID | Name | Status | +--------------------------------------+--------+--------+ | 9969eaa3-0296-48cc-a42e-a02251b778a6 | cirros | active | +--------------------------------------+--------+--------+ [root@linux-node1 ~]# cd /var/lib/glance/images/ [root@linux-node1 images]# ls 9969eaa3-0296-48cc-a42e-a02251b778a6 [root@linux-node1 images]#
我们创建的虚拟机很小，只有2.3M，其实它有个后端文件。40MB
qemu v3格式的，只保存文件变化的部分，剩下的全是一个镜像，这样节省空间，启动也很快
[root@linux-node2 instances]# pwd /var/lib/nova/instances [root@linux-node2 instances]# ls 4e810c6c-5c56-4160-a29c-b240a4550679 _base compute_nodes locks [root@linux-node2 instances]# cd 4e810c6c-5c56-4160-a29c-b240a4550679/ [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls console.log disk disk.info libvirt.xml [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls -lh disk -rw-r--r-- 1 qemu qemu 2.3M Feb 18 22:12 disk [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# file disk disk: QEMU QCOW Image (v3), has backing file (path /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34), 1073741824 bytes [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls -lh /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34 ls: cannot access /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34: No such file or directory [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls -lh /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34e274fcb -rw-r--r-- 1 qemu qemu 40M Feb 18 22:11 /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34e274fcb [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]#
也可以通过下面方式查看磁盘信息
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# cd -
/var/lib/nova/instances
[root@linux-node2 instances]# ls
4e810c6c-5c56-4160-a29c-b240a4550679 _base compute_nodes locks
[root@linux-node2 instances]# cd 4e810c6c-5c56-4160-a29c-b240a4550679/
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls
console.log disk disk.info libvirt.xml
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# qemu-img info disk
image: disk
file format: qcow2
virtual size: 1.0G (1073741824 bytes)
disk size: 2.3M
cluster_size: 65536
backing file: /var/lib/nova/instances/_base/8c31d8bd210f46f11ae77d077c33cff34e274fcb
Format specific information:
compat: 1.1
lazy refcounts: false
[root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]#
虚拟机的xml文件在下面两个位置都能看到，它是动态生成的
[root@linux-node2 qemu]# pwd /etc/libvirt/qemu [root@linux-node2 qemu]# ls instance-00000002.xml networks [root@linux-node2 qemu]# cd - /var/lib/nova/instances/4e810c6c-5c56-4160-a29c-b240a4550679 [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]# ls console.log disk disk.info libvirt.xml [root@linux-node2 4e810c6c-5c56-4160-a29c-b240a4550679]#
只要用openstack管理，这个xml就会自动生成，openstack软重启不一定改变这个xml文件，但是硬重启一定会改变它
早期版本可以virsh手动改，以前是根据模板
openstack再加一层，变成类方式了。不能virsh更改xml文件了
在控制节点可以直接ssh此实例，而不用输入密码。
关于key是怎么跑进去的，通过metadata 元数据机制
[root@linux-node1 images]# ssh cirros@192.168.56.102 $ curl http://169.254.169.254/2009-04-04/meta-data ami-id ami-launch-index ami-manifest-path block-device-mapping/ hostname instance-action instance-id instance-type local-hostname local-ipv4 placement/ public-hostname public-ipv4 public-keys/ reservation-id
获取ipv4的地址。 这就是它的meta-data
$ curl http://169.254.169.254/2009-04-04/meta-data/local-ipv4 192.168.56.102$ $
这个169.254.169.254IP地址来自哪里呢
$ ip ad
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
link/ether fa:16:3e:ff:a2:6e brd ff:ff:ff:ff:ff:ff
inet 192.168.56.102/24 brd 192.168.56.255 scope global eth0
inet6 fe80::f816:3eff:feff:a26e/64 scope link
valid_lft forever preferred_lft forever
有路由
$ ip ro li default via 192.168.56.2 dev eth0 169.254.169.254 via 192.168.56.100 dev eth0 192.168.56.0/24 dev eth0 src 192.168.56.102 $
通过admin用户登录管理页面，找到了
dhcp在用它
登录计算节点去找它，下面命令没找到
[root@linux-node1 images]# ip ad li
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master brqac1b0655-93 state UP qlen 1000
link/ether 00:0c:29:90:3c:7c brd ff:ff:ff:ff:ff:ff
3: virbr0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN
link/ether 52:54:00:fe:ab:0b brd ff:ff:ff:ff:ff:ff
inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
valid_lft forever preferred_lft forever
4: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN qlen 500
link/ether 52:54:00:fe:ab:0b brd ff:ff:ff:ff:ff:ff
5: tap168a2111-98: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master brqac1b0655-93 state UP qlen 1000
link/ether c2:5a:0d:4d:0d:88 brd ff:ff:ff:ff:ff:ff
6: brqac1b0655-93: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP
link/ether 00:0c:29:90:3c:7c brd ff:ff:ff:ff:ff:ff
inet 192.168.56.11/24 brd 192.168.56.255 scope global brqac1b0655-93
valid_lft forever preferred_lft forever
[root@linux-node1 images]#
在namespace命名空间里找
[root@linux-node1 images]# ip netns li qdhcp-ac1b0655-931d-4d6e-ba52-33fd0631e034 [root@linux-node1 images]#
在这个namespace里执行 ip ad li 看到了192.168.56.100
它还有个169.254.169.254
[root@linux-node1 images]# ip netns exec qdhcp-ac1b0655-931d-4d6e-ba52-33fd0631e034 ip ad li
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ns-168a2111-98: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether fa:16:3e:91:55:b6 brd ff:ff:ff:ff:ff:ff
inet 192.168.56.100/24 brd 192.168.56.255 scope global ns-168a2111-98
valid_lft forever preferred_lft forever
inet 169.254.169.254/16 brd 169.254.255.255 scope global ns-168a2111-98
valid_lft forever preferred_lft forever
inet6 fe80::f816:3eff:fe91:55b6/64 scope link
valid_lft forever preferred_lft forever
[root@linux-node1 images]#
虚拟的这个路由来自于dhcp的推送,也是下面配置文件决定的
[root@linux-node1 ~]# grep enable_isolated /etc/neutron/dhcp_agent.ini enable_isolated_metadata = True # enable_isolated_metadata = True. (boolean value) [root@linux-node1 ~]#
你设置为true，它会追加主机路由
# The DHCP server can assist with providing metadata support on isolated # networks. Setting this value to True will cause the DHCP server to append # specific host routes to the DHCP request. The metadata service will only be # activated when the subnet does not contain any router port. The guest # instance must be configured to request host routes via DHCP (Option 121). # This option doesn't have any effect when force_metadata is set to True. # (boolean value) enable_isolated_metadata = True
这个等于true，你从控制节点才可以使用ssh-key登录
虚拟机的key就是自动从169.254.169.254这里获取的
这个命名空间里起了80端口的。之前curl访问元数据通的
[root@linux-node1 ~]# ip netns exec qdhcp-ac1b0655-931d-4d6e-ba52-33fd0631e034 lsof -i:80 COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME neutron-n 25443 neutron 6u IPv4 73251 0t0 TCP *:http (LISTEN) [root@linux-node1 ~]# ip netns exec qdhcp-ac1b0655-931d-4d6e-ba52-33fd0631e034 netstat -lntp Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 25443/python2 tcp 0 0 192.168.56.100:53 0.0.0.0:* LISTEN 25430/dnsmasq tcp 0 0 169.254.169.254:53 0.0.0.0:* LISTEN 25430/dnsmasq tcp6 0 0 fe80::f816:3eff:fe91:53 :::* LISTEN 25430/dnsmasq [root@linux-node1 ~]#
它是通过下面方式获取key的
$ curl http://169.254.169.254/2009-04-04/meta-data/public-keys/0/openssh-key ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA+D1fhI9OjfYFl8Rt5nPPp8r1CfIUUh+RtqbarkBxcBHqP3gtG7dHt3lAt6i822X8YBMO5D3ch1p2SyIXToGPhLOdOC0eBxdmwvJitTCfA3ucAmTYa09HP/jlj leLZ5Qx064AMLCVE/DRR8LlDX9dx49nNVYFUxQbGm6K7Ztx31FZN/O+o2mkb32OQ2y+D7aKtf62OloO8x27MxIM7X3YEVBZvmiszCWejFsX0m4/427iNT0vcGNOTT5oeAtNSH6SmO23ieg2SnIMDKulZ m+Fl2Fx2uVP6R6m5IFD8U7TT6r5Xg0S91LDFcP8VjDCCwYZEFd+txer5fGZVVYR50v7 root@linux-node1.nmap.com $
也可以获取主机名
$ curl http://169.254.169.254/2009-04-04/meta-data/hostname demo.novalocal$ $
业内有个工具，也能帮你做此操作
方式一：cloud-init，帮你初始化，配置cloud-init挺麻烦
方式二：脚本方式也可以，更灵活
我们安装的这个小镜像带cloud-init，你自己做镜像，什么都不加，是无法连的。没法自动获取key的
知识点补充
1、怎么证明nova和glance没问题？执行nova image-list 列出结果，说明上面没问题
2、novncproxy如果挂掉，重启下很快的，没影响
3、600个虚拟机都可以使用单一扁平网络的，几千个虚拟机就不要用单一扁平网络了
4、一些公司现网中配置如下：2颗CPU 64GB内存，一个机器跑7个kvm
5、创建250个实例，30个物理机节点，单一扁平网络也是够的
6、16位的子网掩码，创建的更多
7、sdn（软件定义网络，比如ovs产品）的故障挺多，性能不好。移动架构师一次讲过不推荐
8、vlan最多是4096个，对于公有云远远不够。对于私有欲足够了
9、使用sdn的，主要是公有云
10、分布式路由早晚被干掉，只是把路由转到各个计算节点上
11、Openstack的数据库非常重要，数据库没了没法玩了
12、控制节点一般做高可用
13、单一扁平网络最稳定，中小型的公司足够了
