smp 挂住

[    0.000000] Initializing cgroup subsys cpuset
[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Initializing cgroup subsys cpuacct
[    0.000000] Linux version 3.10.0-957.el7.x86_64 (mockbuild@kbuilder.bsys.centos.org) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-36) (GCC) ) #1 SMP Thu Nov 8 23:39:32 UTC 2018
[    0.000000] Command line: BOOT_IMAGE=/vmlinuz-3.10.0-957.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb quiet LANG=zh_CN.UTF-8
[    0.000000] e820: BIOS-provided physical RAM map:

[4665996.724513] Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 f4 b1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89 
[4666024.724041] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u160:2:32836]
[4666024.724094] Modules linked in: nls_utf8 isofs ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ppdev sg pcspkr virtio_balloon joydev parport_pc parport i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom virtio_blk virtio_console virtio_scsi virtio_net ata_generic pata_acpi serio_raw floppy cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata virtio_pci
[4666024.724142]  virtio_ring virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
[4666024.724150] CPU: 0 PID: 32836 Comm: kworker/u160:2 Kdump: loaded Tainted: G             L ------------   3.10.0-957.el7.x86_64 #1
[4666024.724152] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
[4666024.724160] Workqueue: writeback bdi_writeback_workfn (flush-253:0)
[4666024.724163] task: ffff8b4fa6746180 ti: ffff8b4c2d844000 task.ti: ffff8b4c2d844000
[4666024.724164] RIP: 0010:[<ffffffffb131173a>]  [<ffffffffb131173a>] smp_call_function_many+0x20a/0x270
[4666024.724172] RSP: 0018:ffff8b4c2d847820  EFLAGS: 00000202
[4666024.724173] RAX: 0000000000000003 RBX: 000c0000b127a3d0 RCX: ffff8b4fae2df610
[4666024.724175] RDX: 0000000000000003 RSI: 0000000000000050 RDI: 0000000000000000
[4666024.724176] RBP: ffff8b4c2d847858 R08: ffff8b4cffd71000 R09: ffffffffb15755c9
[4666024.724177] R10: ffff8b4fae21f160 R11: fffff7ac0f34e000 R12: 0000000000000286
[4666024.724179] R13: 0000000000000286 R14: ffff8b4c2d8477d0 R15: ffff8b4c2d8477a0
[4666024.724180] FS:  0000000000000000(0000) GS:ffff8b4fae200000(0000) knlGS:0000000000000000
[4666024.724182] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[4666024.724183] CR2: 00007f90cd151000 CR3: 000000005b610000 CR4: 00000000000006f0
[4666024.724188] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[4666024.724189] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[4666024.724190] Call Trace:
[4666024.724197]  [<ffffffffb127a598>] native_flush_tlb_others+0xb8/0xc0
[4666024.724199]  [<ffffffffb127a734>] flush_tlb_page+0x54/0xa0
[4666024.724203]  [<ffffffffb13fd587>] ptep_clear_flush+0x67/0xa0
[4666024.724207]  [<ffffffffb13f6b66>] page_mkclean_one+0xa6/0xe0
[4666024.724209]  [<ffffffffb13f8983>] rmap_walk+0xf3/0x360
[4666024.724212]  [<ffffffffb13f8d96>] page_mkclean+0x76/0xb0
[4666024.724214]  [<ffffffffb13f6ac0>] ? page_referenced_one+0x180/0x180
[4666024.724216]  [<ffffffffb13f6200>] ? percpu_flush_tlb_batch_pages+0x20/0x20
[4666024.724221]  [<ffffffffb13c1d60>] clear_page_dirty_for_io+0x50/0xd0
[4666024.724223]  [<ffffffffb13c200a>] write_cache_pages+0x22a/0x4e0
[4666024.724254]  [<ffffffffc03d75b0>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[4666024.724259]  [<ffffffffb1544f30>] ? submit_bio+0x70/0x150
[4666024.724273]  [<ffffffffc03d73ab>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[4666024.724276]  [<ffffffffb13c3181>] do_writepages+0x21/0x50
[4666024.724279]  [<ffffffffb146f930>] __writeback_single_inode+0x40/0x260
[4666024.724282]  [<ffffffffb14703c4>] writeback_sb_inodes+0x1c4/0x490
[4666024.724284]  [<ffffffffb147072f>] __writeback_inodes_wb+0x9f/0xd0
[4666024.724287]  [<ffffffffb1470f63>] wb_writeback+0x263/0x2f0
[4666024.724290]  [<ffffffffb145cd7c>] ? get_nr_inodes+0x4c/0x70
[4666024.724292]  [<ffffffffb14718eb>] bdi_writeback_workfn+0x2cb/0x460
[4666024.724296]  [<ffffffffb12b9d4f>] process_one_work+0x17f/0x440
[4666024.724298]  [<ffffffffb12bade6>] worker_thread+0x126/0x3c0
[4666024.724301]  [<ffffffffb12bacc0>] ? manage_workers.isra.25+0x2a0/0x2a0
[4666024.724303]  [<ffffffffb12c1c31>] kthread+0xd1/0xe0
[4666024.724305]  [<ffffffffb12c1b60>] ? insert_kthread_work+0x40/0x40
[4666024.724309]  [<ffffffffb1974c37>] ret_from_fork_nospec_begin+0x21/0x21
[4666024.724311]  [<ffffffffb12c1b60>] ? insert_kthread_work+0x40/0x40

[4666024.724312] Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 f4 b1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89 
[4666052.723825] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u160:2:32836]
[4666052.723883] Modules linked in: nls_utf8 isofs ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ppdev sg pcspkr virtio_balloon joydev parport_pc parport i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom virtio_blk virtio_console virtio_scsi virtio_net ata_generic pata_acpi serio_raw floppy cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata virtio_pci
[4666052.723925]  virtio_ring virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
[4666052.723932] CPU: 0 PID: 32836 Comm: kworker/u160:2 Kdump: loaded Tainted: G             L ------------   3.10.0-957.el7.x86_64 #1
[4666052.723934] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
[4666052.723943] Workqueue: writeback bdi_writeback_workfn (flush-253:0)
[4666052.723945] task: ffff8b4fa6746180 ti: ffff8b4c2d844000 task.ti: ffff8b4c2d844000
[4666052.723947] RIP: 0010:[<ffffffffb131173a>]  [<ffffffffb131173a>] smp_call_function_many+0x20a/0x270
[4666052.723954] RSP: 0018:ffff8b4c2d847820  EFLAGS: 00000202
[4666052.723955] RAX: 0000000000000003 RBX: 000c0000b127a3d0 RCX: ffff8b4fae2df610
[4666052.723957] RDX: 0000000000000003 RSI: 0000000000000050 RDI: 0000000000000000
[4666052.723958] RBP: ffff8b4c2d847858 R08: ffff8b4cffd71000 R09: ffffffffb15755c9
[4666052.723960] R10: ffff8b4fae21f160 R11: fffff7ac0f34e000 R12: 0000000000000286
[4666052.723961] R13: 0000000000000286 R14: ffff8b4c2d8477d0 R15: ffff8b4c2d8477a0
[4666052.723963] FS:  0000000000000000(0000) GS:ffff8b4fae200000(0000) knlGS:0000000000000000
[4666052.723964] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[4666052.723966] CR2: 00007f90cd151000 CR3: 000000005b610000 CR4: 00000000000006f0
[4666052.723970] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[4666052.723971] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[4666052.723972] Call Trace:
[4666052.723978]  [<ffffffffb127a598>] native_flush_tlb_others+0xb8/0xc0
[4666052.723981]  [<ffffffffb127a734>] flush_tlb_page+0x54/0xa0
[4666052.723984]  [<ffffffffb13fd587>] ptep_clear_flush+0x67/0xa0
[4666052.723989]  [<ffffffffb13f6b66>] page_mkclean_one+0xa6/0xe0
[4666052.723991]  [<ffffffffb13f8983>] rmap_walk+0xf3/0x360
[4666052.723993]  [<ffffffffb13f8d96>] page_mkclean+0x76/0xb0
[4666052.723996]  [<ffffffffb13f6ac0>] ? page_referenced_one+0x180/0x180
[4666052.723998]  [<ffffffffb13f6200>] ? percpu_flush_tlb_batch_pages+0x20/0x20
[4666052.724003]  [<ffffffffb13c1d60>] clear_page_dirty_for_io+0x50/0xd0
[4666052.724006]  [<ffffffffb13c200a>] write_cache_pages+0x22a/0x4e0
[4666052.724028]  [<ffffffffc03d75b0>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[4666052.724033]  [<ffffffffb1544f30>] ? submit_bio+0x70/0x150
[4666052.724047]  [<ffffffffc03d73ab>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[4666052.724050]  [<ffffffffb13c3181>] do_writepages+0x21/0x50
[4666052.724053]  [<ffffffffb146f930>] __writeback_single_inode+0x40/0x260
[4666052.724056]  [<ffffffffb14703c4>] writeback_sb_inodes+0x1c4/0x490
[4666052.724058]  [<ffffffffb147072f>] __writeback_inodes_wb+0x9f/0xd0
[4666052.724061]  [<ffffffffb1470f63>] wb_writeback+0x263/0x2f0
[4666052.724064]  [<ffffffffb145cd7c>] ? get_nr_inodes+0x4c/0x70
[4666052.724066]  [<ffffffffb14718eb>] bdi_writeback_workfn+0x2cb/0x460
[4666052.724070]  [<ffffffffb12b9d4f>] process_one_work+0x17f/0x440
[4666052.724072]  [<ffffffffb12bade6>] worker_thread+0x126/0x3c0
[4666052.724074]  [<ffffffffb12bacc0>] ? manage_workers.isra.25+0x2a0/0x2a0
[4666052.724077]  [<ffffffffb12c1c31>] kthread+0xd1/0xe0
[4666052.724079]  [<ffffffffb12c1b60>] ? insert_kthread_work+0x40/0x40
[4666052.724083]  [<ffffffffb1974c37>] ret_from_fork_nospec_begin+0x21/0x21
[4666052.724085]  [<ffffffffb12c1b60>] ? insert_kthread_work+0x40/0x40

[4667060.716964] Code: 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 f4 b1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee 
[4668172.434726] INFO: rcu_sched self-detected stall on CPU { 3}  (t=2211778 jiffies g=20184027 c=20184026 q=28626)
[4668172.435521] Task dump for CPU 3:
[4668172.435524] systemd         R  running task        0     1      0 0x00000008
[4668172.435527] Call Trace:
[4668172.435530]  <IRQ>  [<ffffffffb12d5d88>] sched_show_task+0xa8/0x110
[4668172.435539]  [<ffffffffb12d9c09>] dump_cpu_task+0x39/0x70
[4668172.435544]  [<ffffffffb1352650>] rcu_dump_cpu_stacks+0x90/0xd0
[4668172.435547]  [<ffffffffb1355cf2>] rcu_check_callbacks+0x442/0x730
[4668172.435552]  [<ffffffffb130ad60>] ? tick_sched_do_timer+0x50/0x50
[4668172.435556]  [<ffffffffb12ab136>] update_process_times+0x46/0x80
[4668172.435559]  [<ffffffffb130aad0>] tick_sched_handle+0x30/0x70
[4668172.435561]  [<ffffffffb130ad99>] tick_sched_timer+0x39/0x80
[4668172.435564]  [<ffffffffb12c5f93>] __hrtimer_run_queues+0xf3/0x270
[4668172.435567]  [<ffffffffb12c651f>] hrtimer_interrupt+0xaf/0x1d0
[4668172.435572]  [<ffffffffb125a2cb>] local_apic_timer_interrupt+0x3b/0x60
[4668172.435576]  [<ffffffffb19796c3>] smp_apic_timer_interrupt+0x43/0x60
[4668172.435581]  [<ffffffffb1975df2>] apic_timer_interrupt+0x162/0x170
[4668172.435583]  <EOI>  [<ffffffffb1300fc0>] ? ktime_get_ts64+0xa0/0xf0
[4668172.435592]  [<ffffffffb1457160>] select_estimate_accuracy+0x40/0x120
[4668172.435597]  [<ffffffffb143e03a>] ? __check_object_size+0x1ca/0x250
[4668172.435599]  [<ffffffffb1458351>] do_sys_poll+0x201/0x590
[4668172.435604]  [<ffffffffb18722af>] ? __netlink_sendskb+0x5f/0x180
[4668172.435606]  [<ffffffffb187551b>] ? netlink_unicast+0x1db/0x210
[4668172.435610]  [<ffffffffb1823455>] ? skb_release_data+0xf5/0x140
[4668172.435615]  [<ffffffffb1821697>] ? kfree_skbmem+0x37/0x90
[4668172.435617]  [<ffffffffb18239d4>] ? consume_skb+0x34/0x90
[4668172.435619]  [<ffffffffb18754b8>] ? netlink_unicast+0x178/0x210
[4668172.435621]  [<ffffffffb1875698>] ? netlink_sendmsg+0x148/0x420
[4668172.435623]  [<ffffffffb1456c80>] ? poll_initwait+0x50/0x50
[4668172.435625]  [<ffffffffb18193a6>] ? sock_sendmsg+0xb6/0xf0
[4668172.435628]  [<ffffffffb1819ad1>] ? SYSC_sendto+0x121/0x1c0
[4668172.435630]  [<ffffffffb1300fc0>] ? ktime_get_ts64+0xa0/0xf0
[4668172.435632]  [<ffffffffb14587e4>] SyS_poll+0x74/0x110
[4668172.435635]  [<ffffffffb1974ddb>] system_call_fastpath+0x22/0x27
[4668172.435637]  [<ffffffffb1974d21>] ? system_call_after_swapgs+0xae/0x146

 

posted @ 2021-07-26 14:26  maojun1998  阅读(73)  评论(0编辑  收藏  举报