最近服务器会挂掉 然后磁盘直接损坏无法进入系统 只能进入单用户模式才修复文件系统后才能正常重启
各位大佬有排查思路吗?
下面贴 kdump 信息
KERNEL: /usr/lib/debug/usr/lib/modules/3.10.0-514.el7.x86_64/vmlinux
DUMPFILE: vmcore [PARTIAL DUMP]
CPUS: 2
DATE: Fri Jul 26 14:52:41 2019
UPTIME: 90 days, 16:58:02
LOAD AVERAGE: 0.25, 0.28, 0.31
TASKS: 153
NODENAME: ustack-taicangct55.ustack.qeeyou.com
RELEASE: 3.10.0-514.el7.x86_64
VERSION: #1 SMP Tue Nov 22 16:42:41 UTC 2016
MACHINE: x86_64 (2194 Mhz)
MEMORY: 4 GB
PANIC: "kernel BUG at net/ipv4/inetpeer.c:382!"
PID: 0
COMMAND: "swapper/1"
TASK: ffff880139b81f60 (1 of 2) [THREAD_INFO: ffff880139b8c000]
CPU: 1
STATE: TASK_RUNNING (PANIC)
crash> bt
PID: 0 TASK: ffff880139b81f60 CPU: 1 COMMAND: "swapper/1"
#0 [ffff88013fd03370] machine_kexec at ffffffff81059cdb
#1 [ffff88013fd033d0] __crash_kexec at ffffffff81105182
#2 [ffff88013fd034a0] crash_kexec at ffffffff81105270
#3 [ffff88013fd034b8] oops_end at ffffffff8168ee88
#4 [ffff88013fd034e0] die at ffffffff8102e93b
#5 [ffff88013fd03510] do_trap at ffffffff8168e540
#6 [ffff88013fd03560] do_invalid_op at ffffffff8102b144
#7 [ffff88013fd03610] invalid_op at ffffffff81697e5e
[exception RIP: inet_getpeer+1079]
RIP: ffffffff815afe77 RSP: ffff88013fd036c0 RFLAGS: 00010287
RAX: ffff880138055e40 RBX: 0000000000000001 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 00000000060606df RDI: 00000000060606df
RBP: ffff88013fd03850 R8: ffff880138055e50 R9: ffff88013fd036f8
R10: ffff8800b4d22250 R11: 0000000000000000 R12: ffff8801395fa780
R13: 0000000000000000 R14: ffff88013fd038e0 R15: ffff8800b4d22240
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
#8 [ffff88013fd03738] xfrm_lookup at ffffffff8160f39b
#9 [ffff88013fd037a8] icmp_route_lookup at ffffffff815e613d
#10 [ffff88013fd03888] icmp_send at ffffffff815e6760
#11 [ffff88013fd03a00] nf_send_unreach at ffffffffa032a282 [nf_reject_ipv4]
#12 [ffff88013fd03a20] reject_tg at ffffffffa032f0ca [ipt_REJECT]
#13 [ffff88013fd03a30] ipt_do_table at ffffffffa009fcb0 [ip_tables]
#14 [ffff88013fd03b78] iptable_filter_hook at ffffffffa02ec036 [iptable_filter]
#15 [ffff88013fd03b88] nf_iterate at ffffffff815a6460
#16 [ffff88013fd03bc8] nf_hook_slow at ffffffff815a6548
#17 [ffff88013fd03c00] ip_local_deliver at ffffffff815b0862
#18 [ffff88013fd03c58] ip_rcv_finish at ffffffff815b01aa
#19 [ffff88013fd03c80] ip_rcv at ffffffff815b0b36
#20 [ffff88013fd03ce8] __netif_receive_skb_core at ffffffff8156f582
#21 [ffff88013fd03d50] __netif_receive_skb at ffffffff8156f818
#22 [ffff88013fd03d70] netif_receive_skb_internal at ffffffff8156f8a0
#23 [ffff88013fd03da0] napi_gro_receive at ffffffff81570a28
#24 [ffff88013fd03dc8] virtnet_poll at ffffffffa008d9d5 [virtio_net]
#25 [ffff88013fd03e38] net_rx_action at ffffffff815700b0
#26 [ffff88013fd03eb8] __do_softirq at ffffffff8108f0ff
#27 [ffff88013fd03f28] call_softirq at ffffffff8169805c
#28 [ffff88013fd03f40] do_softirq at ffffffff8102d365
#29 [ffff88013fd03f60] irq_exit at ffffffff8108f495
#30 [ffff88013fd03f78] do_IRQ at ffffffff81698bf8
--- <IRQ stack> ---
#31 [ffff880139b8fde8] ret_from_intr at ffffffff8168dd6d
[exception RIP: native_safe_halt+6]
RIP: ffffffff81060eb6 RSP: ffff880139b8fe98 RFLAGS: 00000286
RAX: 00000000ffffffed RBX: ffff880139b825f8 RCX: 0100000000000000
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000046
RBP: ffff880139b8fe98 R8: 0000000000000000 R9: 0000000000000000
R10: 0000000000000000 R11: 0000000000000000 R12: ffffffff81029569
R13: ffff880139b8fe48 R14: ffffffff810cbcdc R15: ffff880139b8fe10
ORIG_RAX: ffffffffffffffdd CS: 0010 SS: 0018
#32 [ffff880139b8fea0] default_idle at ffffffff8103483f
#33 [ffff880139b8fec0] arch_cpu_idle at ffffffff81035186
#34 [ffff880139b8fed0] cpu_startup_entry at ffffffff810e7c95
#35 [ffff880139b8ff28] start_secondary at ffffffff8104f12a
[7763789.273323] XFS (vda1): page discard on page ffffea00027cf840, inode 0x209215, offset 25886720.
[7763799.300905] XFS (vda1): Internal error XFS_WANT_CORRUPTED_RETURN at line 366 of file fs/xfs/libxfs/xfs_alloc.c. Caller xfs_alloc_ag_vextent_near+0x570/0xab0 [xfs]
[7763799.313824] CPU: 1 PID: 13724 Comm: kworker/u4:2 Tainted: G OE ------------ 3.10.0-514.el7.x86_64 #1
[7763799.313827] Hardware name: RDO OpenStack Compute, BIOS 1.10.2-3.el7_4.1 04/01/2014
[7763799.313838] Workqueue: writeback bdi_writeback_workfn (flush-253:0)
[7763799.313841] ffff8800bbb79000 000000009410b28c ffff8800b4e33460 ffffffff81685fac
[7763799.313844] ffff8800b4e33478 ffffffffa01e115b ffffffffa019eeb0 ffff8800b4e334d0
[7763799.313846] ffffffffa019c764 4c0000b800000020 ffff880035d1d000 00000000a01a0b41
[7763799.313849] Call Trace:
[7763799.313856] [<ffffffff81685fac>] dump_stack+0x19/0x1b
[7763799.313904] [<ffffffffa01e115b>] xfs_error_report+0x3b/0x40 [xfs]
[7763799.313914] [<ffffffffa019eeb0>] ? xfs_alloc_ag_vextent_near+0x570/0xab0 [xfs]
[7763799.313923] [<ffffffffa019c764>] xfs_alloc_fixup_trees+0x2c4/0x370 [xfs]
[7763799.313932] [<ffffffffa019eeb0>] xfs_alloc_ag_vextent_near+0x570/0xab0 [xfs]
[7763799.313951] [<ffffffffa019f4fd>] xfs_alloc_ag_vextent+0x10d/0x150 [xfs]
[7763799.313960] [<ffffffffa01a01c6>] xfs_alloc_vextent+0x446/0x5f0 [xfs]
[7763799.313970] [<ffffffffa01b0663>] xfs_bmap_btalloc+0x3f3/0x780 [xfs]
[7763799.313991] [<ffffffffa01b09fe>] xfs_bmap_alloc+0xe/0x10 [xfs]
[7763799.314002] [<ffffffffa01b1439>] xfs_bmapi_write+0x499/0xab0 [xfs]
[7763799.314018] [<ffffffffa01ecab7>] xfs_iomap_write_allocate+0x177/0x390 [xfs]
[7763799.314032] [<ffffffffa01d68c6>] xfs_map_blocks+0x186/0x240 [xfs]
[7763799.314045] [<ffffffffa01d7f03>] xfs_vm_writepage+0x193/0x5d0 [xfs]
[7763799.314051] [<ffffffff8118b3b3>] __writepage+0x13/0x50
[7763799.314053] [<ffffffff8118bed1>] write_cache_pages+0x251/0x4d0
[7763799.314055] [<ffffffff8118b3a0>] ? global_dirtyable_memory+0x70/0x70
[7763799.314060] [<ffffffff812efe59>] ? blk_flush_plug_list+0xc9/0x230
[7763799.314062] [<ffffffff8118c19d>] generic_writepages+0x4d/0x80
[7763799.314074] [<ffffffffa01d7063>] xfs_vm_writepages+0x53/0x90 [xfs]
[7763799.314077] [<ffffffff8118d24e>] do_writepages+0x1e/0x40
[7763799.314079] [<ffffffff81228730>] __writeback_single_inode+0x40/0x210
[7763799.314082] [<ffffffff8122941e>] writeback_sb_inodes+0x25e/0x420
[7763799.314084] [<ffffffff8122967f>] __writeback_inodes_wb+0x9f/0xd0
[7763799.314086] [<ffffffff81229ec3>] wb_writeback+0x263/0x2f0
[7763799.314088] [<ffffffff8118c8f0>] ? bdi_dirty_limit+0x40/0xe0
[7763799.314091] [<ffffffff8122bdbc>] bdi_writeback_workfn+0x1cc/0x460
[7763799.314095] [<ffffffff810a7f3b>] process_one_work+0x17b/0x470
[7763799.314097] [<ffffffff810a8d76>] worker_thread+0x126/0x410
[7763799.314098] [<ffffffff810a8c50>] ? rescuer_thread+0x460/0x460
[7763799.314103] [<ffffffff810b052f>] kthread+0xcf/0xe0
[7763799.314106] [<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
[7763799.314113] [<ffffffff81696518>] ret_from_fork+0x58/0x90
[7763799.314115] [<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
[7763799.314137] XFS (vda1): page discard on page ffffea00048ae1c0, inode 0x209215, offset 25890816.
[7836803.460242] ------------[ cut here ]------------
[7836803.469540] kernel BUG at net/ipv4/inetpeer.c:382!
[7836803.473564] invalid opcode: 0000 [#1] SMP
[7836803.477139] Modules linked in: ip_set_hash_ipport xt_multiport xt_FULLCONENAT(OE) xt_mark ip_set_hash_net sch_sfq act_mirred cls_u32 sch_ingress sch_htb ifb ppp_async crc_ccitt ppp_generic slhc tun xfrm4_tunnel tunnel4 ipcomp xfrm_ipcomp esp4 ah4 af_key ip_gre ip_tunnel gre xt_REDIRECT nf_nat_redirect ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_set ip_set ipt_REJECT nf_reject_ipv4 xt_TCPMSS nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter iptable_mangle iptable_raw nfnetlink iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd pcspkr virtio_balloon parport_pc parport i2c_piix4 ip_tables xfs libcrc32c virtio_console virtio_blk virtio_net ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel
[7836803.509040] cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm serio_raw drm virtio_pci virtio_ring i2c_core virtio ata_piix floppy libata
[7836803.516803] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G OE ------------ 3.10.0-514.el7.x86_64 #1
[7836803.520704] Hardware name: RDO OpenStack Compute, BIOS 1.10.2-3.el7_4.1 04/01/2014
[7836803.524236] task: ffff880139b81f60 ti: ffff880139b8c000 task.ti: ffff880139b8c000
[7836803.527963] RIP: 0010:[<ffffffff815afe77>] [<ffffffff815afe77>] inet_getpeer.part.3+0x437/0x440
[7836803.531743] RSP: 0018:ffff88013fd036c0 EFLAGS: 00010287
[7836803.535390] RAX: ffff880138055e40 RBX: 0000000000000001 RCX: 0000000000000001
[7836803.539277] RDX: 0000000000000001 RSI: 00000000060606df RDI: 00000000060606df
[7836803.543293] RBP: ffff88013fd03850 R08: ffff880138055e50 R09: ffff88013fd036f8
[7836803.547342] R10: ffff8800b4d22250 R11: 0000000000000000 R12: ffff8801395fa780
[7836803.551371] R13: 0000000000000000 R14: ffff88013fd038e0 R15: ffff8800b4d22240
[7836803.555256] FS: 0000000000000000(0000) GS:ffff88013fd00000(0000) knlGS:0000000000000000
[7836803.559316] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[7836803.562981] CR2: 00007efe2c6d0000 CR3: 00000000b2ae7000 CR4: 00000000003406e0
[7836803.566692] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[7836803.570206] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[7836803.573753] Stack:
[7836803.576108] ffff8801395fa78c 000077fec02fc927 0000000181a9cd80 ffff88013fd03718
[7836803.580009] ffff8801395fa780 ffff8800879ff0c0 ffff8800a13cb900 ffff880138055e48
[7836803.583751] ffff8800b4d22248 ffff880087a146c0 ffff880137be3cc8 ffff88013fd037c0
[7836803.587139] Call Trace:
[7836803.589651] <IRQ>
[7836803.590179]
[7836803.592561] [<ffffffff8160f39b>] ? xfrm_lookup+0x30b/0x3f0
[7836803.595548] [<ffffffff815e613d>] ? icmp_route_lookup.isra.15+0x2ed/0x450
[7836803.598860] [<ffffffff815aff79>] inet_getpeer+0xf9/0x170
[7836803.602460] [<ffffffff815e6760>] icmp_send+0x4c0/0x510
[7836803.606946] [<ffffffff815fd976>] ? fib4_rule_action+0x66/0x70
[7836803.611019] [<ffffffff8158ea1d>] ? fib_rules_lookup+0x10d/0x1c0
[7836803.614040] [<ffffffffa032a282>] nf_send_unreach+0xa2/0xc0 [nf_reject_ipv4]
[7836803.617604] [<ffffffffa032f0ca>] reject_tg+0x6a/0xa0 [ipt_REJECT]
[7836803.620901] [<ffffffffa009fcb0>] ipt_do_table+0x2e0/0x710 [ip_tables]
[7836803.624164] [<ffffffffa02ec036>] iptable_filter_hook+0x36/0x80 [iptable_filter]
[7836803.627421] [<ffffffff815a6460>] nf_iterate+0x70/0xb0
[7836803.630297] [<ffffffff815a6548>] nf_hook_slow+0xa8/0x110
[7836803.633082] [<ffffffff815b0862>] ip_local_deliver+0xb2/0xd0
[7836803.635622] [<ffffffff815b0470>] ? ip_rcv_finish+0x350/0x350
[7836803.638377] [<ffffffff815b01aa>] ip_rcv_finish+0x8a/0x350
[7836803.641076] [<ffffffff815b0b36>] ip_rcv+0x2b6/0x410
[7836803.643314] [<ffffffff815b0120>] ? inet_del_offload+0x40/0x40
[7836803.645869] [<ffffffff8156f582>] __netif_receive_skb_core+0x582/0x800
[7836803.648532] [<ffffffff8156f818>] __netif_receive_skb+0x18/0x60
[7836803.650987] [<ffffffff8156f8a0>] netif_receive_skb_internal+0x40/0xc0
[7836803.653469] [<ffffffff81570a28>] napi_gro_receive+0xd8/0x130
[7836803.655899] [<ffffffffa008d9d5>] virtnet_poll+0x265/0x750 [virtio_net]
[7836803.658469] [<ffffffff815700b0>] net_rx_action+0x170/0x380
[7836803.661079] [<ffffffff8108f0ff>] __do_softirq+0xef/0x280
[7836803.663320] [<ffffffff8169805c>] call_softirq+0x1c/0x30
[7836803.665441] [<ffffffff8102d365>] do_softirq+0x65/0xa0
[7836803.667409] [<ffffffff8108f495>] irq_exit+0x115/0x120
[7836803.669445] [<ffffffff81698bf8>] do_IRQ+0x58/0xf0
[7836803.671524] [<ffffffff8168dd6d>] common_interrupt+0x6d/0x6d
[7836803.673487] <EOI>
[7836803.673861]
[7836803.675162] [<ffffffff81060eb6>] ? native_safe_halt+0x6/0x10
[7836803.677111] [<ffffffff8103483f>] default_idle+0x1f/0xc0
[7836803.679039] [<ffffffff81035186>] arch_cpu_idle+0x26/0x30
[7836803.680913] [<ffffffff810e7c95>] cpu_startup_entry+0x245/0x290
[7836803.682963] [<ffffffff8104f12a>] start_secondary+0x1ba/0x230
[7836803.685076] Code: 90 fe ff ff 4c 89 38 e8 a8 fa ff ff 41 83 44 24 14 01 e9 bb fc ff ff 0f 1f 44 00 00 4c 89 fa e9 92 fe ff ff 0f 0b e8 49 57 ad ff <0f> 0b 0f 1f 80 00 00 00 00 0f 1f 44 00 00 55 48 c7 c0 e0 0e f9
[7836803.693794] RIP [<ffffffff815afe77>] inet_getpeer.part.3+0x437/0x440
[7836803.696787] RSP <ffff88013fd036c0>
这是一个专为移动设备优化的页面(即为了让你能够在 Google 搜索结果里秒开这个页面),如果你希望参与 V2EX 社区的讨论,你可以继续到 V2EX 上打开本讨论主题的完整版本。
V2EX 是创意工作者们的社区,是一个分享自己正在做的有趣事物、交流想法,可以遇见新朋友甚至新机会的地方。
V2EX is a community of developers, designers and creative people.