linux 排查page的状态问题

最近遇到一个page的释放异常的问题，堆栈如下：

[ 1000.691858] BUG: Bad page state in process server.o  pfn:309d22
[ 1000.691859] page:ffffea000c274880 count:0 mapcount:0 mapping:ffff880279688308 index:0x0
[ 1000.691860] page flags: 0x2fffff00020000(mappedtodisk)
[ 1000.691862] page dumped because: non-NULL mapping
[ 1000.691863] Modules linked in: stap_11fa48f04897d7244c07086623507d9_14185(OE) xfs libcrc32c tcp_diag inet_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bridge stp llc dm_mirror dm_region_hash dm_log dm_mod intel_powerclamp snd_hda_intel coretemp ppdev kvm_intel snd_hda_codec snd_hda_core iTCO_wdt gpio_ich iTCO_vendor_support snd_hwdep ioatdma snd_seq parport_pc kvm shpchp parport nfsd snd_seq_device snd_pcm pcspkr sg irqbypass ntb i2c_i801 snd_timer intel_ips snd lpc_ich soundcore auth_rpcgss nfs_acl lockd grace sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic crct10dif_common
[ 1000.691895]  amdkfd amd_iommu_v2 radeon i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ixgbe ahci libahci libata tg3 mdio crc32c_intel dca serio_raw ptp i2c_core pps_core fjes floppy [last unloaded: stap_be77ad5fa9d5c22c253e09b1d6390ba4__1921]
[ 1000.691908] CPU: 3 PID: 29178 Comm: server.o Tainted: G    B      OE  ------------   3.10.0+ #10
[ 1000.691910] Hardware name: To be filled by O.E.M. To be filled by O.E.M./To be filled by O.E.M., BIOS 4.6.3 01/14/2011
[ 1000.691911]  ffffea000c274880 000000001df7af73 ffff88050ee37d08 ffffffff81688527
[ 1000.691913]  ffff88050ee37d30 ffffffff81683751 ffffea000c274880 0000000000000000
[ 1000.691915]  000fffff00000000 ffff88050ee37d78 ffffffff81188d6d fff00000fe000000
[ 1000.691918] Call Trace:
[ 1000.691920]  [<ffffffff81688527>] dump_stack+0x19/0x1b
[ 1000.691922]  [<ffffffff81683751>] bad_page.part.75+0xdf/0xfc
[ 1000.691925]  [<ffffffff81188d6d>] free_pages_prepare+0x16d/0x190
[ 1000.691927]  [<ffffffff811897e4>] free_hot_cold_page+0x74/0x160
[ 1000.691930]  [<ffffffff8118e6a3>] __put_single_page+0x23/0x30
[ 1000.691932]  [<ffffffff8118e6f5>] put_page+0x45/0x60
[ 1000.691934]  [<ffffffff8122cd25>] page_cache_pipe_buf_release+0x15/0x20
[ 1000.691937]  [<ffffffff8122d7a4>] splice_direct_to_actor+0x134/0x200
[ 1000.691940]  [<ffffffff8122d9f0>] ? do_splice_from+0xf0/0xf0
[ 1000.691942]  [<ffffffff8122d8d2>] do_splice_direct+0x62/0x90
[ 1000.691944]  [<ffffffff811fe7c8>] do_sendfile+0x1d8/0x3c0
[ 1000.691947]  [<ffffffff811ffb2e>] SyS_sendfile64+0x5e/0xb0
[ 1000.691949]  [<ffffffff81698b49>] system_call_fastpath+0x16/0x1b
[ 1000.691951] BUG: Bad page state in process server.o  pfn:309d23

可以看出，page释放失败的原因是：non-NULL mapping，也就是释放的时候，page->mapping不为NULL，我们来看check函数：

static inline int free_pages_check(struct page *page)
{
    char *bad_reason = NULL;
    unsigned long bad_flags = 0;

    if (unlikely(page_mapcount(page)))
        bad_reason = "nonzero mapcount";
    if (unlikely(page->mapping != NULL))-------------------page的mapping不为NULL，视为异常。
        bad_reason = "non-NULL mapping";
    if (unlikely(page_ref_count(page) != 0))
        bad_reason = "nonzero _count";
    if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
        bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
        bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
    }
    if (unlikely(mem_cgroup_bad_page_check(page)))
        bad_reason = "cgroup check failed";
    if (unlikely(bad_reason)) {
        bad_page(page, bad_reason, bad_flags);
        return 1;
    }
    page_cpupid_reset_last(page);
    if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
        page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
    return 0;
}

按道理，如果是匿名页，释放的时候，page的mapping会被置为NULL，如下：

static bool free_pages_prepare(struct page *page, unsigned int order)
{
    int i;
    int bad = 0;

    trace_mm_page_free(page, order);
    kmemcheck_free_shadow(page, order);

    if (PageAnon(page))
        page->mapping = NULL;
    for (i = 0; i < (1 << order); i++)
        bad += free_pages_check(page + i);
    if (bad)
        return false;

    if (!PageHighMem(page)) {
        debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order);
        debug_check_no_obj_freed(page_address(page),
                       PAGE_SIZE << order);
    }
    arch_free_page(page, order);
    kernel_map_pages(page, 1 << order, 0);

    return true;
}

既然进入了bad的计数，说明page在释放的时候不是匿名页。我的代码中，page的mapping是指向file的address_space,所以mapping不为NULL。

问题的原因是因为，我自己做的内存池，在管理page的时候，有一个流程没有正常做计数，导致被异常释放的时候，指针还没清理。