Module space空间不足导致insmod失败实例
copy from：http://cukdd.blog.chinaunix.net/uid-25564582-id-5751533.html
在arm平台下移植Atheros Wifi驱动时遇到insmod umac.ko失败的问题；
|
root@OpenWrt:/lib/modules/3.14.43/kernel# ls -l ./drivers/umac.ko -h -rw-r--r-- 1 root root 2.7M Sep 8 06:47 ./drivers/umac.ko |
|
root@OpenWrt:/lib/modules/3.14.43/kernel/drivers# insmod ./umac.ko [ 2072.000717] vmap allocation for size 458752 failed: use vmalloc= to increase size. [ 2072.008447] vmalloc: allocation failure: 454123 bytes [ 2072.012855] insmod: page allocation failure: order:0, mode:0xd0 [ 2072.018885] CPU: 0 PID: 2173 Comm: insmod Tainted: P 3.14.43_cig001 #6 [ 2072.026369] [] (unwind_backtrace) from [] (show_stack+0x10/0x14) [ 2072.034247] [] (show_stack) from [] (dump_stack+0x88/0xcc) [ 2072.041370] [] (dump_stack) from [] (warn_alloc_failed+0xd0/0xfc) [ 2072.049206] [] (warn_alloc_failed) from [] (__vmalloc_node_range+0x1b0/0x1d4) [ 2072.057939] [] (__vmalloc_node_range) from [] (module_alloc+0x30/0x44) [ 2072.066266] [] (module_alloc) from [] (module_alloc_update_bounds+0xc/0x5c) [ 2072.074930] [] (module_alloc_update_bounds) from [] (load_module+0x678/0x1940) [ 2072.083916] [] (load_module) from [] (SyS_init_module+0xc0/0xd8) [ 2072.091602] [] (SyS_init_module) from [] (ret_fast_syscall+0x0/0x30) [ 2072.099663] Mem-info: [ 2072.101819] Normal per-cpu: [ 2072.104602] CPU 0: hi: 90, btch: 15 usd: 27 [ 2072.109556] CPU 1: hi: 90, btch: 15 usd: 83 [ 2072.114153] CPU 2: hi: 90, btch: 15 usd: 10 [ 2072.119006] CPU 3: hi: 90, btch: 15 usd: 36 [ 2072.123746] active_anon:1525 inactive_anon:16 isolated_anon:0 [ 2072.123746] active_file:4500 inactive_file:3187 isolated_file:0 [ 2072.123746] unevictable:0 dirty:0 writeback:0 unstable:0 [ 2072.123746] free:40864 slab_reclaimable:813 slab_unreclaimable:4219 [ 2072.123746] mapped:523 shmem:78 pagetables:95 bounce:0 [ 2072.123746] free_cma:0 [ 2072.154969] Normal free:163456kB min:1944kB low:2428kB high:2916kB active_anon:6100kB inactive_anon:64kB active_file:18000kB inactive_file:12748kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:245760kB managed:236872kB mlocked:0kB dirty:0kB writeback:0kB mapped:2092kB shmem:312kB slab_reclaimable:3252kB slab_unreclaimable:16876kB kernel_stack:608kB pagetables:380kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? no [ 2072.196622] lowmem_reserve[]: 0 0 0 [ 2072.199993] Normal: 0*4kB 8*8kB (UEM) 3*16kB (EM) 1*32kB (U) 2*64kB (UE) 3*128kB (UEM) 2*256kB (UM) 3*512kB (UEM) 1*1024kB (U) 2*2048kB (UM) 38*4096kB (MR) = 163472kB [ 2072.214838] 7765 total pagecache pages [ 2072.218663] 0 pages in swap cache [ 2072.221877] Swap cache stats: add 0, delete 0, find 0/0 [ 2072.227075] Free swap = 0kB [ 2072.230017] Total swap = 0kB [ 2072.242807] 65536 pages of RAM [ 2072.244851] 41172 free pages [ 2072.247715] 6318 reserved pages [ 2072.250933] 2005 slab pages [ 2072.253618] 3711 pages shared [ 2072.256567] 0 pages swap cached |
umac本身有2.7M，比较大；从出现的堆栈看问题出在加载阶段，与ko本身的运行没有关系；
insmod命令实质调用kmodloader；
|
root@OpenWrt:/sbin# ls -l /usr/sbin/insmod lrwxrwxrwx 1 root root 21 Sep 4 07:32 /usr/sbin/insmod -> ../../sbin/kmodloader |
模块加载的内核态实现，由系统调用SyS_init_module开始，其包裹的load_module进行实质的空间分配和模块加载；
|
static int load_module(struct load_info *info, const char __user *uargs, int flags) { struct module *mod; long err;
err = module_sig_check(info); //若module存在signature，则进行检查 if (err) goto free_copy;
err = elf_header_check(info); //elf文件格式检查 if (err) goto free_copy;
/* Figure out module layout, and allocate all the memory. */ mod = layout_and_allocate(info, flags); if (IS_ERR(mod)) { err = PTR_ERR(mod); goto free_copy; } …… } |
|
static struct module *layout_and_allocate(struct load_info *info, int flags) { /* Module within temporary copy. */ struct module *mod; int err;
mod = setup_load_info(info, flags); //记录模块基本信息，以便后续调用； if (IS_ERR(mod)) return mod;
err = check_modinfo(mod, info, flags); if (err) return ERR_PTR(err);
/* Allow arches to frob section contents and sizes. */ err = module_frob_arch_sections(info->hdr, info->sechdrs, info->secstrings, mod); if (err < 0) return ERR_PTR(err);
/* We will do a special allocation for per-cpu sections later. */ info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
/* Determine total sizes, and put offsets in sh_entsize. For now this is done generically; there doesn't appear to be any special cases for the architectures. */ layout_sections(mod, info); layout_symtab(mod, info);
/* Allocate and move to the final place */ err = move_module(mod, info); //在该函数内分配地址空间； if (err) return ERR_PTR(err);
/* Module has been copied to its final place now: return it. */ mod = (void *)info->sechdrs[info->index.mod].sh_addr; kmemleak_load_module(mod, info); return mod; } |
|
static int move_module(struct module *mod, struct load_info *info) { int i; void *ptr;
/* Do the allocs. */ ptr = module_alloc_update_bounds(mod->core_size); …… } |
|
static void *module_alloc_update_bounds(unsigned long size) { void *ret = module_alloc(size);
if (ret) { mutex_lock(&module_mutex); /* Update module bounds. */ if ((unsigned long)ret < module_addr_min) module_addr_min = (unsigned long)ret; if ((unsigned long)ret + size > module_addr_max) module_addr_max = (unsigned long)ret + size; mutex_unlock(&module_mutex); } return ret; } |
|
void *module_alloc(unsigned long size) { return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, GFP_KERNEL, PAGE_KERNEL_EXEC, NUMA_NO_NODE, __builtin_return_address(0)); } |
从上面函数可以看到module加载存在特定的地址范围，[MODULES_VADDR, MODULES_END];
这两个宏定义在；linux-3.14.43/arch/arm/include/asm/memory.h，堆栈信息中提示是空间分配失败导致的，那么这段地址范围有多大呐
|
[wms@localhost linux-3.14.43]$ cat .config | grep CONFIG_PAGE_OFFSET CONFIG_PAGE_OFFSET=0xC0000000 //3G |
|
#define PAGE_OFFSET UL(CONFIG_PAGE_OFFSET) //3G
/* * TASK_SIZE - the maximum size of a user space task. //用户空间最大地址范围； * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area */ #define TASK_SIZE (UL(CONFIG_PAGE_OFFSET) - UL(SZ_16M)) #define TASK_UNMAPPED_BASE ALIGN(TASK_SIZE / 3, SZ_16M)
/* * The module space lives between the addresses given by TASK_SIZE * and PAGE_OFFSET - it must be within 32MB of the kernel text. //注意module space 最大空间为32MB */ #ifndef CONFIG_THUMB2_KERNEL #define MODULES_VADDR (PAGE_OFFSET - SZ_16M) #else /* smaller range for Thumb-2 symbols relocation (2^24)*/ #define MODULES_VADDR (PAGE_OFFSET - SZ_8M) #endif
#if TASK_SIZE > MODULES_VADDR /*若用户态最大地址与module地址存在冲突，编译时报警*/ #error Top of user space clashes with start of module space #endif
/* * The highmem pkmap virtual space shares the end of the module area. */ #ifdef CONFIG_HIGHMEM #define MODULES_END (PAGE_OFFSET - PMD_SIZE) #else #define MODULES_END (PAGE_OFFSET) #endif
|
|
#define PMD_SHIFT 21 #define PGDIR_SHIFT 21
#define PMD_SIZE (1UL << PMD_SHIFT) //2M |
从上面宏定义中可以看出；
用户态最大地址TASK_SIZE为：3G-16M;
Module分配空间地址范围为：[3G-16M, 3G-2M];
必须保证TASK_SIZE与MODULES_VADDR不重叠；
针对上面的问题，我们通过扩展Module的范围来解决；
|
[wms@localhost linux-3.14.43]$ git diff ./arch/arm/include/ diff --git a/linux-3.14.43/arch/arm/include/asm/memory.h b/linux-3.14.43/arch/arm/include/asm/memory.h old mode 100644 new mode 100755 index 4afb376..deca0b2 --- a/linux-3.14.43/arch/arm/include/asm/memory.h +++ b/linux-3.14.43/arch/arm/include/asm/memory.h @@ -39,7 +39,8 @@ * TASK_SIZE - the maximum size of a user space task. * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area */ -#define TASK_SIZE (UL(CONFIG_PAGE_OFFSET) - UL(SZ_16M)) +#define TASK_SIZE (UL(CONFIG_PAGE_OFFSET) - UL(SZ_32M)) #define TASK_UNMAPPED_BASE ALIGN(TASK_SIZE / 3, SZ_16M)
/* @@ -52,7 +53,8 @@ * and PAGE_OFFSET - it must be within 32MB of the kernel text. */ #ifndef CONFIG_THUMB2_KERNEL -#define MODULES_VADDR (PAGE_OFFSET - SZ_16M) +#define MODULES_VADDR (PAGE_OFFSET - SZ_32M) #else /* smaller range for Thumb-2 symbols relocation (2^24)*/ #define MODULES_VADDR (PAGE_OFFSET - SZ_8M) |
