实验二 跟踪分析Linux内核5.0系统调用处理过程

实验二 跟踪分析Linux内核5.0系统调用处理过程

学号293 原创作品转载请注明出处 https://github.com/mengning/linuxkernel/

实验要求

实验环境

Ubuntu 18.04 LTS

gcc 7.3.0

实验步骤

1. 下载内核代码并编译

下载地址

可以直接下载后然后手动解压

也可以按照以下方式下载解压

mkdir ~/LinuxKernel
cd ~/LinuxKernel
wget https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.0.1.tar.xz
xz -d linux-5.0.1.tar.xz
tar -xvf linux-5.0.1.tar
cd linux-5.0.1

然后安装内核编译工具，(可以考虑换源为国内的，参考)

sudo apt install build-essential flex bison libssl-dev libelf-dev libncurses-dev

然后

make menuconfig

选择kernel hacking -> Compile-time checks and compiler options -> [*]compile the kernel with debug info

再make

make -j8

此时已经编译完成，生成的文件地址在 ./arch/x86/boot/bzImage

2.制作根文件系统

cd ~/LinuxKernel
mkdir rootfs
git clone https://github.com/mengning/menu.git
cd menu
sudo apt install gcc-multilib   #不安装，编译时会提示缺少文件，实际上是安装gcc环境不完善导致
gcc -pthread -o init linktable.c menu.c test.c -m32 -static
cd ../rootfs
cp ../menu/init ./
find . | cpio -o -Hnewc |gzip -9 > ../rootfs.img

3.启动MenuOS

qemu-system-i386 -kernel linux-5.0.1/arch/x86/boot/bzImage -initrd rootfs.img

此时显示

尽管我已经装了qemu-system-i386

然后

我于是采用建议下载了qemu-system-x86,这里主要是因为我编译内核时编译的时64位的版本所以不能用
也可以通过重新make i386_defconfig 解决，但是这样在之后的gdb过程中，无法显示断点所在的文件与行数
所以这里我重新使用make menuconfig,去掉勾选64-bit kernel，同时勾选Kernel hacking -> Compile-time checks and compiler options -> Compile the kernel with debug info 使编译成32位的内核并且能方便显示文件位置

再编译

make -j8

此时需要重新生成rootfs.img,为了方便我这里直接修改Makefile中的设置为

然后进行编译

cd ~/LinuxKernel/menu
make rootfs

得到

4.调试跟踪内核启动

cd ..
qemu-system-i386 -kernel linux-5.0.1/arch/x86/boot/bzImage -initrd rootfs.img -S -s -append nokaslr

注意：-append nokaslr选项的说明见知乎。
运行qemu虚拟机后，在当前目录新建一个终端窗口，运行下列命令：

cd linux-5.0
gdb vmlinux

进入gdb界面后连接到qemu,输入

target remote:1234

然后即可正常的进行debug了

5 代码分析

几乎所有的内核模块均会在start_kernel进行初始化.在start_kernel中，会对各项硬件设备进行初始化，包括一些page_address、tick等等，直到最后需要执行的rest_init中，会开始让系统跑起来。

然后在rest_init()过程中，会调用kernel_thread()来创建内核线程kernel_init,它创建用户的init进程，初始化内核，并设置成1号进程，这个进程会继续做相关的系统的初始化。

然后，start_kernel 会调用kernel_thread 并创建kthreadd，负责管理内核中得所有线程，然后进程ID会被设置为2。

最后，会创建idle进程（0号进程），不能被调度，并利用循环来不断调号空闲的CPU时间片，并且从不返回。

参考自：pianogirl123

void __init __weak arch_call_rest_init(void)
{
	rest_init();
}

asmlinkage __visible void __init start_kernel(void)
{
	char *command_line;
	char *after_dashes;

	set_task_stack_end_magic(&init_task);
	smp_setup_processor_id();
	debug_objects_early_init();

	cgroup_init_early();

	local_irq_disable();
	early_boot_irqs_disabled = true;

	/*
	 * Interrupts are still disabled. Do necessary setups, then
	 * enable them.
	 */
	boot_cpu_init();
	page_address_init();
	pr_notice("%s", linux_banner);
	setup_arch(&command_line);
	/*
	 * Set up the the initial canary and entropy after arch
	 * and after adding latent and command line entropy.
	 */
	add_latent_entropy();
	add_device_randomness(command_line, strlen(command_line));
	boot_init_stack_canary();
	mm_init_cpumask(&init_mm);
	setup_command_line(command_line);
	setup_nr_cpu_ids();
	setup_per_cpu_areas();
	smp_prepare_boot_cpu();	/* arch-specific boot-cpu hooks */
	boot_cpu_hotplug_init();

	build_all_zonelists(NULL);
	page_alloc_init();

	pr_notice("Kernel command line: %s\n", boot_command_line);
	parse_early_param();
	after_dashes = parse_args("Booting kernel",
				  static_command_line, __start___param,
				  __stop___param - __start___param,
				  -1, -1, NULL, &unknown_bootoption);
	if (!IS_ERR_OR_NULL(after_dashes))
		parse_args("Setting init args", after_dashes, NULL, 0, -1, -1,
			   NULL, set_init_arg);

	jump_label_init();

	/*
	 * These use large bootmem allocations and must precede
	 * kmem_cache_init()
	 */
	setup_log_buf(0);
	vfs_caches_init_early();
	sort_main_extable();
	trap_init();
	mm_init();

	ftrace_init();

	/* trace_printk can be enabled here */
	early_trace_init();

	/*
	 * Set up the scheduler prior starting any interrupts (such as the
	 * timer interrupt). Full topology setup happens at smp_init()
	 * time - but meanwhile we still have a functioning scheduler.
	 */
	sched_init();
	/*
	 * Disable preemption - early bootup scheduling is extremely
	 * fragile until we cpu_idle() for the first time.
	 */
	preempt_disable();
	if (WARN(!irqs_disabled(),
		 "Interrupts were enabled *very* early, fixing it\n"))
		local_irq_disable();
	radix_tree_init();

	/*
	 * Set up housekeeping before setting up workqueues to allow the unbound
	 * workqueue to take non-housekeeping into account.
	 */
	housekeeping_init();

	/*
	 * Allow workqueue creation and work item queueing/cancelling
	 * early.  Work item execution depends on kthreads and starts after
	 * workqueue_init().
	 */
	workqueue_init_early();

	rcu_init();

	/* Trace events are available after this */
	trace_init();

	if (initcall_debug)
		initcall_debug_enable();

	context_tracking_init();
	/* init some links before init_ISA_irqs() */
	early_irq_init();
	init_IRQ();
	tick_init();
	rcu_init_nohz();
	init_timers();
	hrtimers_init();
	softirq_init();
	timekeeping_init();
	time_init();
	printk_safe_init();
	perf_event_init();
	profile_init();
	call_function_init();
	WARN(!irqs_disabled(), "Interrupts were enabled early\n");

	early_boot_irqs_disabled = false;
	local_irq_enable();

	kmem_cache_init_late();

	/*
	 * HACK ALERT! This is early. We're enabling the console before
	 * we've done PCI setups etc, and console_init() must be aware of
	 * this. But we do want output early, in case something goes wrong.
	 */
	console_init();
	if (panic_later)
		panic("Too many boot %s vars at `%s'", panic_later,
		      panic_param);

	lockdep_init();

	/*
	 * Need to run this when irqs are enabled, because it wants
	 * to self-test [hard/soft]-irqs on/off lock inversion bugs
	 * too:
	 */
	locking_selftest();

	/*
	 * This needs to be called before any devices perform DMA
	 * operations that might use the SWIOTLB bounce buffers. It will
	 * mark the bounce buffers as decrypted so that their usage will
	 * not cause "plain-text" data to be decrypted when accessed.
	 */
	mem_encrypt_init();

#ifdef CONFIG_BLK_DEV_INITRD
	if (initrd_start && !initrd_below_start_ok &&
	    page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {
		pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n",
		    page_to_pfn(virt_to_page((void *)initrd_start)),
		    min_low_pfn);
		initrd_start = 0;
	}
#endif
	kmemleak_init();
	setup_per_cpu_pageset();
	numa_policy_init();
	acpi_early_init();
	if (late_time_init)
		late_time_init();
	sched_clock_init();
	calibrate_delay();
	pid_idr_init();
	anon_vma_init();
#ifdef CONFIG_X86
	if (efi_enabled(EFI_RUNTIME_SERVICES))
		efi_enter_virtual_mode();
#endif
	thread_stack_cache_init();
	cred_init();
	fork_init();
	proc_caches_init();
	uts_ns_init();
	buffer_init();
	key_init();
	security_init();
	dbg_late_init();
	vfs_caches_init();
	pagecache_init();
	signals_init();
	seq_file_init();
	proc_root_init();
	nsfs_init();
	cpuset_init();
	cgroup_init();
	taskstats_init_early();
	delayacct_init();

	check_bugs();

	acpi_subsystem_init();
	arch_post_acpi_subsys_init();
	sfi_init_late();

	/* Do the rest non-__init'ed, we're now alive */
	arch_call_rest_init();   //调用rest_init()
}

rest_init() 函数

void rest_init(void) 
{
    int pid; 
    ……………… 
    kernel_thread(kernel_init, NULL, CLONE_FS);
    numa_default_policy(); 
    pid = kernel_thread(kthreadd, NULL, CLONE_FS | CLONE_FILES);
    rcu_read_lock(); 
    kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns); 
    rcu_read_unlock(); 
    complete(&kthreadd_done); 
    init_idle_bootup_task(current); 
    schedule_preempt_disabled(); 
    cpu_startup_entry(CPUHP_ONLINE); 
}

6.跟踪系统调用

增加系统调用

• 根据学号后两位93，在/usr/include/asm/unistd_32.h中可查得#define __NR_ftruncate 93。
  

• 编写测试 在test.c 中添加两个函数，main函数中添加相应的Menuconfig()

int update(int argc, char *argv[]){
	FILE *out;
	char *file = "93temp";
	int res = -2;
	int fd;
	out = fopen(file,"w+");
	fd = fileno(out);
	if(out == NULL){
		printf("openFailed!!!!!");	
	}
	//printf("res: %d\n",res);
	res = ftruncate(fd, 500);
	fclose(out);
	if(res == 0){
		printf("success!\n");
		out = fopen("93temp","r");
		fseek(out,0L,SEEK_END);  
    		int size=ftell(out);
		printf("size %d\n",size);
		fclose(out);
	}else{
		printf("fail\n");
	}
	return res;
}

int updateAsm(int argc, char *argv[]){
	FILE *out;
	char *file="93temp";
	int fd;
	int res = -2;
	out = fopen(file,"w+");
	if(out == NULL){
		printf("openFailed!!!!!");	
	}
	fd = fileno(out);
	//printf("res: %d\n",res);
	asm volatile(
		"mov $0x5D, %%eax\n\t"
		"int $0x80\n\t"
		"mov %%eax, %0\n\t"
		:"=m"(res)
		:"b"(fd),"c"(200)		
	);
	fclose(out);
	printf("res: %d\n",res);
	if(res == 0){
		printf("Success!\n");
		out = fopen(file, "r");
		fseek(out,0L,SEEK_END);  
    		int size=ftell(out);
		printf("size %d\n",size);
	}else{
		printf("failed!\n");
	}
	fclose(out);
	return res;
}

int main()
{
    ................
    MenuConfig("update","updateFilesize", update);
    MenuConfig("updateAsm","updateFilesizeAsm", updateAsm);
    ExecuteMenu();
}

• 添加Makefile中的开始暂停设置为
  

重新make rootfs

cd ~/LinuxKernel/menu
make rootfs

• 使用gdb跟踪查看

可以看见在使用int 0x80中断之后，CPU会运行arch/x86/entry/entry_32.S中的指令

• 分析entry_32.S代码

  #这段代码就是系统调用处理的过程,其它的中断过程也是与此类似
  #系统调用就是一个特殊的中断，也存在保护现场和回复现场
  ENTRY(system_call)          #这是0x80之后的下一条指令
      RING0_INT_FRAME         # can't unwind into user space anyway
      ASM_CLAC
      pushl_cfi %eax          # save orig_eax
      SAVE_ALL                 #保护现场
      GET_THREAD_INFO(%ebp)
                      # system call tracing in operation / emulation
      testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
      jnz syscall_trace_entry
      cmpl $(NR_syscalls), %eax
      jae syscall_badsys
  syscall_call:
      # 调用了系统调用处理函数，实际的系统调用服务程序
      call *sys_call_table(,%eax,4)#定义的系统调用的表，eax传递过来的就是系统调用号，在例子中就是调用的systime
  syscall_after_call:
      movl %eax,PT_EAX(%esp)      # store the return value
  syscall_exit:
      LOCKDEP_SYS_EXIT
      DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
                      # setting need_resched or sigpending
                      # between sampling and the iret
      TRACE_IRQS_OFF
      movl TI_flags(%ebp), %ecx
      testl $_TIF_ALLWORK_MASK, %ecx  # current->work
      jne syscall_exit_work          #退出之前,syscall_exit_work 
      #进入到syscall_exit_work里边有一个进程调度时机
  
  restore_all:
      TRACE_IRQS_IRET
  restore_all_notrace:        #返回到用户态
  #ifdef CONFIG_X86_ESPFIX32
      movl PT_EFLAGS(%esp), %eax  # mix EFLAGS, SS and CS
      # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
      # are returning to the kernel.
      # See comments in process.c:copy_thread() for details.
      movb PT_OLDSS(%esp), %ah
      movb PT_CS(%esp), %al
      andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
      cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
      CFI_REMEMBER_STATE
      je ldt_ss           # returning to user-space with LDT SS
  #end
      RESTORE_REGS 4          # skip orig_eax/error_code
  irq_return:
      INTERRUPT_RETURN      #iret（宏），系统调用过程到这里结束
  

  实验总结

  ​ 其原理是进程先用适当的值填充寄存器，然后调用一个特殊的指令，这个指令会跳到一个事先定义的内核中的一个位置。在Intel CPU中，这个由中断0x80实现。硬件知道一旦你跳到这个位置，你就不是在限制模式下运行的用户，而是作为操作系统的内核--由用户态转为内核态。

  ​ 进程可以跳转到的内核位置叫做sysem_call。这个过程检查系统调用号，这个号码告诉内核进程请求哪种服务。然后，它查看系统调用表(sys_call_table)找到所调用的内核函数入口地址。接着，就调用函数，等返回后，做一些系统检查，最后返回到进程（或到其他进程，如果这个进程时间用尽）。

  ​ 进程号是由eax寄存器存储的，参数一般是由ebx、ecx、edx、esl、edl、ebp来存储的。