GPU in container
红帽， OpenShift 4.5 - an insider's view for technology partners support GPU sharing
阿里解决方案：
代码地址：
wget http://cgpu.oss-cn-hangzhou.aliyuncs.com/cgpu-0.8.tar.gz tar -xvf cgpu-0.8.tar.gz cd cgpu ls # cgpu-container-wrapper cgpu-km.c cgpu.o cgpu-procfs.c install.sh Makefile os-interface.c README uninstall.sh upgrade.sh version.h
github
git clone https://github.com/lvmxh/cgpu.git cd cgpu
由于nvidia的驱动并不开源， 所以阿里应该进行不了相关源码级别的改造。
我们可以看到，cgpu.o 调用了 filp_open, 猜测阿里的做法，应该是打开了用户态的nv显卡， 截获了对应nv显卡的所有的操作。
但是我们可以导出cgpu的符号表可以大概了解一下，是啥东西。
objdump -t cgpu.o
1 objdump -t cgpu.o 2 3 cgpu.o: file format elf64-x86-64 4 5 SYMBOL TABLE: 6 0000000000000000 l d .text 0000000000000000 .text 7 0000000000000000 l d .text.unlikely 0000000000000000 .text.unlikely 8 0000000000000000 l d .rodata 0000000000000000 .rodata 9 0000000000000000 l d .rodata.str1.1 0000000000000000 .rodata.str1.1 10 0000000000000000 l d .rodata.str1.8 0000000000000000 .rodata.str1.8 11 0000000000000000 l d .data 0000000000000000 .data 12 0000000000000000 l d .bss 0000000000000000 .bss 13 0000000000000248 l .rodata.str1.8 0000000000000000 .LC43 14 0000000000000000 l F .text.unlikely 00000000000000e7 mbedtls_mpi_shift_r 15 0000000000000000 l d __mcount_loc 0000000000000000 __mcount_loc 16 0000000000000000 l d .comment 0000000000000000 .comment 17 0000000000000000 l d .note.GNU-stack 0000000000000000 .note.GNU-stack 18 00000000000045c0 g F .text 000000000000025c cgpu_finalize 19 0000000000002920 g F .text 0000000000000052 cgpu_clear_set_pfn 20 0000000000000000 *UND* 0000000000000000 os_kfree 21 0000000000000000 *UND* 0000000000000000 os_spin_lock 22 0000000000000000 *UND* 0000000000000000 os_writel 23 0000000000000000 *UND* 0000000000000000 os_vma_ops 24 0000000000000000 *UND* 0000000000000000 os_kmalloc 25 0000000000000000 *UND* 0000000000000000 os_wait_event_interruptible_timeout 26 0000000000003af0 g F .text 0000000000000105 cgpu_km_unlocked_ioctl 27 0000000000000000 *UND* 0000000000000000 os_ioremap_nocache 28 0000000000000000 *UND* 0000000000000000 os_iounmap 29 0000000000003c00 g F .text 00000000000000cf cgpu_km_compat_ioctl 30 0000000000000000 *UND* 0000000000000000 os_in_vma_range 31 0000000000000000 *UND* 0000000000000000 os_get_tgid 32 0000000000000000 *UND* 0000000000000000 os_memcpy 33 0000000000000000 *UND* 0000000000000000 os_unmap_range 34 0000000000000000 *UND* 0000000000000000 os_get_page_shift 35 0000000000004d20 g F .text 0000000000000037 group_get_policy 36 0000000000000000 *UND* 0000000000000000 os_get_fops 37 0000000000000000 *UND* 0000000000000000 __fentry__ 38 0000000000004e00 g F .text 0000000000000086 inst_get_meminfo 39 0000000000000000 *UND* 0000000000000000 os_kmalloc_array 40 0000000000000000 *UND* 0000000000000000 os_kthread_run 41 0000000000000000 *UND* 0000000000000000 os_spin_unlock 42 0000000000000000 *UND* 0000000000000000 os_memset 43 0000000000000000 *UND* 0000000000000000 os_ioremap_cache 44 0000000000000000 *UND* 0000000000000000 os_filp_open 45 0000000000000000 *UND* 0000000000000000 __stack_chk_fail 46 0000000000004e90 g F .text 0000000000001858 cgpu_ioctl 47 0000000000000000 *UND* 0000000000000000 os_kthread_stop 48 0000000000000000 *UND* 0000000000000000 put_spin_lock 49 0000000000000000 *UND* 0000000000000000 os_get_system_info 50 0000000000000000 *UND* 0000000000000000 os_cdev_put 51 0000000000004d60 g F .text 0000000000000057 group_set_max_inst 52 0000000000000000 *UND* 0000000000000000 os_put_waitqueue_head 53 0000000000000000 *UND* 0000000000000000 os_virt_to_phys 54 00000000000028e0 g F .text 000000000000003c check_permit 55 0000000000004840 g F .text 000000000000001b inst_get_total_mem 56 0000000000000000 *UND* 0000000000000000 os_spin_lock_init 57 0000000000000000 *UND* 0000000000000000 os_pr_debug 58 0000000000004b70 g F .text 0000000000000126 inst_get_node 59 0000000000003cd0 g F .text 0000000000000047 cgpu_km_poll 60 0000000000003840 g F .text 000000000000007f inst_vma_fault 61 0000000000000000 *UND* 0000000000000000 os_get_vm_file 62 0000000000000000 *UND* 0000000000000000 os_vm_insert_pfn 63 0000000000000000 *UND* 0000000000000000 os_get_zeroed_page 64 0000000000000000 *UND* 0000000000000000 os_get_inode 65 0000000000000000 *UND* 0000000000000000 os_copy_to_user 66 0000000000000000 *UND* 0000000000000000 os_file_inode 67 0000000000004cf0 g F .text 000000000000002f group_set_policy 68 0000000000000000 *UND* 0000000000000000 os_cdev_get 69 0000000000000000 *UND* 0000000000000000 get_rdev 70 0000000000000000 *UND* 0000000000000000 os_set_pgoff 71 0000000000000000 *UND* 0000000000000000 cgpu_km_vma_fault 72 0000000000000000 *UND* 0000000000000000 os_filp_close 73 00000000000038c0 g F .text 0000000000000225 cgpu_km_mmap 74 0000000000000000 *UND* 0000000000000000 os_follow_pfn 75 0000000000000000 *UND* 0000000000000000 os_free_page 76 0000000000000000 *UND* 0000000000000000 os_copy_from_user 77 0000000000004a20 g F .text 0000000000000145 inst_set_weight 78 0000000000004dc0 g F .text 0000000000000037 group_get_max_inst 79 00000000000049f0 g F .text 0000000000000024 inst_get_weight 80 0000000000004860 g F .text 0000000000000148 inst_set_total_mem 81 0000000000004380 g F .text 000000000000023e cgpu_initialize 82 0000000000004ca0 g F .text 000000000000004b inst_set_name 83 0000000000000000 *UND* 0000000000000000 os_file_op 84 0000000000002840 g F .text 0000000000000097 get_system_info 85 0000000000002690 g F .text 00000000000001a3 inst_get_bios 86 00000000000049b0 g F .text 0000000000000037 inst_get_free_weight 87 00000000000040d0 g F .text 00000000000002a2 cgpu_km_close 88 0000000000000000 *UND* 0000000000000000 os_kthread_should_stop 89 0000000000000000 *UND* 0000000000000000 os_ireadcount_inc 90 0000000000000000 *UND* 0000000000000000 os_get_minor 91 0000000000000000 *UND* 0000000000000000 os_printf 92 0000000000004820 g F .text 0000000000000019 inst_get_minor 93 0000000000000000 *UND* 0000000000000000 os_set_filp 94 0000000000000000 *UND* 0000000000000000 get_spin_lock 95 0000000000000000 *UND* 0000000000000000 os_init_waitqueue_head 96 0000000000003d20 g F .text 00000000000003a7 cgpu_km_open 97 0000000000000000 *UND* 0000000000000000 os_minor 98 0000000000000000 *UND* 0000000000000000 os_alloc_file_operations 99 0000000000000000 *UND* 0000000000000000 os_get_device_id 100 0000000000002640 g F .text 000000000000004d _strstr
此外阿里提供内核态参数设置的接口是procfs，并不是sysfs。
打开cgpu-km.c可以看到新驱动提供的文件接口：
grep -n8 file_operations cgpu-km.c 31-extern int cgpu_km_close(struct inode *inode, struct file *filp); 32-extern int cgpu_initialize(void); 33-extern void cgpu_finalize(void); 34-extern int cgpu_km_procfs_init(int); 35-extern int cgpu_km_procfs_deinit(void); 36- 37-int cgpu_major = 0; 38- 39:static struct file_operations cgpu_km_fops = { 40- .owner = THIS_MODULE, 41- .poll = cgpu_km_poll, 42- .unlocked_ioctl = cgpu_km_unlocked_ioctl, 43- .compat_ioctl = cgpu_km_compat_ioctl, 44- .mmap = cgpu_km_mmap, 45- .open = cgpu_km_open, 46- .release = cgpu_km_close, 47-};
我们打开 drm的官方文档，可以看到GPU的驱动实现了大量的ioctl 操作。
通过反汇编，dump 阿里的cgpu.o 可以看到很多ioctl的跳转
objdump -s -d cgpu.o > cgpu.o.txt grep cgpu_ioctl cgpu.o.txt
结果如下：
1 0080 5f696f63 746c0000 00000000 00000000 _ioctl.......... 2 0000000000003af0 <cgpu_km_unlocked_ioctl>: 3 3af0: e8 00 00 00 00 callq 3af5 <cgpu_km_unlocked_ioctl+0x5> 4 3b31: e8 00 00 00 00 callq 3b36 <cgpu_km_unlocked_ioctl+0x46> 5 3b4d: 74 41 je 3b90 <cgpu_km_unlocked_ioctl+0xa0> 6 3b62: e8 00 00 00 00 callq 3b67 <cgpu_km_unlocked_ioctl+0x77> 7 3b77: 75 77 jne 3bf0 <cgpu_km_unlocked_ioctl+0x100> 8 3b97: 75 d1 jne 3b6a <cgpu_km_unlocked_ioctl+0x7a> 9 3ba4: 74 2a je 3bd0 <cgpu_km_unlocked_ioctl+0xe0> 10 3bae: e8 00 00 00 00 callq 3bb3 <cgpu_km_unlocked_ioctl+0xc3> 11 3bb6: 74 18 je 3bd0 <cgpu_km_unlocked_ioctl+0xe0> 12 3bc8: 75 85 jne 3b4f <cgpu_km_unlocked_ioctl+0x5f> 13 3be3: 0f 85 66 ff ff ff jne 3b4f <cgpu_km_unlocked_ioctl+0x5f> 14 3beb: e9 7a ff ff ff jmpq 3b6a <cgpu_km_unlocked_ioctl+0x7a> 15 3bf0: e8 00 00 00 00 callq 3bf5 <cgpu_km_unlocked_ioctl+0x105> 16 0000000000003c00 <cgpu_km_compat_ioctl>: 17 3c00: e8 00 00 00 00 callq 3c05 <cgpu_km_compat_ioctl+0x5> 18 3c3f: e8 00 00 00 00 callq 3c44 <cgpu_km_compat_ioctl+0x44> 19 3c58: 75 54 jne 3cae <cgpu_km_compat_ioctl+0xae> 20 3c61: 74 1d je 3c80 <cgpu_km_compat_ioctl+0x80> 21 3c70: 75 58 jne 3cca <cgpu_km_compat_ioctl+0xca> 22 3c8b: 74 2b je 3cb8 <cgpu_km_compat_ioctl+0xb8> 23 3c95: e8 00 00 00 00 callq 3c9a <cgpu_km_compat_ioctl+0x9a> 24 3c9d: 74 19 je 3cb8 <cgpu_km_compat_ioctl+0xb8> 25 3cac: 74 0a je 3cb8 <cgpu_km_compat_ioctl+0xb8> 26 3cb0: eb b1 jmp 3c63 <cgpu_km_compat_ioctl+0x63> 27 3cc8: eb 99 jmp 3c63 <cgpu_km_compat_ioctl+0x63> 28 3cca: e8 00 00 00 00 callq 3ccf <cgpu_km_compat_ioctl+0xcf> 29 0000000000004e90 <cgpu_ioctl>: 30 ] 31 4e90: e8 00 00 00 00 callq 4e95 <cgpu_ioctl+0x5> 32 4ec5: 74 49 je 4f10 <cgpu_ioctl+0x80> 33 4ecd: 74 41 je 4f10 <cgpu_ioctl+0x80> 34 4ed7: 74 37 je 4f10 <cgpu_ioctl+0x80> 35 4ee1: 77 2d ja 4f10 <cgpu_ioctl+0x80> 36 4ee8: e8 00 00 00 00 callq 4eed <cgpu_ioctl+0x5d> 37 4ef3: 0f 84 c2 0e 00 00 je 5dbb <cgpu_ioctl+0xf2b> 38 4f02: 0f 84 f8 00 00 00 je 5000 <cgpu_ioctl+0x170> 39 4f0c: 75 3d jne 4f4b <cgpu_ioctl+0xbb> 40 4f22: 0f 85 c8 0f 00 00 jne 5ef0 <cgpu_ioctl+0x1060> 41 4f49: 74 c5 je 4f10 <cgpu_ioctl+0x80> 42 4f51: 75 ed jne 4f40 <cgpu_ioctl+0xb0> 43 4f58: 75 b6 jne 4f10 <cgpu_ioctl+0x80> 44 4f7a: 0f 84 c0 06 00 00 je 5640 <cgpu_ioctl+0x7b0> 45 4f8c: 0f 84 cb 06 00 00 je 565d <cgpu_ioctl+0x7cd> 46 4f94: 74 da je 4f70 <cgpu_ioctl+0xe0> 47 4f98: 74 56 je 4ff0 <cgpu_ioctl+0x160> 48 4fa5: 75 c9 jne 4f70 <cgpu_ioctl+0xe0> 49 4fbc: 0f 8e 68 06 00 00 jle 562a <cgpu_ioctl+0x79a> 50 4fd5: 0f 85 9c 06 00 00 jne 5677 <cgpu_ioctl+0x7e7> 51 4fe9: eb 85 jmp 4f70 <cgpu_ioctl+0xe0> 52 4ff6: eb a2 jmp 4f9a <cgpu_ioctl+0x10a> 53 5004: 75 19 jne 501f <cgpu_ioctl+0x18f> 54 5006: e9 aa 08 00 00 jmpq 58b5 <cgpu_ioctl+0xa25> 55 5019: 0f 84 79 06 00 00 je 5698 <cgpu_ioctl+0x808> 56 5025: 75 e9 jne 5010 <cgpu_ioctl+0x180> 57 5032: eb 13 jmp 5047 <cgpu_ioctl+0x1b7> 58 5042: 74 0c je 5050 <cgpu_ioctl+0x1c0> 59 504e: 75 e8 jne 5038 <cgpu_ioctl+0x1a8> 60 505d: 76 13 jbe 5072 <cgpu_ioctl+0x1e2> 61 5062: e8 00 00 00 00 callq 5067 <cgpu_ioctl+0x1d7> 62 506d: e9 a3 fe ff ff jmpq 4f15 <cgpu_ioctl+0x85> 63 5098: e8 00 00 00 00 callq 509d <cgpu_ioctl+0x20d> 64 50a2: e8 00 00 00 00 callq 50a7 <cgpu_ioctl+0x217> 65 50b1: 0f 84 e9 07 00 00 je 58a0 <cgpu_ioctl+0xa10> 66 50ba: 0f 84 75 05 00 00 je 5635 <cgpu_ioctl+0x7a5> 67 50c5: eb 1a jmp 50e1 <cgpu_ioctl+0x251> 68 50db: 0f 84 f1 07 00 00 je 58d2 <cgpu_ioctl+0xa42> 69 50e5: 75 e9 jne 50d0 <cgpu_ioctl+0x240> 70 50f6: e8 00 00 00 00 callq 50fb <cgpu_ioctl+0x26b> 71 5100: e8 00 00 00 00 callq 5105 <cgpu_ioctl+0x275> 72 5119: e8 00 00 00 00 callq 511e <cgpu_ioctl+0x28e> 73 5128: eb 13 jmp 513d <cgpu_ioctl+0x2ad> 74 513b: 74 09 je 5146 <cgpu_ioctl+0x2b6> 75 5144: 75 ea jne 5130 <cgpu_ioctl+0x2a0> 76 516e: c6 05 00 00 00 00 00 movb $0x0,0x0(%rip) # 5175 <cgpu_ioctl+0x2e5> 77 5189: eb 11 jmp 519c <cgpu_ioctl+0x30c> 78 5193: 0f 84 0f 05 00 00 je 56a8 <cgpu_ioctl+0x818> 79 51c0: 75 ce jne 5190 <cgpu_ioctl+0x300> 80 51c8: 0f 84 3a 06 00 00 je 5808 <cgpu_ioctl+0x978> 81 51d1: 0f 84 31 06 00 00 je 5808 <cgpu_ioctl+0x978> 82 51dc: eb 13 jmp 51f1 <cgpu_ioctl+0x361> 83 51eb: 0f 84 d1 06 00 00 je 58c2 <cgpu_ioctl+0xa32> 84 51f5: 75 e9 jne 51e0 <cgpu_ioctl+0x350> 85 5207: e8 00 00 00 00 callq 520c <cgpu_ioctl+0x37c> 86 5229: e8 00 00 00 00 callq 522e <cgpu_ioctl+0x39e> 87 523c: e8 00 00 00 00 callq 5241 <cgpu_ioctl+0x3b1> 88 5274: 74 4d je 52c3 <cgpu_ioctl+0x433> 89 52c1: 75 54 jne 5317 <cgpu_ioctl+0x487> 90 52d7: 75 3e jne 5317 <cgpu_ioctl+0x487> 91 5315: 75 c9 jne 52e0 <cgpu_ioctl+0x450> 92 5326: 74 3e je 5366 <cgpu_ioctl+0x4d6> 93 5364: 75 51 jne 53b7 <cgpu_ioctl+0x527> 94 5372: 75 43 jne 53b7 <cgpu_ioctl+0x527> 95 53b5: 75 c9 jne 5380 <cgpu_ioctl+0x4f0> 96 53bf: 74 4d je 540e <cgpu_ioctl+0x57e> 97 540c: 75 51 jne 545f <cgpu_ioctl+0x5cf> 98 5422: 75 3b jne 545f <cgpu_ioctl+0x5cf> 99 542b: 0f b6 15 00 00 00 00 movzbl 0x0(%rip),%edx # 5432 <cgpu_ioctl+0x5a2> 100 5435: 0f b6 05 00 00 00 00 movzbl 0x0(%rip),%eax # 543c <cgpu_ioctl+0x5ac> 101 544a: 0f b6 05 00 00 00 00 movzbl 0x0(%rip),%eax # 5451 <cgpu_ioctl+0x5c1> 102 5499: 0f 84 1f 02 00 00 je 56be <cgpu_ioctl+0x82e> [39/1693] 103 54ea: 0f 87 99 01 00 00 ja 5689 <cgpu_ioctl+0x7f9> 104 550c: 75 7e jne 558c <cgpu_ioctl+0x6fc> 105 5524: 75 66 jne 558c <cgpu_ioctl+0x6fc> 106 5529: 74 35 je 5560 <cgpu_ioctl+0x6d0> 107 555e: 75 d0 jne 5530 <cgpu_ioctl+0x6a0> 108 5586: 0f 88 50 03 00 00 js 58dc <cgpu_ioctl+0xa4c> 109 55ce: 0f 84 fd 00 00 00 je 56d1 <cgpu_ioctl+0x841> 110 55e0: 0f 84 36 02 00 00 je 581c <cgpu_ioctl+0x98c> 111 55e9: 0f 84 d6 07 00 00 je 5dc5 <cgpu_ioctl+0xf35> 112 55f5: eb 18 jmp 560f <cgpu_ioctl+0x77f> 113 5609: 0f 84 03 02 00 00 je 5812 <cgpu_ioctl+0x982> 114 5615: 75 e9 jne 5600 <cgpu_ioctl+0x770> 115 5620: e8 00 00 00 00 callq 5625 <cgpu_ioctl+0x795> 116 5625: e9 35 fa ff ff jmpq 505f <cgpu_ioctl+0x1cf> 117 5630: e9 3b f9 ff ff jmpq 4f70 <cgpu_ioctl+0xe0> 118 5637: e9 ab fa ff ff jmpq 50e7 <cgpu_ioctl+0x257> 119 5646: 75 17 jne 565f <cgpu_ioctl+0x7cf> 120 5653: e8 00 00 00 00 callq 5658 <cgpu_ioctl+0x7c8> 121 5658: e9 d1 fb ff ff jmpq 522e <cgpu_ioctl+0x39e> 122 5668: e8 00 00 00 00 callq 566d <cgpu_ioctl+0x7dd> 123 5672: e9 9e f8 ff ff jmpq 4f15 <cgpu_ioctl+0x85> 124 5684: e9 e7 f8 ff ff jmpq 4f70 <cgpu_ioctl+0xe0> 125 5693: e9 f4 fe ff ff jmpq 558c <cgpu_ioctl+0x6fc> 126 56a3: e9 85 f9 ff ff jmpq 502d <cgpu_ioctl+0x19d> 127 56ab: 74 08 je 56b5 <cgpu_ioctl+0x825> 128 56b9: e9 07 fb ff ff jmpq 51c5 <cgpu_ioctl+0x335> 129 56cc: e9 3d fe ff ff jmpq 550e <cgpu_ioctl+0x67e> 130 577d: 75 91 jne 5710 <cgpu_ioctl+0x880> 131 57fe: e9 5c f8 ff ff jmpq 505f <cgpu_ioctl+0x1cf> 132 580d: e9 e8 f9 ff ff jmpq 51fa <cgpu_ioctl+0x36a> 133 5817: e9 01 fe ff ff jmpq 561d <cgpu_ioctl+0x78d> 134 5822: 0f 85 be fd ff ff jne 55e6 <cgpu_ioctl+0x756> 135 5833: 0f 85 ad fd ff ff jne 55e6 <cgpu_ioctl+0x756> 136 583f: 0f 85 a1 fd ff ff jne 55e6 <cgpu_ioctl+0x756> 137 584b: 0f 85 95 fd ff ff jne 55e6 <cgpu_ioctl+0x756> 138 5853: 0f 85 8d fd ff ff jne 55e6 <cgpu_ioctl+0x756> 139 585f: 0f 85 81 fd ff ff jne 55e6 <cgpu_ioctl+0x756> 140 5872: 75 07 jne 587b <cgpu_ioctl+0x9eb> 141 588f: 0f 87 9c 09 00 00 ja 6231 <cgpu_ioctl+0x13a1> 142 589c: eb df jmp 587d <cgpu_ioctl+0x9ed> 143 58b0: e9 41 f8 ff ff jmpq 50f6 <cgpu_ioctl+0x266> 144 58bd: e9 6b f7 ff ff jmpq 502d <cgpu_ioctl+0x19d> 145 58cd: e9 28 f9 ff ff jmpq 51fa <cgpu_ioctl+0x36a> 146 58d7: e9 0b f8 ff ff jmpq 50e7 <cgpu_ioctl+0x257> 147 58f4: 0f 8e 92 fc ff ff jle 558c <cgpu_ioctl+0x6fc> 148 5904: 0f 84 82 fc ff ff je 558c <cgpu_ioctl+0x6fc> 149 591c: 0f 88 6a fc ff ff js 558c <cgpu_ioctl+0x6fc> 150 592c: e8 00 00 00 00 callq 5931 <cgpu_ioctl+0xaa1> 151 593b: 0f 84 36 08 00 00 je 6177 <cgpu_ioctl+0x12e7> 152 5977: 75 ea jne 5963 <cgpu_ioctl+0xad3> 153 59ee: e8 00 00 00 00 callq 59f3 <cgpu_ioctl+0xb63> 154 5a10: 0f 87 ef 03 00 00 ja 5e05 <cgpu_ioctl+0xf75> 155 5a1c: 0f 87 13 04 00 00 ja 5e35 <cgpu_ioctl+0xfa5> 156 5a26: 0f 87 39 04 00 00 ja 5e65 <cgpu_ioctl+0xfd5> 157 5aab: 0f 87 1e 03 00 00 ja 5dcf <cgpu_ioctl+0xf3f> 158 5ab8: 0f 87 d7 03 00 00 ja 5e95 <cgpu_ioctl+0x1005> 159 5adf: 0f 87 94 05 00 00 ja 6079 <cgpu_ioctl+0x11e9> 160 5af0: 0f 87 d9 02 00 00 ja 5dcf <cgpu_ioctl+0xf3f> 161 5b04: 0f 87 4f 05 00 00 ja 6059 <cgpu_ioctl+0x11c9> 162 5b1c: 0f 84 dd 04 00 00 je 5fff <cgpu_ioctl+0x116f> 163 5b46: 0f 84 1a 04 00 00 je 5f66 <cgpu_ioctl+0x10d6> 164 5b69: e8 00 00 00 00 callq 5b6e <cgpu_ioctl+0xcde> 165 5b98: 0f 88 9b 03 00 00 js 5f39 <cgpu_ioctl+0x10a9> 166 5bc7: 75 7e jne 5c47 <cgpu_ioctl+0xdb7> 167 5c41: 0f 84 5b 04 00 00 je 60a2 <cgpu_ioctl+0x1212> 168 5c64: 73 4c jae 5cb2 <cgpu_ioctl+0xe22> 169 5c9d: 72 eb jb 5c8a <cgpu_ioctl+0xdfa> 170 5cda: 0f 84 39 02 00 00 je 5f19 <cgpu_ioctl+0x1089> 171 5ce7: e8 00 00 00 00 callq 5cec <cgpu_ioctl+0xe5c> 172 5cf4: 0f 85 92 f8 ff ff jne 558c <cgpu_ioctl+0x6fc> 173 5d13: 0f 87 dc 01 00 00 ja 5ef5 <cgpu_ioctl+0x1065> 174 5d19: 73 51 jae 5d6c <cgpu_ioctl+0xedc> 175 5d3a: 0f 85 ea 01 00 00 jne 5f2a <cgpu_ioctl+0x109a> 176 5d43: eb 1e jmp 5d63 <cgpu_ioctl+0xed3> 177 5d5d: 0f 85 c7 01 00 00 jne 5f2a <cgpu_ioctl+0x109a> 178 5d6a: 77 d9 ja 5d45 <cgpu_ioctl+0xeb5> 179 5d7f: 0f 84 07 f8 ff ff je 558c <cgpu_ioctl+0x6fc> 180 5db4: 75 cf jne 5d85 <cgpu_ioctl+0xef5> 181 5db6: e9 d1 f7 ff ff jmpq 558c <cgpu_ioctl+0x6fc> 182 5dc0: e9 50 f1 ff ff jmpq 4f15 <cgpu_ioctl+0x85> 183 5dca: e9 4e f8 ff ff jmpq 561d <cgpu_ioctl+0x78d> 184 5e00: e9 42 fe ff ff jmpq 5c47 <cgpu_ioctl+0xdb7> 185 5e30: e9 64 fc ff ff jmpq 5a99 <cgpu_ioctl+0xc09> 186 5e60: e9 34 fc ff ff jmpq 5a99 <cgpu_ioctl+0xc09> 187 5e90: e9 04 fc ff ff jmpq 5a99 <cgpu_ioctl+0xc09> 188 5eb4: 0f 84 04 fc ff ff je 5abe <cgpu_ioctl+0xc2e> 189 5ee1: e9 61 fd ff ff jmpq 5c47 <cgpu_ioctl+0xdb7> 190 5ef0: e8 00 00 00 00 callq 5ef5 <cgpu_ioctl+0x1065> 191 5f08: e8 00 00 00 00 callq 5f0d <cgpu_ioctl+0x107d> 192 5f14: e9 56 fe ff ff jmpq 5d6f <cgpu_ioctl+0xedf> 193 5f25: e9 b6 fd ff ff jmpq 5ce0 <cgpu_ioctl+0xe50> 194 5f34: e9 53 f6 ff ff jmpq 558c <cgpu_ioctl+0x6fc> 195 5f5b: 0f 85 e6 fc ff ff jne 5c47 <cgpu_ioctl+0xdb7> 196 5f61: e9 63 fc ff ff jmpq 5bc9 <cgpu_ioctl+0xd39> 197 5f94: 0f 85 ad fc ff ff jne 5c47 <cgpu_ioctl+0xdb7> 198 5fb5: 0f 85 8c fc ff ff jne 5c47 <cgpu_ioctl+0xdb7> 199 5fd9: 0f 85 68 fc ff ff jne 5c47 <cgpu_ioctl+0xdb7> 200 5ff5: e8 00 00 00 00 callq 5ffa <cgpu_ioctl+0x116a> 201 5ffa: e9 6f fb ff ff jmpq 5b6e <cgpu_ioctl+0xcde> 202 6036: 0f 85 0b fc ff ff jne 5c47 <cgpu_ioctl+0xdb7> 203 6054: e9 e5 fa ff ff jmpq 5b3e <cgpu_ioctl+0xcae> 204 606e: 0f 85 4d fe ff ff jne 5ec1 <cgpu_ioctl+0x1031> 205 6074: e9 91 fa ff ff jmpq 5b0a <cgpu_ioctl+0xc7a> 206 6097: 0f 85 1d fe ff ff jne 5eba <cgpu_ioctl+0x102a> 207 609d: e9 43 fa ff ff jmpq 5ae5 <cgpu_ioctl+0xc55> 208 6108: 0f 84 50 01 00 00 je 625e <cgpu_ioctl+0x13ce> 209 6149: 0f 87 a7 fc ff ff ja 5df6 <cgpu_ioctl+0xf66> 210 6153: 0f 87 b8 00 00 00 ja 6211 <cgpu_ioctl+0x1381> 211 616a: 74 1a je 6186 <cgpu_ioctl+0x12f6> 212 6172: e9 d0 fa ff ff jmpq 5c47 <cgpu_ioctl+0xdb7> 213 6181: e9 06 f4 ff ff jmpq 558c <cgpu_ioctl+0x6fc> 214 61b7: 0f 83 3a 02 00 00 jae 63f7 <cgpu_ioctl+0x1567> 215 620f: eb 9f jmp 61b0 <cgpu_ioctl+0x1320> 216 6226: 0f 85 40 ff ff ff jne 616c <cgpu_ioctl+0x12dc> 217 622c: e9 28 ff ff ff jmpq 6159 <cgpu_ioctl+0x12c9> 218 6259: e9 88 f3 ff ff jmpq 55e6 <cgpu_ioctl+0x756> 219 62c7: 75 21 jne 62ea <cgpu_ioctl+0x145a> 220 62d1: 0f 84 ed 01 00 00 je 64c4 <cgpu_ioctl+0x1634> 221 6317: 0f 84 a4 03 00 00 je 66c1 <cgpu_ioctl+0x1831> 222 6320: 0f 84 54 03 00 00 je 667a <cgpu_ioctl+0x17ea> 223 6349: 0f 85 70 ff ff ff jne 62bf <cgpu_ioctl+0x142f> 224 639e: 77 bd ja 635d <cgpu_ioctl+0x14cd> 225 63f2: e9 c8 fe ff ff jmpq 62bf <cgpu_ioctl+0x142f> 226 6437: 0f 83 21 fe ff ff jae 625e <cgpu_ioctl+0x13ce> 227 645b: 0f 87 95 f9 ff ff ja 5df6 <cgpu_ioctl+0xf66> 228 6465: 0f 87 5d 02 00 00 ja 66c8 <cgpu_ioctl+0x1838> 229 6478: 0f 85 ee fc ff ff jne 616c <cgpu_ioctl+0x12dc> 230 64bf: e9 65 ff ff ff jmpq 6429 <cgpu_ioctl+0x1599> 231 6516: 0f 84 9f 00 00 00 je 65bb <cgpu_ioctl+0x172b> 232 6572: 74 3a je 65ae <cgpu_ioctl+0x171e> 233 65b6: e9 54 ff ff ff jmpq 650f <cgpu_ioctl+0x167f> 234 662f: 0f 84 12 f6 ff ff je 5c47 <cgpu_ioctl+0xdb7> 235 663d: 0f 84 04 f6 ff ff je 5c47 <cgpu_ioctl+0xdb7> 236 664d: 0f 84 f4 f5 ff ff je 5c47 <cgpu_ioctl+0xdb7> 237 6675: e9 cd f5 ff ff jmpq 5c47 <cgpu_ioctl+0xdb7> 238 667e: 0f 85 a2 fc ff ff jne 6326 <cgpu_ioctl+0x1496> 239 66bc: e9 fe fb ff ff jmpq 62bf <cgpu_ioctl+0x142f> 240 66c3: e9 f7 fb ff ff jmpq 62bf <cgpu_ioctl+0x142f> 241 66dd: 0f 84 88 fd ff ff je 646b <cgpu_ioctl+0x15db> 242 66e3: e9 84 fa ff ff jmpq 616c <cgpu_ioctl+0x12dc>
此外从"os-interface.c"的源码看：
void *os_init_waitqueue_head(void) { wait_queue_head_t *sched_wq = kmalloc(sizeof(wait_queue_head_t), GFP_KERNEL); if(sched_wq) init_waitqueue_head(sched_wq); return (void*)sched_wq; } void os_put_waitqueue_head(void *sched_wq) { if(sched_wq) kfree(sched_wq); } bool os_kthread_should_stop(void) { return kthread_should_stop(); } void *os_kthread_run(CGPU_THREAD_FN sched_thread_fn, void *data, const char *name) { return (void *)kthread_run((CGPU_THREAD_FN)sched_thread_fn, data, name); } void os_kthread_stop(void *sch) { struct task_struct *sched_ts = (struct task_struct *)sch; if(sched_ts) kthread_stop(sched_ts); }
通过内核线程和queue实现的cGPU调度算法。
----------------------------------------------------------------------------------
Step by Step：
github:
https://github.com/torvalds/linux/
doc:
linux-kernel-labs device_drivers
sphinx-samples/writing_usb_driver
Tutorial on Linux Device Driver Programming Embedded Systems
1. scaffold/skeleton
首先创建一个基本的Device driver， 网上教程或者Example一堆。
Writing device drivers in Linux: A brief tutorial (PDF version)
mkdir GPUDriver && cd GPUDriver cat <<EOF | tee nothing.c #include <linux/module.h> MODULE_LICENSE("Dual BSD/GPL"); EOF cat <<EOF | tee Makefile obj-m := nothing.o EOF RELEASE=`uname -r |cut -d "-" -f 1` # sudo apt-get update && sudo apt-get install -y linux-source-$RELEASE sudo apt-get update && sudo apt-get install -y linux-source # like DKMS install driver https://github.com/IntelRealSense/librealsense/issues/4586# # https://askubuntu.com/questions/554624/how-to-resolve-the-lib-modules-3-13-0-27-generic-build-no-such-file-or-direct make -C /usr/src/linux-headers-$(uname -r) M=`pwd` modules sudo insmod nothing.ko lsmod |grep nothing # sudo modprobe -r nothing sudo rmmod nothing lsmod |grep nothing
rm *
2. scaffold/skeleton- print hello info
1 rm * 2 3 MODULE=hello 4 cat <<EOF | tee ${MODULE}.c 5 #include <linux/init.h> 6 #include <linux/module.h> 7 #include <linux/kernel.h> 8 9 MODULE_LICENSE("Dual BSD/GPL"); 10 11 static int ${MODULE}_init(void) { 12 printk("<1> Hello world!\n"); 13 return 0; 14 } 15 16 static void ${MODULE}_exit(void) { 17 printk("<1> Bye, cruel world\n"); 18 } 19 20 module_init(${MODULE}_init); 21 module_exit(${MODULE}_exit); 22 EOF 23 24 cat <<EOF | tee Makefile 25 obj-m := ${MODULE}.o 26 EOF 27 28 make -C /usr/src/linux-headers-$(uname -r) M=`pwd` modules 29 sudo insmod ${MODULE}.ko 30 lsmod |grep ${MODULE} 31 sudo rmmod ${MODULE} 32 lsmod |grep ${MODULE} 33 34 dmesg |grep world 35 cat /var/log/syslog |grep world
3. scaffold/skeleton- warm up：memory example
device driver
1 rm * 2 MODULE=memory 3 4 # https://stephane.lesimple.fr/blog/kernel-2-6-18-linuxconfig-h-problem/ 5 # https://lzw.me/a/linux-config-no-such-file-or-directory.html 6 # https://bugzilla.redhat.com/show_bug.cgi?id=212463 7 # https://blog.csdn.net/qq_35277038/article/details/80498210 8 # https://www.programmersought.com/article/92814489864/ 9 10 cat <<EOF | tee ${MODULE}.c 11 /* Necessary includes for device drivers */ 12 #include <linux/init.h> 13 #include <linux/config.h> 14 #include <linux/module.h> 15 #include <linux/kernel.h> /* printk() */ 16 #include <linux/slab.h> /* kmalloc() */ 17 #include <linux/fs.h> /* everything... */ 18 #include <linux/errno.h> /* error codes */ 19 #include <linux/types.h> /* size_t */ 20 #include <linux/proc_fs.h> 21 #include <linux/fcntl.h> /* O_ACCMODE */ 22 #include <asm/switch_to.h> /* cli(), *_flags */ 23 #include <asm/uaccess.h> /* copy_from/to_user */ 24 #include <linux/uaccess.h> 25 26 MODULE_LICENSE("Dual BSD/GPL"); 27 28 /* Declaration of ${MODULE}.c functions */ 29 int ${MODULE}_open(struct inode *inode, struct file *filp); 30 int ${MODULE}_release(struct inode *inode, struct file *filp); 31 ssize_t ${MODULE}_read(struct file *filp, char *buf, size_t count, loff_t *f_pos); 32 ssize_t ${MODULE}_write(struct file *filp, const char *buf, size_t count, loff_t *f_pos); 33 void ${MODULE}_exit(void); 34 int ${MODULE}_init(void); 35 36 /* Structure that declares the usual file */ 37 /* access functions */ 38 39 struct file_operations ${MODULE}_fops = { 40 read: ${MODULE}_read, 41 write: ${MODULE}_write, 42 open: ${MODULE}_open, 43 release: ${MODULE}_release 44 }; 45 46 /* Declaration of the init and exit functions */ 47 module_init(${MODULE}_init); 48 module_exit(${MODULE}_exit); 49 50 /* Global variables of the driver */ 51 /* Major number */ 52 int ${MODULE}_major = 60; 53 /* Buffer to store data */ 54 char *${MODULE}_buffer; 55 56 int ${MODULE}_init(void) { 57 int result; 58 59 /* Registering device */ 60 result = register_chrdev(${MODULE}_major, "${MODULE}", &${MODULE}_fops); 61 if (result < 0) { 62 printk( 63 "<1>${MODULE}: cannot obtain major number %d\n", ${MODULE}_major); 64 return result; 65 } 66 67 /* Allocating ${MODULE} for the buffer */ 68 ${MODULE}_buffer = kmalloc(1, GFP_KERNEL); 69 if (!${MODULE}_buffer) { 70 result = -ENOMEM; 71 goto fail; 72 } 73 memset(${MODULE}_buffer, 0, 1); 74 75 printk("<1>Inserting ${MODULE} module\n"); 76 return 0; 77 78 fail: 79 ${MODULE}_exit(); 80 return result; 81 } 82 83 84 void ${MODULE}_exit(void) { 85 /* Freeing the major number */ 86 unregister_chrdev(${MODULE}_major, "${MODULE}"); 87 88 /* Freeing buffer ${MODULE} */ 89 if (${MODULE}_buffer) { 90 kfree(${MODULE}_buffer); 91 } 92 93 printk("<1>Removing ${MODULE} module\n"); 94 } 95 96 int ${MODULE}_open(struct inode *inode, struct file *filp) { 97 /* Success */ 98 return 0; 99 } 100 101 int ${MODULE}_release(struct inode *inode, struct file *filp) { 102 /* Success */ 103 return 0; 104 } 105 106 ssize_t ${MODULE}_read(struct file *filp, char *buf, 107 size_t count, loff_t *f_pos) { 108 /* Transfering data to user space */ 109 copy_to_user(buf,${MODULE}_buffer,1); 110 111 /* Changing reading position as best suits */ 112 if (*f_pos == 0) { 113 *f_pos+=1; 114 return 1; 115 } else { 116 return 0; 117 } 118 } 119 120 ssize_t ${MODULE}_write( struct file *filp, const char *buf, 121 size_t count, loff_t *f_pos) { 122 const char *tmp; 123 124 tmp=buf+count-1; 125 copy_from_user(${MODULE}_buffer,tmp,1); 126 return 1; 127 } 128 EOF 129 130 cat <<EOF | tee Makefile 131 obj-m := ${MODULE}.o 132 EOF 133 134 INPATH=/usr/src/linux-headers-$(uname -r)/include 135 cat <<EOF | sudo tee ${INPATH}/linux/config.h 136 /* WorkAround for linux/config.h */ 137 #ifndef _LINUX_CONFIG_H 138 #define _LINUX_CONFIG_H 139 /* This file is no longer in use and kept only for backward compatibility. 140 * autoconf.h is now included via -imacros on the commandline 141 * https://www.linuxquestions.org/questions/linux-kernel-70/removal-of-include-linux-config-h-file-in-2-6-19-kernel-506363 142 */ 143 #include <generated/autoconf.h> 144 #endif 145 EOF 146 147 # /usr/src/linux-headers-$(uname -r)/include/linux/config.h 148 make -C /usr/src/linux-headers-$(uname -r) M=`pwd` modules 149 sudo insmod ${MODULE}.ko 150 lsmod |grep ${MODULE} 151 cat /proc/modules |grep ${MODULE} 152 sudo mknod /dev/memory c 60 0 153 sudo chmod 666 /dev/memory 154 echo -n abcdef | sudo tee /dev/memory 155 cat /dev/memory 156 157 sudo rmmod ${MODULE} 158 lsmod |grep ${MODULE}
check:
# c code: # static char *name = "world"; ///< An example LKM argument -- default value is "world" # module_param(name, charp, S_IRUGO); ///< Param desc. charp = char ptr, S_IRUGO can be read/not changed # MODULE_PARM_DESC(name, "The name to display in /var/log/kern.log"); ///< parameter description # sudo insmod hello.ko name=Derek # cat /sys/module/memory/name cat /proc/devices |grep memory ls /sys/module/memory/
user app
1 cat <<EOF | tee test.c 2 #include <stdio.h> 3 #include <unistd.h> 4 5 int main() { 6 unsigned char byte,dummy; 7 FILE * DEV; 8 9 /* Opening the device ${MODULE}*/ 10 DEV=fopen("/dev/${MODULE}","w"); 11 12 /* We remove the buffer from the file i/o */ 13 /* setvbuf(DEV,&dummy,_IONBF,1); */ 14 /* Initializing the variable to one */ 15 byte=1; 16 17 /* We make an infinite loop */ 18 while (1) { 19 /* Writing to the parallel port */ 20 /* to turn on a LED */ 21 printf("Write value is %d\n",byte); 22 fwrite(&byte,1,1,DEV); 23 sleep(1); 24 fread(&dummy,1,1,DEV); 25 printf("Read value is %d\n",dummy); 26 /* Updating the byte value */ 27 byte<<=1; 28 if (byte >= 10) 29 break; 30 } 31 fclose(DEV); 32 } 33 EOF 34 35 gcc -o test test.c 36 test
3. scaffold/skeleton- warm up：procfs example
MODULE=driver cat <<EOF | tee ${MODULE}.c /***************************************************************************//** * \file driver.c * * \details Simple Linux device driver (procfs) * * \author EmbeTronicX * * *******************************************************************************/ #include <linux/kernel.h> #include <linux/init.h> #include <linux/module.h> #include <linux/kdev_t.h> #include <linux/fs.h> #include <linux/cdev.h> #include <linux/device.h> #include<linux/slab.h> //kmalloc() #include<linux/uaccess.h> //copy_to/from_user() #include <linux/ioctl.h> #include<linux/proc_fs.h> #define WR_VALUE _IOW('a','a',int32_t*) #define RD_VALUE _IOR('a','b',int32_t*) int32_t value = 0; char etx_array[20]="try_proc_array\n"; static int len = 1; dev_t dev = 0; static struct class *dev_class; static struct cdev etx_cdev; /* ** Function Prototypes */ static int __init etx_driver_init(void); static void __exit etx_driver_exit(void); /*************** Driver Functions **********************/ static int etx_open(struct inode *inode, struct file *file); static int etx_release(struct inode *inode, struct file *file); static ssize_t etx_read(struct file *filp, char __user *buf, size_t len,loff_t * off); static ssize_t etx_write(struct file *filp, const char *buf, size_t len, loff_t * off); static long etx_ioctl(struct file *file, unsigned int cmd, unsigned long arg); /***************** Procfs Functions *******************/ static int open_proc(struct inode *inode, struct file *file); static int release_proc(struct inode *inode, struct file *file); static ssize_t read_proc(struct file *filp, char __user *buffer, size_t length,loff_t * offset); static ssize_t write_proc(struct file *filp, const char *buff, size_t len, loff_t * off); /* ** File operation sturcture */ static struct file_operations fops = { .owner = THIS_MODULE, .read = etx_read, .write = etx_write, .open = etx_open, .unlocked_ioctl = etx_ioctl, .release = etx_release, }; /* ** procfs operation sturcture */ static struct file_operations proc_fops = { .open = open_proc, .read = read_proc, .write = write_proc, .release = release_proc }; /* ** This fuction will be called when we open the procfs file */ static int open_proc(struct inode *inode, struct file *file) { printk(KERN_INFO "proc file opend.....\t"); return 0; } /* ** This fuction will be called when we close the procfs file */ static int release_proc(struct inode *inode, struct file *file) { printk(KERN_INFO "proc file released.....\n"); return 0; } /* ** This fuction will be called when we read the procfs file */ static ssize_t read_proc(struct file *filp, char __user *buffer, size_t length,loff_t * offset) { printk(KERN_INFO "proc file read.....\n"); if(len) len=0; else{ len=1; return 0; } copy_to_user(buffer,etx_array,20); return length;; } /* ** This fuction will be called when we write the procfs file */ static ssize_t write_proc(struct file *filp, const char *buff, size_t len, loff_t * off) { printk(KERN_INFO "proc file wrote.....\n"); copy_from_user(etx_array,buff,len); return len; } /* ** This fuction will be called when we open the Device file */ static int etx_open(struct inode *inode, struct file *file) { printk(KERN_INFO "Device File Opened...!!!\n"); return 0; } /* ** This fuction will be called when we close the Device file */ static int etx_release(struct inode *inode, struct file *file) { printk(KERN_INFO "Device File Closed...!!!\n"); return 0; } /* ** This fuction will be called when we read the Device file */ static ssize_t etx_read(struct file *filp, char __user *buf, size_t len, loff_t *off) { printk(KERN_INFO "Readfunction\n"); return 0; } /* ** This fuction will be called when we write the Device file */ static ssize_t etx_write(struct file *filp, const char __user *buf, size_t len, loff_t *off) { printk(KERN_INFO "Write Function\n"); return 0; } /* ** This fuction will be called when we write IOCTL on the Device file */ static long etx_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch(cmd) { case WR_VALUE: copy_from_user(&value ,(int32_t*) arg, sizeof(value)); printk(KERN_INFO "Value = %d\n", value); break; case RD_VALUE: copy_to_user((int32_t*) arg, &value, sizeof(value)); break; } return 0; } /* ** Module Init function */ static int __init etx_driver_init(void) { /*Allocating Major number*/ if((alloc_chrdev_region(&dev, 0, 1, "etx_Dev")) <0){ printk(KERN_INFO "Cannot allocate major number\n"); return -1; } printk(KERN_INFO "Major = %d Minor = %d \n",MAJOR(dev), MINOR(dev)); /*Creating cdev structure*/ cdev_init(&etx_cdev,&fops); /*Adding character device to the system*/ if((cdev_add(&etx_cdev,dev,1)) < 0){ printk(KERN_INFO "Cannot add the device to the system\n"); goto r_class; } /*Creating struct class*/ if((dev_class = class_create(THIS_MODULE,"etx_class")) == NULL){ printk(KERN_INFO "Cannot create the struct class\n"); goto r_class; } /*Creating device*/ if((device_create(dev_class,NULL,dev,NULL,"etx_device")) == NULL){ printk(KERN_INFO "Cannot create the Device 1\n"); goto r_device; } /*Creating Proc entry*/ proc_create("etx_proc",0666,NULL,&proc_fops); printk(KERN_INFO "Device Driver Insert...Done!!!\n"); return 0; r_device: class_destroy(dev_class); r_class: unregister_chrdev_region(dev,1); return -1; } /* ** Module exit function */ static void __exit etx_driver_exit(void) { remove_proc_entry("etx_proc",NULL); device_destroy(dev_class,dev); class_destroy(dev_class); cdev_del(&etx_cdev); unregister_chrdev_region(dev, 1); printk(KERN_INFO "Device Driver Remove...Done!!!\n"); } module_init(etx_driver_init); module_exit(etx_driver_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("EmbeTronicX <embetronicx@gmail.com>"); MODULE_DESCRIPTION("Simple Linux device driver (procfs)"); MODULE_VERSION("1.6"); EOF cat <<EOF | sed -e "s/\\\t/\t/" | tee Makefile obj-m += ${MODULE}.o KDIR = /lib/modules/\$(shell uname -r)/build all: \tmake -C \$(KDIR) M=\$(shell pwd) modules clean: \tmake -C \$(KDIR) M=\$(shell pwd) clean EOF sudo insmod ${MODULE}.ko lsmod |grep ${MODULE} dmesg |tail ls /proc/etx_proc cat /proc/etx_proc echo "device driver proc" > /proc/etx_proc cat /proc/etx_proc sudo rmmod ${MODULE} lsmod |grep ${MODULE}
3. scaffold/skeleton- warm up：open user space dev example
for open user space file, call filp_open, there are several example in kernel source。 details：oreilly linux device drivers open and release
for GPU ioctl, we can call vfs_ioctl (linux/fs.h), kernel source code ioctl.c/vfs_ioctl, overlayfs also call vfs_ioctl. （more friendly to read https://code.woboq.org/linux/linux/fs/ioctl.c.html）
see more latencytop.c: 70 59433 4897 i915_irq_wait drm_ioctl vfs_ioctl do_vfs_ioctl sys_ioctl
we can also call unlocked_ioctl and compat_ioctl dirctly
A.我们先创建一个新的device设备
这个设备支持ioctl，通过以下脚本创建这个设备，该脚本会加载一个驱动，并生成一个/dev/etx_device
MODULE=driver tee ${MODULE}.c >/dev/null <<EOF /***************************************************************************//** * \file driver.c * * \details Simple Linux device driver (Real Linux Device Driver) * * \author EmbeTronicX * * *******************************************************************************/ #include <linux/kernel.h> #include <linux/init.h> #include <linux/module.h> #include <linux/kdev_t.h> #include <linux/fs.h> #include <linux/cdev.h> #include <linux/device.h> #include<linux/slab.h> //kmalloc() #include<linux/uaccess.h> //copy_to/from_user() #define WR_VALUE _IOW('a','a',int32_t*) #define RD_VALUE _IOR('a','b',int32_t*) #define mem_size 1024 //Memory Size int32_t value = 0; dev_t dev = 0; static struct class *dev_class; static struct cdev etx_cdev; uint8_t *kernel_buffer; /* ** Function Prototypes */ static int __init etx_driver_init(void); static void __exit etx_driver_exit(void); static int etx_open(struct inode *inode, struct file *file); static int etx_release(struct inode *inode, struct file *file); static ssize_t etx_read(struct file *filp, char __user *buf, size_t len,loff_t * off); static ssize_t etx_write(struct file *filp, const char *buf, size_t len, loff_t * off); static long etx_ioctl(struct file *file, unsigned int cmd, unsigned long arg); /* ** File Operations structure */ static struct file_operations fops = { .owner = THIS_MODULE, .read = etx_read, .write = etx_write, .open = etx_open, .unlocked_ioctl = etx_ioctl, .release = etx_release, }; /* ** This fuction will be called when we open the Device file */ static int etx_open(struct inode *inode, struct file *file) { /*Creating Physical memory*/ if((kernel_buffer = kmalloc(mem_size , GFP_KERNEL)) == 0){ printk(KERN_INFO "Cannot allocate memory in kernel\n"); return -1; } printk(KERN_INFO "Device File Opened...!!!\n"); return 0; } /* ** This fuction will be called when we close the Device file */ static int etx_release(struct inode *inode, struct file *file) { kfree(kernel_buffer); printk(KERN_INFO "Device File Closed...!!!\n"); return 0; } /* ** This fuction will be called when we read the Device file */ static ssize_t etx_read(struct file *filp, char __user *buf, size_t len, loff_t *off) { //Copy the data from the kernel space to the user-space copy_to_user(buf, kernel_buffer, mem_size); printk(KERN_INFO "Data Read : Done!\n"); return mem_size; } /* ** This fuction will be called when we write the Device file */ static ssize_t etx_write(struct file *filp, const char __user *buf, size_t len, loff_t *off) { //Copy the data to kernel space from the user-space copy_from_user(kernel_buffer, buf, len); printk(KERN_INFO "Data Write : Done!\n"); return len; } /* ** This fuction will be called when we write IOCTL on the Device file */ static long etx_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch(cmd) { case WR_VALUE: copy_from_user(&value ,(int32_t*) arg, sizeof(value)); sprintf(kernel_buffer, "%s %d, %d, %d, %ld\n","This is a ioctl input. PID, TGID, cmd & value:", current->pid, current->tgid, cmd, arg); printk(KERN_INFO "Value = %d\n", value); break; case RD_VALUE: copy_to_user((int32_t*) arg, &value, sizeof(value)); break; } return 0; } /* ** Module Init function */ static int __init etx_driver_init(void) { /*Allocating Major number*/ if((alloc_chrdev_region(&dev, 0, 1, "etx_Dev")) <0){ printk(KERN_INFO "Cannot allocate major number\n"); return -1; } printk(KERN_INFO "Major = %d Minor = %d \n",MAJOR(dev), MINOR(dev)); /*Creating cdev structure*/ cdev_init(&etx_cdev,&fops); /*Adding character device to the system*/ if((cdev_add(&etx_cdev,dev,1)) < 0){ printk(KERN_INFO "Cannot add the device to the system\n"); goto r_class; } /*Creating struct class*/ if((dev_class = class_create(THIS_MODULE,"etx_class")) == NULL){ printk(KERN_INFO "Cannot create the struct class\n"); goto r_class; } /*Creating device*/ if((device_create(dev_class,NULL,dev,NULL,"etx_device")) == NULL){ printk(KERN_INFO "Cannot create the Device 1\n"); goto r_device; } printk(KERN_INFO "Device Driver Insert...Done!!!\n"); return 0; r_device: class_destroy(dev_class); r_class: unregister_chrdev_region(dev,1); return -1; } /* ** Module exit function */ static void __exit etx_driver_exit(void) { device_destroy(dev_class,dev); class_destroy(dev_class); cdev_del(&etx_cdev); unregister_chrdev_region(dev, 1); printk(KERN_INFO "Device Driver Remove...Done!!!\n"); } module_init(etx_driver_init); module_exit(etx_driver_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("EmbeTronicX <embetronicx@gmail.com>"); MODULE_DESCRIPTION("Simple Linux device driver (Real Linux Device Driver)"); MODULE_VERSION("1.4"); EOF make clean make sudo insmod ${MODULE}.ko lsmod | grep ${MODULE} lsof | grep ${MODULE} # sudo rm /dev/etx_device # sudo rmmod ${MODULE}.ko APP=test_app tee ${APP}.c >/dev/null <<EOF /***************************************************************************//** * \file test_app.c * * \details Userspace application to test the Device driver * * \author EmbeTronicX * * *******************************************************************************/ #include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> #include <unistd.h> #include<sys/ioctl.h> int8_t write_buf[1024]; int8_t read_buf[1024]; #define WR_VALUE _IOW('a','a',int32_t*) #define RD_VALUE _IOR('a','b',int32_t*) int main() { int fd; int32_t value, number; char option; printf("*********************************\n"); printf("*******WWW.EmbeTronicX.com*******\n"); fd = open("/dev/etx_device", O_RDWR); if(fd < 0) { printf("Cannot open device file...\n"); return 0; } while(1) { printf("****Please Enter the Option******\n"); printf(" 1. Write \n"); printf(" 2. Read \n"); printf(" 3. IOCTL Write \n"); printf(" 4. IOCTL Read \n"); printf(" 5. Exit \n"); printf("*********************************\n"); scanf(" %c", &option); printf("Current PID=%d, Your Option = %c\n", getpid(), option); switch(option) { case '1': printf("Enter the string to write into driver :"); scanf(" %[^\t\n]s", write_buf); printf("Data Writing ..."); write(fd, write_buf, strlen(write_buf)+1); printf("Done!\n"); break; case '2': printf("Data Reading ..."); read(fd, read_buf, 1024); printf("Done!\n\n"); printf("Data = %s\n\n", read_buf); break; case '3': printf("Enter the ioctl Value to send\n"); scanf("%d",&number); printf("Writing Value to Driver\n"); ioctl(fd, WR_VALUE, (int32_t*) &number); printf("Done!\n"); break; case '4': printf("Reading ioctl Value from Driver\n"); ioctl(fd, RD_VALUE, (int32_t*) &value); printf("Value is %d\n", value); read(fd, read_buf, 1024); printf("Done!\n\n"); printf("Data = %s\n\n", read_buf); break; case '5': close(fd); exit(1); break; default: printf("Enter Valid option = %c\n",option); break; } } close(fd); } EOF gcc -o ${APP} ${APP}.c sudo ./${APP}
通过运行`sudo ./${APP}` 我肯可以先往这个设备中写一个字符串，然后读一下。
A.我们先创建一个新的device设备
这个设备用来打开/dev/etx_device，并截获（Intercepte）ioctl操作。
通过以下脚本创建这个设别，该脚本会加载一个驱动，并生成一个 /dev/top_intercepte设备
mkdir intercepte && cd intercepte MODULE=intercepte cat <<EOF | sed -e "s/\\\t/\t/" | tee Makefile obj-m += ${MODULE}.o KDIR = /lib/modules/\$(shell uname -r)/build all: \tmake -C \$(KDIR) M=\$(shell pwd) modules clean: \tmake -C \$(KDIR) M=\$(shell pwd) clean EOF SUBDEV=etx_device tee ${MODULE}.c >/dev/null <<EOF /***************************************************************************//** * \file ${MODULE}.c * * \details Simple Linux device driver (Real Linux Device Driver) * * \author EmbeTronicX * * *******************************************************************************/ #include <linux/kernel.h> #include <linux/init.h> #include <linux/module.h> #include <linux/kdev_t.h> #include <linux/fs.h> #include <linux/cdev.h> #include <linux/device.h> #include<linux/slab.h> //kmalloc() #include<linux/uaccess.h> //copy_to/from_user() #define WR_VALUE _IOW('a','a',int32_t*) #define RD_VALUE _IOR('a','b',int32_t*) #define mem_size 1024 //Memory Size int32_t value = 0; dev_t dev = 0; static struct class *dev_class; static struct cdev etx_cdev; uint8_t *kernel_buffer; struct file *file=NULL; /* ** Function Prototypes */ static int __init etx_driver_init(void); static void __exit etx_driver_exit(void); static int etx_open(struct inode *inode, struct file *file); static int etx_release(struct inode *inode, struct file *file); static ssize_t etx_read(struct file *filp, char __user *buf, size_t len,loff_t * off); static ssize_t etx_write(struct file *filp, const char *buf, size_t len, loff_t * off); static long etx_ioctl(struct file *file, unsigned int cmd, unsigned long arg); /* ** File Operations structure */ static struct file_operations fops = { .owner = THIS_MODULE, .read = etx_read, .write = etx_write, .open = etx_open, .unlocked_ioctl = etx_ioctl, .release = etx_release, }; /* ** This fuction will be called when we open the Device file */ static int etx_open(struct inode *inode, struct file *file) { /*Creating Physical memory*/ if((kernel_buffer = kmalloc(mem_size , GFP_KERNEL)) == 0){ printk(KERN_INFO "Cannot allocate memory in kernel\n"); return -1; } printk(KERN_INFO "Device File Opened...!!!\n"); return 0; } /* ** This fuction will be called when we close the Device file */ static int etx_release(struct inode *inode, struct file *file) { kfree(kernel_buffer); printk(KERN_INFO "Device File Closed...!!!\n"); return 0; } /* ** This fuction will be called when we read the Device file */ static ssize_t etx_read(struct file *filp, char __user *buf, size_t len, loff_t *off) { //Copy the data from the kernel space to the user-space copy_to_user(buf, kernel_buffer, mem_size); printk(KERN_INFO "Data Read : Done!\n"); return mem_size; } /* ** This fuction will be called when we write the Device file */ static ssize_t etx_write(struct file *filp, const char __user *buf, size_t len, loff_t *off) { //Copy the data to kernel space from the user-space copy_from_user(kernel_buffer, buf, len); printk(KERN_INFO "Data Write : Done!\n"); return len; } /* ** This fuction will be called when we write IOCTL on the Device file */ static long etx_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { long rc = 0; switch(cmd) { case WR_VALUE: copy_from_user(&value ,(int32_t*) arg, sizeof(value)); sprintf(kernel_buffer, "%s %d, %d, %d, %ld\n","This is a ioctl input. PID, TGID, cmd & value:", current->pid, current->tgid, cmd, arg); rc=file->f_op->unlocked_ioctl(file,cmd,arg); printk(KERN_INFO "Value = %d\n", value); break; case RD_VALUE: copy_to_user((int32_t*) arg, &value, sizeof(value)); break; } return rc; } /* ** Module Init function */ static int __init etx_driver_init(void) { /*Allocating Major number*/ if((alloc_chrdev_region(&dev, 0, 1, "etx_TopDev")) <0){ printk(KERN_INFO "Cannot allocate major number\n"); return -1; } printk(KERN_INFO "Major = %d Minor = %d \n",MAJOR(dev), MINOR(dev)); /*Creating cdev structure*/ cdev_init(&etx_cdev,&fops); /*Adding character device to the system*/ if((cdev_add(&etx_cdev,dev,1)) < 0){ printk(KERN_INFO "Cannot add the device to the system\n"); goto r_class; } /*Creating struct class*/ if((dev_class = class_create(THIS_MODULE,"topetx_class")) == NULL){ printk(KERN_INFO "Cannot create the struct class\n"); goto r_class; } /*Creating device*/ if((device_create(dev_class,NULL,dev,NULL,"top_${MODULE}")) == NULL){ printk(KERN_INFO "Cannot create the Device 1\n"); goto r_device; } /*Creating device*/ file=filp_open("/dev/${SUBDEV}",O_RDWR,0); if(IS_ERR(file)) { printk(KERN_INFO "Cannot Open the Device /dev/${SUBDEV}\n"); goto fail0; } printk(KERN_INFO "Device Driver Insert...Done!!!\n"); return 0; fail0: filp_close(file,NULL); printk("load failed\n"); r_device: class_destroy(dev_class); r_class: unregister_chrdev_region(dev,1); return -1; } /* ** Module exit function */ static void __exit etx_driver_exit(void) { filp_close(file,NULL); device_destroy(dev_class,dev); class_destroy(dev_class); cdev_del(&etx_cdev); unregister_chrdev_region(dev, 1); printk(KERN_INFO "Device Driver Remove...Done!!!\n"); } module_init(etx_driver_init); module_exit(etx_driver_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("EmbeTronicX <embetronicx@gmail.com>"); MODULE_DESCRIPTION("Simple Linux device driver (Real Linux Device Driver)"); MODULE_VERSION("1.4"); EOF make clean make sudo insmod ${MODULE}.ko lsmod | grep ${MODULE} lsof | grep ${MODULE} # sudo rm /dev/etx_device # sudo rmmod ${MODULE}.ko APP=toptest_app tee ${TOPAPP}.c >/dev/null <<EOF /***************************************************************************//** * \file ${TOPAPP}.c * * \details Userspace application to test the Device driver * * \author EmbeTronicX * * *******************************************************************************/ #include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> #include <unistd.h> #include<sys/ioctl.h> int8_t write_buf[1024]; int8_t read_buf[1024]; #define WR_VALUE _IOW('a','a',int32_t*) #define RD_VALUE _IOR('a','b',int32_t*) int main() { int fd; int32_t value, number; char option; printf("*********************************\n"); printf("*******WWW.EmbeTronicX.com*******\n"); fd = open("/dev/top_${MODULE}", O_RDWR); if(fd < 0) { printf("Cannot open device file...\n"); return 0; } while(1) { printf("****Please Enter the Option******\n"); printf(" 1. Write \n"); printf(" 2. Read \n"); printf(" 3. IOCTL Write \n"); printf(" 4. IOCTL Read \n"); printf(" 5. Exit \n"); printf("*********************************\n"); scanf(" %c", &option); printf("Current PID=%d, Your Option = %c\n", getpid(), option); switch(option) { case '1': printf("Enter the string to write into driver :"); scanf(" %[^\t\n]s", write_buf); printf("Data Writing ..."); write(fd, write_buf, strlen(write_buf)+1); printf("Done!\n"); break; case '2': printf("Data Reading ..."); read(fd, read_buf, 1024); printf("Done!\n\n"); printf("Data = %s\n\n", read_buf); break; case '3': printf("Enter the ioctl Value to send\n"); scanf("%d",&number); printf("Writing Value to Driver\n"); ioctl(fd, WR_VALUE, (int32_t*) &number); printf("Done!\n"); break; case '4': printf("Reading ioctl Value from Driver\n"); ioctl(fd, RD_VALUE, (int32_t*) &value); printf("Value is %d\n", value); read(fd, read_buf, 1024); printf("Done!\n\n"); printf("Data = %s\n\n", read_buf); break; case '5': close(fd); exit(1); break; default: printf("Enter Valid option = %c\n",option); break; } } close(fd); } EOF gcc -o ${TOPAPP} ${TOPAPP}.c sudo ./${TOPAPP}
通过运行`sudo ./${TOPAPP}` 我肯可以先往这个设备中执行命令3进行一个ioctl写的命令，然后通过`sudo ./${APP}`的命令3进行一个ioctl读的命令读一下， 看看输出结果有没有发生变化。
REF:
scaffold
Linux Device Drivers, 3rd Edition by Jonathan Corbet, Alessandro Rubini, Greg Kroah-Ha (oreilly)
Linux Device Drivers: Tutorial for Linux Driver Development
Linux Device Driver Tutorial (hello example)
How to write a linux device driver (hello example)
Writing a Linux Driver (good prictures to show differe driver)
How to write a simple Linux device driver? (SPI example)
Writing a Linux Kernel Module — Part 1: Introduction (Total 3 parts, part 3 is GPIO example)
kernel thread
Kernel threads (introduce mechanism)
SysPlay's Blogs Kernel Threads Continued print all tasks info（include process id）， his many blog， CATEGORIES cover many domains：
1 cat <<EOF | tee tasks.c 2 #include <linux/kernel.h> 3 #include <linux/module.h> 4 #include <linux/sched/signal.h> 5 6 char buffer[256]; 7 char * get_task_state(long state) 8 { 9 switch (state) { 10 case TASK_RUNNING: 11 return "TASK_RUNNING"; 12 case TASK_INTERRUPTIBLE: 13 return "TASK_INTERRUPTIBLE"; 14 case TASK_UNINTERRUPTIBLE: 15 return "TASK_UNINTERRUPTIBLE"; 16 case __TASK_STOPPED: 17 return "__TASK_STOPPED"; 18 case __TASK_TRACED: 19 return "__TASK_TRACED"; 20 default: 21 { 22 sprintf(buffer, "Unknown Type:%ld\n", state); 23 return buffer; 24 } 25 } 26 } 27 28 static int test_tasks_init(void) 29 { 30 struct task_struct *task_list; 31 unsigned int process_count = 0; 32 pr_info("%s: In init\n", __func__); 33 for_each_process(task_list) { 34 pr_info("Process: %s\t PID:[%d]\t State:%s\n", 35 task_list->comm, task_list->pid, 36 get_task_state(task_list->state)); 37 process_count++; 38 } 39 pr_info("Number of processes:%u\n", process_count); 40 return 0; 41 } 42 43 static void test_tasks_exit(void) 44 { 45 pr_info("%s: In exit\n", __func__); 46 } 47 48 MODULE_LICENSE("GPL"); 49 module_init(test_tasks_init); 50 module_exit(test_tasks_exit); 51 EOF 52 53 MODULE=tasks 54 cat <<EOF | tee Makefile 55 obj-m := ${MODULE}.o 56 EOF 57 58 make -C /usr/src/linux-headers-$(uname -r) M=`pwd` modules 59 sudo insmod ${MODULE}.ko 60 lsmod |grep ${MODULE} 61 sudo rmmod ${MODULE} 62 lsmod |grep ${MODULE} 63 dmesg |tail
linux-kernel Creation of kernel threads(Learning linux-kernel eBook (PDF))，
Chapters
- Chapter 1: Getting started with linux-kernel
- Chapter 2: Creation and usage of Kernel Threads
- Chapter 3: Event Tracing
- Chapter 4: Fork System call
- Chapter 5: How to find the right person for help.
- Chapter 6: Linux Hello World Device driver
- Chapter 7: Linux: Named Pipes(FIFO)
it show inserting the .ko
1 MODULE=kern_thread 2 cat <<EOF | tee ${MODULE}.c 3 #include <linux/module.h> 4 #include <linux/kernel.h> 5 #include <linux/init.h> 6 #include <linux/kthread.h> 7 #include <linux/sched.h> 8 9 #define AUTHOR "Nachiket Kulkarni" 10 #define DESCRIPTION "Simple module that demonstrates creation of 2 kernel threads" 11 12 static int kthread_func(void *arg) 13 { 14 /* Every kthread has a struct task_struct associated with it which is it's identifier. 15 * Whenever a thread is schedule for execution, the kernel sets "current" pointer to 16 * it's struct task_struct. 17 * current->comm is the name of the command that caused creation of this thread 18 * current->pid is the process of currently executing thread 19 */ 20 printk(KERN_INFO "I am thread: %s[PID = %d] [TGID = %d]\n", current->comm, current->pid, current->tgid); 21 return 0; 22 } 23 24 static int __init init_func(void) 25 { 26 struct task_struct *ts1; 27 struct task_struct *ts2; 28 int err; 29 30 printk(KERN_INFO "Starting 2 threads\n"); 31 32 /*struct task_struct *kthread_create(int (*threadfn)(void *data), void *data, \ 33 * const char *namefmt, ...); 34 * This function creates a kernel thread and starts the thread. 35 */ 36 ts1 = kthread_run(kthread_func, NULL, "thread-1"); 37 if (IS_ERR(ts1)) { 38 printk(KERN_INFO "ERROR: Cannot create thread ts1\n"); 39 err = PTR_ERR(ts1); 40 ts1 = NULL; 41 return err; 42 } 43 44 ts1 = kthread_run(kthread_func, NULL, "thread-1"); 45 if (IS_ERR(ts1)) { 46 printk(KERN_INFO "ERROR: Cannot create thread ts1\n"); 47 err = PTR_ERR(ts1); 48 ts1 = NULL; 49 return err; 50 } 51 52 printk(KERN_INFO "I am thread: %s[PID = %d] [TGID = %d]\n", current->comm, current->pid, current->tgid); 53 return 0; 54 } 55 56 static void __exit exit_func(void) 57 { 58 printk(KERN_INFO "Exiting the module\n"); 59 } 60 61 module_init(init_func); 62 module_exit(exit_func); 63 64 MODULE_AUTHOR(AUTHOR); 65 // MODULE_DESCRIPTION(MODULE_AUTHOR); /* not pass */ 66 MODULE_LICENSE("GPL"); 67 EOF 68 69 # ls /lib/modules/4.15.0-101-generic/build/include/linux/ 70 cat <<EOF | sed -e "s/\\\t/\t/" | tee Makefile 71 obj-m += ${MODULE}.o 72 73 all: 74 \tmake -C /lib/modules/\$(shell uname -r)/build M=\$(PWD) modules 75 clean: 76 \tmake -C /lib/modules/\$(shell uname -r)/build M=\$(PWD) clean 77 EOF 78 79 make -C /usr/src/linux-headers-$(uname -r) M=`pwd` modules 80 sudo insmod ${MODULE}.ko 81 lsmod |grep ${MODULE} 82 sudo rmmod ${MODULE} 83 lsmod |grep ${MODULE} 84 dmesg |tail
and print
[1373814.205950] Starting 2 threads [1373814.206296] I am thread: thread-1[PID = 15204] [TGID = 15204] [1373814.206695] I am thread: insmod[PID = 15203] [TGID = 15203] [1373814.206722] I am thread: thread-1[PID = 15205] [TGID = 15205]
Kernel threads (man ) made easy
Kernel thread to print the running tasks on the CPU every 500ms
MODULE=kthread cat <<EOF | tee ${MODULE}.c #include <linux/kernel.h> #include <linux/module.h> #include <linux/sched/signal.h> #include <linux/kthread.h> #include <linux/delay.h> struct task_struct *print_thread; char buffer[256]; char * get_task_state(long state) { switch (state) { case TASK_RUNNING: return "TASK_RUNNING"; case TASK_INTERRUPTIBLE: return "TASK_INTERRUPTIBLE"; case TASK_UNINTERRUPTIBLE: return "TASK_UNINTERRUPTIBLE"; case __TASK_STOPPED: return "__TASK_STOPPED"; case __TASK_TRACED: return "__TASK_TRACED"; case TASK_IDLE: return "(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)"; case TASK_KILLABLE: return "(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)"; case TASK_STOPPED: return "(TASK_WAKEKILL | __TASK_STOPPED)"; case TASK_TRACED: return "(TASK_WAKEKILL | __TASK_TRACED)"; default: { sprintf(buffer, "Unknown Type:%ld", state); return buffer; } } } static int print_running_thread(void *data) { while(!kthread_should_stop()) { struct task_struct *task_list; for_each_process(task_list) { if (task_list->state == TASK_RUNNING) pr_info("Process: %s\t PID:[%d]\t State:%s\t CPU:%d\n", task_list->comm, task_list->pid, get_task_state(task_list->state), task_list->cpu); } msleep(500); } return 0; } MODULE_LICENSE("GPL"); static int test_tasks_init(void) { pr_info("%s: In init\n", __func__); print_thread = kthread_run(print_running_thread, NULL, "print_running_cpu"); return 0; } static void test_tasks_exit(void) { pr_info("%s: In exit\n", __func__); kthread_stop(print_thread); } module_init(test_tasks_init); module_exit(test_tasks_exit); EOF # ls /lib/modules/4.15.0-101-generic/build/include/linux/ cat <<EOF | sed -e "s/\\\t/\t/" | tee Makefile obj-m += ${MODULE}.o all: \tmake -C /lib/modules/\$(shell uname -r)/build M=\$(PWD) modules clean: \tmake -C /lib/modules/\$(shell uname -r)/build M=\$(PWD) clean EOF sudo insmod ${MODULE}.ko lsmod |grep ${MODULE} sudo rmmod ${MODULE} lsmod |grep ${MODULE} dmesg |tail
Kernel Thread Creation : 1 (schedule with wake_up_process)
MODULE=threads cat <<EOF | tee ${MODULE}.c #include <linux/module.h> #include <linux/kernel.h> #include <linux/kthread.h> // for threads #include <linux/sched.h> // for task_struct #include <linux/time.h> // for using jiffies #include <linux/timer.h> static struct task_struct *thread1; static int thread_fn(void *data) { unsigned long j0,j1; int delay = 60*HZ; printk(KERN_INFO "In thread1"); j0 = jiffies; j1 = j0 + delay; while (time_before(jiffies, j1)){ schedule(); printk(KERN_INFO "jiffies:%ld, I am thread: %s[PID = %d] [TGID = %d]\n", j0, current->comm, current->pid, current->tgid); } return 0; } int thread_init (void) { char our_thread[8]="thread1"; printk(KERN_INFO "in init"); thread1 = kthread_create(thread_fn,NULL,our_thread); if((thread1)) { printk(KERN_INFO "in if"); wake_up_process(thread1); } return 0; } void thread_cleanup(void) { int ret; ret = kthread_stop(thread1); if(!ret) printk(KERN_INFO "Thread stopped"); } MODULE_LICENSE("GPL"); module_init(thread_init); module_exit(thread_cleanup); EOF # ls /lib/modules/4.15.0-101-generic/build/include/linux/ cat <<EOF | sed -e "s/\\\t/\t/" | tee Makefile ifneq (\$(KERNELRELEASE),) \tobj-m := threads.o else KERNELDIR ?= /lib/modules/\$(shell uname -r)/build PWD := \$(shell pwd) default: \t\$(MAKE) -C \$(KERNELDIR) M=\$(PWD) modules endif EOF sudo insmod ${MODULE}.ko lsmod |grep ${MODULE} sudo rmmod ${MODULE} lsmod |grep ${MODULE} dmesg |tail
Linux Device Driver Tutorial Part 19 – Kernel Thread
MODULE=driver cat <<EOF | tee ${MODULE}.c /***************************************************************************//** * \file driver.c * * \details Simple Linux device driver (Kernel Thread) * * \author EmbeTronicX * * *******************************************************************************/ #include <linux/kernel.h> #include <linux/init.h> #include <linux/module.h> #include <linux/kdev_t.h> #include <linux/fs.h> #include <linux/cdev.h> #include <linux/device.h> #include<linux/slab.h> //kmalloc() #include<linux/uaccess.h> //copy_to/from_user() #include <linux/kthread.h> //kernel threads #include <linux/sched.h> //task_struct #include <linux/delay.h> dev_t dev = 0; static struct class *dev_class; static struct cdev etx_cdev; static int __init etx_driver_init(void); static void __exit etx_driver_exit(void); static struct task_struct *etx_thread; /* ** Function Prototypes */ /*************** Driver Fuctions **********************/ static int etx_open(struct inode *inode, struct file *file); static int etx_release(struct inode *inode, struct file *file); static ssize_t etx_read(struct file *filp, char __user *buf, size_t len,loff_t * off); static ssize_t etx_write(struct file *filp, const char *buf, size_t len, loff_t * off); /******************************************************/ int thread_function(void *pv); /* ** Thread */ int thread_function(void *pv) { int i=0; while(!kthread_should_stop()) { printk(KERN_INFO "In EmbeTronicX Thread Function %d\n", i++); msleep(1000); } return 0; } /* ** File operation sturcture */ static struct file_operations fops = { .owner = THIS_MODULE, .read = etx_read, .write = etx_write, .open = etx_open, .release = etx_release, }; /* ** This fuction will be called when we open the Device file */ static int etx_open(struct inode *inode, struct file *file) { printk(KERN_INFO "Device File Opened...!!!\n"); return 0; } /* ** This fuction will be called when we close the Device file */ static int etx_release(struct inode *inode, struct file *file) { printk(KERN_INFO "Device File Closed...!!!\n"); return 0; } /* ** This fuction will be called when we read the Device file */ static ssize_t etx_read(struct file *filp, char __user *buf, size_t len, loff_t *off) { printk(KERN_INFO "Read function\n"); return 0; } /* ** This fuction will be called when we write the Device file */ static ssize_t etx_write(struct file *filp, const char __user *buf, size_t len, loff_t *off) { printk(KERN_INFO "Write Function\n"); return len; } /* ** Module Init function */ static int __init etx_driver_init(void) { /*Allocating Major number*/ if((alloc_chrdev_region(&dev, 0, 1, "etx_Dev")) <0){ printk(KERN_INFO "Cannot allocate major number\n"); return -1; } printk(KERN_INFO "Major = %d Minor = %d \n",MAJOR(dev), MINOR(dev)); /*Creating cdev structure*/ cdev_init(&etx_cdev,&fops); /*Adding character device to the system*/ if((cdev_add(&etx_cdev,dev,1)) < 0){ printk(KERN_INFO "Cannot add the device to the system\n"); goto r_class; } /*Creating struct class*/ if((dev_class = class_create(THIS_MODULE,"etx_class")) == NULL){ printk(KERN_INFO "Cannot create the struct class\n"); goto r_class; } /*Creating device*/ if((device_create(dev_class,NULL,dev,NULL,"etx_device")) == NULL){ printk(KERN_INFO "Cannot create the Device \n"); goto r_device; } etx_thread = kthread_create(thread_function,NULL,"eTx Thread"); if(etx_thread) { wake_up_process(etx_thread); } else { printk(KERN_ERR "Cannot create kthread\n"); goto r_device; } #if 0 /* You can use this method to create and run the thread */ etx_thread = kthread_run(thread_function,NULL,"eTx Thread"); if(etx_thread) { printk(KERN_ERR "Kthread Created Successfully...\n"); } else { printk(KERN_ERR "Cannot create kthread\n"); goto r_device; } #endif printk(KERN_INFO "Device Driver Insert...Done!!!\n"); return 0; r_device: class_destroy(dev_class); r_class: unregister_chrdev_region(dev,1); cdev_del(&etx_cdev); return -1; } /* ** Module exit function */ static void __exit etx_driver_exit(void) { kthread_stop(etx_thread); device_destroy(dev_class,dev); class_destroy(dev_class); cdev_del(&etx_cdev); unregister_chrdev_region(dev, 1); printk(KERN_INFO "Device Driver Remove...Done!!\n"); } module_init(etx_driver_init); module_exit(etx_driver_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("EmbeTronicX <embetronicx@gmail.com>"); MODULE_DESCRIPTION("A simple device driver - Kernel Thread"); MODULE_VERSION("1.14"); EOF cat <<EOF | sed -e "s/\\\t/\t/" | tee Makefile obj-m += ${MODULE}.o KDIR = /lib/modules/\$(shell uname -r)/build all: \tmake -C \$(KDIR) M=\$(shell pwd) modules clean: \tmake -C \$(KDIR) M=\$(shell pwd) clean EOF sudo insmod ${MODULE}.ko lsmod |grep ${MODULE} dmesg |tail sudo rmmod ${MODULE} lsmod |grep ${MODULE}
Githu Linux-Kernel-Examples， stackoverflow Creating Kernel Thread using kernel_thread
ioctl
Linux内核态文件读写相关函数API （vfs_write 和 vfs_read 例子， same 文章来在cnblog）
linux内核态文件操作filp_open/filp_close/vfs_read/vfs_write
内核层读写应用层文件，使用filp_open函数 （vfs_write 和 vfs_read）
Read/write files within a Linux kernel module （也推荐 vfs_write 和 vfs_read）
You should be aware that you should avoid file I/O from within Linux kernel when possible. The main idea is to go "one level deeper" and call VFS level functions instead of the syscall handler directly
Writing to a file from the Kernel （snippet）
linux kernel access file : filp_open – example
Reading Files From The Linux Kernel Space (Module/Driver) (Fedora 14)
Linux Device Driver, kernel thread can't open file? （内核线程打开文件）
目前网上ioctl的例子如下， 编译会有问题，一个是 _struct task_struct_ 不兼容的问题（不要引用asm/uascess.h），一个是ioctl 不再支持的问题， 一个是asm/linkage.h不存在的问题。
# filp_open/sys_open https://developer.aliyun.com/article/449701 cat << EOF | tee chardev.h #include <linux/ioctl.h> #define BAO_IOCTL 't' #define IOCTL_READ _IOR(BAO_IOCTL, 0, int) #define IOCTL_WRITE _IOW(BAO_IOCTL, 1, int) #define BAO_IOCTL_MAXNR 1 EOF # issue: error: dereferencing pointer to incomplete type 'struct task_struct' # REF: https://lkml.org/lkml/2017/10/24/827 # REF: #include <linux/sched.h>, http://biancheng.dnbcw.net/linux/351783.html # REF: #include <linux/uaccess.h>, https://lkml.org/lkml/2017/9/30/189 # https://medium.com/@avenger.v14/hi-when-building-the-enhanced-example-with-character-device-i-encountered-a-build-error-for-55079354f704 # REF: http://dannysun-unknown.blogspot.com/2018/01/error-dereferencing-pointer-to.html MODULE=test UFILE=memory # sudo tee ${MODULE}.c >/dev/null <<EOF cat <<EOF | tee ${MODULE}.c #include <linux/kernel.h> #include <linux/sched.h> #include <linux/module.h> #include <linux/stat.h> #include <linux/fs.h> /* unlocked_ioctl and compat_ioctl, vfs_read, vfs_write */ #include <asm/unistd.h> #include <asm/uaccess.h> #include <linux/types.h> #include <linux/ioctl.h> /* vfs_ioctl */ #include "chardev.h" MODULE_LICENSE("GPL"); //#define __KERNEL_SYSCALLS__ static char buf1[20]; static char buf2[20]; struct file *file=NULL; static int __init testmod_init(void) { mm_segment_t old_fs; ssize_t result; ssize_t ret; sprintf(buf1,"%s","baoqunmin"); file=filp_open("${UFILE}",O_RDWR,0); if(IS_ERR(file)) goto fail0; old_fs=get_fs(); set_fs(get_ds()); ret=file->f_op->write(file,buf1,sizeof(buf1),&file->f_pos); result=file->f_op->read(file,buf2,sizeof(buf2),&file->f_pos); if(result>=0){buf2[19]='\n';printk("buf2-->%s\n",buf2);} else printk("failed\n"); result=file->f_op->unlocked_ioctl(file,IOCTL_READ,1); /* unsupport ioctl any more, hardcode arg=1 */ result=file->f_op->read(file,buf2,sizeof(buf2),&file->f_pos); set_fs(old_fs); filp_close(file,NULL); printk("file loaded\n"); return 0; fail0:{filp_close(file,NULL);printk("load failed\n");} return 1; } static void __exit testmod_cleanup(void) { printk("module exit.................................\n"); } module_init(testmod_init); module_exit(testmod_cleanup); EOF # issue: asm/linkage.h: No such file or directory # REF: https://cnasolution.com/questions/130765/gcc-error-asm-linkage-h-no-such-file-or-directory # REF: 3 ways work around https://github.com/dtaht/sch_cake/issues/110 cat << EOF | sed -e "s/\\\t/\t/" | tee Makefile CC=gcc INCPATH=/lib/modules/\$(shell uname -r)/build/include ARCHPATH=/usr/src/linux-headers-${REL%-*}/arch/x86/include MODCFLAGS := -Wall -DMODULE -D__KERNEL__ -DLINUX -I\$(INCPATH) -I\$(ARCHPATH) test.o :test.c \t\$(CC) \$(MODCFLAGS) -c test.c \techo insmod test.o to turn it on \techo rmmod test to turn it off \techo EOF cat <<EOF | sed -e "s/\\\t/\t/" | tee Makefile obj-m += ${MODULE}.o KDIR = /lib/modules/\$(shell uname -r)/build all: \tmake -C \$(KDIR) M=\$(shell pwd) modules clean: \tmake -C \$(KDIR) M=\$(shell pwd) clean EOF
kernel doc
ascii to string
1 # https://stackoverflow.com/questions/5724761/ascii-hex-convert-in-bash/5725125 2 # echo $'a\tb' 3 # cat <<< "abcdefga" | xxd -c 4 4 # cat <<< "abcdefga" | xxd -ps -c 4 5 # echo Aa | od -t x1 6 # echo -n Aa | hexdump -e '/1 "%02x"' 7 # printf '\301\227\227\223\205\045' | iconv -f ebcdic-uk -t ascii | od -An -vtu1 8 # https://www.xspdf.com/resolution/51314853.html 9 # https://www.tecmint.com/convert-files-to-utf-8-encoding-in-linux/ 10 # https://www.geeksforgeeks.org/iconv-command-in-linux-with-examples/ 11 # https://www.howtoforge.com/linux-cat-command/ 12 # https://stackoverflow.com/questions/5724761/ascii-hex-convert-in-bash/5725125 13 # https://stackoverflow.com/questions/525872/echo-tab-characters-in-bash-script 14 # https://www.linuxquestions.org/questions/linux-newbie-8/tab-in-bash-script-242400/
