smp启动-secondary cpu 启动流程和boot cpu的交互点
上一篇：smp_init 整体流程-新核心执行 secondary_entry
https://www.cnblogs.com/zhangzhiwei122/p/16093602.html
secondary cpu启动流程
secondary_entry // arch/arm64/kernel/head.S
->secondary_startup
->secondary_switched
->secondary_start_kernel // arch/arm64/kernel/smp.c
->cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); // kernel/sched/idle.c
过程分析
（1、先要有 primary_entry 启动过程分析相关知识，2、和primary 启动过程大同小异，3、下面的这篇很好）：
https://www.cnblogs.com/pengdonglin137/p/11925299.html
boot cpu 和 secondary cpu 的交互点
1、cpu_running completion
等待 secondary cpu 设置 online
arch/arm64/kernel/smp.c
a： boot cpu
113int __cpu_up(unsigned int cpu, struct task_struct *idle) 114{ 127 /* Now bring the CPU into our world */ 128 ret = boot_secondary(cpu, idle); 129 if (ret) { 130 pr_err("CPU%u: failed to boot: %d\n", cpu, ret); 131 return ret; 132 } 133 134 /* 135 * CPU was successfully started, wait for it to come online or 136 * time out. 137 */ 138 wait_for_completion_timeout(&cpu_running, 139 msecs_to_jiffies(5000)); 140 if (cpu_online(cpu)) 141 return 0;
128 - boot_secondary
138 - wait for completion_timeout
140 - if secondary cpu online , return 0;
b: secondary cpu
199asmlinkage notrace void secondary_start_kernel(void) 200{ 259 */ 260 pr_info("CPU%u: Booted secondary processor 0x%010lx [0x%08x]\n", 261 cpu, (unsigned long)mpidr, 262 read_cpuid_id()); 263 update_cpu_boot_status(CPU_BOOT_SUCCESS); 264 set_cpu_online(cpu, true); 265 complete(&cpu_running);
199 - 定义函数， secondary_start_kernel
264 - set online
265 - complete cpu running
2、bringup_cpu->bringup_wait_for_ap->wait_for_ap_thread ->wait_for_completion(st->done)
等 secondary cpu 设置(到达） CPUHP_AP_ONLINE_IDLE
kernel/cpu.c
a: boot cpu 流程
242static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup) 243{ 244 struct completion *done = bringup ? &st->done_up : &st->done_down; 245 wait_for_completion(done); 246} 519 520static int bringup_wait_for_ap(unsigned int cpu) 521{ 522 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 523 524 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */ 525 wait_for_ap_thread(st, true); 526 if (WARN_ON_ONCE((!cpu_online(cpu)))) 527 return -ECANCELED; 528 529 /* Unpark the hotplug thread of the target cpu */ 530 kthread_unpark(st->thread); 531 548static int bringup_cpu(unsigned int cpu) 549{ 560 /* Arch-specific enabling code. */ 561 ret = __cpu_up(cpu, idle); 565 return bringup_wait_for_ap(cpu); 566}
b: secondary cpu 流程
arch/arm64/kernel/smp.c
199asmlinkage notrace void secondary_start_kernel(void) 200{ 270 * OK, it's off to the idle thread for us 271 */ 272 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 273}
kernel/sched/idle.c
390void cpu_startup_entry(enum cpuhp_state state) 391{ 392 arch_cpu_idle_prepare(); 393 cpuhp_online_idle(state); 394 while (1) 395 do_idle(); 396}
kernel/cpu.c
248static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup) 249{ 250 struct completion *done = bringup ? &st->done_up : &st->done_down; 251 complete(done); 252}
1175void cpuhp_online_idle(enum cpuhp_state state) 1176{ 1177 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 1189 st->state = CPUHP_AP_ONLINE_IDLE; 1190 complete_ap_thread(st, true); 1191}
3、__cpuhp_kick_ap->wait_for_ap_thread
等 secondary cpu 到达 CPUHP_ONLINE 状态，bringup_noboot_cpus->cpu_up(cpu, CPUHP_ONLINE);给定的 目标状态是 CPUHP_ONLINE
percpu 变量 cpuhp_state 中记录的 各个CPU的state，
从 CPUHP_OFFLINE（即 0 ） 到 CPUHP_AP_ONLINE_IDLE 由 boot cpu 作工作（调用函数完成），然后 bringup_cpu 等待 secondary cpu
进入到 CPUHP_AP_ONLINE_IDLE 。之后，给secondary cpu 执行 unparking cpuhp task, 由 secondary cpu 执行 cpuhp task （ smpboot_thread_fn->cpuhp_thread_fun 函数) 推进 secondary cpu cpuhp_state 的状态，从CPUHP_AP_ONLINE_IDLE 到 CPUHP_ONLINE 。
这儿等待 secondary cpu 状态到达 CPUHP_ONLINE 后返回。
cpuhp_up_callbacks 中的 while(st->state <target ) 循环结束， _cpu_up 函数返回， cpu_up 函数返回， bringup_noboot_cpus 函数返回。
smp_init 函数 收到 bringup_noboot_cpus 函数调用的返回值。
a： boot cpu 流程
kernel/cpu.c
bringup_cpu ->bringup_wait_for_ap ->cpuhp_kick_ap ->__cpuhp_kick_ap ->wait_for_ap_thread 488/* Regular hotplug invocation of the AP hotplug thread */ 489static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st) 490{ 500 st->should_run = true; 501 wake_up_process(st->thread); 502 wait_for_ap_thread(st, st->bringup); 503} 505static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target) 506{ 507 enum cpuhp_state prev_state; 510 prev_state = cpuhp_set_state(st, target); 511 __cpuhp_kick_ap(st); 520static int bringup_wait_for_ap(unsigned int cpu) 521{ 522 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 523 524 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */ 525 wait_for_ap_thread(st, true); 545 return cpuhp_kick_ap(st, st->target); 546}
b: secondary cpu 流程
smpboot_thread_fn // kernel/smpboot.c
->cpuhp_thread_fun // kernel/cpu.c
->complete_ap_thread //kernel/cpu.c
kernel/smpboot.c
107static int smpboot_thread_fn(void *data) 108{ 109 struct smpboot_thread_data *td = data; 110 struct smp_hotplug_thread *ht = td->ht; 111 112 while (1) { 159 if (!ht->thread_should_run(td->cpu)) { 160 preempt_enable_no_resched(); 161 schedule(); 162 } else { 163 __set_current_state(TASK_RUNNING); 164 preempt_enable(); 165 ht->thread_fn(td->cpu); 166 } 167 } 168}
kernel/cpu.c
659static void cpuhp_thread_fun(unsigned int cpu) 660{ 661 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 662 bool bringup = st->bringup; 727 if (!st->should_run) 728 complete_ap_thread(st, bringup); 729} 802static struct smp_hotplug_thread cpuhp_threads = { 803 .store = &cpuhp_state.thread, 804 .create = &cpuhp_create, 805 .thread_should_run = cpuhp_should_run, 806 .thread_fn = cpuhp_thread_fun, 807 .thread_comm = "cpuhp/%u", 808 .selfparking = true, 809};
