pstack 命令
pstack 命令可以打印出进程的线程堆栈信息，使用方法比较简单，
pstack pid
以MySQL 数据为例
[root@mysql ~]# pstack `pidof mysqld`
Thread 30 (Thread 0x7f8770fbe700 (LWP 4704)):
#0 0x00007f8778be658a in sigwaitinfo () from /lib64/libc.so.6
#1 0x0000000000f7bb1b in timer_notify_thread_func (arg=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/mysys/posix_timers.c:89
#2 0x00000000013e1e64 in pfs_spawn_thread (arg=0x25d8be0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/perfschema/pfs.cc:2197
#3 0x00007f877a1f5ea5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f8778cadb0d in clone () from /lib64/libc.so.6
Thread 29 (Thread 0x7f8764eb9700 (LWP 4705)):
#0 0x00007f8779de8644 in __io_getevents_0_4 () from /lib64/libaio.so.1
#1 0x00000000011852d4 in LinuxAIOHandler::collect (this=0x7f8764eb8dd0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2521
#2 0x00000000011874c4 in LinuxAIOHandler::poll (this=0x7f8764eb8dd0, m1=0x7f8764eb8e88, m2=0x7f8764eb8e80, request=0x7f8764eb8e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2681
#3 0x000000000118787c in os_aio_linux_handler (request=0x7f8764eb8e50, m2=0x7f8764eb8e80, m1=0x7f8764eb8e88, global_segment=0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2737
#4 os_aio_handler (segment=0, m1=0x7f8764eb8e88, m2=0x7f8764eb8e80, request=0x7f8764eb8e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:6283
#5 0x00000000013256ad in fil_aio_wait (segment=0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/fil/fil0fil.cc:5858
#6 0x0000000001229960 in io_handler_thread (arg=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/srv/srv0start.cc:319
#7 0x00007f877a1f5ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f8778cadb0d in clone () from /lib64/libc.so.6
Thread 28 (Thread 0x7f87646b8700 (LWP 4706)):
#0 0x00007f8779de8644 in __io_getevents_0_4 () from /lib64/libaio.so.1
#1 0x00000000011852d4 in LinuxAIOHandler::collect (this=0x7f87646b7dd0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2521
#2 0x00000000011874c4 in LinuxAIOHandler::poll (this=0x7f87646b7dd0, m1=0x7f87646b7e88, m2=0x7f87646b7e80, request=0x7f87646b7e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2681
#3 0x000000000118787c in os_aio_linux_handler (request=0x7f87646b7e50, m2=0x7f87646b7e80, m1=0x7f87646b7e88, global_segment=1) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2737
#4 os_aio_handler (segment=1, m1=0x7f87646b7e88, m2=0x7f87646b7e80, request=0x7f87646b7e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:6283
#5 0x00000000013256ad in fil_aio_wait (segment=1) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/fil/fil0fil.cc:5858
#6 0x0000000001229960 in io_handler_thread (arg=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/srv/srv0start.cc:319
#7 0x00007f877a1f5ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f8778cadb0d in clone () from /lib64/libc.so.6
Thread 27 (Thread 0x7f8763eb7700 (LWP 4707)):
#0 0x00007f8779de8644 in __io_getevents_0_4 () from /lib64/libaio.so.1
#1 0x00000000011852d4 in LinuxAIOHandler::collect (this=0x7f8763eb6dd0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2521
#2 0x00000000011874c4 in LinuxAIOHandler::poll (this=0x7f8763eb6dd0, m1=0x7f8763eb6e88, m2=0x7f8763eb6e80, request=0x7f8763eb6e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2681
#3 0x000000000118787c in os_aio_linux_handler (request=0x7f8763eb6e50, m2=0x7f8763eb6e80, m1=0x7f8763eb6e88, global_segment=2) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2737
#4 os_aio_handler (segment=2, m1=0x7f8763eb6e88, m2=0x7f8763eb6e80, request=0x7f8763eb6e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:6283
#5 0x00000000013256ad in fil_aio_wait (segment=2) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/fil/fil0fil.cc:5858
#6 0x0000000001229960 in io_handler_thread (arg=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/srv/srv0start.cc:319
#7 0x00007f877a1f5ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f8778cadb0d in clone () from /lib64/libc.so.6
Thread 26 (Thread 0x7f87636b6700 (LWP 4708)):
#0 0x00007f8779de8644 in __io_getevents_0_4 () from /lib64/libaio.so.1
#1 0x00000000011852d4 in LinuxAIOHandler::collect (this=0x7f87636b5dd0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2521
#2 0x00000000011874c4 in LinuxAIOHandler::poll (this=0x7f87636b5dd0, m1=0x7f87636b5e88, m2=0x7f87636b5e80, request=0x7f87636b5e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2681
#3 0x000000000118787c in os_aio_linux_handler (request=0x7f87636b5e50, m2=0x7f87636b5e80, m1=0x7f87636b5e88, global_segment=3) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2737
#4 os_aio_handler (segment=3, m1=0x7f87636b5e88, m2=0x7f87636b5e80, request=0x7f87636b5e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:6283
#5 0x00000000013256ad in fil_aio_wait (segment=3) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/fil/fil0fil.cc:5858
#6 0x0000000001229960 in io_handler_thread (arg=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/srv/srv0start.cc:319
#7 0x00007f877a1f5ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f8778cadb0d in clone () from /lib64/libc.so.6
Thread 25 (Thread 0x7f8762eb5700 (LWP 4709)):
#0 0x00007f8779de8644 in __io_getevents_0_4 () from /lib64/libaio.so.1
#1 0x00000000011852d4 in LinuxAIOHandler::collect (this=0x7f8762eb4dd0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2521
#2 0x00000000011874c4 in LinuxAIOHandler::poll (this=0x7f8762eb4dd0, m1=0x7f8762eb4e88, m2=0x7f8762eb4e80, request=0x7f8762eb4e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2681
#3 0x000000000118787c in os_aio_linux_handler (request=0x7f8762eb4e50, m2=0x7f8762eb4e80, m1=0x7f8762eb4e88, global_segment=4) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2737
#4 os_aio_handler (segment=4, m1=0x7f8762eb4e88, m2=0x7f8762eb4e80, request=0x7f8762eb4e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:6283
#5 0x00000000013256ad in fil_aio_wait (segment=4) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/fil/fil0fil.cc:5858
#6 0x0000000001229960 in io_handler_thread (arg=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/srv/srv0start.cc:319
#7 0x00007f877a1f5ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f8778cadb0d in clone () from /lib64/libc.so.6
Thread 24 (Thread 0x7f87626b4700 (LWP 4710)):
#0 0x00007f8779de8644 in __io_getevents_0_4 () from /lib64/libaio.so.1
#1 0x00000000011852d4 in LinuxAIOHandler::collect (this=0x7f87626b3dd0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2521
#2 0x00000000011874c4 in LinuxAIOHandler::poll (this=0x7f87626b3dd0, m1=0x7f87626b3e88, m2=0x7f87626b3e80, request=0x7f87626b3e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2681
#3 0x000000000118787c in os_aio_linux_handler (request=0x7f87626b3e50, m2=0x7f87626b3e80, m1=0x7f87626b3e88, global_segment=5) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2737
#4 os_aio_handler (segment=5, m1=0x7f87626b3e88, m2=0x7f87626b3e80, request=0x7f87626b3e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:6283
#5 0x00000000013256ad in fil_aio_wait (segment=5) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/fil/fil0fil.cc:5858
#6 0x0000000001229960 in io_handler_thread (arg=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/srv/srv0start.cc:319
#7 0x00007f877a1f5ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f8778cadb0d in clone () from /lib64/libc.so.6
Thread 23 (Thread 0x7f8761eb3700 (LWP 4711)):
#0 0x00007f8779de8644 in __io_getevents_0_4 () from /lib64/libaio.so.1
#1 0x00000000011852d4 in LinuxAIOHandler::collect (this=0x7f8761eb2dd0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2521
#2 0x00000000011874c4 in LinuxAIOHandler::poll (this=0x7f8761eb2dd0, m1=0x7f8761eb2e88, m2=0x7f8761eb2e80, request=0x7f8761eb2e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2681
#3 0x000000000118787c in os_aio_linux_handler (request=0x7f8761eb2e50, m2=0x7f8761eb2e80, m1=0x7f8761eb2e88, global_segment=6) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:2737
#4 os_aio_handler (segment=6, m1=0x7f8761eb2e88, m2=0x7f8761eb2e80, request=0x7f8761eb2e50) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/os/os0file.cc:6283
#5 0x00000000013256ad in fil_aio_wait (segment=6) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/fil/fil0fil.cc:5858
#6 0x0000000001229960 in io_handler_thread (arg=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/innobase/srv/srv0start.cc:319
#7 0x00007f877a1f5ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007f8778cadb0d in clone () from /lib64/libc.so.6
Thread 22 (Thread 0x7f87616b2700 (LWP 4712)):
......
pstack 进程ID输出内容比较多，如果想要查看具体某个线程的调用情况，获取到线程ID
pstack 线程ID 可打印一个线程的调用堆栈信息
登录数据库执行select sleep(12000)
通过show processlit 获取到会话ID
mysql> show processlist;
+----+------+------------------+-------+---------+------+------------+---------------------+
| Id | User | Host | db | Command | Time | State | Info |
+----+------+------------------+-------+---------+------+------------+---------------------+
| 9 | root | 172.16.1.2:57402 | ceshi | Sleep | 170 | | NULL |
| 10 | root | 172.16.1.2:57403 | ceshi | Sleep | 170 | | NULL |
| 11 | root | 127.0.0.1:52660 | NULL | Query | 9 | User sleep | select sleep(12000) |
| 12 | root | 127.0.0.1:52662 | NULL | Query | 0 | starting | show processlist |
+----+------+------------------+-------+---------+------+------------+---------------------+
4 rows in set (0.00 sec)
通过 performance_schema.threads 表获取操作系统进程ID
mysql> select THREAD_OS_ID from performance_schema.threads where PROCESSLIST_ID=11;
+--------------+
| THREAD_OS_ID |
+--------------+
| 5007 |
+--------------+
1 row in set (0.00 sec)
pstack 5007 输出线程堆栈信息，在排查问题时需要多执行几次，再查看具体卡在哪个函数上
[root@mysql ~]# pstack 5007
Thread 1 (process 5007):
#0 0x00007f877a1f9de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00000000008bab4c in native_cond_timedwait (abstime=0x7f8754773f90, mutex=0x1fc62c0 <LOCK_item_func_sleep>, cond=0x7f8754774000) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/include/thr_cond.h:136
#2 my_cond_timedwait (abstime=0x7f8754773f90, mp=0x1fc62c0 <LOCK_item_func_sleep>, cond=0x7f8754774000) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/include/thr_cond.h:189
#3 inline_mysql_cond_timedwait (src_line=5269, src_file=0x149bc80 "/var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/item_func.cc", abstime=0x7f8754773f90, mutex=0x1fc62c0 <LOCK_item_func_sleep>, that=0x7f8754774000) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/include/mysql/psi/mysql_thread.h:1236
#4 Interruptible_wait::wait (this=0x7f8754774070, cond=0x7f8754774000, mutex=0x1fc62c0 <LOCK_item_func_sleep>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/item_func.cc:5269
#5 0x00000000008cd418 in Item_func_sleep::val_int (this=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/item_func.cc:6070
#6 0x00000000008568f4 in Item::send (this=0x7f872400f008, protocol=0x7f8724008a20, buffer=0x7f8754774430) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/item.cc:7575
#7 0x0000000000cf3013 in THD::send_result_set_row (this=0x7f87240079c0, row_items=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/sql_class.cc:4737
#8 0x0000000000cf30fb in Query_result_send::send_data (this=0x7f872400f1a0, items=...) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/sql_class.cc:2740
#9 0x0000000000d0a5f9 in JOIN::exec (this=0x7f872400f2b8) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/sql_executor.cc:163
#10 0x0000000000d759a0 in handle_query (thd=0x7f87240079c0, lex=0x7f8724009b38, result=0x7f872400f1a0, added_options=1, removed_options=0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/sql_select.cc:191
#11 0x0000000000d36633 in execute_sqlcom_select (thd=0x7f87240079c0, all_tables=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/sql_parse.cc:5156
#12 0x0000000000d3a10a in mysql_execute_command (thd=0x7f87240079c0, first_level=true) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/sql_parse.cc:2827
#13 0x0000000000d3bc8d in mysql_parse (thd=0x7f87240079c0, parser_state=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/sql_parse.cc:5589
#14 0x0000000000d3d4a2 in dispatch_command (thd=0x7f87240079c0, com_data=0x7f8754775da0, command=COM_QUERY) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/sql_parse.cc:1491
#15 0x0000000000d3e074 in do_command (thd=0x7f87240079c0) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/sql_parse.cc:1032
#16 0x0000000000e10bfc in handle_connection (arg=<optimized out>) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/sql/conn_handler/connection_handler_per_thread.cc:313
#17 0x00000000013e1e64 in pfs_spawn_thread (arg=0x2e36390) at /var/lib/pb2/sb_1-2574574-1616743737.84/mysql-5.7.34/storage/perfschema/pfs.cc:2197
#18 0x00007f877a1f5ea5 in start_thread () from /lib64/libpthread.so.0
#19 0x00007f8778cadb0d in clone () from /lib64/libc.so.6
