docker cp 提示Stale file handle
docker 替换 jar 包提示
[root@xxxx merged]# mv sso-server-0.0.1.jar app.jar
mv: overwrite ‘app.jar’? y
mv: cannot move ‘sso-server-0.0.1.jar’ to ‘app.jar’: Stale file handle
排查
用 strace 跟踪，没啥有用的信息。
[root@xxxx merged]# strace mv sso-server-0.0.1.jar app.jar
execve("/usr/bin/mv", ["mv", "sso-server-0.0.1.jar", "app.jar"], [/* 29 vars */]) = 0
brk(NULL) = 0x1688000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd6b349a000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=27070, ...}) = 0
mmap(NULL, 27070, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fd6b3493000
close(3) = 0
open("/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320i\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=155784, ...}) = 0
mmap(NULL, 2255184, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fd6b3053000
mprotect(0x7fd6b3077000, 2093056, PROT_NONE) = 0
mmap(0x7fd6b3276000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7fd6b3276000
mmap(0x7fd6b3278000, 6480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fd6b3278000
close(3) = 0
open("/lib64/libacl.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\37\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=37056, ...}) = 0
mmap(NULL, 2130560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fd6b2e4a000
mprotect(0x7fd6b2e51000, 2097152, PROT_NONE) = 0
mmap(0x7fd6b3051000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7fd6b3051000
close(3) = 0
open("/lib64/libattr.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\23\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19896, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd6b3492000
mmap(NULL, 2113904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fd6b2c45000
mprotect(0x7fd6b2c49000, 2093056, PROT_NONE) = 0
mmap(0x7fd6b2e48000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7fd6b2e48000
close(3) = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20&\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2156160, ...}) = 0
mmap(NULL, 3985888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fd6b2877000
mprotect(0x7fd6b2a3a000, 2097152, PROT_NONE) = 0
mmap(0x7fd6b2c3a000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c3000) = 0x7fd6b2c3a000
mmap(0x7fd6b2c40000, 16864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fd6b2c40000
close(3) = 0
open("/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\25\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=402384, ...}) = 0
mmap(NULL, 2494984, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fd6b2615000
mprotect(0x7fd6b2675000, 2097152, PROT_NONE) = 0
mmap(0x7fd6b2875000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x60000) = 0x7fd6b2875000
close(3) = 0
open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19288, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd6b3491000
mmap(NULL, 2109712, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fd6b2411000
mprotect(0x7fd6b2413000, 2097152, PROT_NONE) = 0
mmap(0x7fd6b2613000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fd6b2613000
close(3) = 0
open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@m\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=142232, ...}) = 0
mmap(NULL, 2208904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fd6b21f5000
mprotect(0x7fd6b220c000, 2093056, PROT_NONE) = 0
mmap(0x7fd6b240b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7fd6b240b000
mmap(0x7fd6b240d000, 13448, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fd6b240d000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd6b3490000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd6b348e000
arch_prctl(ARCH_SET_FS, 0x7fd6b348e840) = 0
mprotect(0x7fd6b2c3a000, 16384, PROT_READ) = 0
mprotect(0x7fd6b240b000, 4096, PROT_READ) = 0
mprotect(0x7fd6b2613000, 4096, PROT_READ) = 0
mprotect(0x7fd6b2875000, 4096, PROT_READ) = 0
mprotect(0x7fd6b2e48000, 4096, PROT_READ) = 0
mprotect(0x7fd6b3051000, 4096, PROT_READ) = 0
mprotect(0x7fd6b3276000, 4096, PROT_READ) = 0
mprotect(0x61d000, 4096, PROT_READ) = 0
mprotect(0x7fd6b349b000, 4096, PROT_READ) = 0
munmap(0x7fd6b3493000, 27070) = 0
set_tid_address(0x7fd6b348eb10) = 26657
set_robust_list(0x7fd6b348eb20, 24) = 0
rt_sigaction(SIGRTMIN, {0x7fd6b21fb820, [], SA_RESTORER|SA_SIGINFO, 0x7fd6b22045f0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7fd6b21fb8b0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7fd6b22045f0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0x7ffdbc6102e0) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7ffdbc6102e0) = -1 ENOENT (No such file or directory)
brk(NULL) = 0x1688000
brk(0x16a9000) = 0x16a9000
open("/proc/filesystems", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd6b3499000
read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tr"..., 1024) = 309
stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7ffdbc60feb0) = -1 ENOENT (No such file or directory)
read(3, "", 1024) = 0
close(3) = 0
munmap(0x7fd6b3499000, 4096) = 0
access("/etc/selinux/config", F_OK) = 0
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=106075056, ...}) = 0
mmap(NULL, 106075056, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fd6abccb000
close(3) = 0
geteuid() = 0
ioctl(0, TCGETS, {B38400 opost isig icanon echo ...}) = 0
stat("app.jar", {st_mode=S_IFREG|0644, st_size=81917312, ...}) = 0
lstat("sso-server-0.0.1.jar", {st_mode=S_IFREG|0644, st_size=81919007, ...}) = 0
lstat("app.jar", {st_mode=S_IFREG|0644, st_size=81917312, ...}) = 0
geteuid() = 0
renameat2(AT_FDCWD, "sso-server-0.0.1.jar", AT_FDCWD, "app.jar", 0) = -1 ESTALE (Stale file handle)
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2502, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd6b3499000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2502
read(3, "", 4096) = 0
close(3) = 0
munmap(0x7fd6b3499000, 4096) = 0
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib64/charset.alias", O_RDONLY|O_NOFOLLOW) = -1 ENOENT (No such file or directory)
write(2, "mv: ", 4mv: ) = 4
write(2, "cannot move \342\200\230sso-server-0.0.1."..., 55cannot move ‘sso-server-0.0.1.jar’ to ‘app.jar’) = 55
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
write(2, ": Stale file handle", 19: Stale file handle) = 19
write(2, "\n", 1
) = 1
lseek(0, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
close(0) = 0
close(1) = 0
close(2) = 0
exit_group(1) = ?
+++ exited with 1 +++
fuser查看被啥占用了，无效信息，这俩进程怎么kill。。。
[root@xxxx merged]# fuser -m sso-server-0.0.1.jar
/var/lib/docker/overlay2/a90ef58db2a05a8003c6bf8098841c5d58ffaf81177886353d0096fdb5fea809/merged/sso-server-0.0.1.jar: 26198c 27091rce
[root@xxxx merged]# fuser -m app.jar
/var/lib/docker/overlay2/a90ef58db2a05a8003c6bf8098841c5d58ffaf81177886353d0096fdb5fea809/merged/app.jar: 26198c 27091rce
[root@xxxx merged]# fuser -m -v app.jar
USER PID ACCESS COMMAND
/var/lib/docker/overlay2/a90ef58db2a05a8003c6bf8098841c5d58ffaf81177886353d0096fdb5fea809/merged/app.jar:
root kernel mount /var/lib/docker/overlay2/a90ef58db2a05a8003c6bf8098841c5d58ffaf81177886353d0096fdb5fea809/merged
root 26198 ..c.. bash
root 27091 Frce. java
[root@xxxx merged]# ps -ef|grep 26198
root 26198 26193 0 11:25 pts/3 00:00:00 -bash
root 27239 26198 0 11:43 pts/3 00:00:00 ps -ef
root 27240 26198 0 11:43 pts/3 00:00:00 grep --color=auto 26198
[root@xxxx merged]# ps -ef|grep 27091
root 27091 27070 99 11:42 ? 00:01:14 java -server -Xmx2g -Xms2g -Xmn256m -XX:PermSize=256m -Xss256k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -jar /app.jar
root 27244 26198 0 11:43 pts/3 00:00:00 grep --color=auto 27091
github上看到这个，https://github.com/coreos/bugs/issues/2152 ，下面有人说升级内核解决问题。开动
解决方案
升级内核网上的教程都有点问题了，会提示Error getting repository data for elrepo-kernel, repository not found，最后去官网看下。http://elrepo.org/tiki/HomePage，正确步骤如下：
导入 public key
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
安装ELRepo
yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm
同时切换到清华大学源 https://mirrors.tuna.tsinghua.edu.cn/help/elrepo/
编辑 /etc/yum.repos.d/elrepo.repo 文件，在 mirrorlist= 开头的行前面加 # 注释掉；并将 elrepo.org/linux 替换为 mirrors.tuna.tsinghua.edu.cn/elrepo。
最后，更新软件包缓存
sudo yum makecache
安装主线版内核
yum --enablerepo=elrepo-kernel install kernel-ml -y
发现还是很慢，于是到清华大学源搜索包名下载安装了。
wget https://mirrors.tuna.tsinghua.edu.cn/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-5.10.2-1.el7.elrepo.x86_64.rpm
yum install kernel-ml-5.10.2-1.el7.elrepo.x86_64.rpm -y
更新grub，不管GRUB_TIMEOUT=0原来是啥，改为0
[root@xxxx ~]# cat /etc/default/grub
GRUB_TIMEOUT=0
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto rhgb quiet net.ifnames=0 vga=792 console=tty0 console=ttyS0,115200n8"
GRUB_DISABLE_RECOVERY="true"
重新生成
[root@xxxx ~]# grub2-mkconfig -o /boot/grub2/grub.cfg
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-5.10.2-1.el7.elrepo.x86_64
Found initrd image: /boot/initramfs-5.10.2-1.el7.elrepo.x86_64.img
Found linux image: /boot/vmlinuz-3.10.0-1062.18.1.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-1062.18.1.el7.x86_64.img
Found linux image: /boot/vmlinuz-3.10.0-693.11.1.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-693.11.1.el7.x86_64.img
Found linux image: /boot/vmlinuz-3.10.0-693.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-693.el7.x86_64.img
Found linux image: /boot/vmlinuz-0-rescue-201803271148b62cae480a0002020050
Found initrd image: /boot/initramfs-0-rescue-201803271148b62cae480a0002020050.img
done
重启服务器
验证
找到目录
cd $(docker inspect --format='{{json .GraphDriver.Data.MergedDir}}' sso|sed 's/\"//g')
替换
[root@xxxx merged]# mv sso-server-0.0.1.jar app.jar
mv: overwrite ‘app.jar’? y
[root@xxxx merged]#
搞定！
