hadoop3 无法启动 查找内存消耗原因
[root@hadoop3 hadoop]# xlo
Starting namenodes on [hadoop3]
上一次登录：三 12月 27 15:07:11 CST 2017pts/24 上
hadoop3: ERROR: Cannot set priority of namenode process 9025
Starting datanodes
上一次登录：三 12月 27 15:16:59 CST 2017pts/24 上
hadoop3: ERROR: Cannot set priority of datanode process 9236
hadoop1: ERROR: Cannot set priority of datanode process 22612
Starting secondary namenodes [hadoop3]
上一次登录：三 12月 27 15:17:01 CST 2017pts/24 上
Starting resourcemanager
上一次登录：三 12月 27 15:17:04 CST 2017pts/24 上
Starting nodemanagers
上一次登录：三 12月 27 15:17:08 CST 2017pts/24 上
[root@hadoop3 hadoop]#
[root@hadoop3 hadoop]# cat ~/.bashrc
# .bashrc
# User specific aliases and functions
alias rm='rm -i'
alias cp='cp -i'
alias mv='mv -i'
alias xlpy='ps -aux|grep python|grep -v grep'
alias xlfg='free -g'
alias xlfm='free -m'
alias xlce='crontab -e'
alias xlcl='crontab -l'
alias xlrp='python '
alias xlhdfsf='/usr/local/hadoop/bin/hdfs namenode -format'
alias xlo='/usr/local/hadoop/sbin/start-all.sh'
alias xlc='/usr/local/hadoop/sbin/stop-all.sh'
# Source global definitions
if [ -f /etc/bashrc ]; then
. /etc/bashrc
fi
[root@hadoop3 hadoop]#
[root@hadoop3 ~]# xlfg
total used free shared buff/cache available
Mem: 15 2 0 0 12 12
Swap: 7 0 7
[root@hadoop3 ~]#
[root@hadoop3 ~]# ps -eo rss,pmem,vsize,args | sort -k 1 -r -n
484708 2.9 6007736 /usr/local/jdk/bin/java -Dproc_namenode -Djava.net.preferIPv4Stack=true -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,RFAS -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-namenode-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-namenode-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml org.apache.hadoop.hdfs.server.namenode.NameNode
383044 2.3 6177364 /usr/local/jdk/bin/java -Dproc_resourcemanager -Djava.net.preferIPv4Stack=true -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-resourcemanager-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-resourcemanager-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
381180 2.3 6043360 /usr/local/jdk/bin/java -Dproc_nodemanager -Djava.net.preferIPv4Stack=true -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-nodemanager-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-nodemanager-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.yarn.server.nodemanager.NodeManager
375868 2.3 5962452 /usr/local/jdk/bin/java -Dproc_secondarynamenode -Djava.net.preferIPv4Stack=true -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,RFAS -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-secondarynamenode-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-secondarynamenode-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode
335036 2.0 5998380 /usr/local/jdk/bin/java -Dproc_datanode -Djava.net.preferIPv4Stack=true -Dhadoop.security.logger=ERROR,RFAS -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-datanode-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-datanode-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml org.apache.hadoop.hdfs.server.datanode.DataNode
68436 0.4 110564 /usr/lib/systemd/systemd-journald
59936 0.3 3077632 /usr/local/phantomjs/bin/phantomjs --cookies-file=/tmp/tmprwyetcbf --webdriver=49908
40744 0.2 552392 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
39820 0.2 352972 /usr/sbin/rsyslogd -n
38052 0.2 478916 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
36612 0.2 478660 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
36112 0.2 2838472 /usr/local/phantomjs/bin/phantomjs --cookies-file=/tmp/tmp8_stcm62 --webdriver=33070
35688 0.2 478660 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
34256 0.2 552392 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
33968 0.2 404672 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
33572 0.2 404672 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
33220 0.2 2839092 /usr/local/phantomjs/bin/phantomjs --cookies-file=/tmp/tmpq9k5ga_d --webdriver=37099
12152 0.0 529480 /usr/lib/polkit-1/polkitd --no-debug
8132 0.0 439592 /usr/sbin/NetworkManager --no-daemon
6424 0.0 144460 sshd: root@notty
5724 0.0 143932 sshd: root@pts/28,pts/29,pts/30,pts/31,pts/32
5712 0.0 143932 sshd: root@pts/26,pts/21,pts/22
5708 0.0 144064 sshd: root@pts/24,pts/6,pts/7
5704 0.0 144092 sshd: root@notty
5700 0.0 143932 sshd: root@pts/1
5684 0.0 144068 sshd: root@pts/0
5676 0.0 143932 sshd: root@pts/2
5668 0.0 143932 sshd: root@pts/8
5668 0.0 143932 sshd: root@pts/27
5668 0.0 143932 sshd: root@pts/23
4024 0.0 49748 /usr/lib/systemd/systemd --switched-root --system --deserialize 21
3800 0.0 83840 /usr/sbin/sendmail -FCronDaemon -i -odi -oem -oi -t -f root
3792 0.0 83840 /usr/sbin/sendmail -s zh_cn_xiaomi_buqiang_jiankong g1e2t3@qq.com
3664 0.0 80936 /usr/sbin/sshd -D
2700 0.0 51404 /usr/sbin/wpa_supplicant -u -f /var/log/wpa_supplicant.log -c /etc/wpa_supplicant/wpa_supplicant.conf -u -f /var/log/wpa_supplicant.log -P /var/run/wpa_supplicant.pid
2532 0.0 176076 /usr/sbin/CROND -n
2488 0.0 176076 /usr/sbin/CROND -n
2488 0.0 176076 /usr/sbin/CROND -n
2488 0.0 176076 /usr/sbin/CROND -n
2488 0.0 176076 /usr/sbin/CROND -n
2488 0.0 176076 /usr/sbin/CROND -n
2488 0.0 176076 /usr/sbin/CROND -n
2488 0.0 176076 /usr/sbin/CROND -n
2488 0.0 176076 /usr/sbin/CROND -n
2356 0.0 43704 /usr/lib/systemd/systemd-udevd
2264 0.0 115636 -bash
2252 0.0 146328 top
2224 0.0 115528 -bash
2212 0.0 22008 nload
2204 0.0 146316 top
2204 0.0 115608 -bash
2192 0.0 115652 -bash
2188 0.0 115608 -bash
2184 0.0 115528 -bash
2184 0.0 115480 -bash
2144 0.0 115528 -bash
2136 0.0 53248 /usr/libexec/openssh/sftp-server
2136 0.0 115516 -bash
2116 0.0 115480 -bash
2116 0.0 115480 -bash
2112 0.0 115480 -bash
2112 0.0 115400 -bash
2112 0.0 115400 -bash
2108 0.0 115400 -bash
2108 0.0 115400 -bash
2104 0.0 115400 -bash
1792 0.0 24724 /bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation
1724 0.0 24208 /usr/lib/systemd/systemd-logind
1632 0.0 124192 /usr/sbin/crond -n
1584 0.0 49020 /sbin/auditd -n
1400 0.0 192564 /usr/sbin/lvmetad -f
1396 0.0 113132 /bin/sh -c cd /home/data/xl_project;cat r1 > rtmp;date >r1; free -g >>r1;cat rtmp>>r1;/usr/sbin/sendmail -s 'zh_cn_xiaomi_buqiang_jiankong' g1e2t3@qq.com < r1;
1276 0.0 137392 ps -eo rss,pmem,vsize,args
1228 0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
1228 0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
1228 0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
1224 0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
1224 0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
1224 0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
1220 0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
1192 0.0 21480 /usr/sbin/irqbalance --foreground
920 0.0 126696 sort -k 1 -r -n
824 0.0 110048 /sbin/agetty --noclear tty1 linux
RSS %MEM VSZ COMMAND
0 0.0 0 [xfs_mru_cache]
0 0.0 0 [xfs-data/sda1]
0 0.0 0 [xfs-data/dm-2]
0 0.0 0 [xfs-data/dm-0]
0 0.0 0 [xfs-conv/sda1]
0 0.0 0 [xfs-conv/dm-2]
0 0.0 0 [xfs-conv/dm-0]
0 0.0 0 [xfs-cil/sda1]
0 0.0 0 [xfs-cil/dm-2]
0 0.0 0 [xfs-cil/dm-0]
0 0.0 0 [xfs-buf/sda1]
0 0.0 0 [xfs-buf/dm-2]
0 0.0 0 [xfs-buf/dm-0]
0 0.0 0 [xfsalloc]
0 0.0 0 [xfsaild/sda1]
0 0.0 0 [xfsaild/dm-2]
0 0.0 0 [xfsaild/dm-0]
0 0.0 0 [writeback]
0 0.0 0 [watchdog/7]
0 0.0 0 [watchdog/6]
0 0.0 0 [watchdog/5]
0 0.0 0 [watchdog/4]
0 0.0 0 [watchdog/3]
0 0.0 0 [watchdog/2]
0 0.0 0 [watchdog/1]
0 0.0 0 [watchdog/0]
0 0.0 0 [ttm_swap]
0 0.0 0 [scsi_tmf_5]
0 0.0 0 [scsi_tmf_4]
0 0.0 0 [scsi_tmf_3]
0 0.0 0 [scsi_tmf_2]
0 0.0 0 [scsi_tmf_1]
0 0.0 0 [scsi_tmf_0]
0 0.0 0 [scsi_eh_5]
0 0.0 0 [scsi_eh_4]
0 0.0 0 [scsi_eh_3]
0 0.0 0 [scsi_eh_2]
0 0.0 0 [scsi_eh_1]
0 0.0 0 [scsi_eh_0]
0 0.0 0 [rcu_sched]
0 0.0 0 [rcuos/9]
0 0.0 0 [rcuos/8]
0 0.0 0 [rcuos/7]
0 0.0 0 [rcuos/6]
0 0.0 0 [rcuos/5]
0 0.0 0 [rcuos/4]
0 0.0 0 [rcuos/3]
0 0.0 0 [rcuos/2]
0 0.0 0 [rcuos/15]
0 0.0 0 [rcuos/14]
0 0.0 0 [rcuos/13]
0 0.0 0 [rcuos/12]
0 0.0 0 [rcuos/11]
0 0.0 0 [rcuos/10]
0 0.0 0 [rcuos/1]
0 0.0 0 [rcuos/0]
0 0.0 0 [rcuob/9]
0 0.0 0 [rcuob/8]
0 0.0 0 [rcuob/7]
0 0.0 0 [rcuob/6]
0 0.0 0 [rcuob/5]
0 0.0 0 [rcuob/4]
0 0.0 0 [rcuob/3]
0 0.0 0 [rcuob/2]
0 0.0 0 [rcuob/15]
0 0.0 0 [rcuob/14]
0 0.0 0 [rcuob/13]
0 0.0 0 [rcuob/12]
0 0.0 0 [rcuob/11]
0 0.0 0 [rcuob/10]
0 0.0 0 [rcuob/1]
0 0.0 0 [rcuob/0]
0 0.0 0 [rcu_bh]
0 0.0 0 [phantomjs] <defunct>
0 0.0 0 [phantomjs] <defunct>
0 0.0 0 [phantomjs] <defunct>
0 0.0 0 [phantomjs] <defunct>
0 0.0 0 [perf]
0 0.0 0 [netns]
0 0.0 0 [migration/7]
0 0.0 0 [migration/6]
0 0.0 0 [migration/5]
0 0.0 0 [migration/4]
0 0.0 0 [migration/3]
0 0.0 0 [migration/2]
0 0.0 0 [migration/1]
0 0.0 0 [migration/0]
0 0.0 0 [md]
0 0.0 0 [kworker/u32:2]
0 0.0 0 [kworker/u32:1]
0 0.0 0 [kworker/7:2]
0 0.0 0 [kworker/7:1H]
0 0.0 0 [kworker/7:0H]
0 0.0 0 [kworker/7:0]
0 0.0 0 [kworker/6:1H]
0 0.0 0 [kworker/6:1]
0 0.0 0 [kworker/6:0H]
0 0.0 0 [kworker/6:0]
0 0.0 0 [kworker/5:1H]
0 0.0 0 [kworker/5:1]
0 0.0 0 [kworker/5:0H]
0 0.0 0 [kworker/5:0]
0 0.0 0 [kworker/4:2H]
0 0.0 0 [kworker/4:2]
0 0.0 0 [kworker/4:1]
0 0.0 0 [kworker/4:0H]
0 0.0 0 [kworker/3:2H]
0 0.0 0 [kworker/3:1H]
0 0.0 0 [kworker/3:1]
0 0.0 0 [kworker/3:0]
0 0.0 0 [kworker/2:2H]
0 0.0 0 [kworker/2:1H]
0 0.0 0 [kworker/2:1]
0 0.0 0 [kworker/2:0]
0 0.0 0 [kworker/1:2]
0 0.0 0 [kworker/1:1H]
0 0.0 0 [kworker/1:0H]
0 0.0 0 [kworker/1:0]
0 0.0 0 [kworker/0:2]
0 0.0 0 [kworker/0:1H]
0 0.0 0 [kworker/0:0H]
0 0.0 0 [kworker/0:0]
0 0.0 0 [kthrotld]
0 0.0 0 [kthreadd]
0 0.0 0 [kswapd0]
0 0.0 0 [ksoftirqd/7]
0 0.0 0 [ksoftirqd/6]
0 0.0 0 [ksoftirqd/5]
0 0.0 0 [ksoftirqd/4]
0 0.0 0 [ksoftirqd/3]
0 0.0 0 [ksoftirqd/2]
0 0.0 0 [ksoftirqd/1]
0 0.0 0 [ksoftirqd/0]
0 0.0 0 [ksmd]
0 0.0 0 [kpsmoused]
0 0.0 0 [kmpath_rdacd]
0 0.0 0 [kipmi0]
0 0.0 0 [kintegrityd]
0 0.0 0 [khungtaskd]
0 0.0 0 [khugepaged]
0 0.0 0 [khelper]
0 0.0 0 [kdmflush]
0 0.0 0 [kdmflush]
0 0.0 0 [kdmflush]
0 0.0 0 [kdevtmpfs]
0 0.0 0 [kblockd]
0 0.0 0 [kauditd]
0 0.0 0 [ipv6_addrconf]
0 0.0 0 [fsnotify_mark]
0 0.0 0 [events_power_ef]
0 0.0 0 [edac-poller]
0 0.0 0 [deferwq]
0 0.0 0 [crypto]
0 0.0 0 [bioset]
0 0.0 0 [bioset]
0 0.0 0 [bioset]
0 0.0 0 [bioset]
0 0.0 0 [ata_sff]
[root@hadoop3 ~]#
http://www.cnblogs.com/JemBai/archive/2011/06/21/2086184.html
ps - 按进程消耗内存多少排序
ps -eo rss,pmem,pcpu,vsize,args | sort -k 1 -r -n | less
解析一下：
ps 都知道，是linux,unix显示进程信息的, -e 是显示所有进程, -o是定制显示信息的格式
rss: resident set size, 表示进程占用RAM(内存)的大小，单位是KB
pmem: %M, 占用内存的百分比
pcpu：%C，占用cpu的百分比
vsize: 表示进程占用的虚拟内存的大小，KB
args：进程名（command）
sort命令对ps结果进行排序
-k 1 :按第一个参数 rss进行排序
-r：逆序
-n：numeric，按数字来排序
1：在命令行提示符执行top命令
2：输入大写P，则结果按CPU占用降序排序。输入大写M，结果按内存占用降序排序。（注：大写P可以在capslock状态输入p，或者按Shift+p）
ps -aux | grep phantomjs | cut -c 10-15 | xargs kill -9
手动杀死进程
free内存变为2G，namenode可以启动，但datanode仍然不可以启动，此处说明“腾出内存”方向正确，进一步查内存消耗：已经不能通过手动杀死进程来实现腾出内存，故因为缓存占用大量内存考虑释放缓存。
