数仓数据
修改两个文件
安装目录：
/mnt/home/work/hadoop-3.3.1/bin
[root@test11.bj.sm etc]# cd hadoop/ [root@test11.bj.sm hadoop]# ls capacity-scheduler.xml hadoop-metrics2.properties httpfs-log4j.properties log4j.properties ssl-client.xml.example yarnservice-log4j.properties configuration.xsl hadoop-policy.xml httpfs-site.xml mapred-env.cmd ssl-server.xml.example yarn-site.xml container-executor.cfg hadoop-user-functions.sh.example kms-acls.xml mapred-env.sh user_ec_policies.xml.template core-site.xml hdfs-rbf-site.xml kms-env.sh mapred-queues.xml.template workers hadoop-env.cmd hdfs-site.xml kms-log4j.properties mapred-site.xml yarn-env.cmd hadoop-env.sh httpfs-env.sh kms-site.xml shellprofile.d yarn-env.sh [root@test11.bj.sm hadoop]# mv hdfs-site.xml hdfs-site.xmlbak [root@test11.bj.sm hadoop]# mv core-site.xml core-site.xmlbak [root@test11.bj.sm hadoop]# rz -E rz waiting to receive. [root@test11.bj.sm hadoop]# rz -E
修改host
root@test11.bj.sm hadoop]# hadoop fs -ls / ^C[root@test11.bj.sm hadoop]# hadoop fs -ls /user/data 2021-08-17 21:03:54,780 INFO retry.RetryInvocationHandler: org.apache.hadoop.net.ConnectTimeoutException: Call From java.net.UnknownHostException: test11.bj.sm: test11.bj.sm: 未知的名称或服务 to master2.bjtx.ishumei.com:8020 failed on socket timeout exception: org.apache.hadoop.net.ConnectTimeoutException: 20000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=master2.bjtx.ishumei.com/140.143.158.40:8020]; For more details see: http://wiki.apache.org/hadoop/SocketTimeout, while invoking ClientNamenodeProtocolTranslatorPB.getFileInfo over master2.bjtx.ishumei.com/140.143.158.40:8020 after 1 failover attempts. Trying to failover after sleeping for 1462ms. ^Cls: Interrupted [root@test11.bj.sm hadoop]# vim /etc/hosts [root@test11.bj.sm hadoop]# ping 10.163.45.80 PING 10.163.45.80 (10.163.45.80) 56(84) bytes of data. 64 bytes from 10.163.45.80: icmp_seq=1 ttl=64 time=3.93 ms 64 bytes from 10.163.45.80: icmp_seq=2 ttl=64 time=3.88 ms 64 bytes from 10.163.45.80: icmp_seq=3 ttl=64 time=3.88 ms ^C --- 10.163.45.80 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2023ms rtt min/avg/max/mdev = 3.881/3.900/3.939/0.077 ms [root@test11.bj.sm hadoop]# vim /etc/hosts [root@test11.bj.sm hadoop]# hadoop fs -ls /user/data Found 157 items drwxr-xr-x - xudeqing supergroup 0 2021-06-11 13:03 /user/data/ExtractToken drwxrwxrwx - azkaban supergroup 0 2021-08-17 12:41 /user/data/OfflineClipsMonitor drwxrwxrwx - duanjiaojiao supergroup 0 2021-07-24 18:01 /user/data/StrategyInspectionTool drwxrwxrwx - yangguang supergroup 0 2021-07-25 10:21 /user/data/advanced_features drwxrwxrwx - liangkun supergroup 0 2020-03-16 17:57 /user/data/archives drwxrwxr-x - shaoli asrtext 0 2021-05-09 22:09 /user/data/asr_text drwxrwxrwx - hdfs supergroup 0 2021-04-29 14:25 /user/data/asr_train_text
文件操作
[root@test11.bj.sm hadoop]# hadoop fs -ls /user/tmp/hjc/dt=20210817/bj_text
Found 2 items
-rw-r--r-- 2 huangjincheng supergroup 0 2021-08-17 21:03 /user/tmp/hjc/dt=20210817/bj_text/_SUCCESS
-rw-r--r-- 2 huangjincheng supergroup 878018961 2021-08-17 21:03 /user/tmp/hjc/dt=20210817/bj_text/part-00000-540292af-76df-4bc0-92e7-038465869363.txt
[root@test11.bj.sm hadoop]# hadoop fs -du -h /user/tmp/hjc/dt=20210817/bj_text
0 0 /user/tmp/hjc/dt=20210817/bj_text/_SUCCESS
837.3 M 1.6 G /user/tmp/hjc/dt=20210817/bj_text/part-00000-540292af-76df-4bc0-92e7-038465869363.txt
[root@test11.bj.sm hadoop]# cd
[root@test11.bj.sm ~]# ls
1 bash_new Conf download financeTask.log packages pkg_zabbix tmp zookeeper-3.4.9 zookeeper-3.4.9.tar.gz zook.sh zook.txt
[root@test11.bj.sm ~]# cd /home/
[root@test11.bj.sm hmee]# ls
centos compile dongming huangyunpeng log nginx putpkg weipingshun work yangxueyi zhangjuntao zhangyaping zuohaitao
[root@test11.bj.sm hmee]# cd huangyunpeng/
[root@test11.bj.sm huangyunpeng]# hadoop fs -get /user/tmp/hjc/dt=20210817/bj_text
[root@test11.bj.sm huangyunpeng]# cd bj_text/
[root@test11.bj.sm bj_text]# ls
part-00000-540292af-76df-4bc0-92e7-038465869363.txt _SUCCESS
[root@test11.bj.sm bj_text]# du -h -s *
839M part-00000-540292af-76df-4bc0-92e7-038465869363.txt
0 _SUCCESS
[root@test11.bj.sm bj_text]# cat part-00000-540292af-76df-4bc0-92e7-038465869363.txt | head -n 2
/v2/saas/anti_fraud/text {"data":{"role":"USER","tokenId":"51C3245477349583010038124B","nickname":"hm心心","channel":"FX_SEARCH","text":"","room":"0"},"accessKey":"C5hq0GNPeaa2fBDoBY04","type":"ZHIBO"}
/v2/saas/anti_fraud/text {"accessKey":"binqUxtuzA8MZXwBcPpH","appId":"default","data":{"channel":"SIXINSHENHE","ip":"182.88.141.8","nickname":"","receiveTokenId":"747984577","text":"睡觉啦","tokenId":"744103320"},"type":"ZHIBO"}
