hadoop使用

• 启动hadoop，浏览器访问：http://192.168.128.78:9870/
  

• 显示目录结构

[root@master ~]# hadoop dfs -ls -R /
WARNING: Use of this script to execute dfs is deprecated.
WARNING: Attempting to execute replacement "hdfs dfs" instead.
drwxr-xr-x   - root supergroup          0 2027-11-30 19:23 /hadoop
drwx-wx-wx   - root supergroup          0 2027-12-01 09:26 /tmp
drwx-wx-wx   - root supergroup          0 2027-12-01 09:26 /tmp/hive
drwx------   - root supergroup          0 2027-12-03 19:52 /tmp/hive/root
drwx------   - root supergroup          0 2027-12-03 19:51 /tmp/hive/root/d3e00e50-57a5-4146-8441-5615a3bdec7b
drwx------   - root supergroup          0 2027-12-03 19:51 /tmp/hive/root/d3e00e50-57a5-4146-8441-5615a3bdec7b/_tmp_space.db
drwx------   - root supergroup          0 2027-12-03 19:52 /tmp/hive/root/dcd27fc9-0e38-4f20-8fdf-52c548e9494b
drwx------   - root supergroup          0 2027-12-03 19:52 /tmp/hive/root/dcd27fc9-0e38-4f20-8fdf-52c548e9494b/_tmp_space.db
drwxr-xr-x   - root supergroup          0 2027-12-01 09:27 /user
drwxr-xr-x   - root supergroup          0 2027-12-01 09:27 /user/hive
drwxr-xr-x   - root supergroup          0 2027-12-01 09:27 /user/hive/warehouse
drwxr-xr-x   - root supergroup          0 2027-12-03 20:09 /user/hive/warehouse/bigdata.db
drwxr-xr-x   - root supergroup          0 2027-12-03 20:09 /user/hive/warehouse/bigdata.db/province_sale
 
[root@master ~]# hdfs dfs -ls -R /
drwxr-xr-x   - root supergroup          0 2027-11-30 19:23 /hadoop
drwx-wx-wx   - root supergroup          0 2027-12-01 09:26 /tmp
drwx-wx-wx   - root supergroup          0 2027-12-01 09:26 /tmp/hive
drwx------   - root supergroup          0 2027-12-03 19:52 /tmp/hive/root
drwx------   - root supergroup          0 2027-12-03 19:51 /tmp/hive/root/d3e00e50-57a5-4146-8441-5615a3bdec7b
drwx------   - root supergroup          0 2027-12-03 19:51 /tmp/hive/root/d3e00e50-57a5-4146-8441-5615a3bdec7b/_tmp_space.db
drwx------   - root supergroup          0 2027-12-03 19:52 /tmp/hive/root/dcd27fc9-0e38-4f20-8fdf-52c548e9494b
drwx------   - root supergroup          0 2027-12-03 19:52 /tmp/hive/root/dcd27fc9-0e38-4f20-8fdf-52c548e9494b/_tmp_space.db
drwxr-xr-x   - root supergroup          0 2027-12-01 09:27 /user
drwxr-xr-x   - root supergroup          0 2027-12-01 09:27 /user/hive
drwxr-xr-x   - root supergroup          0 2027-12-01 09:27 /user/hive/warehouse
drwxr-xr-x   - root supergroup          0 2027-12-03 20:09 /user/hive/warehouse/bigdata.db
drwxr-xr-x   - root supergroup          0 2027-12-03 20:09 /user/hive/warehouse/bigdata.db/province_sale

• 创建目录

hdfs dfs -mkdir /data

• 复制本地文件到hadoop

hdfs dfs -put [本地文件路径] [hadoop目录]

查看详情

[root@hadoop01 software]# hdfs dfs -put /home/software/a.txt /data
[root@hadoop01 software]# hdfs dfs -ls -R /
drwxr-xr-x   - root supergroup          0 2023-12-05 19:47 /data
-rw-r--r--   1 root supergroup          0 2023-12-05 19:47 /data/a.txt

• 复制本地目录到hadoop

hdfs dfs -put [本地目录路径] [hadoop目录]

查看详情

[root@hadoop01 test]# pwd
/home/software/test
[root@hadoop01 test]# ls
a.txt
[root@hadoop01 test]# hdfs dfs -put /home/software/test/ /data
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x   - root supergroup          0 2023-12-05 19:52 /data
-rw-r--r--   1 root supergroup          0 2023-12-05 19:47 /data/a.txt
drwxr-xr-x   - root supergroup          0 2023-12-05 19:52 /data/test
-rw-r--r--   1 root supergroup          0 2023-12-05 19:52 /data/test/a.txt

• 查看指定目录下的内容

hdfs dfs -ls [hadoop目录]

查看详情

[root@hadoop01 test]# hdfs dfs -ls /data
Found 2 items
-rw-r--r--   1 root supergroup          0 2023-12-05 19:47 /data/a.txt
drwxr-xr-x   - root supergroup          0 2023-12-05 19:52 /data/test

• 查看文件内容

hdfs dfs -cat [hadoop文件路径]

查看详情

[root@hadoop01 test]# hdfs dfs -cat /data/test/a.txt
2023-12-05 20:32:28,115 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
aaa
bbb
ccc

• 在指定路径下创建文件

hdfs dfs -touchz [/hadoop目录/文件名]

查看详情

[root@hadoop01 test]# hdfs dfs -touchz /data/test/b.txt
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x   - root supergroup          0 2023-12-05 20:06 /data
-rw-r--r--   1 root supergroup          3 2023-12-05 20:06 /data/a.txt
drwxr-xr-x   - root supergroup          0 2023-12-05 20:36 /data/test
-rw-r--r--   1 root supergroup         19 2023-12-05 20:32 /data/test/a.txt
-rw-r--r--   1 root supergroup          0 2023-12-05 20:36 /data/test/b.txt

• 修改文件名称

[root@hadoop01 test]# hdfs dfs -mv /data/test/a.txt /data/test/c.txt
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x   - root supergroup          0 2023-12-05 20:06 /data
-rw-r--r--   1 root supergroup          3 2023-12-05 20:06 /data/a.txt
drwxr-xr-x   - root supergroup          0 2023-12-05 20:41 /data/test
-rw-r--r--   1 root supergroup          0 2023-12-05 20:36 /data/test/b.txt
-rw-r--r--   1 root supergroup         19 2023-12-05 20:32 /data/test/c.txt

• 将hadoop目录的文件下载到本地

[root@hadoop01 test]# hdfs dfs -get /data/test/c.txt /home/software/test/
2023-12-05 20:43:22,715 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
[root@hadoop01 test]# pwd
/home/software/test
[root@hadoop01 test]# ls
a.txt  c.txt

• 删除文件、目录

[root@hadoop01 test]# hdfs dfs -rm /data/test/c.txt
Deleted /data/test/c.txt
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x   - root supergroup          0 2023-12-05 20:06 /data
-rw-r--r--   1 root supergroup          3 2023-12-05 20:06 /data/a.txt
drwxr-xr-x   - root supergroup          0 2023-12-05 21:00 /data/test
-rw-r--r--   1 root supergroup          0 2023-12-05 20:36 /data/test/b.txt
 
[root@hadoop01 test]# hdfs dfs -rm -r /data/test
Deleted /data/test
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x   - root supergroup          0 2023-12-05 21:00 /data
-rw-r--r--   1 root supergroup          3 2023-12-05 20:06 /data/a.txt

• 将hadoop指定目录下所有文件保存为一个文件，同时down到本地

hdfs dfs -getmerge [hadoop目录] [/本地目录/文件名]

查看详情

[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x   - root supergroup          0 2023-12-05 21:06 /data
-rw-r--r--   1 root supergroup          3 2023-12-05 20:06 /data/a.txt
-rw-r--r--   1 root supergroup          7 2023-12-05 21:06 /data/b.txt
[root@hadoop01 test]# hdfs dfs -cat /data/a.txt
2023-12-05 21:20:53,987 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
aaa
[root@hadoop01 test]# hdfs dfs -cat /data/b.txt
2023-12-05 21:21:20,902 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
bbb
ccc
 
[root@hadoop01 test]# hdfs dfs -getmerge /data /home/software/c.txt
2023-12-05 21:08:08,098 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
[root@hadoop01 test]# cat /home/software/c.txt 
aaabbb
ccc