hadoop使用
-
启动hadoop，浏览器访问：http://192.168.128.78:9870/
-
显示目录结构
[root@master ~]# hadoop dfs -ls -R /
WARNING: Use of this script to execute dfs is deprecated.
WARNING: Attempting to execute replacement "hdfs dfs" instead.
drwxr-xr-x - root supergroup 0 2027-11-30 19:23 /hadoop
drwx-wx-wx - root supergroup 0 2027-12-01 09:26 /tmp
drwx-wx-wx - root supergroup 0 2027-12-01 09:26 /tmp/hive
drwx------ - root supergroup 0 2027-12-03 19:52 /tmp/hive/root
drwx------ - root supergroup 0 2027-12-03 19:51 /tmp/hive/root/d3e00e50-57a5-4146-8441-5615a3bdec7b
drwx------ - root supergroup 0 2027-12-03 19:51 /tmp/hive/root/d3e00e50-57a5-4146-8441-5615a3bdec7b/_tmp_space.db
drwx------ - root supergroup 0 2027-12-03 19:52 /tmp/hive/root/dcd27fc9-0e38-4f20-8fdf-52c548e9494b
drwx------ - root supergroup 0 2027-12-03 19:52 /tmp/hive/root/dcd27fc9-0e38-4f20-8fdf-52c548e9494b/_tmp_space.db
drwxr-xr-x - root supergroup 0 2027-12-01 09:27 /user
drwxr-xr-x - root supergroup 0 2027-12-01 09:27 /user/hive
drwxr-xr-x - root supergroup 0 2027-12-01 09:27 /user/hive/warehouse
drwxr-xr-x - root supergroup 0 2027-12-03 20:09 /user/hive/warehouse/bigdata.db
drwxr-xr-x - root supergroup 0 2027-12-03 20:09 /user/hive/warehouse/bigdata.db/province_sale
[root@master ~]# hdfs dfs -ls -R /
drwxr-xr-x - root supergroup 0 2027-11-30 19:23 /hadoop
drwx-wx-wx - root supergroup 0 2027-12-01 09:26 /tmp
drwx-wx-wx - root supergroup 0 2027-12-01 09:26 /tmp/hive
drwx------ - root supergroup 0 2027-12-03 19:52 /tmp/hive/root
drwx------ - root supergroup 0 2027-12-03 19:51 /tmp/hive/root/d3e00e50-57a5-4146-8441-5615a3bdec7b
drwx------ - root supergroup 0 2027-12-03 19:51 /tmp/hive/root/d3e00e50-57a5-4146-8441-5615a3bdec7b/_tmp_space.db
drwx------ - root supergroup 0 2027-12-03 19:52 /tmp/hive/root/dcd27fc9-0e38-4f20-8fdf-52c548e9494b
drwx------ - root supergroup 0 2027-12-03 19:52 /tmp/hive/root/dcd27fc9-0e38-4f20-8fdf-52c548e9494b/_tmp_space.db
drwxr-xr-x - root supergroup 0 2027-12-01 09:27 /user
drwxr-xr-x - root supergroup 0 2027-12-01 09:27 /user/hive
drwxr-xr-x - root supergroup 0 2027-12-01 09:27 /user/hive/warehouse
drwxr-xr-x - root supergroup 0 2027-12-03 20:09 /user/hive/warehouse/bigdata.db
drwxr-xr-x - root supergroup 0 2027-12-03 20:09 /user/hive/warehouse/bigdata.db/province_sale
- 创建目录
hdfs dfs -mkdir /data
- 复制本地文件到hadoop
hdfs dfs -put [本地文件路径] [hadoop目录]
查看详情
[root@hadoop01 software]# hdfs dfs -put /home/software/a.txt /data
[root@hadoop01 software]# hdfs dfs -ls -R /
drwxr-xr-x - root supergroup 0 2023-12-05 19:47 /data
-rw-r--r-- 1 root supergroup 0 2023-12-05 19:47 /data/a.txt
- 复制本地目录到hadoop
hdfs dfs -put [本地目录路径] [hadoop目录]
查看详情
[root@hadoop01 test]# pwd
/home/software/test
[root@hadoop01 test]# ls
a.txt
[root@hadoop01 test]# hdfs dfs -put /home/software/test/ /data
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x - root supergroup 0 2023-12-05 19:52 /data
-rw-r--r-- 1 root supergroup 0 2023-12-05 19:47 /data/a.txt
drwxr-xr-x - root supergroup 0 2023-12-05 19:52 /data/test
-rw-r--r-- 1 root supergroup 0 2023-12-05 19:52 /data/test/a.txt
- 查看指定目录下的内容
hdfs dfs -ls [hadoop目录]
查看详情
[root@hadoop01 test]# hdfs dfs -ls /data
Found 2 items
-rw-r--r-- 1 root supergroup 0 2023-12-05 19:47 /data/a.txt
drwxr-xr-x - root supergroup 0 2023-12-05 19:52 /data/test
- 查看文件内容
hdfs dfs -cat [hadoop文件路径]
查看详情
[root@hadoop01 test]# hdfs dfs -cat /data/test/a.txt
2023-12-05 20:32:28,115 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
aaa
bbb
ccc
- 在指定路径下创建文件
hdfs dfs -touchz [/hadoop目录/文件名]
查看详情
[root@hadoop01 test]# hdfs dfs -touchz /data/test/b.txt
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x - root supergroup 0 2023-12-05 20:06 /data
-rw-r--r-- 1 root supergroup 3 2023-12-05 20:06 /data/a.txt
drwxr-xr-x - root supergroup 0 2023-12-05 20:36 /data/test
-rw-r--r-- 1 root supergroup 19 2023-12-05 20:32 /data/test/a.txt
-rw-r--r-- 1 root supergroup 0 2023-12-05 20:36 /data/test/b.txt
- 修改文件名称
[root@hadoop01 test]# hdfs dfs -mv /data/test/a.txt /data/test/c.txt
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x - root supergroup 0 2023-12-05 20:06 /data
-rw-r--r-- 1 root supergroup 3 2023-12-05 20:06 /data/a.txt
drwxr-xr-x - root supergroup 0 2023-12-05 20:41 /data/test
-rw-r--r-- 1 root supergroup 0 2023-12-05 20:36 /data/test/b.txt
-rw-r--r-- 1 root supergroup 19 2023-12-05 20:32 /data/test/c.txt
- 将hadoop目录的文件下载到本地
[root@hadoop01 test]# hdfs dfs -get /data/test/c.txt /home/software/test/
2023-12-05 20:43:22,715 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
[root@hadoop01 test]# pwd
/home/software/test
[root@hadoop01 test]# ls
a.txt c.txt
- 删除文件、目录
[root@hadoop01 test]# hdfs dfs -rm /data/test/c.txt
Deleted /data/test/c.txt
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x - root supergroup 0 2023-12-05 20:06 /data
-rw-r--r-- 1 root supergroup 3 2023-12-05 20:06 /data/a.txt
drwxr-xr-x - root supergroup 0 2023-12-05 21:00 /data/test
-rw-r--r-- 1 root supergroup 0 2023-12-05 20:36 /data/test/b.txt
[root@hadoop01 test]# hdfs dfs -rm -r /data/test
Deleted /data/test
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x - root supergroup 0 2023-12-05 21:00 /data
-rw-r--r-- 1 root supergroup 3 2023-12-05 20:06 /data/a.txt
- 将hadoop指定目录下所有文件保存为一个文件，同时down到本地
hdfs dfs -getmerge [hadoop目录] [/本地目录/文件名]
查看详情
[root@hadoop01 test]# hdfs dfs -ls -R /
drwxr-xr-x - root supergroup 0 2023-12-05 21:06 /data
-rw-r--r-- 1 root supergroup 3 2023-12-05 20:06 /data/a.txt
-rw-r--r-- 1 root supergroup 7 2023-12-05 21:06 /data/b.txt
[root@hadoop01 test]# hdfs dfs -cat /data/a.txt
2023-12-05 21:20:53,987 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
aaa
[root@hadoop01 test]# hdfs dfs -cat /data/b.txt
2023-12-05 21:21:20,902 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
bbb
ccc
[root@hadoop01 test]# hdfs dfs -getmerge /data /home/software/c.txt
2023-12-05 21:08:08,098 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
[root@hadoop01 test]# cat /home/software/c.txt
aaabbb
ccc
