HDFS api操作

1.获取FileSystem对象方式1
1 public void getFileSystem1() throws IOException {
2     Configuration conf = new Configuration();
3     conf.set("fs.defaultFS", "hdfs://master:8020");
4     FileSystem fileSystem = FileSystem.get(conf);
5     System.out.println(fileSystem);
6 
7 }
2.获取FileSystem对象方式2 
1 public void getFileSystem2() throws URISyntaxException, IOException {
2     FileSystem fileSystem = FileSystem.get(new URI("hdfs://master:8020"), new Configuration());
3     System.out.println(fileSystem);
4 }

3.获取FileSystem对象方式

1 public void getFileSystem3() throws IOException {
2     Configuration configuration = new Configuration();
3     configuration.set("fs.defaultFS", "hdfs://master:8020");
4     FileSystem fileSystem = FileSystem.newInstance(configuration);
5     System.out.println(fileSystem);
6 }

4.获取FileSystem对象方式4

1 public void getFileSystem4() throws URISyntaxException, IOException {
2     FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://master:9000"), new Configuration());
3     System.out.println(fileSystem);
4 }

5.遍历HDFS目录

 1 public void listMyfile() throws URISyntaxException, IOException {
 2     FileSystem fileSystem = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
 3     RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/"), true);//true表示递归遍历
 4     while (files.hasNext()) {
 5         LocatedFileStatus next = files.next();
 6         System.out.println(next.getPath() + "---" + next.getPath().getName());
 7     }
 8     fileSystem.close();
 9 
10 }
6.创建文件及文件 
1  public void mkdir() throws Exception {
2         FileSystem fileSystem = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
3         boolean mkdirs = fileSystem.mkdirs(new Path("/a/b/c"));//可以递归创键文件夹
4         System.out.println(mkdirs);
5         fileSystem.create(new Path("/aa/bb/cc/a.txt"));//创建文件，当文件夹不存在可以递归创键
6         fileSystem.close();
7     }
7.获取HDFS存储的文件方式1
1 public void downloadFile1() throws IOException, URISyntaxException {
2     FileSystem fileSystem = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
3     FSDataInputStream open = fileSystem.open(new Path("/a.txt"));
4     FileOutputStream fos = new FileOutputStream(new File("E:\\a.txt"));
5     IOUtils.copy(open, fos);
6     IOUtils.closeQuietly(open);
7     IOUtils.closeQuietly(fos);
8     fileSystem.close();
9 }
8.获取HDFS存储的文件方式2 
1 public void downloadFile2() throws IOException, URISyntaxException {
2     FileSystem fileSystem = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
3     fileSystem.copyToLocalFile(new Path("/a.txt"), new Path("E://b.txt"));
4     fileSystem.close();
5 }
9.上传文件到HDFS 
1 public void uploadFile() throws URISyntaxException, IOException {
2     FileSystem fileSystem = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
3     fileSystem.copyFromLocalFile(new Path("E:\\students.txt"), new Path("/students.txt"));
4     fileSystem.close();
5 }
10.合并多个小文件到HDFS
 1 /***
 2  * 合并多个小文件
 3  */
 4 @Test
 5 public void mergeFile() throws URISyntaxException, IOException {
 6     //1.获取FileSystem(分布式文件系统)
 7     FileSystem fileSystem = FileSystem.get(new URI("hdfs://master:9000"), new Configuration());
 8     //2.获取hdfs大文件的输出流
 9     FSDataOutputStream outputStream = fileSystem.create(new Path("/big.txt"));
10     //3.获取本地文件系统
11     LocalFileSystem localFileSystem = FileSystem.getLocal(new Configuration());
12     //4.获取本地文件夹下所有文件的详情
13     FileStatus[] fileStatuses = localFileSystem.listStatus(new Path("E:\\temp\\blocks"));
14     //5.遍历每个文件，获取每个文件的输入流
15     for (FileStatus fileStatus : fileStatuses) {
16         FSDataInputStream inputStream = localFileSystem.open(fileStatus.getPath());
17         //6.将小文件复制到大文件
18         IOUtils.copy(inputStream,outputStream);
19         IOUtils.closeQuietly(inputStream);
20     }
21     //7.关闭流
22     IOUtils.closeQuietly(outputStream);
23     localFileSystem.close();
24     fileSystem.close();
25 
26 }

 

posted @ 2022-07-01 17:02  荒野拾粪人  阅读(45)  评论(0)  编辑  收藏  举报