HDFS_JAVA API

测试环境 :  win

服务环境 : centos7.5

版本 : hadoop3.1.3

1.pom.xml

  <dependencies>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client</artifactId>
      <version>3.1.3</version>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.12</version>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-log4j12</artifactId>
      <version>1.7.30</version>
    </dependency>
  </dependencies>

2.在resources 添加 log4j.properties 

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

3.客户端代码

package com.hadoop.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;

/**
 * @author cun.gao
 * @create 2021/8/22 - 9:08
 */
public class HdfsClient {
  Configuration configuration ;
  FileSystem fs ;
  /*
  *注意事项：
  *   1. 参数优先级: （1）客户端代码中设置的值 >
  *                 （2）ClassPath 下的用户自定义配置文件 >
  *                 （3）然后是服务器的自定义配置（xxx-site.xml）>
  *                 （4）服务器的默认配置（xxx-default.xml）

  * */
  @Before
  public void getFs() throws URISyntaxException, IOException, InterruptedException {
    // 参数设置
    String URL = "hdfs://hadoop01:8020";
    String UserName = "root";
    // 创建 Configuration 对象,读取相关配置文件
    configuration = new Configuration();
    //设置参数 设置副本数
    //configuration.set("dfs.replication", "20");
    fs = FileSystem.get(new URI(URL), configuration,UserName);
  }
  //关闭资源
  @After
  public void close() throws IOException {
    fs.close();
    System.out.println("fs已关闭...");
  }

  //创建目录
  @Test
  public void testMkdirs() throws IOException {
    // 1 创建目录
    fs.mkdirs(new Path("/java_api/demo1/"));
  }

  //上传文件
  @Test
  public void testcopyFromLocalFile() throws IOException {
    // 2 上传文件
    /*
    boolean delSrc 是否删除源文件,
    boolean overwrite 是否覆盖hdfs文件,
    Path src 源文件路径,
    Path dst hdfs路径
    */
    Path srcPath = new Path("d:/tmp/gaocun.txt");
    Path dfsPath = new Path("/java_api/");
    fs.copyFromLocalFile(false,false,srcPath,dfsPath);
    System.out.println(srcPath.getName()+"文件已上传");
  }

  //下载文件
  @Test
  public void testcopyToLocalFile() throws IOException {
    // 3 下载文件
    /*
    boolean delSrc 是否删除源文件,
    Path src hdfs路径,
    Path dst 本地路径,
    boolean useRawLocalFileSystem 是否开启文件校验
    * */
    Path srcPath = new Path("d:/tmp/download.txt");
    Path dfsPath = new Path("/java_api/gaocun.txt");
    fs.copyToLocalFile(true,dfsPath,srcPath,true);
    System.out.println(dfsPath.getName()+"文件已下载");
  }

  //文件更名和移动
  @Test
  public void testRename() throws IOException {
    fs.rename(new Path("/java_api/demo"),new Path("/java_api/demoNew"));
    System.out.println("文件已改名");
  }

  //删除文件和目录
  @Test
  public void testdelete() throws IOException {
    fs.delete(new Path("/java_api/demoNew"),true);
    System.out.println("文件已删除");
  }

  //文件详情查看
  @Test
  public void testFindAll() throws IOException {
    RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/sanguo/wuguo.txt"), true);
    while (listFiles.hasNext()) {
      LocatedFileStatus fileStatus = listFiles.next();
      System.out.println("========" + fileStatus.getPath() + "=========");
      System.out.println(fileStatus.getPermission());
      System.out.println(fileStatus.getOwner());
      System.out.println(fileStatus.getGroup());
      System.out.println(fileStatus.getLen());
      System.out.println(fileStatus.getModificationTime());
      System.out.println(fileStatus.getReplication());
      System.out.println(fileStatus.getBlockSize());
      System.out.println(fileStatus.getPath().getName());
      // 获取块信息
      BlockLocation[] blockLocations = fileStatus.getBlockLocations();
      System.out.println(Arrays.toString(blockLocations));
    }
  }

}