HDFS API 测试用例

增加依赖

        <!--hadoop-->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.9.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.9.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.9.2</version>
        </dependency>

 

编写测试用例

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

@Slf4j
public class HdfsTest {

    private static final String HDFS_PATH = "hdfs://192.168.3.127:8020";

    /**
     * 解决无权限访问,设置远程hadoop的linux用户名称
     */
    private static final String USER = "root";

    FileSystem fileSystem = null;
    Configuration configuration = null;

    @Before
    public void setUp() throws URISyntaxException, IOException, InterruptedException {

        System.setProperty("hadoop.home.dir", "D:\\app\\hadoop-2.9.2");

        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, USER);
    }

    @After
    public void tearDown(){

        fileSystem = null;
        configuration = null;
    }

    @Test
    public void mkdirTest() throws IOException {

        String home = System.getProperty("hadoop.home.dir");
        log.info("====>hadoop_home is {}", home);
        String dirPath = "/hdfsapi/test";
        fileSystem.mkdirs(new Path(dirPath));
    }

    @Test
    public void putTest() throws IOException {

        long start = System.currentTimeMillis();

        String filePath = "E:\\ThunderDownload\\hadoop-2.9.2.tar.gz";

        String digestPaht = "/";
        fileSystem.copyFromLocalFile(new Path(filePath), new Path(digestPaht));
        long end = System.currentTimeMillis();
        log.info("====>put file cost {}", end - start);
    }
}