HDFS客户端操作

1.创建一个maven工程hdfs-demo，并导入相关的依赖jar，以下是pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>hdfs-demo</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.1.3</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-slf4j-impl</artifactId>
            <version>2.12.0</version>
        </dependency>
    </dependencies>

</project>

2.在项目的src/main/resources目录下，新建一个文件log4j2.xml，以下是log4j2.xml

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="error" strict="true" name="XMLConfig">
    <Appenders>
        <!-- 类型名为Console，名称为必须属性 -->
        <Appender type="Console" name="STDOUT">
            <!-- 布局为PatternLayout的方式，
            输出样式为[INFO] [2018-01-22 17:34:01][org.test.Console]I'm here -->
            <Layout type="PatternLayout"
                    pattern="[%p] [%d{yyyy-MM-dd HH:mm:ss}][%c{10}]%m%n" />
        </Appender>

    </Appenders>

    <Loggers>
        <!-- 可加性为false -->
        <Logger name="test" level="info" additivity="false">
            <AppenderRef ref="STDOUT" />
        </Logger>

        <!-- root loggerConfig设置 -->
        <Root level="info">
            <AppenderRef ref="STDOUT" />
        </Root>
    </Loggers>
</Configuration>

3.在src\main\java下创建创建一个HdfsClient类，以下是HdfsClient.java

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;

public class HdfsClient {
    private URI uri;
    private Configuration conf;
    private String user;
    private FileSystem fs;
    @Before
    public void init() throws IOException, InterruptedException {
        /*参数解释说明:
                uri:指定需要连接的地址 hdfs://hadoop102:8020
                conf:指的是配置对象
                user:指定使用客户端的对象
        */
        uri = URI.create("hdfs://hadoop102:8020");
        conf = new Configuration();
        user="atguigu";
        //创建客户端对象
        fs = FileSystem.get(uri, conf, user);
    }
    @After
    public void close() throws IOException {
        //关闭客户端
        fs.close();
    }
    @Test
    public void mkdirs() throws IOException {
        fs.mkdirs(new Path("/java1/aaa"));
    }
    //上传文件
    @Test
    public void put() throws IOException {
        // 参数说明
        // boolean delSrc 指是否将原文件删除(windows)
        //boolean overwrite 指是否覆盖已有文件
        // Path src 指原上传的文件路径(windowd)
        // Path dst 指将目标文件路径
        fs.copyFromLocalFile(false,false,new Path("E:\\学习\\简单求和.pdf"),new Path("/"));
    }
    //下载文件
    @Test
    public void get() throws IOException {
        // 参数说明
        // boolean delSrc 指是否将原文件删除
        // Path src 指要下载的文件路径
        // Path dst 指将文件下载到的路径
        // boolean useRawLocalFileSystem 是否开启文件校验
        fs.copyToLocalFile(false,new Path("/简单求和.pdf"),new Path("E:\\学习\\废品"),true);
    }
    //删除文件或目录
    @Test
    public void delete() throws IOException {
        fs.delete(new Path("/java"),true);
    }
    //文件或目录的更名和移动
    @Test
    public void rename() throws IOException {
        //文件的更名
        //fs.rename(new Path("/简单求和.pdf"),new Path("/求和.pdf"));
        //文件的移动
        //fs.rename(new Path("/求和.pdf"),new Path("/java1"));
        //文件的更名和移动
        fs.rename(new Path("/java1/求和.pdf"),new Path("/简单求和.pdf"));
    }
    //获取文件详情
    @Test
    public void listFile() throws IOException {
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fs.listFiles(new Path("/"), false);
        while (locatedFileStatusRemoteIterator.hasNext()) {
            LocatedFileStatus File = locatedFileStatusRemoteIterator.next();
            System.out.println("------------" + File.getPath() + "-----------");
            System.out.println(File.getPermission());
            System.out.println(File.getOwner());
            System.out.println(File.getGroup());
            System.out.println(File.getLen());
            SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            System.out.println(simpleDateFormat.format(File.getModificationTime()));
            System.out.println(File.getReplication());
            System.out.println(File.getBlockSize());
            System.out.println(File.getPath().getName());
            BlockLocation[] blockLocations = File.getBlockLocations();
            System.out.println(Arrays.toString(blockLocations));
        }
    }
    //判断是文件还是目录
    @Test
    public void fileOrDir() throws IOException {
        FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
        for (FileStatus status : fileStatuses) {
            if (status.isFile()) {
                System.out.println("File "+status.getPath());
            }else{
                System.out.println("Dir "+status.getPath());
            }
        }
    }
    //通过IO上传
    @Test
    public void pubByIo() throws IOException {
        FileInputStream fis = new FileInputStream(new File("E:\\学习\\废品\\马原复习重点.docx"));
        FSDataOutputStream fdos = fs.create(new Path("/马原复习重点.docx"));
        IOUtils.copyBytes(fis,fdos,conf);
        IOUtils.closeStreams(fdos,fis);
    }
    //通过IO下载
    @Test
    public void getByIo() throws IOException {
        FSDataInputStream fdis = fs.open(new Path("/马原复习重点.docx"));
        FileOutputStream fos = new FileOutputStream(new File("E:\\学习\\废品\\马原复习重点1.docx"));
        IOUtils.copyBytes(fdis,fos,conf);
        IOUtils.closeStreams(fos,fdis);
    }
}