HDFS API操作

一、HDFS 文件上传

    @Test
    public void testCopyFromLocal() throws URISyntaxException, IOException, InterruptedException {
        // 1. 获取 fs 对象
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://wt1:9000"), conf, "tom");
        // 2. 执行上传
        fs.copyFromLocalFile(new Path("D:\\test.txt"), new Path("/"));
        // 3. 关闭资源
        fs.close();
    }

优先级 由高到底

1、代码

Configuration conf = new Configuration();
// 设置备份
conf.set("dfs.replication", "2");

2、项目配置文件

src\main\resources\hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

3、节点配置文件

二、文件下载

    @Test
    public void testCopyToLocal() throws URISyntaxException, IOException, InterruptedException {
        // 1. 获取 fs 对象
        Configuration conf = new Configuration();
        // 设置备份
        FileSystem fs = FileSystem.get(new URI("hdfs://wt1:9000"), conf, "tom");
        // 2. 执行下载
        fs.copyToLocalFile(new Path("/wt/li/test.txt"), new Path("E:\\abc"));
        // 3. 关闭资源
        fs.close();
    }

三、文件删除

    @Test
    public void testDeleteFile() throws URISyntaxException, IOException, InterruptedException {
        // 获取 fs 对象
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://wt1:9000"), conf, "tom");
        // 执行删除
        fs.delete(new Path("/wt/li/test.txt"), true);

        // 关闭 fs
        fs.close();
    }

 四、修改文件名

    @Test
    public void testRename() throws URISyntaxException, IOException, InterruptedException {
        // 1. 获取fs对象
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://wt1:9000"), conf, "tom");
        // 2.重命名
        fs.rename(new Path("/test.txt"), new Path("/abc.txt"));
        // 3.关闭
        fs.close();
    }

五、HDFS查看文件详情

 @Test
    public void testFileInfo() throws URISyntaxException, IOException, InterruptedException {
        // 1. 获取fs对象
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://wt1:9000"), conf, "tom");
        //
        RemoteIterator<LocatedFileStatus> RemoteIterator = fs.listFiles(new Path("/"), true);
        while (RemoteIterator.hasNext()){
            LocatedFileStatus fileStatus = RemoteIterator.next();
            // 文件名称
            System.out.println(fileStatus.getPath().getName());
            // 文件权限
            System.out.println(fileStatus.getPermission());
            // 文件长度
            System.out.println(fileStatus.getLen());
            // 块信息
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            for (BlockLocation blockLocation : blockLocations) {
                String[] hosts = blockLocation.getHosts();
                for (String host : hosts) {
                    System.out.println(host);
                }
            }
            System.out.println("===============");
        }
    }

 六、判断文件 文件夹

    @Test
    public void testFileDir() throws URISyntaxException, IOException, InterruptedException {
        // 1. 获取fs对象
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://wt1:9000"), conf, "tom");
        FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
        for (FileStatus fileStatus : fileStatuses) {
            if(fileStatus.isFile()){
                // 文件
                System.out.println("f:"+fileStatus.getPath().getName());
            }else {
                // 目录
                System.out.println("d:"+fileStatus.getPath().getName());
            }
        }
    }