Hadoop之HDFS的FileSystem类的使用

1.导入jar包

解压hadoop-2.7.7.tar.gz，并在如下路径找到图中所示三个jar包

 

下图所示路径所有jar包

 

还要用到下面三个jar包

 

 

 下图所示路径所有jar包

 把以上jar包全部加入到项目中

 

2.查看文件信息

@Test
    public void connectHDFS()
    {
        Configuration conf = new Configuration();   //做一个配置
        conf.set("fs.defaultFS", "hdfs://192.168.0.51:9000");//服务器的地址  端口号
        try {
            FileSystem fileSystem = FileSystem.get(conf);
            FileStatus fileStatus = fileSystem.getFileStatus(new Path("/upload/hello.txt"));
            System.out.println(fileStatus.isFile()); //判断是不是一个文件
            System.out.println(fileStatus.isDirectory()); //判断是不是一个文件夹
            System.out.println(fileStatus.getPath());    //获取文件路径
            System.out.println(fileStatus.getLen());    //获取文件大小
            
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    
    }

 

3.修改文件名

@Test
public void rename()
{
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", "hdfs://192.168.0.51:9000");//服务器的地址 端口号
Properties properties = System.getProperties(); //这两行告诉hadoop访问时的用户为root
properties.setProperty("HADOOP_USER_NAME", "root");
try {
FileSystem fileSystem = FileSystem.get(configuration);
boolean rename = fileSystem.rename(new Path("/upload/jdk-8u221-linux-x64.tar.gz"),new Path("/upload/jdk1.8.tar.gz"));
System.out.println(rename?"修改成功！":"修改失败！");
fileSystem.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}

 

4.创建文件夹

@Test
    public void mkdir()
    {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", "hdfs://192.168.0.51:9000");
        Properties properties = System.getProperties();     //这两行告诉hadoop访问时的用户为root
        properties.setProperty("HADOOP_USER_NAME", "root");
        try {
            FileSystem fileSystem = FileSystem.get(configuration);
            fileSystem.mkdirs(new Path("/user"));
            fileSystem.mkdirs(new Path("/bailiban/yw"));
            fileSystem.close();   //关闭
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

 

5.文件上传

@Test
    public void upload()
    {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", "hdfs://192.168.0.51:9000");
        Properties properties = System.getProperties();     //这两行告诉hadoop访问时的用户为root
        properties.setProperty("HADOOP_USER_NAME", "root");
        try {
            FileSystem fileSystem = FileSystem.get(configuration);
            FSDataOutputStream out = fileSystem.create(new Path("/doupo.txt"));
            FileInputStream in = new FileInputStream("F:\\doupo.txt");
            byte [] b = new byte[1024];
            int len = 0;
            while((len=in.read(b))!=-1)
            {
                out.write(b,0,len);
            }
            in.close();
            out.close();
            fileSystem.close();   //关闭
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

 

6.文件下载

@Test
    public void download()
    {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", "hdfs://192.168.0.51:9000");
        Properties properties = System.getProperties();     //这两行告诉hadoop访问时的用户为root
        properties.setProperty("HADOOP_USER_NAME", "root");
        try {
            FileSystem fileSystem = FileSystem.get(configuration);
            FSDataInputStream in = fileSystem.open(new Path("/upload/hello.txt"));
            FileOutputStream out = new FileOutputStream(new File("F:\\hello.txt"));
            byte [] b = new byte [1024];
            int len = 0;
            while((len=in.read(b))!=-1)
            {
                out.write(b,0,len);
            }
            in.close();
            out.close();
            fileSystem.close();   //关闭
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

 

7.递归遍历所有文件和文件夹

@Test
    public void fileList()
    {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", "hdfs://192.168.0.51:9000");
        Properties properties = System.getProperties();     //这两行告诉hadoop访问时的用户为root
        properties.setProperty("HADOOP_USER_NAME", "root");
        try {
            FileSystem fileSystem = FileSystem.get(configuration);
            FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
            for (FileStatus fileStatus : listStatus) 
            {
                digui(fileStatus,fileSystem);
            }    
            fileSystem.close();   //关闭
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }


    private void digui(FileStatus fileStatus, FileSystem fileSystem) {
        String name = "";
        if(fileStatus.isDirectory())
        {
            name = fileStatus.getPath()+"";
            System.out.println("文件夹："+name.split("9000")[1]);
            try {
                FileStatus[] listStatus = fileSystem.listStatus(new Path(name.split("9000")[1]));
                if(listStatus.length!=0)
                {
                    for (FileStatus fileStatus2 : listStatus) 
                    {
                        digui(fileStatus2,fileSystem);
                    }
                }
            } catch (FileNotFoundException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            
            
        }
        else
        {
            System.out.println("文件:"+(fileStatus.getPath()+"").split("9000")[1]);
        }
        
    }