package com.bigdata.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;
/**
* 使用Java API 操作HDFS文件系统
*/
public class HDFSApp {
public static final String HDFS_PATH = "hdfs://hadoop000:8020";
FileSystem fileSystem = null;
Configuration configuration = null;
@Before
public void setUp() throws Exception {
configuration = new Configuration();
/**
* 为了后面访问client访问datanNode(open,write等操作)
* 这里把datanode访问方式设置为用hostname访问,client端上建立了hadoop000和阿里云服务器的外网映射
* 否则,由于云服务器上搭建的伪分布式系统中,namenode和datanode是同一个机器,采用内网连接,
* nameNode返回给Client dataNode的地址来写数据时候,此时返回的dataNode地址是内网地址,client无法连接
*/
configuration.set("dfs.client.use.datanode.hostname","true");
/**
* 伪分布式,修改默认的副本系数
*/
configuration.set("dfs.replication","1");
/**
* 构造一个访问指定HDFS系统的客户端对象
* 第一个参数:HDFS的URI
* 第二个参数:客户端指定的配置参数
* 第三个参数:客户端的身份,用户名
*/
fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "guoqiang");
System.out.println("--------setUp--------");
}
/**
* 创建文件夹
* 只涉及到nameNode的操作,不回连接到dataNode
*/
@Test
public void mkdir() throws Exception {
fileSystem.mkdirs(new Path("/jUnitTest/testB"));
}
/**
* 查看HDFS内容
*/
@Test
public void text() throws Exception {
FSDataInputStream in = fileSystem.open(new Path("/cdh_version.properties"));
IOUtils.copyBytes(in, System.out, 1024);
}
/**
* 创建文件
*/
@Test
public void create() throws Exception {
FSDataOutputStream out = fileSystem.create(new Path("/jUnitTest/testB/a.txt"));
out.writeUTF("Hello GQ");
out.flush();
out.close();
}
/**
* 文件名更改
*/
@Test
public void rename() throws Exception {
Path oldPath = new Path("/jUnitTest/testB/a.txt");
Path newPath = new Path("/jUnitTest/testB/b.txt");
boolean result = fileSystem.rename(oldPath,newPath);
System.out.println(result);
}
/**
* 拷贝本地文件到fileSystem
*/
@Test
public void copyFromLocalFile() throws Exception {
Path src = new Path("/Users/mac126/code/javaProgram/InsertionSort.java");
Path dst = new Path("/jUnitTest/InsertionSort.java");
fileSystem.copyFromLocalFile(src, dst);
}
/**
* 拷贝大文件,带进度条
*/
@Test
public void copyFromLocalBigFile() throws Exception {
InputStream in = new BufferedInputStream(new FileInputStream(new File("/Users/mac126/Downloads/94011351-1-80.flv")));
FSDataOutputStream out = fileSystem.create(new Path("/graduateVideo/zyz.flv"), new Progressable() {
@Override
public void progress() {
System.out.print("#");
}
});
IOUtils.copyBytes(in, out, 4096);
}
/**
* 拷贝HDFS文件到本地,即下载
*/
@Test
public void copyToLocalFile() throws Exception {
Path src = new Path("/jUnitTest/InsertionSort.java");
Path dst = new Path("/Users/mac126/blog/javaProgram.java");
fileSystem.copyToLocalFile(src, dst);
}
/**
* 查看目标文件夹下的所有文件
*/
@Test
public void listFiles() throws Exception {
FileStatus[] statuses = fileSystem.listStatus(new Path("/jUnitTest/"));
for (FileStatus file : statuses) {
String isDir = file.isDirectory() ? "文件夹":"文件";
String permission = file.getPermission().toString();
short replication = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir + "\t" + permission + "\t" + permission + "\t" +
replication + "\t" + length + "\t" + path);
}
}
/**
* 递归查看目标文件夹下的所有文件
*/
@Test
public void listFilesRecursion() throws Exception {
RemoteIterator<LocatedFileStatus> iterator = fileSystem.listFiles(new Path("/jUnitTest/"), true);
while(iterator.hasNext()) {
LocatedFileStatus file = iterator.next();
String isDir = file.isDirectory() ? "文件夹":"文件";
String permission = file.getPermission().toString();
short replication = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir + "\t" + permission + "\t" + permission + "\t" +
replication + "\t" + length + "\t" + path);
}
}
/**
* 查看文件块信息
*/
@Test
public void getFileBlockLocations() throws Exception {
FileStatus status = fileSystem.getFileStatus(new Path("/graduateVideo/zyz.flv"));
BlockLocation[] blockLocations = fileSystem.getFileBlockLocations(status, 0, status.getLen());
for(BlockLocation blockLocation : blockLocations) {
for(String name : blockLocation.getNames()) {
System.out.println(name + " :" + blockLocation.getOffset() + " : " + blockLocation.getLength());
}
}
}
/**
* 删除操作
*/
@Test
public void delete() throws Exception {
boolean result = fileSystem.delete(new Path("/jUnitTest/InsertionSort.java"), false);
System.out.println(result);
}
@After
public void tearDown() {
configuration = null;
fileSystem = null;
System.out.println("--------tearDown--------");
}
// public static void main(String[] args) throws Exception {
// Configuration configuration = new Configuration();
// FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop000:8020"), configuration, "guoqiang");
// boolean result = fileSystem.mkdirs(new Path("/hdfsApi/test"));
// System.out.println(result);
// }
}