<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.1.1</version>
</dependency>
package com.imooc.bigdata;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.*;
import java.net.URI;
/**
* Unit test for simple App.
*/
public class HDFSAppTest {
public static final String HDFS_PATH = "hdfs://rocketmq-nameserver1:9000";
FileSystem fileSystem = null;
Configuration configuration = null;
@Before
public void setUp() throws Exception {
System.out.println("--------setup-------");
configuration = new Configuration();
configuration.set("dfs.replication", "1");//设置副本为1
fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "root");
}
@Test
public void mkdir() throws Exception{
Path path = new Path("/hdfsapi/test");
boolean result = fileSystem.mkdirs(path);
System.out.println(result);
}
/**
* 查看HDFS内容
*/
@Test
public void text() throws Exception {
FSDataInputStream in = fileSystem.open(new Path("/a.txt"));
IOUtils.copyBytes(in, System.out, 1024);
}
/**
* 创建文件
*/
@Test
public void create() throws Exception {
FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/b.txt"));
out.writeUTF("hello hdfs b");
out.flush();
out.close();
}
/**
* 修改文件名
* @throws Exception
*/
@Test
public void rename() throws Exception {
Path oldPath = new Path("/hdfsapi/test/b.txt");
Path newPath = new Path("/hdfsapi/test/c.txt");
boolean result = fileSystem.rename(oldPath, newPath);
System.out.println(result);
}
/**
* 拷贝本地文件到hdfs文件系统
* @throws Exception
*/
@Test
public void copyFromLocal() throws Exception {
Path src = new Path("D:/test.txt");
Path dst = new Path("/hdfsapi/test");
fileSystem.copyFromLocalFile(src, dst);
}
/**
* 拷贝本地大文件到hdfs文件系统，进行进度显示
* @throws Exception
*/
@Test
public void copyBigFromLocal() throws Exception {
InputStream in = new BufferedInputStream(new FileInputStream(new File("E:\\test.zip")));
FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/vm.zip"), new Progressable () {
public void progress() {
System.out.print(".");
}
});
IOUtils.copyBytes(in, out, 4096);
}
/**
* 从hdfs拷贝文件到本地
* @throws Exception
*/
@Test
public void copyToLocalFile() throws Exception {
Path src = new Path("/hdfsapi/test/c.txt");
Path dst = new Path("E:/sh");
fileSystem.copyToLocalFile(false, src, dst, true);
}
/**
* 获取文件夹下的所有文件
* @throws Exception
*/
@Test
public void listFile() throws Exception {
FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi/test"));
for (FileStatus status : statuses) {
System.out.println(status.getPermission().toString());
System.out.println(status.isDirectory());
System.out.println(status.getBlockSize());
System.out.println(status.getOwner());
System.out.println(status.isFile());
System.out.println(status.getLen());
System.out.println(status.getPath());
System.out.println("----------------");
}
System.out.println(statuses);
}
/**
* 递归获取文件夹下的所有文件
* @throws Exception
*/
@Test
public void listFileRecusive() throws Exception {
RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/hdfsapi/test"), true);
while (files.hasNext()){
LocatedFileStatus status = files.next();
System.out.println(status.getPermission().toString());
System.out.println(status.isDirectory());
System.out.println(status.getBlockSize());
System.out.println(status.getOwner());
System.out.println(status.isFile());
System.out.println(status.getLen());
System.out.println(status.getPath());
System.out.println("----------------");
}
}
/**
* 查看文件块信息
* @throws Exception
*/
@Test
public void getFileBlockLocations() throws Exception {
FileStatus fileStatus = fileSystem.getFileStatus(new Path("/jdk-8u211-linux-x64.tar.gz"));
BlockLocation[] blockLocations = fileSystem.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
for (BlockLocation blockLocation : blockLocations) {
for(String name : blockLocation.getNames()){
System.out.println(name + " : " + blockLocation.getOffset() + " : " + blockLocation.getLength());
}
}
}
@Test
public void delete() throws Exception {
boolean result = fileSystem.delete(new Path("/jdk-8u211-linux-x64.tar.gz"), true);
System.out.println(result);
}
@After
public void tearDown(){
configuration = null;
fileSystem = null;
System.out.println("-------shutdown-----------");
}
// public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException {
// Configuration configuration = new Configuration();
// FileSystem fileSystem = FileSystem.get(new URI("hdfs://rocketmq-nameserver1:9000"), configuration, "root");
// Path path = new Path("/hdfsapi/test");
// boolean result = fileSystem.mkdirs(path);
// System.out.println(result);
// }
}
JAVA API连接HA(High Available) Hadoop集群
package com.imooc.bigdata;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
/**
* Created by Administrator on 2020/4/15.
*/
public class HDFSTest {
public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
Configuration conf=new Configuration(false);
String nameservices = "mycluster";
String[] namenodesAddr = {"rocketmq-nameserver1:8020","rocketmq-nameserver2:8020"};
String[] namenodes = {"nn1","nn2"};
conf.set("fs.defaultFS", "hdfs://" + nameservices);
conf.set("dfs.nameservices",nameservices);
conf.set("dfs.ha.namenodes." + nameservices, namenodes[0]+","+namenodes[1]);
conf.set("dfs.namenode.rpc-address." + nameservices + "." + namenodes[0], namenodesAddr[0]);
conf.set("dfs.namenode.rpc-address." + nameservices + "." + namenodes[1], namenodesAddr[1]);
conf.set("dfs.client.failover.proxy.provider." + nameservices,"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
String hdfsRPCUrl = "hdfs://" + nameservices + ":" + 8020;
// DistributedFileSystem dfs = new DistributedFileSystem();
// dfs.initialize(URI.create(hdfsRPCUrl),conf);
FileSystem dfs = FileSystem.get(new URI(hdfsRPCUrl), conf, "root");
try {
Path path = new Path("/dfs");
boolean result = dfs.mkdirs(path);
System.out.println(result);
} catch (IOException e) {
e.printStackTrace();
} finally{
try {
dfs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
