1.maven 创建项目项目环境
2.配置hadoop环境依赖
https://mvnrepository.com/ // maven 网上仓库
配置 pom.xml
<properties>
<!--定义hadoop 版本-->
<hadoop.version>2.6.0-cdh5.9.0</hadoop.version>
</properties>
<repositories>
<repository>
<!--添加 cdh hadoop 仓库地址-->
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<!-- 添加junit 依赖包 -->
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<!-- 添加hadoop 依赖包 -->
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<!--<version>2.6.0-cdh5.9.0</version>-->
<version>${hadoop.version}</version>
</dependency>
</dependencies>
3.Hadoop Java AIP基本操作
package com.imooc.bigdata.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;
/**
* 使用Java API操作HDFS文件系统
*
* 关键点：
* 1）创建Configuration
* 2）获取FileSystem
* 3）...就是你的HDFS API的操作
*/
public class HDFSApp {
public static final String HDFS_PATH = "hdfs://hadoop000:8020";
FileSystem fileSystem = null;
Configuration configuration = null;
@Before
public void setUp() throws Exception {
System.out.println("--------setUp---------");
configuration = new Configuration();
configuration.set("dfs.replication","1");
/**
* 构造一个访问指定HDFS系统的客户端对象
* 第一个参数：HDFS的URI
* 第二个参数：客户端指定的配置参数
* 第三个参数：客户端的身份，说白了就是用户名
*/
fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "hadoop");
}
/**
* 创建HDFS文件夹
*/
@Test
public void mkdir() throws Exception {
fileSystem.mkdirs(new Path("/hdfsapi/test"));
}
/**
* 查看HDFS内容
*/
@Test
public void text()throws Exception {
FSDataInputStream in = fileSystem.open(new Path("/cdh_version.properties"));
IOUtils.copyBytes(in, System.out, 1024);
}
/**
* 创建文件
*/
@Test
public void create()throws Exception {
// FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/a.txt"));
FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/b.txt"));
out.writeUTF("hello pk: replication 1");
out.flush();
out.close();
}
/**
* 测试文件名更改
* @throws Exception
*/
@Test
public void rename() throws Exception {
Path oldPath = new Path("/hdfsapi/test/b.txt");
Path newPath = new Path("/hdfsapi/test/c.txt");
boolean result = fileSystem.rename(oldPath, newPath);
System.out.println(result);
}
/**
* 拷贝本地文件到HDFS文件系统
*/
@Test
public void copyFromLocalFile() throws Exception {
Path src = new Path("/Users/rocky/data/hello.txt");
Path dst = new Path("/hdfsapi/test/");
fileSystem.copyFromLocalFile(src,dst);
}
/**
* 拷贝大文件到HDFS文件系统：带进度
*/
@Test
public void copyFromLocalBigFile() throws Exception {
InputStream in = new BufferedInputStream(new FileInputStream(new File("/Users/rocky/tmp/software/jdk-8u91-linux-x64.tar.gz")));
FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/jdk.tgz"),
new Progressable() {
public void progress() {
System.out.print(".");
}
});
IOUtils.copyBytes(in, out ,4096);
}
/**
* 拷贝HDFS文件到本地：下载
*/
@Test
public void copyToLocalFile() throws Exception {
Path src = new Path("/hdfsapi/test/hello.txt");
Path dst = new Path("/Users/rocky/tmp/software");
fileSystem.copyToLocalFile(src, dst);
}
/**
* 查看目标文件夹下的所有文件
*/
@Test
public void listFiles() throws Exception {
FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi/test"));
for(FileStatus file : statuses) {
String isDir = file.isDirectory() ? "文件夹" : "文件";
String permission = file.getPermission().toString();
short replication = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir + "\t" + permission
+ "\t" + replication + "\t" + length
+ "\t" + path
);
}
}
/**
* 递归查看目标文件夹下的所有文件
*/
@Test
public void listFilesRecursive() throws Exception {
RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/hdfsapi/test"), true);
while (files.hasNext()) {
LocatedFileStatus file = files.next();
String isDir = file.isDirectory() ? "文件夹" : "文件";
String permission = file.getPermission().toString();
short replication = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir + "\t" + permission
+ "\t" + replication + "\t" + length
+ "\t" + path
);
}
}
/**
* 查看文件块信息
*/
@Test
public void getFileBlockLocations() throws Exception {
FileStatus fileStatus = fileSystem.getFileStatus(new Path("/hdfsapi/test/jdk.tgz"));
BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus,0,fileStatus.getLen());
for(BlockLocation block : blocks) {
for(String name: block.getNames()) {
System.out.println(name +" : " + block.getOffset() + " : " + block.getLength() + " : " + block.getHosts());
}
}
}
/**
* 删除文件
*/
@Test
public void delete() throws Exception {
boolean result = fileSystem.delete(new Path("/hdfsapi/test/jdk.tgz"), true);
System.out.println(result);
}
@Test
public void testReplication() {
System.out.println(configuration.get("dfs.replication"));
}
@After
public void tearDown() {
configuration = null;
fileSystem = null;
System.out.println("--------tearDown---------");
}
}
