Eclipse+hadoop伪态式分布+API

Eclipse+hadoop伪态式分布+API

1、在eclipse中新建一个Maven文件，下载对应jar包

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>2.9.2</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>2.9.2</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-hdfs</artifactId>
    <version>2.9.2</version>
</dependency>

2、编写脚本代码（浏览hdfs目录）

public class Test {
	public static void main(String[] args) throws IOException {
		// TODO Auto-generated method stub
		String dir = "hdfs://192.168.80.128:9000/";
		try {
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(dir),conf);
			FileStatus[] status = fs.listStatus(new Path(dir));
			List<String> names = new ArrayList<String>();
			for (int i = 0; i < status.length; ++i) {
				if (status[i].isFile()) {
					names.add(status[i].getPath().toString());
					System.out.println(status[i].getPath().toString());
				}else if (status[i].isDirectory()) {
					names.add(status[i].getPath().toString());
					System.out.println(status[i].getPath().toString());
				}
			}
			fs.close();			
		} catch (Exception e) {
			// TODO: handle exception
		}		
	}
}

3、如果出现一下报错

在src下创建文件log4j.properties

log4j.rootLogger=WARN, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n

4、即可使用java编程来对hadoop进行操作

API操作

1、浏览hdfs目录文件（文件夹和文件）

public class ListFileAndFolder {

	public static void main(String[] args) throws IOException {
		// TODO Auto-generated method stub
		String dir = "hdfs://192.168.80.128:9000/";//地址
		try {
			Configuration conf = new Configuration();//加载配置文件
			FileSystem fs = FileSystem.get(URI.create(dir),conf);//获取文件系统实例
			FileStatus[] status = fs.listStatus(new Path("/test"));//待获取目录
			List<String> names = new ArrayList<String>();//存放获取的目录链接
			for (int i = 0; i < status.length; ++i) {
				if (status[i].isFile()) {//文件
					names.add(status[i].getPath().toString());
					System.out.println(status[i].getPath().toString());
				}else if (status[i].isDirectory()) {//文件夹
					names.add(status[i].getPath().toString());
					System.out.println(status[i].getPath().toString());
				}
			}
			fs.close();			
		} catch (Exception e) {
			// TODO: handle exception
			System.out.println("失败");
		}
	}
}

2、创建文件夹

public class CreateFolder {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String uri = "hdfs://192.168.80.128:9000/";//端口9000
		try {
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//使用root用户才有权限
			String pathString = "/wujf";//文件夹名
			boolean exists = fs.exists(new Path(pathString));
			if(!exists){//文件夹不存在则创建
			boolean result = fs.mkdirs(new Path(pathString));
				System.out.println(result);
			}
			fs.close();	
		} catch (Exception e) {
			// TODO: handle exception
			System.out.println("失败");
		}
	}
}

3、创建文件并且写入内容

public class WriteFile {
	public static void main(String[] args) {
		String uri = "hdfs://192.168.80.128:9000/";
		try {
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//root用户
			String pathString = "/wujf/wujf"+".txt";//路径以及文件名
			FSDataOutputStream fsDataOutputStream = fs.create(new Path(pathString));
			String inpuString = "I am wujf";//写入的内容
			IOUtils.copyBytes(new ByteArrayInputStream(inpuString.getBytes()),fsDataOutputStream,conf,true);
			fs.close();	
		} catch (Exception e) {
			// TODO: handle exception
			System.out.println("失败");
		}
	}
}

4、复制文件

public class CopyFile {
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String uri = "hdfs://192.168.80.128:9000/";
		try {
			Configuration conf = new Configuration();
            //root用户才有权限创建文件
			FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");
			String pathString = "/output/write1"+".txt";//复制后的文件路径以及文件名
			FSDataOutputStream fsDataOutputStream = fs.create(new Path(pathString));
			String inpuString = "/input/test1.txt";//待复制的文件
			IOUtils.copyBytes(new ByteArrayInputStream(inpuString.getBytes()),fsDataOutputStream,conf,true);
            //参数1：输入流，参数2：输出流；参数3：配置对象，参数4：是否关闭流
			fs.close();	
		} catch (Exception e) {
			// TODO: handle exception
			System.out.println("失败");
		}
	}
}

5、读取文件内容

public class ReadFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String uri = "hdfs://192.168.80.128:9000/";
		try {
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");
			String pathString = "/output/write2"+".txt";//待读取的文件
			FSDataInputStream fsDataInputStream = fs.open(new Path(pathString));
			IOUtils.copyBytes(fsDataInputStream, System.out, conf ,true);
			fs.close();
		} catch (Exception e) {
			// TODO: handle exception
			System.out.print("失败");
		}
	}
}

6、删除文件

public class DeleteFolder {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String uri = "hdfs://192.168.80.128:9000/";
		try {
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");
			String pathString = "/output/write"+".txt";
			System.out.print(fs.deleteOnExit(new Path(pathString)));
			fs.close();
		} catch (Exception e) {
			// TODO: handle exception
			System.out.print("失败");
		}
	}
}

7、上传本地文件

public class UploadFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String uri = "hdfs://192.168.80.128:9000/";
		try {
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//root用户权限高，不容易出现其他问题
			Path srcPath = new Path("D:\\temp\\abc.txt");//本地文件路径
			Path upPath=new Path("/");//上传文件的保存位置,存放于根目录
			fs.copyFromLocalFile(srcPath, upPath);
		} catch (Exception e) {
			// TODO: handle exception
			System.out.println("失败");
		}
	}

}

8、下载文件到本地

public class DownloadFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String uri = "hdfs://192.168.80.128:9000/";
		try {
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//root用户权限高，不容易出现其他问题
			
			Path inputStream = new Path("/write.txt");//待下载文件
			Path outputStream = new Path("D:\\temp");//存放路径
			//参数1：是否删除源文件，参数2：待下载文件路径，参数3：存放路径，参数4:是否开启文件校验
			fs.copyToLocalFile(false, inputStream, outputStream, true);
		} catch (Exception e) {
			// TODO: handle exception
			System.out.println("失败");
		}
	}
}