编程操作hdfs文件
可实现对文件的访问,如创建、修改、删除等。
工具是IDEA
代码如下:
import java.io.*; import java.util.Scanner; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.*; public class hdfs { //判断路径是否存在 public static boolean test(Configuration conf,String path) throws IOException { FileSystem fs=FileSystem.get(conf); return fs.exists(new Path(path)); } //创建目录 public static boolean mkdir (Configuration conf ,String remoteDir)throws IOException { FileSystem fs=FileSystem.get(conf); Path dirPath=new Path(remoteDir); boolean result=fs.mkdirs(dirPath); fs.close(); return result; } //创建文件 public static void touchz(Configuration conf,String remoteFilePath )throws IOException { FileSystem fs=FileSystem.get(conf); Path remotePath=new Path(remoteFilePath); FSDataOutputStream outputStream =fs.create(remotePath); outputStream.close(); fs.close(); } //删除文件 public static boolean rm(Configuration conf,String remoteFilePath)throws IOException { FileSystem fs=FileSystem.get(conf); Path remotePath=new Path(remoteFilePath); boolean result=fs.delete(remotePath,false); fs.close(); return result; } //读文件 public static void cat(Configuration conf,String FilePath)throws IOException { FileSystem fs=FileSystem.get(conf); Path file=new Path(FilePath); FSDataInputStream getIt=fs.open(file); BufferedReader d=new BufferedReader(new InputStreamReader(getIt)); String content=d.readLine(); System.out.println(content); d.close(); fs.close(); } //追加文件内容 public static void appendContentToFile(Configuration conf,String content,String remoteFilePath)throws IOException { System.out.println("追加的位置:1.开头 2.结尾"); Scanner tt=new Scanner(System.in); int cc=tt.nextInt(); //输入整型 if(cc==1){ //追加到开头 //先获取文件内容,再字符串拼接。 FileSystem fs=FileSystem.get(conf); Path file=new Path(remoteFilePath); FSDataInputStream getIt=fs.open(file); BufferedReader d=new BufferedReader(new InputStreamReader(getIt)); String cont=d.readLine(); fs.close(); hdfs.rm(conf,remoteFilePath);//删除 hdfs.touchz(conf, remoteFilePath);//新建 //字符串拼接 cont=content+cont; FileSystem fs1=FileSystem.get(conf); Path file1=new Path(remoteFilePath); FSDataOutputStream out=fs1.append(file1); out.write(cont.getBytes()); out.close(); fs1.close(); }else if(cc==2){ //追加到结尾 FileSystem fs=FileSystem.get(conf); Path remotePath=new Path(remoteFilePath); FSDataOutputStream out=fs.append(remotePath); out.write(content.getBytes()); out.close(); fs.close(); }else{ System.out.println("请重新输入!"); } } //将文件1写入文件2 public static void appendContentToFile2(Configuration conf,String remoteFilePath,String remoteFilePath2)throws IOException { FileSystem fs=FileSystem.get(conf); Path file=new Path(remoteFilePath); FSDataInputStream getIt=fs.open(file); BufferedReader d=new BufferedReader(new InputStreamReader(getIt)); String content1=d.readLine(); Path remotePath=new Path(remoteFilePath2); FSDataOutputStream out=fs.append(remotePath); out.write(content1.getBytes()); d.close(); out.close(); fs.close(); } public static void main(String[] args) { Configuration conf=new Configuration(); conf.set("dfs.client.block.write.replace-datanode-on-failure.policy","NEVER"); conf.set("dfs.client.block.write.replace-datanode-on-failure.enable","true"); conf.set("fs.default.name", "hdfs://192.168.58.128:9000"); conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem"); String remoteDir ="/user/hadoop/LZY";//HDFS目录 String remoteFilePath="/user/hadoop/LZY/hdfstest1.txt"; String content="信1805-820183542刘子煜HDFS课堂测试"; //创建目录 int i=1; while(i!=0){ Scanner input=new Scanner(System.in); System.out.println("请输入:"); System.out.println("1.新建文本目录 2.新建文本文件 3.编辑"); System.out.println("4.打开文本文件 5.删除文件 6.退出"); int contents=input.nextInt(); //输入整型 if(contents==1){ System.out.println("请输入目录路径"); Scanner input1=new Scanner(System.in); remoteDir =input1.next(); try { if(!hdfs.test(conf, remoteDir)) { hdfs.mkdir(conf, remoteDir); System.out.println("创建目录"+remoteDir); } else { System.out.println(remoteDir+"目录已存在"); } } catch (IOException e) { e.printStackTrace(); } }else if(contents==2){ System.out.println("请输入文件名字和具体路径"); Scanner input1=new Scanner(System.in); remoteFilePath =input1.next(); try { //创建文件 if (!hdfs.test(conf, remoteFilePath)) { hdfs.touchz(conf, remoteFilePath); System.out.println("创建文件" + remoteFilePath+"成功"); } else { System.out.println(remoteFilePath + "已存在,创建失败!"); } }catch (IOException e) { e.printStackTrace(); } }else if(contents==3){ try { //向文件内输入 System.out.println("请输入文本内容:"); Scanner input1=new Scanner(System.in); content =input1.next(); System.out.println("请输入文件目录:"); Scanner input2=new Scanner(System.in); remoteFilePath =input2.next(); hdfs.appendContentToFile(conf, content, remoteFilePath); hdfs.cat(conf, remoteFilePath); }catch (IOException e) { e.printStackTrace(); } }else if(contents==4){ try { System.out.println("请输入要打开的目录及文件名:"); Scanner input2=new Scanner(System.in); remoteFilePath =input2.next(); hdfs.cat(conf,remoteFilePath); }catch (IOException e) { e.printStackTrace(); } }else if(contents==5){ try { System.out.println("请输入要删除的目录及文件名:"); Scanner input2=new Scanner(System.in); remoteFilePath =input2.next(); hdfs.rm(conf,remoteFilePath); System.out.println(remoteFilePath+"已删除成功!"); }catch (IOException e) { e.printStackTrace(); } }else if(contents==6){ i=0; } } } }
运行如下: