java操作hdfs
1 package hdfs; 2 3 import java.io.FileNotFoundException; 4 import java.io.IOException; 5 import java.net.URI; 6 7 import org.apache.hadoop.conf.Configuration; 8 import org.apache.hadoop.fs.FileStatus; 9 import org.apache.hadoop.fs.FileSystem; 10 import org.apache.hadoop.fs.LocatedFileStatus; 11 import org.apache.hadoop.fs.Path; 12 import org.apache.hadoop.fs.RemoteIterator; 13 import org.junit.Before; 14 import org.junit.Test; 15 16 public class HdfsClientEasy { 17 18 private FileSystem fs = null; 19 20 @Before 21 public void getFs() throws Exception { 22 23 // 拿到一个配置参数的封装对象,构造函数中就会对classpath下的xxx-site.xml文件进行解析 24 // 真实的项目工程中就应该把xxx-site.xml文件加入到工程中来 25 Configuration conf = new Configuration(); 26 // to set a parameter, figure out the filesystem is hdfs 27 conf.set("fs.defaultFS", "hdfs://localhost:8000/"); 28 conf.set("dfs.replication", "1"); 29 30 // 获取到一个具体文件系统的客户端实例对象,产生的实例究竟是哪一种文件系统的客户端,是根据conf中的相关参数来决定 31 // fs = FileSystem.get(conf); 32 33 //这种获取fs的方法可以指定访问hdfs的客户端身份 34 fs = FileSystem.get(new URI("hdfs://localhost:8000/"), conf, "wochu"); 35 36 } 37 38 /** 39 * 上传文件 40 * 41 * @throws IllegalArgumentException 42 * @throws IOException 43 */ 44 @Test 45 public void testUpload() throws IllegalArgumentException, IOException { 46 fs.copyFromLocalFile(new Path("/Users/wochu/Desktop/tool/hdfs/test.txt"), new Path("/xue")); 47 48 } 49 50 /** 51 * 删除文件 52 * 53 * @throws IllegalArgumentException 54 * @throws IOException 55 */ 56 @Test 57 public void testRmfile() throws IllegalArgumentException, IOException { 58 59 boolean res = fs.delete(new Path("/xue"), true); 60 61 System.out.println(res ? "delete is successfully :)" 62 : "it is failed :("); 63 64 } 65 66 /** 67 * 创建文件夹 68 * 69 * @throws IllegalArgumentException 70 * @throws IOException 71 */ 72 @Test 73 public void testMkdir() throws IllegalArgumentException, IOException { 74 fs.mkdirs(new Path("/xue")); 75 76 } 77 78 /** 79 * 重命名文件 80 * 81 * @throws IllegalArgumentException 82 * @throws IOException 83 */ 84 @Test 85 public void testRename() throws IllegalArgumentException, IOException { 86 87 fs.rename(new Path("/xue/test.txt"), new Path("/xue/rename.txt")); 88 89 } 90 91 /** 92 * 列出目录下的文件信息 93 * 94 * @throws FileNotFoundException 95 * @throws IllegalArgumentException 96 * @throws IOException 97 */ 98 @Test 99 public void testListFiles() throws FileNotFoundException, 100 IllegalArgumentException, IOException { 101 102 // 递归列出文件 103 RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles( 104 new Path("/"), true); 105 106 while (listFiles.hasNext()) { 107 108 LocatedFileStatus file = listFiles.next(); 109 110 System.out.println(file.getPath().getName()); 111 112 } 113 114 System.out.println("--------------------------------------------"); 115 116 // 列出文件及文件夹 117 FileStatus[] status = fs.listStatus(new Path("/")); 118 for (FileStatus file : status) { 119 120 System.out.println(file.getPath().getName() + " " 121 + (file.isDirectory() ? "d" : "f")); 122 123 } 124 125 } 126 127 /** 128 * 从hdfs中下载数据到本地 129 * 130 * @throws IllegalArgumentException 131 * @throws IOException 132 */ 133 @Test 134 public void testDownload() throws IllegalArgumentException, IOException { 135 136 fs.copyToLocalFile(new Path("/xue/rename.txt"), new Path("/Users/wochu/Desktop/tool/hdfs/name.txt")); 137 138 } 139 140 }