将存储在本地的大量分散的小文件，合并并保存在hdfs文件系统中

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90

import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URI;    import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path;    public class PutMerge {        private InputStream in = null;     private OutputStream out = null;     private String localPath;     private String hdfsPath;        public PutMerge(String localPath, String hdfsPath) {         this.localPath = localPath;         this.hdfsPath = hdfsPath;     }        public void start() throws Exception {         File file = new File(localPath);         Configuration conf = new Configuration();            FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);         out = fs.create(new Path(hdfsPath));            work(file);            close();     }        /*      * 1.根据用户定义的参数设置本地目录和HDFS的目标文件      *      * 2.创建一个输出流写入到HDFS文件      *      * 3.遍历本地目录中的每个文件，打开文件，并读取文件内容，将文件的内容写到HDFS文件中。      */     private void work(File file) throws Exception {         if (file.isFile()) {             int byteRead = 0;             byte[] buffer = new byte[256];             while ((byteRead = in.read(buffer)) > 0) {                 out.write(buffer, 0, byteRead);             }         } else if (file.isDirectory()) {             File[] files = file.listFiles();             for (int i = 0; i < files.length; i++) {                 if (files[i].isFile()) {                     in = new BufferedInputStream(new FileInputStream(files[i]));                     int byteRead = 0;                     byte[] buffer = new byte[256];                     while ((byteRead = in.read(buffer)) > 0) {                         out.write(buffer, 0, byteRead);                     }                 } else {                     work(files[i]);                 }             }         }     }        private void close() throws IOException {         if (in != null) {             in.close();         }            if (out != null) {             out.close();         }     }        /**      * @param args      * @throws IOException      */     public static void main(String[] args) throws Exception {         if (args.length < 2) {             System.out.println("Usage:\n\t " + PutMerge.class.getName()                     + " [LocalPath] [HDFSPath]");             System.exit(1);         }         new PutMerge(args[0], args[1]).start();     } }