Java:大文件拆分工具

java大文件拆分工具(过滤掉表头)

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;

public class SplitTools {
    public static void main(String[] args) throws IOException {
        // "E:\\eutranrelation.csv";
        String bigFilePath = args[0];
        // "E:\\eutranrelation-%s.csv";
        String splitFileParttern = args[1];
        // 32
        int splitFileCount = Integer.valueOf(args[2]);

        BufferedReader reader = new BufferedReader(new FileReader(bigFilePath));
        String header = reader.readLine();
        int totalLine = 0;
        if (header != null) {
            String line = null;
            while ((line = reader.readLine()) != null) {
                totalLine++;
            }
        }

        reader.close();

        int splitFileTotalLines = totalLine / splitFileCount;
        int generateFileIdx = 0;

        BufferedWriter bufferedWriter = null;
        reader = new BufferedReader(new FileReader(bigFilePath));
        header = reader.readLine();
        int currentIdx = 0;
        if (header != null) {
            String line = null;
            while ((line = reader.readLine()) != null) {
                if (currentIdx == 0) {
                    String filePath = String.format(splitFileParttern,
                            String.valueOf(generateFileIdx));
                    bufferedWriter = new BufferedWriter(
                            new FileWriter(filePath));
                }

                bufferedWriter.write(line + "\r\n");

                currentIdx++;

                if (currentIdx == splitFileTotalLines) {
                    generateFileIdx++;
                    if (generateFileIdx != splitFileCount) {
                        currentIdx = 0;
                        bufferedWriter.flush();
                        bufferedWriter.close();
                    }
                }
            }

            if (bufferedWriter != null) {
                bufferedWriter.flush();
                bufferedWriter.close();
            }
        }

        reader.close();

        System.out.println("The total number of documents is:" + totalLine);
    }
}

 

posted @ 2018-10-18 09:55  cctext  阅读(2003)  评论(0编辑  收藏  举报