大文件读写
import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; public class FileSplit { public static void main(String[] args) throws IOException { long timer = System.currentTimeMillis(); int bufferSize = 20 * 1024 * 1024;//设读取文件的缓存为20MB //建立缓冲文本输入流 File file = new File("/media/Data/毕业设计/kdd cup/数据/userid_profile.txt"); FileInputStream fileInputStream = new FileInputStream(file); BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream); InputStreamReader inputStreamReader = new InputStreamReader(bufferedInputStream); BufferedReader input = new BufferedReader(inputStreamReader, bufferSize); int splitNum = 112-1;//要分割的块数减一 int fileLines = 23669283;//输入文件的行数 long perSplitLines = fileLines / splitNum;//每个块的行数 for (int i = 0; i <= splitNum; ++i) { //分割 //每个块建立一个输出 FileWriter output = new FileWriter("/home/haoqiong/part" + i + ".txt"); String line = null; //逐行读取，逐行输出 for (long lineCounter = 0; lineCounter < perSplitLines && (line = input.readLine()) != null; ++lineCounter) { output.append(line + "\r"); } output.flush(); output.close(); output = null; } input.close(); timer = System.currentTimeMillis() - timer; System.out.println("处理时间：" + timer); } }
方法2
void largeFileIO(String inputFile, String outputFile) { try { BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(inputFile))); BufferedReader in = new BufferedReader(new InputStreamReader(bis, "utf-8"), 10 * 1024 * 1024);//10M缓存 FileWriter fw = new FileWriter(outputFile); while (in.ready()) { String line = in.readLine(); fw.append(line + " "); } in.close(); fw.flush(); fw.close(); } catch (IOException ex) { ex.printStackTrace(); }
//NIO 读写
package com.netty.demo1.vera.demo; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Date; import java.util.List; interface Callback { void action(String line); } /** * Hello world! * */ public class NIOReadline { public static void main(String[] args) throws IOException { int bufSize = 1000000;//一次读取的字节长度 File fin = new File("C:\\Devs\\abc.txt");//读取的文件 File fout = new File("C:\\Devs\\out.txt");//写出的文件 Date startDate = new Date(); FileChannel fcin = new RandomAccessFile(fin, "r").getChannel(); ByteBuffer rBuffer = ByteBuffer.allocate(bufSize); FileChannel fcout = new RandomAccessFile(fout, "rws").getChannel(); ByteBuffer wBuffer = ByteBuffer.allocateDirect(bufSize); readFileByLine(bufSize, fcin, rBuffer, fcout, wBuffer); Date endDate = new Date(); System.out.print(startDate+"|"+endDate);//测试执行时间 if(fcin.isOpen()){ fcin.close(); } if(fcout.isOpen()){ fcout.close(); } } public static void readFileByLine(int bufSize, FileChannel fcin, ByteBuffer rBuffer, FileChannel fcout, ByteBuffer wBuffer) { String enter = "\n"; List<String> dataList = new ArrayList<String>();//存储读取的每行数据 byte[] lineByte = new byte[0]; String encode = "GBK"; // String encode = "UTF-8"; try { //temp：由于是按固定字节读取，在一次读取中，第一行和最后一行经常是不完整的行，因此定义此变量来存储上次的最后一行和这次的第一行的内容， //并将之连接成完成的一行，否则会出现汉字被拆分成2个字节，并被提前转换成字符串而乱码的问题 byte[] temp = new byte[0]; while (fcin.read(rBuffer) != -1) {//fcin.read(rBuffer)：从文件管道读取内容到缓冲区(rBuffer) int rSize = rBuffer.position();//读取结束后的位置，相当于读取的长度 byte[] bs = new byte[rSize];//用来存放读取的内容的数组 rBuffer.rewind();//将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法 rBuffer.get(bs);//相当于rBuffer.get(bs,0,bs.length())：从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域 rBuffer.clear(); int startNum = 0; int LF = 10;//换行符 int CR = 13;//回车符 boolean hasLF = false;//是否有换行符 for(int i = 0; i < rSize; i++){ if(bs[i] == LF){ hasLF = true; int tempNum = temp.length; int lineNum = i - startNum; lineByte = new byte[tempNum + lineNum];//数组大小已经去掉换行符 System.arraycopy(temp, 0, lineByte, 0, tempNum);//填充了lineByte[0]~lineByte[tempNum-1] temp = new byte[0]; System.arraycopy(bs, startNum, lineByte, tempNum, lineNum);//填充lineByte[tempNum]~lineByte[tempNum+lineNum-1] String line = new String(lineByte, 0, lineByte.length, encode);//一行完整的字符串(过滤了换行和回车) dataList.add(line); // System.out.println(line); writeFileByLine(fcout, wBuffer, line + enter); //过滤回车符和换行符 if(i + 1 < rSize && bs[i + 1] == CR){ startNum = i + 2; }else{ startNum = i + 1; } } } if(hasLF){ temp = new byte[bs.length - startNum]; System.arraycopy(bs, startNum, temp, 0, temp.length); }else{//兼容单次读取的内容不足一行的情况 byte[] toTemp = new byte[temp.length + bs.length]; System.arraycopy(temp, 0, toTemp, 0, temp.length); System.arraycopy(bs, 0, toTemp, temp.length, bs.length); temp = toTemp; } } if(temp != null && temp.length > 0){//兼容文件最后一行没有换行的情况 String line = new String(temp, 0, temp.length, encode); dataList.add(line); // System.out.println(line); writeFileByLine(fcout, wBuffer, line + enter); } } catch (IOException e) { e.printStackTrace(); } } /** * 写到文件上 * @param fcout * @param wBuffer * @param line */ @SuppressWarnings("static-access") public static void writeFileByLine(FileChannel fcout, ByteBuffer wBuffer, String line) { try { fcout.write(wBuffer.wrap(line.getBytes("UTF-8")), fcout.size()); } catch (IOException e) { e.printStackTrace(); } } }
