关于BufferedWriter.write超过30W条数据写入过慢问题。
原创文章，转载请注明出处！
------------------------------------------------------------
今天接到一个项目需求变更，是关于从数据库查询到30W条数据放到一个List中，然后使用StringBuffer把List取出放入到StringBuffer中。最后使用 BufferedWriter.write(buffer.toString());添加到文件中，此时会消耗几个小时才能把数据存放到文件中。
项目源码：
1 public void generateCommonSMSFile(String localFilePath,String date) throws Exception{ 2 try { 3 List smsList = getSingleDataSMS("03",date); 4 List listid = new ArrayList(); 5 List subList = new ArrayList(); 6 StringBuffer buffer = new StringBuffer("S|2|S0355||||"); 7 buffer.append(smsList.size() + "|"); 8 buffer.append("\r\n"); 9 if(smsList != null && smsList.size() != 0) { 10 for(int i = 0;i < smsList.size();i++) { 11 OmsSingleDataDTO msdd = (OmsSingleDataDTO)smsList.get(i); 12 String transDate = msdd.getTransDate(); 13 String leastDate = msdd.getLeastDate(); 14 buffer.append(String.valueOf(i+1)); 15 buffer.append("|"); 16 buffer.append(msdd.getMobile()); 17 buffer.append("|"); 18 buffer.append(msdd.getCustNo()); 19 buffer.append("|"); 20 buffer.append(msdd.getAdvMemo1()); 21 buffer.append("|"); 22 buffer.append(CommonUtil.deleteZero(transDate.substring(5, 7))+"月"+CommonUtil.deleteZero(transDate.substring(8))+"日"); 23 buffer.append("|"); 24 buffer.append(msdd.getTransAmount()); 25 buffer.append("|"); 26 buffer.append(CommonUtil.deleteZero(leastDate.substring(4, 6))+"月"+CommonUtil.deleteZero(leastDate.substring(6))+"日"); 27 buffer.append("|"); 28 buffer.append("FQ"+msdd.getConsumeCode()); 29 buffer.append("|"); 30 buffer.append(msdd.getInstNum()); 31 buffer.append("|"); 32 buffer.append(msdd.getFirstAmount()); 33 buffer.append("|"); 34 buffer.append(msdd.getEachAmount()); 35 buffer.append("|"); 36 buffer.append(msdd.getTotalPoundage()); 37 buffer.append("|"); 38 buffer.append(msdd.getAdvMemo2()); 39 buffer.append("|"); 40 buffer.append("\r\n"); 41 subList.add(msdd.getSeqId()); 42 if((i+1)%5000==0){ 43 listid.add(subList); 44 subList=new ArrayList(); 45 } 46 } 47 listid.add(subList); 48 } 49 File file1 = FileUtils.createFile(localFilePath); 50 BufferedWriter bw = FileUtils.getWrite(file1); 51 bw.write(buffer.toString()); 52 bw.close(); 53 if(listid != null && listid.size() >= 1) { 54 for(int i=0;i<listid.size();i++) { 55 OMSLogger.info("普通消费类 - 第"+ (i+1) +"次运行开始"); 56 long msBefore = System.currentTimeMillis(); 57 58 updateMosSingleData((List)listid.get(i)); 59 60 long msAfter = System.currentTimeMillis(); 61 OMSLogger.info("普通消费类 - 第"+ (i+1) +"次运行结束，共耗时" + (msAfter - msBefore) + "毫秒"); 62 } 63 } 64 } catch (Exception e) { 65 OMSLogger.error("生成单笔分期批量短信文件异常"); 66 throw new Exception("生成单笔分期批量短信文件异常",e); 67 } 68 69 }
查询数据库，返回list供上面的数据写入文件的方法使用
1 private List getSingleDataSMS(String instFlag,String vdate) throws PafaDAOException { 2 3 List SmsList = new ArrayList(); 4 List list = new ArrayList(); 5 OmsSingleDataDTO msdd = null; 6 // list = executeSqlDao.queryForList("SELECT-SMS-SingleData-LIST",null); 7 HashMap map =new HashMap(); 8 map.put("instFlag", instFlag); 9 map.put("vdate", vdate); 10 int total = 0; 11 total = ((Integer)executeSqlDao.queryForObject("SELECT-SMS-SingleData-COUNT", map)).intValue(); 12 // 防止大于1万条时出错,循环处理 13 for(int i=0; i<total; i+=5000){ 14 HashMap paramMap =new HashMap(); 15 List rs = new ArrayList(); 16 paramMap.put("startNum", String.valueOf(i+1)); 17 paramMap.put("endNum", String.valueOf(i+5000)); 18 paramMap.put("instFlag", instFlag); 19 paramMap.put("vDate", vdate); 20 rs = executeSqlDao.queryForList("SELECT-SMS-SingleData-LIST",paramMap); 21 if (rs!=null){ 22 for(int j=0; j<rs.size(); j++){ 23 list.add(rs.get(j)); 24 } 25 } 26 } 27 28 if(list != null && list.size() >= 1) { 29 Set set = commonService.getLogoValue(null); 30 for(int m = 0;m < list.size();m++) { 31 msdd = (OmsSingleDataDTO)list.get(m); 32 if(!OrderBaseService.getLogIsValid(msdd.getCardNo(),"14",set)) { 33 // 更新处理状态 34 this.updateMosSingleData2(msdd.getSeqId()); 35 continue; 36 } 37 String custNo = msdd.getCustNo(); 38 custNo = custNo.length() >= 12 ? custNo.substring(custNo.length() - 12):custNo; 39 msdd.setCustNo(custNo); 40 SmsList.add(msdd); 41 } 42 } 43 44 return SmsList; 45 }
以上是使用的一次全部查询出数据，然后一次写入到文件中，但是这种方法会用几个小时才能完全写入。下面是自己优化后的方案：
1 public void generateSMSFile(String localFilePath,String date) throws Exception{ 2 try { 3 File file1 = FileUtils.createFile(localFilePath); 4 BufferedWriter bw = FileUtils.getWrite(file1); 5 StringBuffer buffer = new StringBuffer("S|1|S0180||||"); 6 7 List smslistCount = getSingleDataSMS("03", date); 8 buffer.append(smslistCount.size() + "|"); 9 10 buffer.append("\r\n"); 11 12 List subList = new ArrayList(); 13 List listid = new ArrayList(); 14 int count = 1; 15 int total = 0; 16 total = generateSMSFilestpeSize(date); 17 18 //防止大于1万条时出错,循环处理 19 for(int k=0; k<total; k+=5000){ 20 List smsList = getSingleDataSMSstep("03", date, k, k+5000); 21 if(smsList != null && smsList.size() != 0) { 22 for(int i = 0;i < smsList.size();i++) { 23 OmsSingleDataDTO msdd = (OmsSingleDataDTO)smsList.get(i); 24 String transDate = msdd.getTransDate(); 25 String leastDate = msdd.getLeastDate(); 26 count = count +1; 27 buffer.append(String.valueOf(count)); 28 buffer.append("|"); 29 buffer.append(msdd.getMobile()); 30 buffer.append("|"); 31 buffer.append(msdd.getCustNo()); 32 buffer.append("|"); 33 buffer.append(transDate.substring(5,7)); 34 buffer.append("|"); 35 buffer.append(transDate.substring(8,10)); 36 buffer.append("|"); 37 buffer.append(msdd.getTransAmount()); 38 buffer.append("|"); 39 buffer.append(msdd.getFirstAmount()); 40 buffer.append("|"); 41 buffer.append(msdd.getEachAmount()); 42 buffer.append("|"); 43 buffer.append(leastDate.substring(4,6)); 44 buffer.append("|"); 45 buffer.append(leastDate.substring(6,8)); 46 buffer.append("|"); 47 buffer.append(msdd.getConsumeCode()); 48 buffer.append("|"); 49 buffer.append("\r\n"); 50 subList.add(msdd.getSeqId()); 51 if((i+1)%5000==0){ 52 listid.add(subList); 53 subList=new ArrayList(); 54 } 55 } 56 listid.add(subList); 57 } 58 bw.write(buffer.toString()); 59 buffer = new StringBuffer(); 60 if(listid != null && listid.size() >= 1) { 61 for(int i=0;i<listid.size();i++) { 62 OMSLogger.info("保费类 - 第"+ (count) +"次运行开始"); 63 long msBefore = System.currentTimeMillis(); 64 65 updateMosSingleData((List)listid.get(i)); 66 67 long msAfter = System.currentTimeMillis(); 68 OMSLogger.info("保费类 - 第"+ (count) +"次运行结束，共耗时" + (msAfter - msBefore) + "毫秒"); 69 } 70 } 71 } 72 bw.close(); 73 74 } catch (Exception e) { 75 OMSLogger.error("生成单笔分期批量短信文件异常"); 76 throw new Exception("生成单笔分期批量短信文件异常",e); 77 } 78 }
每次获取5000条数据返回list，写入文件之后再次执行该方法在获取5000条数据
1 private List getSingleDataSMSstep(String instFlag,String vdate,int startNum,int endNum) throws PafaDAOException { 2 3 List SmsList = new ArrayList(); 4 List list = new ArrayList(); 5 OmsSingleDataDTO msdd = null; 6 HashMap paramMap =new HashMap(); 7 List rs = new ArrayList(); 8 paramMap.put("startNum", startNum); 9 paramMap.put("endNum", endNum); 10 paramMap.put("instFlag", instFlag); 11 paramMap.put("vDate", vdate); 12 rs = executeSqlDao.queryForList("SELECT-SMS-SingleData-LIST",paramMap); 13 if (rs!=null){ 14 for(int j=0; j<rs.size(); j++){ 15 list.add(rs.get(j)); 16 } 17 } 18 19 if(list != null && list.size() >= 1) { 20 Set set = commonService.getLogoValue(null); 21 for(int m = 0;m < list.size();m++) { 22 msdd = (OmsSingleDataDTO)list.get(m); 23 if(!OrderBaseService.getLogIsValid(msdd.getCardNo(),"14",set)) { 24 // 更新处理状态 25 this.updateMosSingleData2(msdd.getSeqId()); 26 continue; 27 } 28 String custNo = msdd.getCustNo(); 29 custNo = custNo.length() >= 12 ? custNo.substring(custNo.length() - 12):custNo; 30 msdd.setCustNo(custNo); 31 SmsList.add(msdd); 32 } 33 } 34 35 return SmsList; 36 }
补充一个自己写的小例子：
1 package myTempTest; 2 3 import java.io.BufferedWriter; 4 import java.io.File; 5 import java.io.FileWriter; 6 import java.io.IOException; 7 8 public class ioTestBuffer { 9 public static void main(String[] args) throws IOException { 10 File file1 = new File("d:\\io\\out.txt");//最终写入文件地址 11 BufferedWriter bw = new BufferedWriter(new FileWriter(file1)); 12 StringBuffer buffer = new StringBuffer(""); 13 for (int j = 0; j < 30; j++) { 14 //之所以I循环1W次就写出一次，是因为buffer超过1W次append之后极容易在次append的时候报错。 15 for (int i = 0; i < 10000; i++) { 16 buffer.append(i+""+i+""+i+""+i+""+i+""+i+""+i+""+i+""+i+""+i+""); 17 buffer.append("\r\n");//换行 18 } 19 bw.write(buffer.toString());//1W次I循环结束 写入到文件中 20 buffer = new StringBuffer("");//格式化buffer。 21 buffer.append(j+"-------------------------------------");//第 j 此循环加上个记号 22 buffer.append("\r\n");//一次J循环结束 换行 23 } 24 bw.close(); 25 } 26 }
