读网文《将20M文件从30秒压缩到1秒,我是如何做到的?》做实验
先在微信公众号上看到网文《将20M文件从30秒压缩到1秒,我是如何做到的?》,然后在网上搜索了一下,看到了原文:https://www.jianshu.com/p/2e46ccb125ef
很惊奇他把时间压缩到了三十分之一,于是就有了做实验验证的想法。
我的实验对象是apache-tomcat-9.0.30.zip,Redis-x64-3.2.100.msi,Redis-x64-3.2.100.zip这三个文件,加起来21M,比作者的十张2M图片略多。
任务是把它们三个压缩成一个result.zip文件。
首先我实验的是rugularZip方法,也是作者提到的第一种方法:
// time elapsed:1s652ms private boolean rugularZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); byte[] buffer=new byte[BUFFER_SIZE]; int readLen=0; try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip)); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) { zipOut.write(buffer,0,readLen); } inputStream.close(); } } zipOut.close(); }catch(Exception e) { e.printStackTrace(); return false; } return true; }
用时1秒652毫秒。
接下来实验的是bufferOuputZip方法,也就是作者提到的第二种方法:
// time elapsed:1s207ms private boolean bufferOuputZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); byte[] buffer=new byte[BUFFER_SIZE]; int readLen=0; try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(zipOut) ; for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip)); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) { bufferedOutputStream.write(buffer,0,readLen); } inputStream.close(); } } bufferedOutputStream.flush(); zipOut.close(); return true; }catch(Exception e) { e.printStackTrace(); return false; } }
用时1秒207毫秒,少了四分之一。
接下来实验作者提出的第三种方法:
// elapsed:1s188ms private boolean nioChannalZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; WritableByteChannel wChannel=Channels.newChannel(zipOut); for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { FileChannel readChannel=new FileInputStream(fileWillZip).getChannel(); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); readChannel.transferTo(0, readChannel.size(),wChannel) ; readChannel.close(); } } wChannel.close(); zipOut.close(); return true; }catch(Exception e) { e.printStackTrace(); return false; } }
一秒188毫秒,没少多少。
接下来再看作者又提到了内存映射文件,他又说和第三种方法速度差不多哦,算了我就不试了。
作者最后又提到pip,我一看原来是用线程,这当然快了,因为主线程只要调用压缩线程就可以返回了,压缩线程消耗的时间不计算在主线程运行时间内,运行当然是秒回。
当然pip里面有个阻塞回调的过程,这消耗了一些时间。
更新DB,写文件都可以另起线程运行,这也是以空间换时间的方法之一。
完整程序:
package zip; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.InputStream; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.WritableByteChannel; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; // Used to zip a file public class FileZipper { private static final int BUFFER_SIZE = 1024; public boolean compressFilesToZip(String[] files,String zipfile) { return threadZip(files,zipfile); } private boolean threadZip(String[] fromFiles,String toFile) { new ZipThread(fromFiles,toFile).start();; return true; } // elapsed:1s188ms private boolean nioChannalZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; WritableByteChannel wChannel=Channels.newChannel(zipOut); for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { FileChannel readChannel=new FileInputStream(fileWillZip).getChannel(); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); readChannel.transferTo(0, readChannel.size(),wChannel) ; readChannel.close(); } } wChannel.close(); zipOut.close(); return true; }catch(Exception e) { e.printStackTrace(); return false; } } // time elapsed:1s207ms private boolean bufferOuputZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); byte[] buffer=new byte[BUFFER_SIZE]; int readLen=0; try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(zipOut) ; for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip)); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) { bufferedOutputStream.write(buffer,0,readLen); } inputStream.close(); } } bufferedOutputStream.flush(); zipOut.close(); return true; }catch(Exception e) { e.printStackTrace(); return false; } } // time elapsed:1s652ms private boolean rugularZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); byte[] buffer=new byte[BUFFER_SIZE]; int readLen=0; try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip)); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) { zipOut.write(buffer,0,readLen); } inputStream.close(); } } zipOut.close(); }catch(Exception e) { e.printStackTrace(); return false; } return true; } /** * change seconds to DayHourMinuteSecond format * * @param startMs * @param endMs * @return */ private static String ms2DHMS(long startMs, long endMs) { String retval = null; long secondCount = (endMs - startMs) / 1000; String ms = (endMs - startMs) % 1000 + "ms"; long days = secondCount / (60 * 60 * 24); long hours = (secondCount % (60 * 60 * 24)) / (60 * 60); long minutes = (secondCount % (60 * 60)) / 60; long seconds = secondCount % 60; if (days > 0) { retval = days + "d" + hours + "h" + minutes + "m" + seconds + "s"; } else if (hours > 0) { retval = hours + "h" + minutes + "m" + seconds + "s"; } else if (minutes > 0) { retval = minutes + "m" + seconds + "s"; } else if(seconds > 0) { retval = seconds + "s"; }else { return ms; } return retval + ms; } public static String calculateElaspedTime(long startMs) { long endMs = System.currentTimeMillis(); return ms2DHMS(startMs,endMs); } public static void main(String[] args) { String[] files= {"D:\\usr\\apache-tomcat-9.0.30.zip", "D:\\usr\\Redis-x64-3.2.100.msi", "D:\\usr\\Redis-x64-3.2.100.zip"}; String zipfile="D:\\usr\\result.zip"; long startMs = System.currentTimeMillis(); FileZipper fz=new FileZipper(); boolean isCreated=fz.compressFilesToZip(files, zipfile); if(isCreated) { System.out.println("File:'"+zipfile+"' created,time elapsed:"+calculateElaspedTime(startMs)); } } }
package zip; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.WritableByteChannel; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; public class ZipThread extends Thread{ private String[] files; private String zipfile; public ZipThread(String[] files,String zipfile) { this.files=files; this.zipfile=zipfile; } public void run() { nioChannalZip(this.files,this.zipfile); } private boolean nioChannalZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; WritableByteChannel wChannel=Channels.newChannel(zipOut); for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { FileChannel readChannel=new FileInputStream(fileWillZip).getChannel(); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); readChannel.transferTo(0, readChannel.size(),wChannel) ; readChannel.close(); } } wChannel.close(); zipOut.close(); return true; }catch(Exception e) { e.printStackTrace(); return false; } } }
--END-- 2020-01-06 14:54
分类:
Java.文本文件操作
【推荐】国内首个AI IDE,深度理解中文开发场景,立即下载体验Trae
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步
· Linux系列:如何用heaptrack跟踪.NET程序的非托管内存泄露
· 开发者必知的日志记录最佳实践
· SQL Server 2025 AI相关能力初探
· Linux系列:如何用 C#调用 C方法造成内存泄露
· AI与.NET技术实操系列(二):开始使用ML.NET
· 无需6万激活码!GitHub神秘组织3小时极速复刻Manus,手把手教你使用OpenManus搭建本
· C#/.NET/.NET Core优秀项目和框架2025年2月简报
· Manus爆火,是硬核还是营销?
· 终于写完轮子一部分:tcp代理 了,记录一下
· 【杭电多校比赛记录】2025“钉耙编程”中国大学生算法设计春季联赛(1)
2017-01-06 一次性上传多个文件到服务器端(一)