读网文《将20M文件从30秒压缩到1秒，我是如何做到的？》做实验

先在微信公众号上看到网文《将20M文件从30秒压缩到1秒，我是如何做到的？》，然后在网上搜索了一下，看到了原文：https://www.jianshu.com/p/2e46ccb125ef

很惊奇他把时间压缩到了三十分之一，于是就有了做实验验证的想法。

我的实验对象是apache-tomcat-9.0.30.zip，Redis-x64-3.2.100.msi，Redis-x64-3.2.100.zip这三个文件，加起来21M，比作者的十张2M图片略多。

任务是把它们三个压缩成一个result.zip文件。

首先我实验的是rugularZip方法，也是作者提到的第一种方法：

// time elapsed:1s652ms
    private boolean rugularZip(String[] fromFiles,String toFile) {
        File zipFile=new File(toFile);
        byte[] buffer=new byte[BUFFER_SIZE];
        int readLen=0;
        
        try {
            ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ;

            for(String file:fromFiles) {
                File fileWillZip=new File(file);
                
                if(fileWillZip.exists()) {
                    InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip));
                    String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed
                    zipOut.putNextEntry(new ZipEntry(entryName));
    
                    while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) {
                        zipOut.write(buffer,0,readLen);
                    }
                    inputStream.close();    

                }
            }

            zipOut.close();
        }catch(Exception e) {
            e.printStackTrace();
            return false;
        }
        
        return true;
    }

用时1秒652毫秒。

 

接下来实验的是bufferOuputZip方法，也就是作者提到的第二种方法：

// time elapsed:1s207ms
    private boolean bufferOuputZip(String[] fromFiles,String toFile) {
        File zipFile=new File(toFile);
        byte[] buffer=new byte[BUFFER_SIZE];
        int readLen=0;
        
        try {
            ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ;
            BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(zipOut) ;

            for(String file:fromFiles) {
                File fileWillZip=new File(file);
                
                if(fileWillZip.exists()) {
                    InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip));
                    String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed
                    zipOut.putNextEntry(new ZipEntry(entryName));
    
                    while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) {
                        bufferedOutputStream.write(buffer,0,readLen);
                    }
                    inputStream.close();    
                }
            }
            
            bufferedOutputStream.flush();
            zipOut.close();
            
            return true;
        }catch(Exception e) {
            e.printStackTrace();
            return false;
        }
    }

用时1秒207毫秒，少了四分之一。

 

接下来实验作者提出的第三种方法：

// elapsed:1s188ms
    private boolean nioChannalZip(String[] fromFiles,String toFile) {
        File zipFile=new File(toFile);
        
        try {
            ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ;
            WritableByteChannel wChannel=Channels.newChannel(zipOut);

            for(String file:fromFiles) {
                File fileWillZip=new File(file);
                
                if(fileWillZip.exists()) {
                    FileChannel readChannel=new FileInputStream(fileWillZip).getChannel(); 
                    
                    String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed
                    zipOut.putNextEntry(new ZipEntry(entryName));
    
                    readChannel.transferTo(0, readChannel.size(),wChannel)    ;
                    readChannel.close();
                }
            }

            wChannel.close();
            zipOut.close();
            
            return true;
        }catch(Exception e) {
            e.printStackTrace();
            return false;
        }
    }

一秒188毫秒，没少多少。

 

接下来再看作者又提到了内存映射文件，他又说和第三种方法速度差不多哦，算了我就不试了。

 

作者最后又提到pip，我一看原来是用线程，这当然快了，因为主线程只要调用压缩线程就可以返回了，压缩线程消耗的时间不计算在主线程运行时间内，运行当然是秒回。

当然pip里面有个阻塞回调的过程，这消耗了一些时间。

更新DB，写文件都可以另起线程运行，这也是以空间换时间的方法之一。

 

完整程序：

package zip;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

// Used to zip a file
public class FileZipper {
    private static final int BUFFER_SIZE = 1024;
    
    public boolean compressFilesToZip(String[] files,String zipfile) {
        return threadZip(files,zipfile);
    }
    
    private boolean threadZip(String[] fromFiles,String toFile) {
        new ZipThread(fromFiles,toFile).start();;
        return true;
    }
    
    // elapsed:1s188ms
    private boolean nioChannalZip(String[] fromFiles,String toFile) {
        File zipFile=new File(toFile);
        
        try {
            ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ;
            WritableByteChannel wChannel=Channels.newChannel(zipOut);

            for(String file:fromFiles) {
                File fileWillZip=new File(file);
                
                if(fileWillZip.exists()) {
                    FileChannel readChannel=new FileInputStream(fileWillZip).getChannel(); 
                    
                    String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed
                    zipOut.putNextEntry(new ZipEntry(entryName));
    
                    readChannel.transferTo(0, readChannel.size(),wChannel)    ;
                    readChannel.close();
                }
            }

            wChannel.close();
            zipOut.close();
            
            return true;
        }catch(Exception e) {
            e.printStackTrace();
            return false;
        }
    }
    
    // time elapsed:1s207ms
    private boolean bufferOuputZip(String[] fromFiles,String toFile) {
        File zipFile=new File(toFile);
        byte[] buffer=new byte[BUFFER_SIZE];
        int readLen=0;
        
        try {
            ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ;
            BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(zipOut) ;

            for(String file:fromFiles) {
                File fileWillZip=new File(file);
                
                if(fileWillZip.exists()) {
                    InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip));
                    String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed
                    zipOut.putNextEntry(new ZipEntry(entryName));
    
                    while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) {
                        bufferedOutputStream.write(buffer,0,readLen);
                    }
                    inputStream.close();    
                }
            }
            
            bufferedOutputStream.flush();
            zipOut.close();
            
            return true;
        }catch(Exception e) {
            e.printStackTrace();
            return false;
        }
    }
    
    // time elapsed:1s652ms
    private boolean rugularZip(String[] fromFiles,String toFile) {
        File zipFile=new File(toFile);
        byte[] buffer=new byte[BUFFER_SIZE];
        int readLen=0;
        
        try {
            ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ;

            for(String file:fromFiles) {
                File fileWillZip=new File(file);
                
                if(fileWillZip.exists()) {
                    InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip));
                    String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed
                    zipOut.putNextEntry(new ZipEntry(entryName));
    
                    while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) {
                        zipOut.write(buffer,0,readLen);
                    }
                    inputStream.close();    

                }
            }

            zipOut.close();
        }catch(Exception e) {
            e.printStackTrace();
            return false;
        }
        
        return true;
    }
    
    /**
     * change seconds to DayHourMinuteSecond format
     * 
     * @param startMs
     * @param endMs
     * @return
     */
    private static String ms2DHMS(long startMs, long endMs) {
        String retval = null;
        long secondCount = (endMs - startMs) / 1000;
        String ms = (endMs - startMs) % 1000 + "ms";

        long days = secondCount / (60 * 60 * 24);
        long hours = (secondCount % (60 * 60 * 24)) / (60 * 60);
        long minutes = (secondCount % (60 * 60)) / 60;
        long seconds = secondCount % 60;

        if (days > 0) {
            retval = days + "d" + hours + "h" + minutes + "m" + seconds + "s";
        } else if (hours > 0) {
            retval = hours + "h" + minutes + "m" + seconds + "s";
        } else if (minutes > 0) {
            retval = minutes + "m" + seconds + "s";
        } else if(seconds > 0) {
            retval = seconds + "s";
        }else {
            return ms;
        }

        return retval + ms;
    }
    
    public static String calculateElaspedTime(long startMs) {
        long endMs = System.currentTimeMillis();
        return ms2DHMS(startMs,endMs);
    }
    
    
    public static void main(String[] args) {
        String[] files= {"D:\\usr\\apache-tomcat-9.0.30.zip",
                         "D:\\usr\\Redis-x64-3.2.100.msi",
                         "D:\\usr\\Redis-x64-3.2.100.zip"};
        String zipfile="D:\\usr\\result.zip";
        
        long startMs = System.currentTimeMillis();
        FileZipper fz=new FileZipper();
        boolean isCreated=fz.compressFilesToZip(files, zipfile);
        if(isCreated) {
            System.out.println("File:'"+zipfile+"' created,time elapsed:"+calculateElaspedTime(startMs));
        }
    }
}

 

package zip;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

public class ZipThread extends Thread{
    private String[] files;
    private String zipfile;
    
    public ZipThread(String[] files,String zipfile) {
        this.files=files;
        this.zipfile=zipfile;
    }
    
    public void run() {
        nioChannalZip(this.files,this.zipfile);
    }
    
    private boolean nioChannalZip(String[] fromFiles,String toFile) {
        File zipFile=new File(toFile);
        
        try {
            ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ;
            WritableByteChannel wChannel=Channels.newChannel(zipOut);

            for(String file:fromFiles) {
                File fileWillZip=new File(file);
                
                if(fileWillZip.exists()) {
                    FileChannel readChannel=new FileInputStream(fileWillZip).getChannel(); 
                    
                    String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed
                    zipOut.putNextEntry(new ZipEntry(entryName));
    
                    readChannel.transferTo(0, readChannel.size(),wChannel)    ;
                    readChannel.close();
                }
            }

            wChannel.close();
            zipOut.close();
            
            return true;
        }catch(Exception e) {
            e.printStackTrace();
            return false;
        }
    }
}

--END-- 2020-01-06 14:54