Java并发下载文件&修改保存【IO】【多线程】

java并发下载文件、修改保存；【IO】【多线程】

1）java实现用url下载文件，储存到指定文件夹

package com.company;

import java.io.BufferedInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;


public class Main {

    public static void main(String[] args) throws IOException {
        System.out.println("2199");
        //https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png
        //C:\Users\jinqingyang\Desktop\javaIO\download
        String targetRoute = "C:\\Users\\jinqingyang\\Desktop\\javaIO\\download\\";

        ////调用轮子，进行下载
        long start = System.currentTimeMillis();
        String url001 = "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png";
        String fileName = "MyPic001.png".toString();
        String fileRoute = new StringBuffer(targetRoute + fileName).toString();
        downloadUsingStream(url001, fileRoute);
        long finish = System.currentTimeMillis();
        System.out.println("time=" + (finish-start));

        ////从已有的列表中下载多个文件
        String[] strs = {"https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402154152536-1698601120.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155257738-1040878756.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155324774-928819452.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155345098-544304226.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155404336-904923055.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155431135-1032860349.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155501392-595613994.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155538301-612543310.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155550422-295230732.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155618222-1104112179.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155714229-1850007920.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155804886-802000350.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155840534-212230918.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155910567-747754721.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155938007-1903923944.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160009903-1294768132.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160038620-1691751684.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png"
        };
        long startList = System.currentTimeMillis();
        for(int i=0; i<strs.length; ++i){
            String thisFileRoute = new StringBuffer(targetRoute + "List\\MyPic" + i + ".png").toString();
            downloadUsingStream(strs[i], thisFileRoute);
        }
        long finishList = System.currentTimeMillis();
        System.out.println("time=" + (finishList-startList));
    }

    //用流来下载文件，是一个底层的轮子；
    public static void downloadUsingStream(String urlStr, String fileRoute) throws IOException {
        URL url = new URL(urlStr);
        BufferedInputStream bis = new BufferedInputStream(url.openStream());
        FileOutputStream fis = new FileOutputStream(fileRoute);
        byte[] buffer = new byte[1024];
        int count = 0;
        while ((count = bis.read(buffer, 0, 1024)) != -1) {
            fis.write(buffer, 0, count);
        }
        fis.close();
        bis.close();
    }
}

分别进行单文件、多文件的下载，然后计时如下：

单个图片和多个图片分别用时992ms、4386ms

2）java实现多线程下载（记录下载时间）

使用基本的线程池ThreadPoolExecutor，重写Runnable方法

package com.company;

import java.io.BufferedInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import static com.company.Main.downloadUsingStream;


public class Main {

    public static void main(String[] args) throws IOException {
        System.out.println("2199");
        //https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png
        //C:\Users\jinqingyang\Desktop\javaIO\download
        String targetRoute = "C:\\Users\\jinqingyang\\Desktop\\javaIO\\download\\";

        ////调用轮子，进行下载
        long start = System.currentTimeMillis();
        String url001 = "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png";
        String fileName = "MyPic001.png".toString();
        String fileRoute = new StringBuffer(targetRoute + fileName).toString();
        downloadUsingStream(url001, fileRoute);
        long finish = System.currentTimeMillis();
        System.out.println("time=" + (finish - start) + "ms");

        ////从已有的列表中下载多个文件
        //待会新建文件夹、考虑下去重的问题
        String[] strs = {"https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402154152536-1698601120.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155257738-1040878756.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155324774-928819452.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155345098-544304226.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155404336-904923055.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155431135-1032860349.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155501392-595613994.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155538301-612543310.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155550422-295230732.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155618222-1104112179.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155714229-1850007920.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155804886-802000350.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155840534-212230918.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155910567-747754721.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155938007-1903923944.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160009903-1294768132.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160038620-1691751684.png",
                "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png"
        };

        long startList = System.currentTimeMillis();
        ThreadPoolExecutor executor = new ThreadPoolExecutor(20, 40, 500, TimeUnit.MILLISECONDS,
                new ArrayBlockingQueue<Runnable>(20));////这个需要调参适配，corePoolSize越大越好；此案例达到20即可达到性能极限

        for (int i = 0; i < strs.length; ++i) {
            MyTask myTask = new MyTask(targetRoute, i, strs[i]);
            executor.execute(myTask);
        }

        executor.shutdown();
        long finishList = System.currentTimeMillis();
        System.out.println("timeList=" + (finishList - startList) + "ms");

    }

    //用流来下载文件，是一个底层的轮子；
    public static void downloadUsingStream(String urlStr, String fileRoute) throws IOException {
        URL url = new URL(urlStr);
        BufferedInputStream bis = new BufferedInputStream(url.openStream());
        FileOutputStream fis = new FileOutputStream(fileRoute);
        byte[] buffer = new byte[1024];
        int count = 0;
        while ((count = bis.read(buffer, 0, 1024)) != -1) {
            fis.write(buffer, 0, count);
        }
        fis.close();
        bis.close();
    }
}

class MyTask implements Runnable {
    String targetRoute;
    int i;
    String str;

    MyTask(String targetRoute, int i, String str){
        this.str = str;
        this.i = i;
        this .targetRoute = targetRoute;
    }

    @Override
    public void run() {
            String thisFileRoute = new StringBuffer(targetRoute + "List\\MyPic" + (i + 1) + ".png").toString();
            try {
                downloadUsingStream(str, thisFileRoute);
                System.out.println("正在下载第" + i);
            } catch (IOException e) {
                e.printStackTrace();
        }
    }
}

 

 

 这个time=3ms是主线程的时间，而不是完成任务的时间

》》》》现在多线程确实是加快了速度，但是暂未定量获取【完成任务的总时间】

 

3）java读取json文件并正则化匹配修改【JSON】【正则化】【java读+写文件】

 

package com.company;

import java.io.BufferedInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;


import java.io.BufferedReader;  //import io系列的多个依赖,这里不合并的目的是要展开看看熟悉一下
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.io.FileInputStream;


public class Main {

    public static void main(String[] args) throws IOException {
        System.out.println("2199");
        //[1]使用java生态接口读取JSON(本地文件，来源可以是python/JS从页面扒下来)，并打印到控制台，注意utf-8解码(读出来发现没问题，就不管了)
        String path = "C:\\Users\\jinqingyang\\Desktop\\java_IO\\download\\Data.json";//这里测试发现无论是.txt还是.json都可以读取
//        readJsonAndPrint(path);
        //[2]正则化匹配所需的图片url，并存储到String[]数组中
//        getUrlsAndSave(path);
        //[3]并发下载图片文件
//        download(getUrlsAndSave(path));
        //[One]使用java生态写文件，该文件专门存储urls,在文件开头写上一个汉字简介，来测试utf-8的使用情况
        writeToFile(getUrlsAndSave(path));
        //[Two]使用java修改本地txt(读+写)
        String txtName = "C:\\Users\\jinqingyang\\Desktop\\java_IO\\download\\URLs.txt";
        modifyTxt(txtName);
    }


    public static String txt2String(File file){  ////这是个轮子
        StringBuilder result = new StringBuilder();
        try{
            // 构造一个BufferedReader类来读取文件
            BufferedReader br = new BufferedReader(new FileReader(file));
            String s = null;
            // 使用readLine方法，一次读一行
            while((s = br.readLine())!=null){
                result.append(System.lineSeparator()+s);
            }
            br.close();
        }catch(Exception e){
            e.printStackTrace();
        }
        return result.toString();
    }

    public static void readJsonAndPrint(String pathName){
        File file0 = new File(pathName);
        String string0 = txt2String(file0);
        System.out.println(string0);//打印全文
    }

    public static ArrayList<String> getUrlsAndSave(String pathName){
        ArrayList<String> urls = new ArrayList<>();
        File file = new File(pathName);
        try{
            // 构造一个BufferedReader类来读取文件
            BufferedReader br = new BufferedReader(new FileReader(file));
            String s = null;
            // 使用readLine方法，一次读一行
            while((s = br.readLine())!=null){
                //进行匹配，如果匹配成功就加入urls数组
                String url = "";
                if(s.contains("img src=\"https://img2020.cnblogs.com/blog/1806053")){
                    url = s.substring(s.indexOf("https://img"),3+s.lastIndexOf("png"));
//                    System.out.println(url);
                    urls.add(url);
                }
            }
            br.close();
        }catch(Exception e){
            e.printStackTrace();
        }
        return urls;
    }

    public static void writeToFile(ArrayList<String> strs) throws IOException {
        //写入文件介绍（汉字+英文+数字）[两行]
        File myFile = new File("C:\\Users\\jinqingyang\\Desktop\\java_IO\\download\\URLs.txt");
        myFile.createNewFile();//还要再创建新文件
        BufferedWriter out = new BufferedWriter(new FileWriter(myFile));

        String str0 = "这个txt的内容是我的cnblog里面的21个图片的url！！！\r\n下面是正文内容：\r\n";// \r\n即为换行
        out.write(str0);

        for(int i=0; i<strs.size(); ++i){
            //将每个url写入，注意换行符的事情
            out.write(strs.get(i)+"\r\n");
        }

        out.flush(); // 把缓存区内容压入文件
        out.close(); // 最后记得关闭文件
    }

    public static void modifyTxt(String txtName){//要修改的内容，我在里面自定义
        try { // 防止文件建立或读取失败，用catch捕捉错误并打印，也可以throw

            /* 读取txt文件 */
            File modifyFile = new File(txtName); // 要读取以上路径的input.txt文件
            InputStreamReader reader = new InputStreamReader(
                    new FileInputStream(modifyFile)); // 建立一个输入流对象reader
            BufferedReader br = new BufferedReader(reader); // 建立一个对象，它把文件内容转成计算机能读懂的语言
            String line = br.readLine();//读取第一行
            BufferedWriter out = new BufferedWriter(new FileWriter(modifyFile));
            while (line != null) {
                //modify出新的内容
                String modifyLine = line+"》》》【这里是修改的内容】》》》》\r\n";
                //写入buffer
                out.write(modifyLine);//积攒buffer里面的内容
                // 一次读入一行数据
                line = br.readLine();
            }
            out.write("已修改！！");
            /* 写进txt文件 */
            out.flush(); // 把缓存区内容压入文件【这里是把前面write的内容一次性flush写进去！！！！】
            out.close(); // 最后记得关闭文件

        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    //用流来下载文件，是一个底层的轮子；
    public static void downloadUsingStream(String urlStr, String fileRoute) throws IOException {
        URL url = new URL(urlStr);
        BufferedInputStream bis = new BufferedInputStream(url.openStream());
        FileOutputStream fis = new FileOutputStream(fileRoute);
        byte[] buffer = new byte[1024];
        int count = 0;
        while ((count = bis.read(buffer, 0, 1024)) != -1) {
            fis.write(buffer, 0, count);
        }
        fis.close();
        bis.close();
    }

    public static void download(ArrayList<String> strs) throws IOException {  //从已有的列表中下载多个文件
        //https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png
        //C:\Users\jinqingyang\Desktop\javaIO\download
        String targetRoute = "C:\\Users\\jinqingyang\\Desktop\\java_IO\\download\\";

        ////调用轮子，进行下载
        long start = System.currentTimeMillis();
        String url001 = "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png";
        String fileName = "MyPic001.png".toString();
        String fileRoute = new StringBuffer(targetRoute + fileName).toString();
        downloadUsingStream(url001, fileRoute);
        long finish = System.currentTimeMillis();
        System.out.println("time=" + (finish - start) + "ms");

        long startList = System.currentTimeMillis();
        ThreadPoolExecutor executor = new ThreadPoolExecutor(25, 50, 500, TimeUnit.MILLISECONDS,
                new ArrayBlockingQueue<Runnable>(25));////这个需要调参适配，corePoolSize越大越好；此案例达到25即可达到性能极限

        for (int i = 0; i < strs.size(); ++i) {
            MyTask myTask = new MyTask(targetRoute, i, strs.get(i));
            executor.execute(myTask);
        }

        executor.shutdown();
        long finishList = System.currentTimeMillis();
        System.out.println("timeList=" + (finishList - startList) + "ms");
    }
}

class MyTask implements Runnable {
    String targetRoute;
    int i;
    String str;

    MyTask(String targetRoute, int i, String str){
        this.str = str;
        this.i = i;
        this .targetRoute = targetRoute;
    }

    @Override
    public void run() {
        String thisFileRoute = new StringBuffer(targetRoute + "List\\MyPic" + (i + 1) + ".png").toString();
        try {
            Main.downloadUsingStream(str, thisFileRoute);
            System.out.println("正在下载第" + i);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}