多线程爬虫Java调用wget下载文件，独立线程读取输出缓冲区

写了个抓取appstore的，要抓取大量的app，本来是用httpclient，但是效果不理想，于是直接调用wget下载，但是由于标准输出、错误输出的原因会导致卡住，另外wget也会莫名的卡住。

所以我采用：

一、独立线程读取输出信息；

二、自己实现doWaitFor方法来代替api提供的waitFor()方法，避免子进程卡死。

三、设置超时，杀死wget子进程，没有正确返回的话，重试一次，并把超时时间加倍；

 有了以上操作，wget不会卡死，就算卡住了也会因为超时被干掉再重试一次，所以绝大部分的app可以被抓取下来。

import com.google.common.io.Files;
import com.xxx.appstore.service.crawler.CalcMD5Service;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.RandomUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;


public class CrawlerUtils {

    public static final String APK_DOWNLOAD_PATH = "/data/appstore/category/";
    private static Logger LOGGER = LoggerFactory.getLogger(CrawlerUtils.class);

    /**
     * 使用wget下载文件
     *
     * @param displayName  appName
     * @param category     分类
     * @param download_url 下载地址
     * @return 成功返回文件路径，失败返回null
     */
    public static String downloadFileByWget(String displayName, String category, String download_url) {
        if (StringUtils.isBlank(displayName) || StringUtils.isBlank(category) || StringUtils.isBlank(download_url)) {
            LOGGER.info("downloadFileByWget ERROR, displayName:{}, category:{}, download_url:{}", new Object[]{displayName, category, download_url});
            return null;
        }
        String fileName = CalcMD5Service.encoder(displayName + RandomUtils.nextInt(1000));
        String seed = CalcMD5Service.encoder(category);
        String midPath = StringUtils.left(seed, 10);
        String filePath = APK_DOWNLOAD_PATH + midPath + "/" + fileName + ".apk";
        File file = new File(filePath);
        try {
            Files.createParentDirs(file);
        } catch (IOException e) {
            LOGGER.warn("IOException", e);
            return null;
        }
        int retry = 2;
        int res = -1;
        int time = 1;
        while (retry-- > 0) {
            ProcessBuilder pb = new ProcessBuilder("wget", download_url, "-t", "2", "-T", "10", "-O", filePath);
            LOGGER.info("wget shell: {}", pb.command());
            Process ps = null;
            try {
                ps = pb.start();
            } catch (IOException e) {
                LOGGER.error("IOException", e);
            }
            res = doWaitFor(ps, 30 * time++);
            if (res != 0) {
                LOGGER.warn("Wget download failed...");
            } else {
                break;
            }
        }
        if (res != 0) {
            return null;
        }
        return filePath;
    }


    /**
     * @param ps      sub process
     * @param timeout 超时时间，SECONDS
     * @return 正常结束返回0
     */
    private static int doWaitFor(Process ps, int timeout) {
        int res = -1;
        if (ps == null) {
            return res;
        }
        List<String> stdoutList = new ArrayList<>();
        List<String> erroroutList = new ArrayList<>();
        boolean finished = false;
        int time = 0;
        ThreadUtil stdoutUtil = new ThreadUtil(ps.getInputStream(), stdoutList);
        ThreadUtil erroroutUtil = new ThreadUtil(ps.getErrorStream(), erroroutList);
        //启动线程读取缓冲区数据
        stdoutUtil.start();
        erroroutUtil.start();
        while (!finished) {
            time++;
            if (time >= timeout) {
                LOGGER.info("Process wget timeout 30s, destroyed!");
                ps.destroy();
                break;
            }
            try {
                res = ps.exitValue();
                finished = true;
            } catch (IllegalThreadStateException e) {
                try {
                    TimeUnit.SECONDS.sleep(1);
                } catch (InterruptedException e1) {

                }
            }
        }
        return res;
    }
}

 

import org.apache.commons.io.Charsets;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;


public class ThreadUtil implements Runnable {
    // 设置读取的字符编码
    private String character = Charsets.UTF_8.displayName();
    private List<String> list;
    private InputStream inputStream;

    public ThreadUtil(InputStream inputStream, List<String> list) {
        this.inputStream = inputStream;
        this.list = list;
    }

    public void start() {
        Thread thread = new Thread(this);
        thread.setDaemon(true);//将其设置为守护线程
        thread.start();
    }

    public void run() {
        BufferedReader br = null;
        try {
            br = new BufferedReader(new InputStreamReader(inputStream, character));
            String line = null;
            while ((line = br.readLine()) != null) {
                list.add(line);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                //释放资源
                inputStream.close();
                if (br != null) {
                    br.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

}