JAVA多线程处理大量数据(一)

背景说明：要对服务器上一个目录进行全量文件读取，采用传统的单线程性能较差，耗时严重。

1、多线程执行类--FileThreadUtils.java

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.io.FileUtil;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;

/**
 * @Description: 文件多线程工具类
 * @Date: 2023/3/7 14:12
 * @Version: 1.0
 */
@Slf4j
public class FileThreadUtils{

    //待处理数据
    private List<String> fileList;
    //每个线程处理的数据量
    private int perCount = 1000;
    //待处理数据总数量
    private int count = 0;


    /**
     * @description: 设置文件数据 
     * @date: 2023/3/8 10:24
     * @param list
     * @return void
     */
    public void setFileList(List<String> list) {
        this.fileList = list;
        this.count = list.size();
    }

    /**
     * @description:  处理多线程
     * @date: 2023/3/8 10:21
     * @param
     * @return void
     */
    public void multiThread(){
        // 一千条为基准为一个线程处理
        List<List<String>> groupList = ListUtils.partition(fileList, perCount);
        CountDownLatch countDownLatch = new CountDownLatch(groupList.size());
        ExecutorService executorService = Executors.newFixedThreadPool(groupList.size());

        for (int i = 0; i < groupList.size(); i++) {
            int finalI = i;
            executorService.execute(() -> {
                List<String> subFileList = groupList.get(finalI);
                for (String filePath : subFileList) {
                    filePath = filePath.replaceAll("\\\\", "/");
                    log.info(Thread.currentThread().getName() + " 线程开始处理文件 ：" + filePath);
                    //这里放置你的业务代码
                    //这里放置你的业务代码
                    //这里放置你的业务代码
                }
                countDownLatch.countDown();
            });
        }
        try {
            countDownLatch.await();
            executorService.shutdown();   //关闭线程池
        } catch (InterruptedException e) {
            //throw new RuntimeException(e);
            log.info("线程池关闭异常: ",e.getMessage());
        }
    }

}

 

2、数据分组工具类-- ListUtils.java

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

/**
 * @Description: TODO
 * @Date: 2023/3/8 9:39
 * @Version: 1.0
 */
public class ListUtils {
    /**
     * 集合按长度分组
     *
     * @param list
     * @param size
     * @param <T>
     * @return
     */
    public static <T> List<List<T>> partition(final List<T> list, final int size) {
        if (list == null) {
            throw new IllegalArgumentException("List must not be null");


        }
        if (size <= 0) {
            throw new IllegalArgumentException("Size must be greater than 0");
        }
        List<List<T>> result = new ArrayList<>();
        Iterator<T> it = list.iterator();
        List<T> subList = null;
        while (it.hasNext()) {
            if (subList == null) {
                subList = new ArrayList<>();
            }
            T t = it.next();
            subList.add(t);
            if (subList.size() == size) {
                result.add(subList);
                subList = null;
            }
        }
        //补充最后一页
        if (subList != null) {
            result.add(subList);
        }
        return result;

    }
}