JAVA多线程处理大量数据(二)--推荐
背景说明:要对服务器上一个目录进行全量文件读取
1、多线程执行类--FileThreadUtils.java
import cn.hutool.core.collection.CollUtil; import cn.hutool.core.io.FileUtil; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import java.io.IOException; import java.util.*; import java.util.concurrent.*; import java.util.stream.Collectors; /** * @Description: 文件多线程工具类 * @Date: 2023/3/9 14:12 * @Version: 1.0 */ @ Slf4j@ Component public class FileThreadUtils { /** * @description: 根据文件数量动态创建线程数 * @date: 2023/3/10 15:55 * @param fileNumber * @return java.lang.Long */ public Integer getThreadSize(int fileNumber) { int prcessors = Runtime.getRuntime().availableProcessors(); prcessors *= 2; prcessors += 1; return (fileNumber + prcessors - 1) / prcessors; } /** * @description: 使用多线程处理文件 * @date: 2023/3/10 16:00 * @param scanDTO * @return java.lang.String */ public void scanAllFile(List < String > fileList) throws InterruptedException, ExecutionException { //所少条数据开启一个子线程 Integer perCount = getThreadSize(fileList.size()); List < List < String >> groupList = ListUtils.partition(fileList, perCount); //初始化线程池方式1 //ExecutorService es = Executors.newFixedThreadPool(groupList.size()); //初始化线程池方式2(推荐) ThreadPoolExecutor es = new ThreadPoolExecutor(groupList.size(), groupList.size(), 1, TimeUnit.SECONDS, new ArrayBlockingQueue < > (1)); List < FileTask > tasks = new ArrayList < > (); for (int i = 0; i < groupList.size(); i++) { log.info("一批数据加入线程池: " + i); tasks.add(new FileTask(groupList.get(i))); } List < Future < Map < String, Object >>> futures = es.invokeAll(tasks); es.shutdown(); //汇总各个子线程的数据结果 //这里处理线程结果 for (int i = 0; i < futures.size(); i++) { Future < Map < String, Object >> mapFuture = futures.get(i); //System.out.println("index:" + i + ",future:"+ futures.get(i).get()); Map < String, Object > stringObjectMap = mapFuture.get(); } } /** * @description: 文件处理单线程 * @date: 2023/3/10 16:01 * @return */ public static class FileTask implements Callable < Map < String, Object >> { List < String > subFileList; public FileTask(List < String > list) { this.subFileList = list; } @Override public Map < String, Object > call() throws Exception { for (String filePath: subFileList) { filePath = filePath.replaceAll("\\\\", "/"); log.info(Thread.currentThread().getName() + " 线程开始处理文件 :" + filePath); } Map < String, Object > subResultMap = new HashMap < > (); //log.info("subMap{}",subResultMap); return subResultMap; } } }
2、数据分组工具类-- ListUtils.java
import java.util.ArrayList; import java.util.Iterator; import java.util.List; /** * @Description: TODO * @Date: 2023/3/8 9:39 * @Version: 1.0 */ public class ListUtils { /** * 集合按长度分组 * * @param list * @param size * @param <T> * @return */ public static <T> List<List<T>> partition(final List<T> list, final int size) { if (list == null) { throw new IllegalArgumentException("List must not be null"); } if (size <= 0) { throw new IllegalArgumentException("Size must be greater than 0"); } List<List<T>> result = new ArrayList<>(); Iterator<T> it = list.iterator(); List<T> subList = null; while (it.hasNext()) { if (subList == null) { subList = new ArrayList<>(); } T t = it.next(); subList.add(t); if (subList.size() == size) { result.add(subList); subList = null; } } //补充最后一页 if (subList != null) { result.add(subList); } return result; } }
Tips: 异步处理数据
ExecutorService executor = Executors.newFixedThreadPool(1); CompletableFuture<Map> map1Future = CompletableFuture.supplyAsync(() -> {return new HashMap();}, executor); CompletableFuture.allOf(map1Future).get();