多线程并行_countDown

/**
 * 首次启动加载数据至缓存
 */
public class ApplicationStartTask {

    private static Logger logger = LoggerFactory.getLogger(ApplicationStartTask.class);
    private AtomicInteger loadNum = new AtomicInteger(0);

    @Autowired
    private FaceProducer cacheSynchronizationProducer;
    @Autowired
    private FaceConsumer cacheSynchronizationConsumer;

    @Autowired
    private FaceInfoService faceInfoService;

    /**
     * 当提交一个任务到线程池时，线程池会创建一个线程来执行任务，即使其他空闲的基本线程能够执行新任务也会创建线程，
     * <br>等到需要执行的任务数大于线程池基本大小时就不再创建
     */
    private static final int POOL_CORE_SIZE = Runtime.getRuntime().availableProcessors();

    /**
     * 线程池允许创建的最大线程数。如果队列满了，并且已创建的线程数小于最大线程数，则线程池会再创建新的线程执行任务
     */
    private static final int POOL_MAX_SIZE = Runtime.getRuntime().availableProcessors();

    /**
     * 线程池的工作线程空闲后，保持存活的时间 <br>
     * 单位:分
     */
    private static final int POOL_KEEP_ALIVE_TIME = 2;

    @Autowired
    @Qualifier("sqliteManagerImpl")
    private SqlieteManager sqlieteManager;

    @Autowired
    private FaceCompareManager faceCompareManager;

    @Value("${local.sqlite.sqlname}")
    private String sqlName;

    @Bean
    public ApplicationStartTask loadCache() {
        logger.info("[启动任务]首次启动载入数据至缓存...");
        checkSqLiteFile();
        List<String> belongIds = sqlieteManager.getAllBelongIds();
        if (ArrayUtil.isNotEmpty(belongIds)) {

            loadFromSqlite(belongIds);

            initMqServer();

            return new ApplicationStartTask();
        }
        loadFromDB();

        initMqServer();
        return new ApplicationStartTask();

    }

    /**
     * 开启MQ服务
     */
    private void initMqServer() {

        logger.info("开始开启MQ服务...");
        cacheSynchronizationProducer.producerStart();
        cacheSynchronizationConsumer.getMessage();
    }


    /**
     * 直接从MYSQL载入数据
     */
    private void loadFromDB() {
        logger.info("当前CPU内核线程数:{}", POOL_CORE_SIZE);

        long beginTimsamp = System.currentTimeMillis();

        List<String> allBelongId = faceCompareManager.getAllBelongId();
        int size = 0;
        for (String belongId : allBelongId) {
            FaceInfoSyncDataQuery query = new FaceInfoSyncDataQuery(belongId);
            query.setPageSize(1000);
            query.setDeleteStatus(0);
            Result<Integer> result = faceInfoService.getTotalPage(query);
            if (!result.isSuccess() || result.getModel() == 0) {
                continue;
            }

            int totalPage = result.getModel();

            size += totalPage;

        }
        logger.info("总任务数:{}", size);
        ThreadPoolExecutor poolExecutor = new ThreadPoolExecutor(POOL_CORE_SIZE, POOL_MAX_SIZE, POOL_KEEP_ALIVE_TIME,
                TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(size));
        CountDownLatch latch = new CountDownLatch(size);

        for (String belongId : allBelongId) {

            FaceInfoSyncDataQuery query = new FaceInfoSyncDataQuery();
            query.setBelongId(belongId);
            query.setDeleteStatus(0);
            query.setPageSize(1000);
            Result<FaceInfoSyncDataVO> result = faceInfoService.getFaces(query);
            if (!result.isSuccess() || result.getModel() == null || result.getModel().getTotalPages() == 0) {
                continue;
            }
            FaceInfoSyncDataVO faceInfoSyncDataVO = result.getModel();

            int totalPage = faceInfoSyncDataVO.getTotalPages();
            //第一页

            poolExecutor.execute(new CacheLoadTask1(1, belongId, faceInfoSyncDataVO.getFaces(), latch));
            if (totalPage == 1) {
                continue;
            }
            for (int i = 2; i <= totalPage; i++) {
                query = new FaceInfoSyncDataQuery();
                query.setBelongId(belongId);
                query.setPageSize(1000);
                query.setPageNo(i);
                query.setDeleteStatus(0);
                result = faceInfoService.getFaces(query);
                if (!result.isSuccess() || result.getModel() == null || ArrayUtil.isEmpty(result.getModel().getFaces())) {
                    continue;
                }
                faceInfoSyncDataVO = result.getModel();

                poolExecutor.execute(new CacheLoadTask1(i, belongId, faceInfoSyncDataVO.getFaces(), latch));


            }
        }
        try {

            latch.await();// 同步阻塞，直到所有线程工作完成
            // long endTime = System.currentTimeMillis(); // 获取结束时间

            poolExecutor.shutdown();// 不允许提交新任务，等待当前任务及队列中的任务全部执行完毕后退出
            // // 支持等待以前提交的任务停止执行
            // // 所有任务关闭请求或线程中断或超时，阻塞取消
            // poolExecutor.awaitTermination(20, TimeUnit.MINUTES);
            // poolExecutor.shutdownNow();//通过Thread.interrupt试图停止所有正在执行的线程,并不再处理还在队列中等待的任务...
        } catch (Exception e) {
            logger.error("[启动任务]线程池异常打印:{}", e);
            poolExecutor.shutdownNow();

        }
//        logger.info("[启动任务]从DB中载入数据至缓存完毕!共载入{}条人脸数据", loadNum.get());

        logger.info("[启动任务]完毕！全文索引{}条数据,共耗时:{}s", loadNum.get(), (System.currentTimeMillis() - beginTimsamp) / 1000);


    }

    class CacheLoadTask1 implements Runnable {


        private CountDownLatch latch;
        List<FaceInfoDO> faceInfoDOs = null;
        int putCacheNum = 0;
        String belongId = null;
        int pageNo = 0;

        public CacheLoadTask1(int pageNo, String belongId, List<FaceInfoDO> faceInfoDOs, CountDownLatch latch) {
            this.latch = latch;
            this.faceInfoDOs = faceInfoDOs;
            this.belongId = belongId;
            this.pageNo = pageNo;
            // this.index = index;
        }

        @Override
        public void run() {
            Long beginTime = System.currentTimeMillis();
            try {
                for (FaceInfoDO faceInfoDO : faceInfoDOs) {
                    boolean isPutCache = FaceInfoCacheManage.put(faceInfoDO);
                    if (isPutCache) {
                        putCacheNum++;
                        loadNum.incrementAndGet();

                    }
                }
                logger.info("[启动任务]线程{}处理完belongId-{}第{}页数据,共载入{}条人脸数据,耗时:{}ms", Thread.currentThread().getName(),
                        belongId, pageNo, putCacheNum, System.currentTimeMillis() - beginTime);

            } catch (Exception e) {
                logger.error("[启动任务]载入sqlite任务[CacheLoadTask]出错:{}", e);
            } finally {
                // TODO: handle finally clause
                latch.countDown();
            }
        }

    }

    /**
     * 从sqlite数据库中加载数据
     */
    private void loadFromSqlite(List<String> belongIds) {

        logger.info("当前CPU内核线程数:{}", POOL_CORE_SIZE);
        ThreadPoolExecutor poolExecutor = new ThreadPoolExecutor(POOL_CORE_SIZE, POOL_MAX_SIZE, POOL_KEEP_ALIVE_TIME,
                TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(belongIds.size()));
        CountDownLatch latch = new CountDownLatch(belongIds.size());
        for (String belongId : belongIds) {
            List<FaceInfoDO> faceInfoDOs = sqlieteManager.selectAllInfoByBelongId(sqlName, belongId);

            poolExecutor.execute(new CacheLoadTask(belongId, faceInfoDOs, latch));

        }

        try {

            latch.await();// 同步阻塞，直到所有线程工作完成
            // long endTime = System.currentTimeMillis(); // 获取结束时间
            logger.info("[启动任务]从sqlite中载入数据至缓存完毕!共载入{}条人脸数据", loadNum.get());

            poolExecutor.shutdown();// 不允许提交新任务，等待当前任务及队列中的任务全部执行完毕后退出
            // // 支持等待以前提交的任务停止执行
            // // 所有任务关闭请求或线程中断或超时，阻塞取消
            // poolExecutor.awaitTermination(20, TimeUnit.MINUTES);
            // poolExecutor.shutdownNow();//通过Thread.interrupt试图停止所有正在执行的线程,并不再处理还在队列中等待的任务...
        } catch (Exception e) {
            logger.error("[启动任务]线程池异常打印:{}", e);
            poolExecutor.shutdownNow();

        }

    }


    private void checkSqLiteFile() {
        File sqliteFile = new File(sqlName + ".sqlite3");
        if (!sqliteFile.exists()) {
            logger.info("[启动任务]sqlite数据库未创建，开始创建...");
            // 第一次启动创建本地表
            try {
                sqlieteManager.createTable(sqlName);
            } catch (Exception e) {
                logger.error(e.getMessage());
            }

        }
    }

}