ListSplitUtil (批量数据处理)
public class ListSplitUtil {
/**
* 拆分
*
* @param allList allList
* @param limitCount limitCount
* @param <T> T
* @return List
*/
public static <T> List<List<T>> splitList(List<T> allList, int limitCount){
if (CollectionUtils.isEmpty(allList)) {
return new ArrayList<>();
}
List<List<T>> appSplitList = new ArrayList<>();
long total = allList.size();
long remain = total % limitCount;
long times = total / limitCount;
long realTimes = remain == 0 ? times : times + 1;
for (long i = 0; i < realTimes; i++) {
List<T> batchList = allList.stream().skip(i * limitCount).limit(limitCount).collect(Collectors.toList());
appSplitList.add(batchList);
}
return appSplitList;
}
}
// 使用实例
List<List<PredictModelDuConfig>> lists = ListSplitUtil.splitList(newConfigList, 100);
for (List<PredictModelDuConfig> configList : lists) {
regressionModelDao.batchInsertPredictModelDuConfig(configList);
}
public int batchInsertPredictModelDuConfig(List<PredictModelDuConfig> duConfigList) {
if (CollectionUtils.isEmpty(duConfigList)) {
return 0;
}
Map<String, Object> param = new HashMap<>();
param.put("duConfigList", duConfigList);
return dalClient.execute(SQL_NAMESPACE + "batchInsertPredictModelDuConfig", param);
}
<sql id="batchInsertPredictModelDuConfig" isRead="true">
<![CDATA[
insert into predict_model_du_config(DU_ID,TYPE,MODEL_ID,STATUS,MODEL_STATUS,EXCEPT_START_TIME,EXCEPT_END_TIME,PRESSURE_DATA_RATIO,XAIXS_CUT_COUNT,CLEAR_DATA_HIGH_QUANTILE,CLEAR_DATA_LOW_QUANTILE,INTERCEPT_OFFSET_ABSOLUTE)
values
<#list duConfigList as item>
('${item.duId}',${item.type},'${item.modelId}',${item.status},${item.modelStatus},'${item.exceptStartTime}','${item.exceptEndTime}','${item.pressureDataRatio}','${item.xaixsCutCount}','${item.clearDataHighQuantile}','${item.clearDataLowQuantile}','${item.interceptOffsetAbsolute}')
<#if item_has_next>
,
</#if>
</#list>
]]>
</sql>
每天一点点,惊喜不间断