方案设计 8大批量方案

目录

• 1.1　　　　脏页处理(1k条数据处理最慢3s)

 

---

1　　　　数据量万级别内

回到顶部

1.1　　　　脏页处理(1k条数据处理最慢3s)

• 使用场景

数据量上万

 

• 技术栈

DDD设计领域，springboot，jpa,mapStruct,mysql数据库

 

• 代码存储位置

阿里云盘

 

• 可优化点

可以在配置文件中配置脏页每次缓存多少数据，再刷

 

• pom依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>lockDemo</artifactId>
        <groupId>com.example</groupId>
        <version>0.0.1-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>ddwei-service</artifactId>
    <version>1.1</version>

    <dependencies>
        <dependency>
            <groupId>com.ddwei</groupId>
            <artifactId>ddwei-api</artifactId>
            <version>1.1</version>
        </dependency>
        <dependency>
            <groupId>commons-lang</groupId>
            <artifactId>commons-lang</artifactId>
            <version>2.3</version>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.junit.jupiter</groupId>
            <artifactId>junit-jupiter-api</artifactId>
            <version>5.3.2</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-jpa</artifactId>
            <version>2.1.7.RELEASE</version>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.46</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid-spring-boot-starter</artifactId>
            <version>1.1.10</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
            <version>1.18.2</version>
        </dependency>

        <dependency>
            <groupId>org.mapstruct</groupId>
            <!-- jdk8以下就使用mapstruct -->
            <artifactId>mapstruct-jdk8</artifactId>
            <version>1.2.0.Final</version>
        </dependency>
        <dependency>
            <groupId>org.mapstruct</groupId>
            <artifactId>mapstruct-processor</artifactId>
            <version>1.2.0.Final</version>
        </dependency>

    </dependencies>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>

</project>

 

• 启动类：

package com.ddwei;

import com.ddwei.infrastructure.moduleA.repository.impl.CommonRepositoryImpl;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;

@SpringBootApplication
@ComponentScan(basePackages = "com.ddwei.*")
@EnableJpaRepositories(repositoryBaseClass = CommonRepositoryImpl.class)
public class LockDemoApplication {

    public static void main(String[] args) {
        SpringApplication.run(LockDemoApplication.class, args);
    }

}

 

dto层之Service

package com.ddwei.api.bff.moduleA.service;

import com.ddwei.api.bff.moduleA.dto.request.CallServerParentQuery;
import com.ddwei.api.bff.moduleA.dto.request.CallServerQuery;
import com.ddwei.api.bff.moduleA.dto.response.CallServerDto;
import com.ddwei.api.bff.moduleA.dto.response.CallServerParentDto;
import com.ddwei.api.exception.DBusinessException;
import com.ddwei.api.temp.jarBao.PageParm;
import com.ddwei.api.temp.jarBao.PageResult;

import java.util.List;

public interface CallServerService {
    CallServerDto methodA(CallServerQuery query) throws DBusinessException;

    List<CallServerDto> methodB(List<CallServerQuery> list) throws DBusinessException;

    PageResult<CallServerDto> methodC(PageParm<CallServerQuery> pageParm) throws DBusinessException;

    void save(CallServerDto dto) throws DBusinessException;

    void methodCSavAllBatch(CallServerParentDto dto)throws DBusinessException;

    void methodCUpdateAllBatch(CallServerParentDto dto)throws DBusinessException;


    void blank() throws DBusinessException;

    String params(String string,Integer integer) throws DBusinessException;

    void fanxing(CallServerParentQuery query) throws DBusinessException;
}

 

dto层之ServiceImpl

package com.ddwei.application.moduleA.service.impl;


import com.ddwei.api.bff.moduleA.dto.request.CallServerParentQuery;
import com.ddwei.api.bff.moduleA.dto.request.CallServerQuery;
import com.ddwei.api.bff.moduleA.dto.response.CallServerDto;
import com.ddwei.api.bff.moduleA.dto.response.CallServerParentDto;
import com.ddwei.api.bff.moduleA.service.CallServerService;
import com.ddwei.api.exception.DBusinessException;
import com.ddwei.api.temp.jarBao.PageParm;
import com.ddwei.api.temp.jarBao.PageResult;
import com.ddwei.domain.moduleA.domainobject.DdweiEntity;
import com.ddwei.domain.moduleA.repository.DdweiRepository;
import com.ddwei.infrastructure.moduleA.converter.DDweiConverter;
import com.ddwei.infrastructure.referenceservice.ComcACLService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ObjectUtils;

import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;

@Service
@Slf4j
public class CallServerServiceImpl implements CallServerService {
    @Autowired private ComcACLService comcACLService;
    @Autowired private DdweiRepository ddweiRepository;
    @Autowired private DDweiConverter dDweiConverter;
    @Override
    public CallServerDto methodA(CallServerQuery query) throws DBusinessException {
        try {
            return comcACLService.methodA(query);
            //1.异常一般只在Service层抛出
            //2.三方返回指定的错误
        } catch (DBusinessException e) {
            //3.转成给bff抛出规定好的错误
            throw new DBusinessException("1111",e.getMessage(),"ddwei");
        }
    }

    @Override
    public List<CallServerDto> methodB(List<CallServerQuery> list) throws DBusinessException {
        try {
            CallServerQuery query = CallServerQuery.builder().inputPropertyA("1").build();
            CallServerDto callServerDto = comcACLService.methodA(query);
            return comcACLService.methodB(list);
            //1.异常一般只在Service层抛出
            //2.三方返回指定的错误
        } catch (DBusinessException e) {
            //3.转成给bff抛出规定好的错误
            throw new DBusinessException("1111",e.getMessage(),"ddwei");
        }
    }

    @Override
    public PageResult<CallServerDto> methodC(PageParm<CallServerQuery> pageParm) throws DBusinessException {
        return null;
    }

    public static void main(String[] args) {
        for(int i = 1;i<1000;i++){
            System.out.println("{\"inputPropertyA\":\""+i+"\",\"inputPrepertyB\":\"刘备\",\"outputPropertyA\":\"张飞\"},");
        }
    }

    @Override
    public void save(CallServerDto dto) throws DBusinessException {
        //这种不规范，通常这里都是用repostioy一次查到所有的do,为了简化，直接用Converter转了，
        DdweiEntity ddweiEntity = dDweiConverter.dtoToDomainObject(dto);
        ddweiRepository.saveOrUpdate(ddweiEntity);
    }

    /**
     * insert插值  见 CommonRepositoryImpl
     * @author weidoudou
     * @date 2023/7/21 6:21
     * @param dto 请添加参数描述
     * @return void
     **/
    @Override
    public void methodCSavAllBatch(CallServerParentDto dto) throws DBusinessException {
        log.info("=============start");
        long startTime = System.nanoTime();

        if(ObjectUtils.isEmpty(dto)|| CollectionUtils.isEmpty(dto.getList())){
            throw new DBusinessException("1111111","参数为空","DDWEISYS");
        }

        //这种不规范，通常这里都是用repostioy一次查到所有的do,为了简化，直接用Converter转了，
        List<DdweiEntity> doList = dto.getList().stream().map(
                dDweiConverter::dtoToDomainObject
        ).collect(Collectors.toList());

        ddweiRepository.saveAllBatch(doList);

        long endTime = System.nanoTime();
        log.info("=============end");
        double resultTime = (endTime-startTime)/1000000000.0;
        log.info("=============用时"+resultTime+"s");
    }

    /**
     * 先查再更
     * @author weidoudou
     * @date 2023/7/21 6:20
     * @param dto 请添加参数描述
     * @return void
     **/
    @Override
    public void methodCUpdateAllBatch(CallServerParentDto dto) throws DBusinessException {
        log.info("=============start");
        long startTime = System.nanoTime();

        if(ObjectUtils.isEmpty(dto)|| CollectionUtils.isEmpty(dto.getList())){
            throw new DBusinessException("1111111","参数为空","DDWEISYS");
        }

        //这种不规范，通常这里都是用repostioy一次查到所有的do,为了简化，直接用Converter转了，
        List<DdweiEntity> doList = dto.getList().stream().map(
                dDweiConverter::dtoToDomainObject
        ).collect(Collectors.toList());

        ddweiRepository.updateAllBatch(doList);

        long endTime = System.nanoTime();
        log.info("=============end");
        double resultTime = (endTime-startTime)/1000000000.0;
        log.info("=============用时"+resultTime+"s");
    }

    @Override
    public void blank() throws DBusinessException {
        System.out.println("blank");
    }

    @Override
    public String params(String string, Integer integer) throws DBusinessException {
        return "aaa";
    }

    @Override
    public void fanxing(CallServerParentQuery query) throws DBusinessException {
        System.out.println("fanxing");

    }
}

 

• dto层之Dto

@Getter
@Setter
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class CallServerQuery implements Serializable {
    /**
     * 属性A
     */
    @NotNull(message = "属性A不能为空")
    private String inputPropertyA;
    /**
     * 属性B
     */
    private String inputPropertyB;
}

@Getter
@Setter
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class CallServerParentQuery {
    private String colum1;
    private CallServerQuery data;
}

@Getter
@Setter
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class CallServerDto implements Serializable {
    /**
     * 主键
     */
    private String inputPropertyA;
    /**
     * 属性A
     */
    private String outputPropertyA;
    /**
     * 属性B
     */
    private String inputPrepertyB;
}

@Getter
@Setter
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class CallServerParentDto {
    private String aaa;
    private List<CallServerDto> list;
}

 

• do层之Repository

package com.ddwei.domain.moduleA.repository;

import com.ddwei.domain.moduleA.domainobject.DdweiEntity;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.domain.Specification;

import java.util.List;

public interface DdweiRepository {

    /**
     *根据主键查询
     * @author weidoudou
     * @date 2022/2/19 12:50
     * @param id 请添加参数描述
     * @return com.ddwei.domain.moduleA.domainobject.DdweiEntity
     **/
    DdweiEntity findByInputPropertyA(String id);

    /**
     * 保存do
     * @author weidoudou
     * @date 2022/2/19 12:50
     * @param entity 请添加参数描述
     * @return com.ddwei.domain.moduleA.domainobject.DdweiEntity
     **/
    DdweiEntity saveOrUpdate(DdweiEntity entity);

    /**
     * 批量新增保存
     * @author weidoudou
     * @date 2023/7/20 8:05
     * @param list 请添加参数描述
     * @return void
     **/
    void saveAllBatch(List<DdweiEntity> list);

    /**
     * 批量修改保存
     * @author weidoudou
     * @date 2023/7/20 8:06
     * @param list 请添加参数描述
     * @return void
     **/
    void updateAllBatch(List<DdweiEntity> list);
    
    /**
     * 根据do查找dolist
     * @author weidoudou
     * @date 2022/8/7 17:35
     * @param entity 请添加参数描述
     * @return java.util.List<com.ddwei.domain.moduleA.domainobject.DdweiEntity>
     **/
    List<DdweiEntity> findAllList(DdweiEntity entity);

    /**
     *分页 ，查询所有数据
     * @author weidoudou
     * @date 2022/8/7 17:29
     * @param specification 请添加参数描述
     * @param  pageable 请添加参数描述
     * @return org.springframework.data.domain.Page<com.ddwei.infrastructure.moduleA.dao.jpa.entity.TDdwei>
     **/
    Page<DdweiEntity> findAll(Specification specification, Pageable pageable);

}

 

• do层之RepositoryImpl

package com.ddwei.infrastructure.moduleA.repository.impl;

import com.ddwei.domain.moduleA.domainobject.DdweiEntity;
import com.ddwei.domain.moduleA.repository.DdweiRepository;
import com.ddwei.infrastructure.moduleA.converter.DDweiConverter;
import com.ddwei.infrastructure.moduleA.dao.jpa.dao.TDdweiDao;
import com.ddwei.infrastructure.moduleA.dao.jpa.entity.TDdwei;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.stereotype.Service;

import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;

@Service
public class DdweiRepositoryImpl implements DdweiRepository {
    @Autowired
    private TDdweiDao tDdweiDao;
    @Autowired
    private DDweiConverter dDweiConverter;

    @Override
    public DdweiEntity findByInputPropertyA(String id) {
        //查询出po
        TDdwei tDdwei = tDdweiDao.findByInputPropertyA(id);
        return dDweiConverter.toDomainObject(tDdwei);
    }

    @Override
    public DdweiEntity saveOrUpdate(DdweiEntity entity) {
        TDdwei tDdwei = dDweiConverter.toPo(entity);

        return dDweiConverter.toDomainObject(tDdweiDao.saveAndFlush(tDdwei));
    }

    @Override
    public void saveAllBatch(List<DdweiEntity> list) {
        List<TDdwei> poList = Optional.ofNullable(list).map(doList->{
            return doList.stream().map(
                    e->{
                        return dDweiConverter.toPo(e);
                    }
            ).collect(Collectors.toList());
        }).orElse(null);
        tDdweiDao.saveAllBatch(poList);
    }

    @Override
    public void updateAllBatch(List<DdweiEntity> list) {
        List<TDdwei> poList = Optional.ofNullable(list).map(doList->{
            return doList.stream().map(
                    e->{
                        return dDweiConverter.toPo(e);
                    }
            ).collect(Collectors.toList());
        }).orElse(null);
        tDdweiDao.updateAllBatch(poList);
    }

    @Override
    public List<DdweiEntity> findAllList(DdweiEntity entity) {
        List<TDdwei> list = tDdweiDao.findAll(new Specification() {
            @Override
            public Predicate toPredicate(Root root, CriteriaQuery criteriaQuery, CriteriaBuilder criteriaBuilder) {
                List<Predicate> preList = new ArrayList<>();

                if(StringUtils.isNotBlank(entity.getInputPropertyA())){
                    preList.add(criteriaBuilder.equal(root.get("inputPropertyA").as(String.class),entity.getInputPropertyA()));
                }
                if(StringUtils.isNotBlank(entity.getInputPropertyB())){
                    preList.add(criteriaBuilder.equal(root.get("inputPropertyB").as(String.class),entity.getInputPropertyB()));
                }

                criteriaQuery.where(preList.toArray(new Predicate[preList.size()]));
                return criteriaQuery.getRestriction();
            }
        });

        //判断非空,po转do
        List<DdweiEntity> entityList = Optional.ofNullable(list).map(
                listE->{
                    return listE.stream().map(DDweiConverter.INSTANCE::toDomainObject).collect(Collectors.toList());
                }
                ).orElse(new ArrayList<>());
        return entityList;
    }

    @Override
    public Page<DdweiEntity> findAll(Specification specification, Pageable pageable) {
        return null;
    }
}

 

• do层之do

@Getter
@Setter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DdweiEntity {
    /**
     * 属性A
     */
    private String inputPropertyA;
    /**
     * 属性B
     */
    private String inputPropertyB;
    /**
     * 属性C
     */
    private String outputPropertyA;

}

 

• po层之dao

package com.ddwei.infrastructure.moduleA.dao.jpa.dao;

import com.ddwei.domain.moduleA.repository.CommonRepository;
import com.ddwei.infrastructure.moduleA.dao.jpa.entity.TDdwei;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.domain.Specification;

import java.util.List;

public interface TDdweiDao extends CommonRepository<TDdwei,String> {

    /**
     * 根据主键查询，最开始用的是findById，后来因为某种原因废弃掉了
     * @author weidoudou
     * @date 2022/8/4 7:52
     * @param inputPropertyA 请添加参数描述
     * @return com.ddwei.infrastructure.moduleA.dao.jpa.entity.TDdwei
     **/
    TDdwei findByInputPropertyA(String inputPropertyA);

    /**
     * 分页 ，查询所有数据
     * @param specification
     * @param pageable
     * @return
     */
    Page<TDdwei> findAll(Specification specification, Pageable pageable);

    /**
     * 查询列表
     * @author weidoudou
     * @date 2022/8/7 17:31
     * @param specification 请添加参数描述
     * @return java.util.List<com.ddwei.infrastructure.moduleA.dao.jpa.entity.TDdwei>
     **/
    List<TDdwei> findAll(Specification specification);
}

 

• po层之核心中的核心 CommonRepository

package com.ddwei.domain.moduleA.repository;

import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.repository.CrudRepository;
import org.springframework.data.repository.NoRepositoryBean;

import java.util.List;

/**
 * 封装JPA基类，所有dao的接口需要继承这个类
 * 如果启用此类，需要启动类加上下边的注解，@EnableJpaRepositories(repositoryBaseClass = CommonRepositoryImpl.class)
 * @author weidoudou
 * @date 2023/6/29 8:02
 * T 查询实体类型
 * ID 查询主键类型
 **/
@NoRepositoryBean
public interface CommonRepository<T,ID> extends JpaRepository<T,ID>, CrudRepository<T,ID>, JpaSpecificationExecutor<T> {

    /**
     * 批量新增，只做新增操作，不判断数据是否已存在
     * @author weidoudou
     * @date 2023/6/26 8:04
     * @param entities 请添加参数描述
     * @return java.util.List<S>
     **/
    <S extends T> List<S> saveAllBatch(Iterable<S> entities);

    /**
     * 批量更新
     * @author weidoudou
     * @date 2023/6/29 8:07
     * @param entities 请添加参数描述
     * @return void
     **/
    void updateAllBatch(Iterable<T> entities);

}

 

• po层之核心中的核心 CommonRepositoryImpl

package com.ddwei.infrastructure.moduleA.repository.impl;

import com.ddwei.domain.moduleA.repository.CommonRepository;
import org.springframework.data.jpa.repository.support.JpaEntityInformation;
import org.springframework.data.jpa.repository.support.SimpleJpaRepository;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.Assert;

import javax.persistence.EntityManager;
import java.util.ArrayList;
import java.util.List;

public class CommonRepositoryImpl<T,ID> extends SimpleJpaRepository<T,ID> implements CommonRepository<T,ID> {
    private final  EntityManager entityManager;

    public CommonRepositoryImpl(JpaEntityInformation<T,?> entityInformation, EntityManager entityManager){
        super(entityInformation,entityManager);
        this.entityManager = entityManager;
    }
    @Override
    @Transactional(rollbackFor = Exception.class)
    public <S extends T> List<S> saveAllBatch(Iterable<S> entities) {
        Assert.notNull(entities,"Entites must not be null");
        List<S> result = new ArrayList<S>();
        for(S entity :entities){
            entityManager.persist(entity);//如果数据库存在该主键则抛异常，否则存在实体中，等待insert
            result.add(entity);
        }
        entityManager.flush();//写入数据库
        entityManager.clear();
        return result;
    }

    @Override
    @Transactional(rollbackFor = Exception.class)
    public void updateAllBatch(Iterable<T> entities) {
        Assert.notNull(entities,"Entites must not be null");
        for(T entity :entities){
            entityManager.merge(entity);
        }
        entityManager.flush();
        entityManager.clear();
    }

}

 

• po层之po

@Entity
@Setter
@Getter
@Table(name = "T_DDWEI")
public class TDdwei {
    /**
     * 属性A
     */
    @Id
    @Column(name = "INPUT_PROPERTY_A")
    private String inputPropertyA;
    /**
     * 属性B
     */
    @Column(name = "INPUT_PROPERTY_B")
    private String inputPropertyB;
    /**
     * 属性C
     */
    @Column(name = "OUTPUT_PROPERTY_A")
    private String outputPropertyA;

}

 

• 测试结果1000条数据

2023-07-21 06:11:07.898  INFO 12828 --- [nio-8080-exec-4] c.d.a.m.s.impl.CallServerServiceImpl     : =============start
Fri Jul 21 06:11:07 CST 2023 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
2023-07-21 06:11:08.577  INFO 12828 --- [nio-8080-exec-4] c.d.a.m.s.impl.CallServerServiceImpl     : =============end
2023-07-21 06:11:08.577  INFO 12828 --- [nio-8080-exec-4] c.d.a.m.s.impl.CallServerServiceImpl     : =============用时0.6759568s

 

 

 

N　　　　常见报错

N.1　　　　@Transactional with readonly = true

saveAllBatch 刷脏页方法上没有加事务导致