Mybatis-Plus大批量插入数据到MySQL

MyBatis-Plus的saveBatch方法

@GetMapping("/save1")
public void save1() {
    // 数据准备
    List<MallOrder> orderList = getMallOrderList();

    // mybatis-plus
    long start = System.currentTimeMillis();
    mallOrderService.saveBatch(orderList);
    System.out.println("mybatis-plus的【savaBatch】插入数据，耗时：" + (System.currentTimeMillis() - start) + "ms");
}

MyBatis-Plus的xml方式

@GetMapping("/save2")
public void save2() {
    // 数据准备
    List<MallOrder> orderList = getMallOrderList();

    // xml
    long start = System.currentTimeMillis();
    mallOrderService.saveBatchXml(orderList);
    System.out.println("mybatis-plus的【xml拼接sql】插入数据，耗时：" + (System.currentTimeMillis() - start) + "ms");
}

<insert id="saveBatch">
    insert into mall_order (order_id,
    customer_id,
    order_status,
    payment_method,
    total_amount,
    shipping_fee,
    coupon_discount,
    order_date,
    payment_time,
    shipping_address,
    receiver_name,
    receiver_phone)
    values
    <foreach collection="orderList" item="item" separator=",">
        (#{item.orderId},
        #{item.customerId},
        #{item.orderStatus},
        #{item.paymentMethod},
        #{item.totalAmount},
        #{item.shippingFee},
        #{item.couponDiscount},
        #{item.orderDate},
        #{item.paymentTime},
        #{item.shippingAddress},
        #{item.receiverName},
        #{item.receiverPhone})
    </foreach>
</insert>

MyBatis-Plus的批量插入器

@GetMapping("/save3")
public void save3() {
    // 数据准备
    List<MallOrder> orderList = getMallOrderList();

    // insertBatchSomeColumn
    long start = System.currentTimeMillis();
    mallOrderService.insertBatchSomeColumn(orderList);
    System.out.println("mybatis-plus的【insertBatchSomeColumn】插入数据，耗时：" + (System.currentTimeMillis() - start) + "ms");
}

package com.qiangesoft.batchsave.config;

import com.baomidou.mybatisplus.core.injector.AbstractMethod;
import com.baomidou.mybatisplus.core.injector.DefaultSqlInjector;
import com.baomidou.mybatisplus.core.metadata.TableInfo;
import com.baomidou.mybatisplus.extension.injector.methods.InsertBatchSomeColumn;

import java.util.List;

/**
 * sql注入器
 *
 * @author qiangesoft
 * @date 2024-04-11
 */
public class InsertBatchSqlInjector extends DefaultSqlInjector {

    @Override
    public List<AbstractMethod> getMethodList(Class<?> mapperClass, TableInfo tableInfo) {
        // super.getMethodList() 保留 Mybatis Plus 自带的方法
        List<AbstractMethod> methodList = super.getMethodList(mapperClass, tableInfo);
        // 添加自定义方法：批量插入，方法名为 insertBatchSomeColumn
        // bean mapper中的方法名也是insertBatchSomeColumn 须和内部定义好的方法名保持一致。
        methodList.add(new InsertBatchSomeColumn());
        return methodList;
    }

}

package com.qiangesoft.batchsave.config;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * mybatis-plus配置
 *
 * @author qiangesoft
 * @date 2024-04-11
 */
@Configuration
public class MybatisPlusConfig {

    @Bean
    public InsertBatchSqlInjector insertBatchSqlInjector() {
        return new InsertBatchSqlInjector();
    }

}

package com.qiangesoft.batchsave.mapper;

import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.qiangesoft.batchsave.entity.MallOrder;
import org.apache.ibatis.annotations.Param;

import java.util.List;

/**
 * <p>
 * 订单信息表 Mapper 接口
 * </p>
 *
 * @author qiangesoft
 * @since 2024-05-07
 */
public interface MallOrderMapper extends BaseMapper<MallOrder> {

    /**
     * 批量插入 仅适用于mysql
     *
     * @param orderList
     */
    Integer insertBatchSomeColumn(List<MallOrder> orderList);
}

SqlSession手动提交

@GetMapping("/save4")
public void save4() {
    // 数据准备
    List<MallOrder> orderList = getMallOrderList();

    // 手动提交
    long start = System.currentTimeMillis();
    mallOrderService.manualCommit(orderList);
    System.out.println("sqlSession的【手动提交】插入数据100000条，耗时：" + (System.currentTimeMillis() - start) + "ms");
}

@Autowired
private SqlSessionFactory sqlSessionFactory;

@Override
public void manualCommit(List<MallOrder> orderList) {
    // 关闭自动提交
    SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
    MallOrderMapper mallOrderMapper = sqlSession.getMapper(MallOrderMapper.class);
    for (MallOrder mallOrder : orderList) {
        mallOrderMapper.insert(mallOrder);
    }

    sqlSession.commit();
    sqlSession.clearCache();
    sqlSession.close();
}

循环执行MyBatis-Plus的save方法

@GetMapping("/save5")
public void save5() {
    // 数据准备
    List<MallOrder> orderList = getMallOrderList();

    // 循环插入
    long start = System.currentTimeMillis();
    for (MallOrder mallOrder : orderList) {
        mallOrderService.save(mallOrder);
    }
    System.out.println("mybatis-plus的【循环】插入数据100000条，耗时：" + (System.currentTimeMillis() - start) + "ms");
}

Statement批量执行

@GetMapping("/save6")
public void save6() {
    // 数据准备
    List<MallOrder> orderList = getMallOrderList();

    // statement.executeBatch
    long start = System.currentTimeMillis();
    mallOrderService.executeBatch(orderList);
    System.out.println("statement的【executeBatch】插入数据100000条，耗时：" + (System.currentTimeMillis() - start) + "ms");
}

@Override
public void executeBatch(List<MallOrder> orderList) {
    String sql = "insert into mall_order (" +
            "customer_id," +
            "order_status," +
            "payment_method," +
            "total_amount," +
            "shipping_fee," +
            "coupon_discount," +
            "order_date," +
            "payment_time," +
            "shipping_address," +
            "receiver_name," +
            "receiver_phone) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);";

    SqlSession sqlSession = null;
    Connection connection = null;
    PreparedStatement statement = null;
    try {
        sqlSession = sqlSessionFactory.openSession();
        connection = sqlSession.getConnection();
        connection.setAutoCommit(false);
        statement = connection.prepareStatement(sql);

        for (MallOrder mallOrder : orderList) {
            // 主键自增不设置
//                statement.setLong(0, mallOrder.getOrderId());
            statement.setLong(1, mallOrder.getCustomerId());
            statement.setInt(2, mallOrder.getOrderStatus());
            statement.setInt(3, mallOrder.getPaymentMethod());
            statement.setBigDecimal(4, mallOrder.getTotalAmount());
            statement.setBigDecimal(5, mallOrder.getShippingFee());
            statement.setBigDecimal(6, mallOrder.getCouponDiscount());
            statement.setObject(7, mallOrder.getOrderDate());
            statement.setObject(8, mallOrder.getPaymentTime());
            statement.setString(9, mallOrder.getShippingAddress());
            statement.setString(10, mallOrder.getReceiverName());
            statement.setString(11, mallOrder.getReceiverPhone());
            statement.addBatch();
        }
        statement.executeBatch();
        connection.commit();
    } catch (SQLException e) {
        throw new RuntimeException(e);
    } finally {
        if (statement != null) {
            try {
                statement.close();
            } catch (SQLException e) {
                throw new RuntimeException(e);
            }
        }
        if (connection != null) {
            try {
                connection.close();
            } catch (SQLException e) {
                throw new RuntimeException(e);
            }
        }
        if (sqlSession != null) {
            sqlSession.close();
        }
    }
}

MyBatis-Plus的saveBatch方法+异步任务

@GetMapping("/save7")
public void save7() {
    // 数据准备
    List<MallOrder> orderList = getMallOrderList();

    // 异步任务处理
    long start = System.currentTimeMillis();
    mallOrderService.saveBatchAsync(orderList);
    System.out.println("mybatis-plus的【异步任务处理】插入数据100000条，耗时：" + (System.currentTimeMillis() - start) + "ms");
}

@Autowired
private ThreadPoolTaskExecutor threadPoolTaskExecutor;
@Autowired
private PlatformTransactionManager transactionManager;

@Override
public void saveBatchAsync(List<MallOrder> orderList) {
    int count = orderList.size();
    // 每批次插入的数据量
    int pageSize = 1000;
    // 线程数
    int threadNum = count % pageSize == 0 ? (count / pageSize) : (count / pageSize + 1);
    CountDownLatch countDownLatch = new CountDownLatch(threadNum);

    for (int i = 0; i < threadNum; i++) {
        int startIndex = i * pageSize;
        int endIndex = Math.min(count, (i + 1) * pageSize);
        List<MallOrder> subList = orderList.subList(startIndex, endIndex);
        threadPoolTaskExecutor.execute(() -> {
            DefaultTransactionDefinition transactionDefinition = new DefaultTransactionDefinition();
            TransactionStatus status = transactionManager.getTransaction(transactionDefinition);
            try {
                // 保存数据
                this.saveBatch(subList);
                transactionManager.commit(status);
            } catch (Exception exception) {
                transactionManager.rollback(status);
                throw exception;
            } finally {
                countDownLatch.countDown();
            }
        });
    }
    try {
        countDownLatch.await();
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    }
}

测试

CREATE TABLE `mall_order`
(
    `order_id`         BIGINT         NOT NULL AUTO_INCREMENT COMMENT '订单ID（主键）',
    `customer_id`      BIGINT         NOT NULL COMMENT '客户ID（关联customer表）',
    `order_status`     tinyint(4)     NOT NULL DEFAULT 1 COMMENT '订单状态 1-待支付 2-已支付 3-待发货 4-已发货 5-已完成 6-已取消',
    `payment_method`   tinyint(4)     NULL     DEFAULT null COMMENT '支付方式; 1-现金 2-支付宝 3-微信 4-银行卡',
    `total_amount`     DECIMAL(10, 2) NOT NULL COMMENT '订单总金额',
    `shipping_fee`     DECIMAL(10, 2) NOT NULL DEFAULT 0 COMMENT '运费',
    `coupon_discount`  DECIMAL(10, 2) NOT NULL DEFAULT 0 COMMENT '优惠券减免金额',
    `order_date`       DATETIME       NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '下单日期',
    `payment_time`     DATETIME                DEFAULT NULL COMMENT '支付时间',
    `shipping_address` VARCHAR(255)   NULL COMMENT '收货地址',
    `receiver_name`    VARCHAR(50)    NULL COMMENT '收货人姓名',
    `receiver_phone`   VARCHAR(20)    NULL COMMENT '收货人电话',
    PRIMARY KEY (`order_id`)
) ENGINE = InnoDB
  DEFAULT CHARSET = utf8mb4 COMMENT ='订单信息表';

package com.qiangesoft.batchsave.entity;

import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.io.Serializable;
import java.math.BigDecimal;
import java.time.LocalDateTime;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;

/**
 * <p>
 * 订单信息表
 * </p>
 *
 * @author qiangesoft
 * @since 2024-05-07
 */
@Getter
@Setter
@Accessors(chain = true)
@TableName("mall_order")
public class MallOrder implements Serializable {

    private static final long serialVersionUID = 1L;

    /**
     * 订单ID（主键）
     */
    @TableId(value = "order_id", type = IdType.INPUT)
    private Long orderId;

    /**
     * 客户ID（关联customer表）
     */
    private Long customerId;

    /**
     * 订单状态 1-待支付 2-已支付 3-待发货 4-已发货 5-已完成 6-已取消
     */
    private Integer orderStatus;

    /**
     * 支付方式; 1-现金 2-支付宝 3-微信 4-银行卡
     */
    private Integer paymentMethod;

    /**
     * 订单总金额
     */
    private BigDecimal totalAmount;

    /**
     * 运费
     */
    private BigDecimal shippingFee;

    /**
     * 优惠券减免金额
     */
    private BigDecimal couponDiscount;

    /**
     * 下单日期
     */
    private LocalDateTime orderDate;

    /**
     * 支付时间
     */
    private LocalDateTime paymentTime;

    /**
     * 收货地址
     */
    private String shippingAddress;

    /**
     * 收货人姓名
     */
    private String receiverName;

    /**
     * 收货人电话
     */
    private String receiverPhone;


}

1000条

/**
 * 构建数据
 *
 * @return
 */
private static List<MallOrder> getMallOrderList() {
    List<MallOrder> orderList = new ArrayList<>();
    for (int i = 0; i < 10000; i++) {
        MallOrder mallOrder = new MallOrder();
        mallOrder.setCustomerId(1L);
        mallOrder.setOrderStatus(1);
        mallOrder.setPaymentMethod(1);
        mallOrder.setTotalAmount(BigDecimal.valueOf(12));
        mallOrder.setShippingFee(BigDecimal.valueOf(1));
        mallOrder.setCouponDiscount(BigDecimal.valueOf(0));
        mallOrder.setOrderDate(LocalDateTime.now());
        mallOrder.setPaymentTime(LocalDateTime.now());
        mallOrder.setShippingAddress("哈哈");
        mallOrder.setReceiverName("暂时");
        mallOrder.setReceiverPhone("13211111111");
        orderList.add(mallOrder);
    }
    return orderList;
}

在这里插入图片描述

10000条

/**
 * 构建数据
 *
 * @return
 */
private static List<MallOrder> getMallOrderList() {
    List<MallOrder> orderList = new ArrayList<>();
    for (int i = 0; i < 1000; i++) {
        MallOrder mallOrder = new MallOrder();
        mallOrder.setCustomerId(1L);
        mallOrder.setOrderStatus(1);
        mallOrder.setPaymentMethod(1);
        mallOrder.setTotalAmount(BigDecimal.valueOf(12));
        mallOrder.setShippingFee(BigDecimal.valueOf(1));
        mallOrder.setCouponDiscount(BigDecimal.valueOf(0));
        mallOrder.setOrderDate(LocalDateTime.now());
        mallOrder.setPaymentTime(LocalDateTime.now());
        mallOrder.setShippingAddress("哈哈");
        mallOrder.setReceiverName("暂时");
        mallOrder.setReceiverPhone("13211111111");
        orderList.add(mallOrder);
    }
    return orderList;
}