SpringBoot快速插入1000萬條數據

導讀

　　有時候爲了驗證系統瓶頸，需要往數據庫表中插入大量數據，可以寫sheel腳本插入，前幾天爲了插入100萬條數據，走的sheel腳本(點我直達)，插入速度簡直無法直視，花了3小時，才插入了10萬條，後來沒轍了，多跑幾次sheel腳本(算是多線程操作吧)，最終花了4個多小時才插入100萬條記錄。今天使用java程序快速插入1000萬條數據，最終只需要3分鐘多一點就搞定了，好了下面開始吧~

添加依賴

        <!--lombok-->
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.16</version>
        </dependency>
        <!--mysql-->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
        </dependency>
        <!--mybatis plus和spring boot整合-->
        <dependency>
            <groupId>com.baomidou</groupId>
            <artifactId>mybatis-plus-boot-starter</artifactId>
            <version>3.4.0</version>
        </dependency>
        <!--Druid連接池-->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid-spring-boot-starter</artifactId>
            <version>1.1.10</version>
        </dependency>
        <!-- guava -->
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>31.0.1-jre</version>
        </dependency>

application.properties

# 端口號
server.port=9999
#===========數據庫相關=============
#spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver
#spring.datasource.url=jdbc:mysql://127.0.0.1/shop?useUnicode=true&characterEncoding=utf-8&useSSL=false
#spring.datasource.username=root
#spring.datasource.password=root
# mybatis plus下劃線轉駝峯
mybatis-plus.configuration.map-underscore-to-camel-case=true
# 配置mybatis plus打印sql日誌
#mybatis-plus.configuration.log-impl=org.apache.ibatis.logging.stdout.StdOutImpl
spring.datasource.name=mysql_test
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
# druid配置
# 監控統計攔截的filters
spring.datasource.druid.filters=stat
spring.datasource.druid.driver-class-name=com.mysql.cj.jdbc.Driver
spring.datasource.druid.url=jdbc:mysql://127.0.0.1/yb_mysql?useUnicode=true&characterEncoding=utf-8&useSSL=false
spring.datasource.druid.username=root
spring.datasource.druid.password=root
#配置初始化大小/最小/最大
spring.datasource.druid.initial-size=1
spring.datasource.druid.min-idle=1
spring.datasource.druid.max-active=20
#獲取連接等待超時時間
spring.datasource.druid.max-wait=60000
#間隔多久進行一次檢測，檢測需要關閉的空閒連接
spring.datasource.druid.min-evictable-idle-time-millis=300000
spring.datasource.druid.validation-query= SELECT 'x'
spring.datasource.druid.test-while-idle=true
spring.datasource.druid.test-on-borrow=true
spring.datasource.druid.test-on-return=false

數據庫表結構

 

實體類

package com.ybchen.domain;

import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.EqualsAndHashCode;

import java.io.Serializable;

/**
 * <p>
 * 用戶表
 * </p>
 *
 * @author chenyanbin
 * @since 2021-11-07
 */
@Data
@EqualsAndHashCode(callSuper = false)
@TableName("user")
public class UserDO implements Serializable {

    private static final long serialVersionUID = 1L;

    /**
     * 主鍵
     */
    @TableId(value = "id", type = IdType.AUTO)
    private Long id;

    /**
     * 姓名
     */
    private String name;

    /**
     * 備註
     */
    private String remark;


}

Mapper

package com.ybchen.mapper;

import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.ybchen.domain.UserDO;
import org.apache.ibatis.annotations.Param;

import java.util.List;

public interface UserMapper extends BaseMapper<UserDO> {
    /**
     * 批量插入
     * @param userList
     * @return
     */
    int batchInsert(@Param("listUser") List<UserDO> userList);
}

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ybchen.mapper.UserMapper">

    <!-- 通用查詢映射結果 -->
    <resultMap id="BaseResultMap" type="com.ybchen.domain.UserDO">
        <id column="id" property="id"/>
        <result column="name" property="name"/>
        <result column="remark" property="remark"/>
    </resultMap>

    <!-- 通用查詢結果列 -->
    <sql id="Base_Column_List">
        id, name, remark
    </sql>

    <!-- 批量插入 -->
    <insert id="batchInsert">
        insert into user (`name`,`remark`)
        values
        <foreach collection="listUser" item="item" separator=",">
            (#{item.name},#{item.remark})
        </foreach>
    </insert>
</mapper>

Controller

package com.ybchen.controller;

import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.google.common.collect.Lists;
import com.ybchen.domain.UserDO;
import com.ybchen.mapper.UserMapper;
import com.ybchen.utils.JsonData;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;

/**
 * @Author：chenyanbin
 */
@RestController
public class UserController {
    @Autowired
    UserMapper userMapper;

    @GetMapping("batchInsert")
    public JsonData batchInsert() {
        int num = 2000;
        CountDownLatch latch = new CountDownLatch(1);
        List<UserDO> userList = new ArrayList<>();
        new Thread(() -> {
            for (int i = 0; i < 10000000; i++) {
                UserDO userDO = new UserDO();
                userDO.setName("陳彥斌====》" + i);
                userDO.setRemark("博客地址：https://www.cnblogs.com/chenyanbin/");
                userList.add(userDO);
            }
            latch.countDown();
        }).start();
        try {
            latch.await();
        } catch (InterruptedException e) {
        }
        //2000條爲一批，插入1000萬條
        List<List<UserDO>> partition = Lists.partition(userList, num);
        partition.stream().forEach(user -> {
            int rows = userMapper.batchInsert(user);
            System.err.println("插入數據成功，rows:" + rows);
        });
        return JsonData.buildSuccess();
    }

    @GetMapping("all")
    public JsonData all(){
        return JsonData.buildSuccess(userMapper.selectList(new LambdaQueryWrapper<>()));
    }
}

項目結構

 

演示

最終耗時