記錄一個mysql大量數據快速插入的方法

經過數月的歷練與開發，終於在這個月手上負責的一個項目即將交付了，看着這一兩個月讓我偶感壓力的繁雜的事情一件件處理完畢，原本動盪的心靜也漸漸平靜了些許。此時再回過頭過去看那些困擾我N多個日夜的某些問題其實也不過如此。有些事情自己無法改變，只能樂觀地坦然地面對。面對充滿未知的未來努力讓自己保持每一天都快樂纔是王道，哈哈！

在本週將一個需要處理上千萬數據量的項目部署到UAT環境後，發現數據插入到mysql(mariadb)的時間太慢了，儘管已經根據項目需要將原來的innodb存儲引擎修改爲了更適合大量數據插入和查詢效率更高的myisam引擎。通過對代碼再次進行分析後發現是由於代碼對於數據插入到mysql的過程中耗費了大量的時間，由於myisam又不支持事務，不能設置批量插入，思量後決定將之前的單條insert語句修改爲多條數據的insert語句或許性能就會有所提升，正好趁着今天放假將這塊的代碼先寫個DEMO給測試下。

測試代碼

測試代碼如下：

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;

@RequestMapping(value = "mysql-test")
@RestController
public class MysqlTestController {

    @Autowired
    private JdbcTemplate jdbcTemplate;


    @GetMapping(value = "mulTest")
    public String mulTest(Integer size, @RequestParam(defaultValue = "false") boolean sharding) {
        initUserInfoTable();
        BlockingQueue<UserInfo> userInfoQueue = initUserInfoQueue(size);
        return size + " 條記錄插入，耗時" + processQueue(userInfoQueue, sharding);
    }

    /**
     * 初始化userInfo表
     */
    private void initUserInfoTable() {
        String userInfoTableDdlStr = "create table if not exists  `userinfo` (\n" +
                "  `userId` int(11) DEFAULT NULL,\n" +
                "  `userName` varchar(255) DEFAULT NULL,\n" +
                "  `gender` varchar(255) DEFAULT NULL,\n" +
                "  `phone` varchar(255) DEFAULT NULL,\n" +
                "  `address` varchar(255) DEFAULT NULL,\n" +
                "  `birthDay` bigint(20) DEFAULT NULL,\n" +
                "  `school` varchar(255) DEFAULT NULL,\n" +
                "  `hobby` varchar(255) DEFAULT NULL,\n" +
                "  KEY `index_userid` (`userId`)\n" +
                ") ENGINE=MyISAM DEFAULT CHARSET=utf8";
        jdbcTemplate.execute(userInfoTableDdlStr);
    }

    private long processQueue(BlockingQueue<UserInfo> userInfoQueue, boolean sharding) {
        int mulSize = 200;
        long beginTime = System.currentTimeMillis();
        try {
            if (sharding) {
                while (!userInfoQueue.isEmpty()) {
                    UserInfo userInfo = userInfoQueue.take();
                    StringBuffer singleInserSql = new StringBuffer();
                    singleInserSql.append("INSERT INTO `userInfo` (`userId`, `userName`, `gender`, `phone`, `address`, `birthDay`, `school`, `hobby`) VALUES (?,?,?,?,?,?,?,?)");
                    jdbcTemplate.update(singleInserSql.toString(), new Object[]{
                            userInfo.getUserId(), userInfo.getUserName(), userInfo.getGender(), userInfo.getPhone(), userInfo.getAddress(), userInfo.getBirthDay().getTime() / 1000, userInfo.getSchool(), userInfo.getHobby()
                    });
                }
            } else {
                String insertSqlStr = genMulFixSqlStr(mulSize);
                List<UserInfo> userInfoList = new ArrayList<>();
                while (!userInfoQueue.isEmpty()) {
                    UserInfo userInfo = userInfoQueue.take();
                    userInfoList.add(userInfo);
                    if (userInfoList.size() == mulSize) {
                        Object[] params = getMulParams(userInfoList, userInfoList.size());
                        jdbcTemplate.update(insertSqlStr, params);
                        userInfoList.clear();
                    }
                }
                //插入剩餘的數據
                if (!userInfoList.isEmpty()) {
                    String restInsertSql = genMulFixSqlStr(userInfoList.size());
                    Object[] params = getMulParams(userInfoList, userInfoList.size());
                    jdbcTemplate.update(restInsertSql, params);
                    userInfoList.clear();
                }
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        long endTime = System.currentTimeMillis();
        return endTime - beginTime;
    }

    private Object[] getMulParams(List<UserInfo> userInfoList, int mulSize) {
        Object[] params = new Object[mulSize * 8];
        for (int i = 0; i < mulSize; i++) {
            UserInfo userInfo = userInfoList.get(i);
            params[8 * i] = userInfo.getUserId();
            params[8 * i + 1] = userInfo.getUserName();
            params[8 * i + 2] = userInfo.getGender();
            params[8 * i + 3] = userInfo.getPhone();
            params[8 * i + 4] = userInfo.getAddress();
            params[8 * i + 5] = userInfo.getBirthDay().getTime() / 1000;
            params[8 * i + 6] = userInfo.getSchool();
            params[8 * i + 7] = userInfo.getHobby();
        }
        return params;
    }

    private String genMulFixSqlStr(int mulSize) {
        StringBuffer mulInserSql = new StringBuffer();
        mulInserSql.append("INSERT INTO `userInfo` (`userId`, `userName`, `gender`, `phone`, `address`, `birthDay`, `school`, `hobby`) VALUES (?,?,?,?,?,?,?,?)");
        for (int i = 1; i < mulSize; i++) {
            mulInserSql.append(",(?,?,?,?,?,?,?,?)");
        }
        return mulInserSql.toString();
    }

    private BlockingQueue<UserInfo> initUserInfoQueue(Integer size) {
        BlockingQueue<UserInfo> userInfoQueue = new LinkedBlockingQueue<>();
        for (int i = 0; i <= size; i++) {
            UserInfo userInfo = new UserInfo();
            userInfo.setAddress("shanghai" + i);
            userInfo.setBirthDay(new Date());
            userInfo.setGender(i % 2 == 0 ? "male" : "famale");
            userInfo.setHobby("programing,basketball,reading,driving");
            userInfo.setPhone("15700" + i);
            userInfo.setSchool("njCoolege");
            userInfo.setUserId(i);
            userInfo.setUserName("ping" + i);

            userInfoQueue.add(userInfo);
        }
        return userInfoQueue;
    }


    private class UserInfo {
        private Integer userId;
        private String userName;
        private String gender;
        private String phone;
        private String address;
        private Date birthDay;
        private String school;
        private String hobby;

        public Integer getUserId() {
            return userId;
        }

        public void setUserId(Integer userId) {
            this.userId = userId;
        }

        public String getUserName() {
            return userName;
        }

        public void setUserName(String userName) {
            this.userName = userName;
        }

        public String getGender() {
            return gender;
        }

        public void setGender(String gender) {
            this.gender = gender;
        }

        public String getPhone() {
            return phone;
        }

        public void setPhone(String phone) {
            this.phone = phone;
        }

        public String getAddress() {
            return address;
        }

        public void setAddress(String address) {
            this.address = address;
        }

        public Date getBirthDay() {
            return birthDay;
        }

        public void setBirthDay(Date birthDay) {
            this.birthDay = birthDay;
        }

        public String getSchool() {
            return school;
        }

        public void setSchool(String school) {
            this.school = school;
        }

        public String getHobby() {
            return hobby;
        }

        public void setHobby(String hobby) {
            this.hobby = hobby;
        }
    }


}

快速敲完後，立即試試插入1萬條數據試試效率如何

分別訪問：A.http://localhost:8080/mysql-test/mulTest?size=10000&sharding=true

B.http://localhost:8080/mysql-test/mulTest?size=10000&sharding=true

兩個URL後發現之前的單條插入的sql語句插入10000條時間花費了9721多毫秒

而修改後的多條SQL插入10000條數據僅花費了373毫秒

通過簡單的對比測試後，可知多條數據批量插入的效率確實比單條數據插入的數據高上許多倍，比想象之中的預期還要高上許多。

有了上面的實踐數據作爲有力的理論後盾，得了，爲了項目能更好地上線，再在家裏加個班吧，寫代碼去了。加油！！！

總結

最後還是再總結一下

對於需要向mysql中插入大量數據可通過如下方式進行：

1.將表的存儲引擎修改爲myisam

2.插入數據時採用批量插入方式進行