mysql分库分表、分库分表算法、分库分表实战与批处理高命中同库表率算法

说明：

timestamp时间即将耗尽，使用datetime 替换 timestamp：
1)区别：时区影响：对于TIMESTAMP，它把客户端插入的时间从当前时区转化为UTC（世界标准时间）进行存储。查询时，将其又转化为客户端当前时区进行返回；而对于DATETIME，不做任何改变，基本上是原样输入和输出。
        存储空间：timestamp占用4个字节，datetime存储占用8个字节
        时间范围：
              timestamp可表示范围:1970-01-01 00:00:00~2038-01-09 03:14:07，
          datetime支持的范围更宽1000-01-01 00:00:00 ~ 9999-12-31 23:59:59
2)替换原因：timestamp会在2038年耗尽，timestamp的DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP属性，
   datetime完美兼容多时区业务，不必考虑时区影响, datetime类型列的默认值需在业务代码中指定。

一.分库分表算法:
1.算法(零散均匀分库)：
1.1)分库分表规则
中间变量=USER_ID%(分库数量*每个库的表数量)
库=中间变量/每个库的表数量
表=中间变量%每个库的表数量

1.2)查看用户所在的库、表
SET @userId = 65;
SET @dbCount = 8;
SET @tablePerDB = 256;
SET @centerNum = @userId % (@tablePerDB * @dbCount);
SET @dbNum = convert(@centerNum / @tablePerDB,signed) ;
SET @tableNum = (@centerNum % @tablePerDB);
SELECT @dbNum,@tableNum;

2.算法(同尾号均匀分配同库)：
2.1)分库分表规则
中间变量=USER_ID%(分库数量*每个库的表数量)
库=中间变量%分库数量
表=中间变量/分库数量

3.2)查看用户所在的库、表
SET @userId = 65;
SET @dbCount = 8;
SET @tablePerDB = 256;
SET @centerNum = @userId % (@tablePerDB * @dbCount);
SET @dbNum = (@centerNum % @dbCount);
SET @tableNum = convert(@centerNum / @dbCount,signed) ;
SELECT @dbNum,@tableNum;
注意：该方式dbCount=10时,能根据userId尾号，能肉眼区分在那个库。

二.相关存储过程

1.分库分表：根据指定业务列创建,id,inserttime,updatetime,isactive字段为规范字段,库与表序号从0开始.
1.1)创建分库分表存储过程
use mysql;
drop procedure if EXISTS split_db_split_table1;
create procedure split_db_split_table1(in logicDb varchar(100),in logicTable varchar(100),in dbCount int(11),in tableCountPerDb int(11),in tableFieldSql text)
begin
DECLARE ddlSql text;
DECLARE dbName varchar(100);
DECLARE tableName varchar(100);
DECLARE dbStartSeq int(11);
DECLARE dbEndSeq int(11);
DECLARE tableStartSeq int(11);
DECLARE tableEndSeq int(11);

 #设置库开始序号
set dbStartSeq = 0;
 #设置库结束序号
set dbEndSeq = dbCount-1;

 #循环建库,dbCount=1表示不分库
while dbStartSeq <= dbEndSeq do
 #检查数据库是否存在,不存在创建,dbCount=1表示不分库
 IF dbCount=1 THEN 
  set dbName = logicDb;
 ELSE
  set dbName = CONCAT(logicDb,"_",LPAD(dbStartSeq,2,'0'));
 END IF;

 set ddlSql = CONCAT(" CREATE DATABASE IF NOT EXISTS ",dbName);
 set @dynamicSql = ddlSql;
 prepare stmt from @dynamicSql;
 execute stmt;     

 #设置表开始序号
 set tableStartSeq = 0;
 #设置表结束序号
 set tableEndSeq = tableCountPerDb-1;
 #循环创建表
 while tableStartSeq <= tableEndSeq do
   set tableName = CONCAT(logicTable,"_",LPAD(tableStartSeq,4,'0'));
  set tableName = CONCAT(" ",dbName,".",tableName);
   # 删除表
   set ddlSql = CONCAT(" DROP TABLE IF EXISTS ",tableName);
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;       

   # 创建表    
   SET ddlSql = CONCAT(" CREATE TABLE IF NOT EXISTS ",tableName," (
               id bigint(20) AUTO_INCREMENT PRIMARY KEY NOT NULL comment '唯一Id',
               user_id bigint(20) NOT NULL COMMENT '用户ID' ,"
               ,IFNULL(tableFieldSql,""),
               "
               inserttime timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '插入时间',
               updatetime timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
               isactive tinyint(1) NOT NULL DEFAULT '1' COMMENT '是否有效'
          ); "
     );
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;              
      
   # 创建idx_user_id索引  
   set ddlSql = CONCAT(" create index idx_user_id on ",tableName," (user_id); ");
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;

   # 创建idx_inserttime索引  
   set ddlSql = CONCAT(" create index idx_inserttime on ",tableName," (inserttime); ");
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;

   # 创建idx_updatetime索引  
   set ddlSql = CONCAT(" create index idx_updatetime on ",tableName," (updatetime); ");
  set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;

   set tableStartSeq = tableStartSeq+1;
 end while;
 set dbStartSeq = dbStartSeq+1;
end while;
end;

1.2)调用分库分表储存过程
use sys;
SET @tableFieldSql = "
age int(4) NOT NULL COMMENT '年龄' ,
";
call split_db_split_table1('pms_manager','tb_user',8,256,@tableFieldSql);

2.分库分表：根据表全部字段语句创建,注意:库与表序号从0开始
2.1)创建分库分表存储过程
use mysql;
drop procedure if EXISTS split_db_split_table2;
create procedure split_db_split_table2(in logicDb varchar(100),in logicTable varchar(100),in dbCount int(11),in tableCountPerDb int(11),in tableFieldSql text)
begin
DECLARE ddlSql text;
DECLARE dbName varchar(100);
DECLARE tableName varchar(100);
DECLARE dbStartSeq int(11);
DECLARE dbEndSeq int(11);
DECLARE tableStartSeq int(11);
DECLARE tableEndSeq int(11);

 #设置库开始序号
set dbStartSeq = 0;
 #设置库结束序号
set dbEndSeq = dbCount-1;

 #循环建库,dbCount=1表示不分库
while dbStartSeq <= dbEndSeq do
 #检查数据库是否存在,不存在创建,dbCount=1表示不分库
 IF dbCount=1 THEN 
  set dbName = logicDb;
 ELSE
  set dbName = CONCAT(logicDb,"_",LPAD(dbStartSeq,2,'0'));
 END IF;

 set ddlSql = CONCAT(" CREATE DATABASE IF NOT EXISTS ",dbName);
 set @dynamicSql = ddlSql;
 prepare stmt from @dynamicSql;
 execute stmt;     

 #设置表开始序号
 set tableStartSeq = 0;
 #设置表结束序号
 set tableEndSeq = tableCountPerDb-1;
 #循环创建表
 while tableStartSeq <= tableEndSeq do
   set tableName = CONCAT(logicTable,"_",LPAD(tableStartSeq,4,'0'));
  set tableName = CONCAT(" ",dbName,".",tableName);
   # 删除表
   set ddlSql = CONCAT(" DROP TABLE IF EXISTS ",tableName);
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;       

   # 创建表    
   SET ddlSql = CONCAT(" CREATE TABLE IF NOT EXISTS ",tableName,IFNULL(tableFieldSql,""));
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;     
  
   set tableStartSeq = tableStartSeq+1;
 end while;
 set dbStartSeq = dbStartSeq+1;
end while;
end;

2.2)调用分库分表储存过程
use sys;
SET @tableFieldSql = "

(
id bigint(20) AUTO_INCREMENT PRIMARY KEY NOT NULL comment '唯一Id',
user_id bigint(20) NOT NULL COMMENT '用户ID' ,
inserttime timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '插入时间',
updatetime timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
isactive tinyint(1) NOT NULL DEFAULT '1' COMMENT '是否有效'
)
ENGINE=InnoDB
DEFAULT CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci
COMMENT='用户表'
AUTO_INCREMENT=1
ROW_FORMAT=DYNAMIC;

";
call split_db_split_table2('pms_manager','tb_user',8,256,@tableFieldSql);

3.删除分库分表：根据参数删库删表，库与表序号从0开始
3.1)创建分库分表存储过程
use mysql;
drop procedure if EXISTS destory_db_split_table;
create procedure destory_db_split_table(in logicDb varchar(100),in logicTable varchar(100),in dbCount int(11),in tableCountPerDb int(11),in isDeleteDb tinyint(1))
begin
DECLARE ddlSql text;
DECLARE dbName varchar(100);
DECLARE tableName varchar(100);
DECLARE dbStartSeq int(11);
DECLARE dbEndSeq int(11);
DECLARE tableStartSeq int(11);
DECLARE tableEndSeq int(11);

 #设置库开始序号
set dbStartSeq = 0;
 #设置库结束序号
set dbEndSeq = dbCount-1;

 #循环建库,dbCount=1表示不分库
while dbStartSeq <= dbEndSeq do
 #检查数据库是否存在,不存在创建,dbCount=1表示不分库
 IF dbCount=1 THEN 
  set dbName = logicDb;
 ELSE
  set dbName = CONCAT(logicDb,"_",LPAD(dbStartSeq,2,'0'));
 END IF;

 #设置表开始序号
 set tableStartSeq = 0;
 #设置表结束序号
 set tableEndSeq = tableCountPerDb-1;
 #循环创建表
 while tableStartSeq <= tableEndSeq do
   set tableName = CONCAT(logicTable,"_",LPAD(tableStartSeq,4,'0'));
  set tableName = CONCAT(" ",dbName,".",tableName);
   # 删除表
   set ddlSql = CONCAT(" DROP TABLE IF EXISTS ",tableName," ; ");
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;       
  
   set tableStartSeq = tableStartSeq+1;
 end while;

 If isDeleteDb THEN
  set ddlSql = CONCAT(" DROP DATABASE IF EXISTS ",dbName," ; ");
  set @dynamicSql = ddlSql;
  prepare stmt from @dynamicSql;
  execute stmt;     
 END IF;
 set dbStartSeq = dbStartSeq+1;
end while;
end;

3.2)调用删库删表储存过程
use sys;
call destory_db_split_table('pms_manager','tb_user',8,256,false);

三.用户id批量排序，高命中率分库分表算法
/**
     * 根据用户Id取2位进行反转再排序，已提升同库分表命中概率(兼容按尾号２位、尾号１位分表)
     * @param userList
     */
    private void sortByUserIdEndHitRate(List<UserInfo> userList){
        Collections.sort(userList, new Comparator<UserInfo>() {
            @Override
            public int compare(UserInfo o1, UserInfo o2) {
                Long end1 = Long.valueOf(reverse(o1.getUserId()));
                Long end2 = Long.valueOf(reverse(o2.getUserId()));
                return end1.compareTo(end2);
            }
        });
    }

    /**
     * 根据用户Id字符串反转
     * @param userId
     * @return
     */
    private String reverse(String userId) {
        char[] s = userId.trim().toCharArray();
        char[] ns = new char[2];
        int n = s.length - 1;
        if(s.length == 0){
            ns[0] = '0';
            ns[1] = '0';
        }else if(s.length == 1){
            ns[0] = s[n];
            ns[1] = '0';
        }else {
            ns[0] = s[n];
            ns[1] = s[n - 1];
        }
        return new String(ns);
    }