mysql分庫分表、分庫分表算法、分庫分表實戰與批處理高命中同庫表率算法

說明：

timestamp時間即將耗盡，使用datetime 替換 timestamp：
1)區別：時區影響：對於TIMESTAMP，它把客戶端插入的時間從當前時區轉化爲UTC（世界標準時間）進行存儲。查詢時，將其又轉化爲客戶端當前時區進行返回；而對於DATETIME，不做任何改變，基本上是原樣輸入和輸出。
        存儲空間：timestamp佔用4個字節，datetime存儲佔用8個字節
        時間範圍：
              timestamp可表示範圍:1970-01-01 00:00:00~2038-01-09 03:14:07，
          datetime支持的範圍更寬1000-01-01 00:00:00 ~ 9999-12-31 23:59:59
2)替換原因：timestamp會在2038年耗盡，timestamp的DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP屬性，
   datetime完美兼容多時區業務，不必考慮時區影響, datetime類型列的默認值需在業務代碼中指定。

一.分庫分表算法:
1.算法(零散均勻分庫)：
1.1)分庫分表規則
中間變量=USER_ID%(分庫數量*每個庫的表數量)
庫=中間變量/每個庫的表數量
表=中間變量%每個庫的表數量

1.2)查看用戶所在的庫、表
SET @userId = 65;
SET @dbCount = 8;
SET @tablePerDB = 256;
SET @centerNum = @userId % (@tablePerDB * @dbCount);
SET @dbNum = convert(@centerNum / @tablePerDB,signed) ;
SET @tableNum = (@centerNum % @tablePerDB);
SELECT @dbNum,@tableNum;

2.算法(同尾號均勻分配同庫)：
2.1)分庫分表規則
中間變量=USER_ID%(分庫數量*每個庫的表數量)
庫=中間變量%分庫數量
表=中間變量/分庫數量

3.2)查看用戶所在的庫、表
SET @userId = 65;
SET @dbCount = 8;
SET @tablePerDB = 256;
SET @centerNum = @userId % (@tablePerDB * @dbCount);
SET @dbNum = (@centerNum % @dbCount);
SET @tableNum = convert(@centerNum / @dbCount,signed) ;
SELECT @dbNum,@tableNum;
注意：該方式dbCount=10時,能根據userId尾號，能肉眼區分在那個庫。

二.相關存儲過程

1.分庫分表：根據指定業務列創建,id,inserttime,updatetime,isactive字段爲規範字段,庫與表序號從0開始.
1.1)創建分庫分表存儲過程
use mysql;
drop procedure if EXISTS split_db_split_table1;
create procedure split_db_split_table1(in logicDb varchar(100),in logicTable varchar(100),in dbCount int(11),in tableCountPerDb int(11),in tableFieldSql text)
begin
DECLARE ddlSql text;
DECLARE dbName varchar(100);
DECLARE tableName varchar(100);
DECLARE dbStartSeq int(11);
DECLARE dbEndSeq int(11);
DECLARE tableStartSeq int(11);
DECLARE tableEndSeq int(11);

 #設置庫開始序號
set dbStartSeq = 0;
 #設置庫結束序號
set dbEndSeq = dbCount-1;

 #循環建庫,dbCount=1表示不分庫
while dbStartSeq <= dbEndSeq do
 #檢查數據庫是否存在,不存在創建,dbCount=1表示不分庫
 IF dbCount=1 THEN 
  set dbName = logicDb;
 ELSE
  set dbName = CONCAT(logicDb,"_",LPAD(dbStartSeq,2,'0'));
 END IF;

 set ddlSql = CONCAT(" CREATE DATABASE IF NOT EXISTS ",dbName);
 set @dynamicSql = ddlSql;
 prepare stmt from @dynamicSql;
 execute stmt;     

 #設置表開始序號
 set tableStartSeq = 0;
 #設置表結束序號
 set tableEndSeq = tableCountPerDb-1;
 #循環創建表
 while tableStartSeq <= tableEndSeq do
   set tableName = CONCAT(logicTable,"_",LPAD(tableStartSeq,4,'0'));
  set tableName = CONCAT(" ",dbName,".",tableName);
   # 刪除表
   set ddlSql = CONCAT(" DROP TABLE IF EXISTS ",tableName);
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;       

   # 創建表    
   SET ddlSql = CONCAT(" CREATE TABLE IF NOT EXISTS ",tableName," (
               id bigint(20) AUTO_INCREMENT PRIMARY KEY NOT NULL comment '唯一Id',
               user_id bigint(20) NOT NULL COMMENT '用戶ID' ,"
               ,IFNULL(tableFieldSql,""),
               "
               inserttime timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '插入時間',
               updatetime timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新時間',
               isactive tinyint(1) NOT NULL DEFAULT '1' COMMENT '是否有效'
          ); "
     );
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;              
      
   # 創建idx_user_id索引  
   set ddlSql = CONCAT(" create index idx_user_id on ",tableName," (user_id); ");
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;

   # 創建idx_inserttime索引  
   set ddlSql = CONCAT(" create index idx_inserttime on ",tableName," (inserttime); ");
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;

   # 創建idx_updatetime索引  
   set ddlSql = CONCAT(" create index idx_updatetime on ",tableName," (updatetime); ");
  set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;

   set tableStartSeq = tableStartSeq+1;
 end while;
 set dbStartSeq = dbStartSeq+1;
end while;
end;

1.2)調用分庫分表儲存過程
use sys;
SET @tableFieldSql = "
age int(4) NOT NULL COMMENT '年齡' ,
";
call split_db_split_table1('pms_manager','tb_user',8,256,@tableFieldSql);

2.分庫分表：根據表全部字段語句創建,注意:庫與表序號從0開始
2.1)創建分庫分表存儲過程
use mysql;
drop procedure if EXISTS split_db_split_table2;
create procedure split_db_split_table2(in logicDb varchar(100),in logicTable varchar(100),in dbCount int(11),in tableCountPerDb int(11),in tableFieldSql text)
begin
DECLARE ddlSql text;
DECLARE dbName varchar(100);
DECLARE tableName varchar(100);
DECLARE dbStartSeq int(11);
DECLARE dbEndSeq int(11);
DECLARE tableStartSeq int(11);
DECLARE tableEndSeq int(11);

 #設置庫開始序號
set dbStartSeq = 0;
 #設置庫結束序號
set dbEndSeq = dbCount-1;

 #循環建庫,dbCount=1表示不分庫
while dbStartSeq <= dbEndSeq do
 #檢查數據庫是否存在,不存在創建,dbCount=1表示不分庫
 IF dbCount=1 THEN 
  set dbName = logicDb;
 ELSE
  set dbName = CONCAT(logicDb,"_",LPAD(dbStartSeq,2,'0'));
 END IF;

 set ddlSql = CONCAT(" CREATE DATABASE IF NOT EXISTS ",dbName);
 set @dynamicSql = ddlSql;
 prepare stmt from @dynamicSql;
 execute stmt;     

 #設置表開始序號
 set tableStartSeq = 0;
 #設置表結束序號
 set tableEndSeq = tableCountPerDb-1;
 #循環創建表
 while tableStartSeq <= tableEndSeq do
   set tableName = CONCAT(logicTable,"_",LPAD(tableStartSeq,4,'0'));
  set tableName = CONCAT(" ",dbName,".",tableName);
   # 刪除表
   set ddlSql = CONCAT(" DROP TABLE IF EXISTS ",tableName);
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;       

   # 創建表    
   SET ddlSql = CONCAT(" CREATE TABLE IF NOT EXISTS ",tableName,IFNULL(tableFieldSql,""));
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;     
  
   set tableStartSeq = tableStartSeq+1;
 end while;
 set dbStartSeq = dbStartSeq+1;
end while;
end;

2.2)調用分庫分表儲存過程
use sys;
SET @tableFieldSql = "

(
id bigint(20) AUTO_INCREMENT PRIMARY KEY NOT NULL comment '唯一Id',
user_id bigint(20) NOT NULL COMMENT '用戶ID' ,
inserttime timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '插入時間',
updatetime timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新時間',
isactive tinyint(1) NOT NULL DEFAULT '1' COMMENT '是否有效'
)
ENGINE=InnoDB
DEFAULT CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci
COMMENT='用戶表'
AUTO_INCREMENT=1
ROW_FORMAT=DYNAMIC;

";
call split_db_split_table2('pms_manager','tb_user',8,256,@tableFieldSql);

3.刪除分庫分表：根據參數刪庫刪表，庫與表序號從0開始
3.1)創建分庫分表存儲過程
use mysql;
drop procedure if EXISTS destory_db_split_table;
create procedure destory_db_split_table(in logicDb varchar(100),in logicTable varchar(100),in dbCount int(11),in tableCountPerDb int(11),in isDeleteDb tinyint(1))
begin
DECLARE ddlSql text;
DECLARE dbName varchar(100);
DECLARE tableName varchar(100);
DECLARE dbStartSeq int(11);
DECLARE dbEndSeq int(11);
DECLARE tableStartSeq int(11);
DECLARE tableEndSeq int(11);

 #設置庫開始序號
set dbStartSeq = 0;
 #設置庫結束序號
set dbEndSeq = dbCount-1;

 #循環建庫,dbCount=1表示不分庫
while dbStartSeq <= dbEndSeq do
 #檢查數據庫是否存在,不存在創建,dbCount=1表示不分庫
 IF dbCount=1 THEN 
  set dbName = logicDb;
 ELSE
  set dbName = CONCAT(logicDb,"_",LPAD(dbStartSeq,2,'0'));
 END IF;

 #設置表開始序號
 set tableStartSeq = 0;
 #設置表結束序號
 set tableEndSeq = tableCountPerDb-1;
 #循環創建表
 while tableStartSeq <= tableEndSeq do
   set tableName = CONCAT(logicTable,"_",LPAD(tableStartSeq,4,'0'));
  set tableName = CONCAT(" ",dbName,".",tableName);
   # 刪除表
   set ddlSql = CONCAT(" DROP TABLE IF EXISTS ",tableName," ; ");
   set @dynamicSql = ddlSql;
   prepare stmt from @dynamicSql;
   execute stmt;       
  
   set tableStartSeq = tableStartSeq+1;
 end while;

 If isDeleteDb THEN
  set ddlSql = CONCAT(" DROP DATABASE IF EXISTS ",dbName," ; ");
  set @dynamicSql = ddlSql;
  prepare stmt from @dynamicSql;
  execute stmt;     
 END IF;
 set dbStartSeq = dbStartSeq+1;
end while;
end;

3.2)調用刪庫刪表儲存過程
use sys;
call destory_db_split_table('pms_manager','tb_user',8,256,false);

三.用戶id批量排序，高命中率分庫分表算法
/**
     * 根據用戶Id取2位進行反轉再排序，已提升同庫分表命中概率(兼容按尾號２位、尾號１位分表)
     * @param userList
     */
    private void sortByUserIdEndHitRate(List<UserInfo> userList){
        Collections.sort(userList, new Comparator<UserInfo>() {
            @Override
            public int compare(UserInfo o1, UserInfo o2) {
                Long end1 = Long.valueOf(reverse(o1.getUserId()));
                Long end2 = Long.valueOf(reverse(o2.getUserId()));
                return end1.compareTo(end2);
            }
        });
    }

    /**
     * 根據用戶Id字符串反轉
     * @param userId
     * @return
     */
    private String reverse(String userId) {
        char[] s = userId.trim().toCharArray();
        char[] ns = new char[2];
        int n = s.length - 1;
        if(s.length == 0){
            ns[0] = '0';
            ns[1] = '0';
        }else if(s.length == 1){
            ns[0] = s[n];
            ns[1] = '0';
        }else {
            ns[0] = s[n];
            ns[1] = s[n - 1];
        }
        return new String(ns);
    }