概述
- ShardingSphere,它由Sharding-JDBC、Sharding-Proxy和Sharding-Sidecar(計劃中)這3款相互獨立的產品組成。定位爲輕量級Java框架。其實就是一個增強版的JDBC驅動,完全兼容JDBC和各種ORM框架。內部改寫了SQL的添加和查詢規則。適用於任何基於Java的ORM框架,如:JPA, Hibernate, Mybatis, Spring JDBC Template或直接使用JDBC。
- 目前已經進入Apache孵化器。以4.x版本爲新的發佈開始
如何單庫分表集成
- 首先集成一個不分庫只分表的模式。創建一個springboot項目,這裏使用Sharding-JDBC3.0版本。使用
sharding-jdbc-spring-boot-starter
集成
pom.xml配置
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.4.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.example</groupId>
<artifactId>shatding-springboot-mybatis-generator</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>shatding-springboot-mybatis-generator</name>
<description>SpringBoot2.x整合MyBatisGenerator 以及分庫分表插件</description>
<properties>
<sharding.jdbc.version>3.0.0</sharding.jdbc.version>
<mybatis.version>1.3.0</mybatis.version>
<druid.version>1.1.10</druid.version>
<mysql.version>5.1.38</mysql.version>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>${mybatis.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>${druid.version}</version>
</dependency>
<dependency>
<groupId>io.shardingsphere</groupId>
<artifactId>sharding-jdbc-spring-boot-starter</artifactId>
<version>${sharding.jdbc.version}</version>
</dependency>
創建測試數據局test_order
。分別創建三張表t_address
, t_user0
,t_user1
。這裏假設t_user這個預計隨着系統的運行。公司發展很好,以後數據量會暴增。所以提前進行水平分片存儲。相對於垂直分片,它不再將數據根據業務邏輯分類,而是通過某個字段(或某幾個字段),根據某種規則將數據分散至多個庫或表中,每個分片僅包含數據的一部分。這樣單表數據量降下來了,mysql的B+樹的檢索效率就提高了
創建測試數據
CREATE TABLE `t_address` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`code` varchar(64) DEFAULT NULL COMMENT '編碼',
`name` varchar(64) DEFAULT NULL COMMENT '名稱',
`pid` varchar(64) NOT NULL DEFAULT '0' COMMENT '父id',
`type` int(11) DEFAULT NULL COMMENT '1國家2省3市4縣區',
`lit` int(11) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=21 DEFAULT CHARSET=utf8;
CREATE TABLE `t_user0` (
`id` bigint(20) NOT NULL,
`name` varchar(64) DEFAULT NULL COMMENT '名稱',
`city_id` int(12) DEFAULT NULL COMMENT '城市',
`sex` tinyint(1) DEFAULT NULL COMMENT '性別',
`phone` varchar(32) DEFAULT NULL COMMENT '電話',
`email` varchar(32) DEFAULT NULL COMMENT '郵箱',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '創建時間',
`password` varchar(32) DEFAULT NULL COMMENT '密碼',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `t_user1` (
`id` bigint(20) NOT NULL,
`name` varchar(64) DEFAULT NULL COMMENT '名稱',
`city_id` int(12) DEFAULT NULL COMMENT '城市',
`sex` tinyint(1) DEFAULT NULL COMMENT '性別',
`phone` varchar(32) DEFAULT NULL COMMENT '電話',
`email` varchar(32) DEFAULT NULL COMMENT '郵箱',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '創建時間',
`password` varchar(32) DEFAULT NULL COMMENT '密碼',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
下面就是基本的curd騷操作配置了
添加實體類
public class User {
private Long id;
private String name;
private String phone;
private String email;
private String password;
private Integer cityId;
private Date createTime;
private Integer sex;
}
public class Address {
private Long id;
private String code;
private String name;
private String pid;
private Integer type;
private Integer lit;
}
創建Mapper
@Mapper
public interface AddressMapper {
/** * 保存 */
void save(Address address);
/** * 查詢 * @param id * @return */
Address get(Long id);
}
@Mapper
public interface UserMapper {
/** * 保存 */
void save(User user);
/** * 查詢 * @param id * @return */
User get(Long id);
}
添加Controller
UserController
@Controller
public class UserController {
@Autowired
private UserMapper userMapper;
@RequestMapping("/user/save")
@ResponseBody
public String save() {
for (int i = 0; i <10 ; i++) {
User user=new User();
user.setName("test"+i);
user.setCityId(1%2==0?1:2);
user.setCreateTime(new Date());
user.setSex(i%2==0?1:2);
user.setPhone("11111111"+i);
user.setEmail("xxxxx");
user.setCreateTime(new Date());
user.setPassword("eeeeeeeeeeee");
userMapper.save(user);
}
return "success";
}
@RequestMapping("/user/get/{id}")
@ResponseBody
public User get(@PathVariable Long id) {
User user = userMapper.get(id);
System.out.println(user.getId());
return user;
}
}
AddressController
@Controller
public class AddressController {
@Autowired
private AddressMapper addressMapper;
@RequestMapping("/address/save")
@ResponseBody
public String save() {
for (int i = 0; i <10 ; i++) {
Address address=new Address();
address.setCode("code_"+i);
address.setName("name_"+i);
address.setPid(i+"");
address.setType(0);
address.setLit(i%2==0?1:2);
addressMapper.save(address);
}
return "success";
}
@RequestMapping("/address/get/{id}")
@ResponseBody
public Address get(@PathVariable Long id) {
return addressMapper.get(id);
}
}
AddressMapper.xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.example.mapper.AddressMapper">
<insert id="save" parameterType="Address">
INSERT INTO t_address(code,name,pid,type,lit)
VALUES
(
#{code},#{name},#{pid},#{type},#{lit}
)
</insert>
<select id="get" parameterType="long" resultType="Address">
select * from t_address where id = #{id}
</select>
</mapper>
UserMapper.xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.example.mapper.UserMapper">
<insert id="save" parameterType="User">
INSERT INTO t_user(name,phone,email,city_id,sex,password)
VALUES
(
#{name},#{phone},#{email},#{cityId},#{sex},#{password}
)
</insert>
<select id="get" parameterType="long" resultType="User">
select * from t_user where id = #{id}
</select>
</mapper>
設置分片規則
- application.yml中配置具體要進行分片存儲的表規則
- 行表達式標識符可以使用 ->{…},但前者與Spring本身的屬性文件佔位符衝突,因此在Spring環境中使用行表達式標識符建議使用$->{…}。
server:
port: 8080
spring:
application:
name: shatding-springboot-mybatis
mybatis:
mapper-locations: classpath:mybatis/mapper/*.xml type-aliases-package: com.example.entity sharding: jdbc: datasource: names: ds0 # 數據源ds0 ds0: type: com.alibaba.druid.pool.DruidDataSource driver-class-name: com.mysql.jdbc.Driver url: jdbc:mysql://localhost:3306/test_order username: root password: 123456 config: sharding: props: sql.show: true tables: t_user: #t_user表 key-generator-column-name: id #主鍵 actual-data-nodes: ds0.t_user${0..1} #數據節點,均勻分佈 table-strategy: #分表策略 inline: #行表達式 sharding-column: sex algorithm-expression: t_user${sex % 2} #按模運算分配
一個簡單的水平分片單庫分表就完成了。進行測試就發現數據分別存儲到t_user0和t_user1兩個表中。這裏採用的事按照字段sex
取模分片存儲。
如何集成分庫又分表
- 既分庫又分表其實只需要在配置文件修改一個分片規則即可,不用修改業務任何代碼。分庫分表的數據表不能用自增主鍵,Sharding-JDBC會自動分配一個id,默認使用雪花算法(snowflake)生成64bit的長整型數據。
CREATE TABLE `t_address` (
`id` bigint(20) NOT NULL,
`code` varchar(64) DEFAULT NULL COMMENT '編碼',
`name` varchar(64) DEFAULT NULL COMMENT '名稱',
`pid` varchar(64) NOT NULL DEFAULT '0' COMMENT '父id',
`type` int(11) DEFAULT NULL COMMENT '1國家2省3市4縣區',
`lit` int(11) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `t_user0` (
`id` bigint(20) NOT NULL,
`name` varchar(64) DEFAULT NULL COMMENT '名稱',
`city_id` int(12) DEFAULT NULL COMMENT '城市',
`sex` tinyint(1) DEFAULT NULL COMMENT '性別',
`phone` varchar(32) DEFAULT NULL COMMENT '電話',
`email` varchar(32) DEFAULT NULL COMMENT '郵箱',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '創建時間',
`password` varchar(32) DEFAULT NULL COMMENT '密碼',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `t_user1` (
`id` bigint(20) NOT NULL,
`name` varchar(64) DEFAULT NULL COMMENT '名稱',
`city_id` int(12) DEFAULT NULL COMMENT '城市',
`sex` tinyint(1) DEFAULT NULL COMMENT '性別',
`phone` varchar(32) DEFAULT NULL COMMENT '電話',
`email` varchar(32) DEFAULT NULL COMMENT '郵箱',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '創建時間',
`password` varchar(32) DEFAULT NULL COMMENT '密碼',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
修改分配規則
server:
port: 9900
spring:
application:
name: shard-jdbc-starter
# mybatis
mybatis:
mapper-locations: classpath:mybatis/mapper/*.xml type-aliases-package: com.example.entity ###數據源名稱,多數據源以逗號分隔 sharding: jdbc: datasource: names: ds0,ds1 # 數據源ds0 ds0: type: com.alibaba.druid.pool.DruidDataSource driver-class-name: com.mysql.jdbc.Driver url: jdbc:mysql://localhost:3306/order1 username: root password: 123456 # 數據源ds1 ds1: type: com.alibaba.druid.pool.DruidDataSource driver-class-name: com.mysql.jdbc.Driver url: jdbc:mysql://localhost:3306/order2 username: root password: 123456 config: sharding: props: sql.show: true tables: t_user: #t_user表 key-generator-column-name: id #主鍵 actual-data-nodes: ds${0..1}.t_user${0..1} #數據節點,均勻分佈 database-strategy: #分庫策略 inline: #行表達式 sharding-column: city_id #列名稱,多個列以逗號分隔 algorithm-expression: ds${city_id % 2} #按模運算分配 table-strategy: #分表策略 inline: #行表達式 sharding-column: sex algorithm-expression: t_user${sex % 2} t_address: key-generator-column-name: id actual-data-nodes: ds${0..1}.t_address database-strategy: inline: sharding-column: lit algorithm-expression: ds${lit % 2}
分庫又分表就完成了,測試over。
官方配置示例
數據分片
sharding.jdbc.datasource.names=ds0,ds1
sharding.jdbc.datasource.ds0.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.ds0.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.ds0.url=jdbc:mysql://localhost:3306/ds0
sharding.jdbc.datasource.ds0.username=root
sharding.jdbc.datasource.ds0.password=
sharding.jdbc.datasource.ds1.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.ds1.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.ds1.url=jdbc:mysql://localhost:3306/ds1
sharding.jdbc.datasource.ds1.username=root
sharding.jdbc.datasource.ds1.password=
sharding.jdbc.config.sharding.tables.t_order.actual-data-nodes=ds$->{0..1}.t_order$->{0..1}
sharding.jdbc.config.sharding.tables.t_order.table-strategy.inline.sharding-column=order_id
sharding.jdbc.config.sharding.tables.t_order.table-strategy.inline.algorithm-expression=t_order$->{order_id % 2}
sharding.jdbc.config.sharding.tables.t_order.key-generator-column-name=order_id
sharding.jdbc.config.sharding.tables.t_order_item.actual-data-nodes=ds$->{0..1}.t_order_item$->{0..1}
sharding.jdbc.config.sharding.tables.t_order_item.table-strategy.inline.sharding-column=order_id
sharding.jdbc.config.sharding.tables.t_order_item.table-strategy.inline.algorithm-expression=t_order_item$->{order_id % 2}
sharding.jdbc.config.sharding.tables.t_order_item.key-generator-column-name=order_item_id
sharding.jdbc.config.sharding.binding-tables=t_order,t_order_item
sharding.jdbc.config.sharding.broadcast-tables=t_config
sharding.jdbc.config.sharding.default-database-strategy.inline.sharding-column=user_id
sharding.jdbc.config.sharding.default-database-strategy.inline.algorithm-expression=ds$->{user_id % 2}
讀寫分離
harding.jdbc.datasource.names=master,slave0,slave1
sharding.jdbc.datasource.master.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.master.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.master.url=jdbc:mysql://localhost:3306/master
sharding.jdbc.datasource.master.username=root
sharding.jdbc.datasource.master.password=
sharding.jdbc.datasource.slave0.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.slave0.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.slave0.url=jdbc:mysql://localhost:3306/slave0
sharding.jdbc.datasource.slave0.username=root
sharding.jdbc.datasource.slave0.password=
sharding.jdbc.datasource.slave1.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.slave1.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.slave1.url=jdbc:mysql://localhost:3306/slave1
sharding.jdbc.datasource.slave1.username=root
sharding.jdbc.datasource.slave1.password=
sharding.jdbc.config.masterslave.load-balance-algorithm-type=round_robin
sharding.jdbc.config.masterslave.name=ms
sharding.jdbc.config.masterslave.master-data-source-name=master
sharding.jdbc.config.masterslave.slave-data-source-names=slave0,slave1
sharding.jdbc.config.props.sql.show=true
數據分片 + 讀寫分離
sharding.jdbc.datasource.names=master0,master1,master0slave0,master0slave1,master1slave0,master1slave1
sharding.jdbc.datasource.master0.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.master0.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.master0.url=jdbc:mysql://localhost:3306/master0
sharding.jdbc.datasource.master0.username=root
sharding.jdbc.datasource.master0.password=
sharding.jdbc.datasource.master0slave0.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.master0slave0.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.master0slave0.url=jdbc:mysql://localhost:3306/master0slave0
sharding.jdbc.datasource.master0slave0.username=root
sharding.jdbc.datasource.master0slave0.password=
sharding.jdbc.datasource.master0slave1.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.master0slave1.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.master0slave1.url=jdbc:mysql://localhost:3306/master0slave1
sharding.jdbc.datasource.master0slave1.username=root
sharding.jdbc.datasource.master0slave1.password=
sharding.jdbc.datasource.master1.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.master1.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.master1.url=jdbc:mysql://localhost:3306/master1
sharding.jdbc.datasource.master1.username=root
sharding.jdbc.datasource.master1.password=
sharding.jdbc.datasource.master1slave0.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.master1slave0.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.master1slave0.url=jdbc:mysql://localhost:3306/master1slave0
sharding.jdbc.datasource.master1slave0.username=root
sharding.jdbc.datasource.master1slave0.password=
sharding.jdbc.datasource.master1slave1.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.master1slave1.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.master1slave1.url=jdbc:mysql://localhost:3306/master1slave1
sharding.jdbc.datasource.master1slave1.username=root
sharding.jdbc.datasource.master1slave1.password=
sharding.jdbc.config.sharding.tables.t_order.actual-data-nodes=ds$->{0..1}.t_order$->{0..1}
sharding.jdbc.config.sharding.tables.t_order.table-strategy.inline.sharding-column=order_id
sharding.jdbc.config.sharding.tables.t_order.table-strategy.inline.algorithm-expression=t_order$->{order_id % 2}
sharding.jdbc.config.sharding.tables.t_order.key-generator-column-name=order_id
sharding.jdbc.config.sharding.tables.t_order_item.actual-data-nodes=ds$->{0..1}.t_order_item$->{0..1}
sharding.jdbc.config.sharding.tables.t_order_item.table-strategy.inline.sharding-column=order_id
sharding.jdbc.config.sharding.tables.t_order_item.table-strategy.inline.algorithm-expression=t_order_item$->{order_id % 2}
sharding.jdbc.config.sharding.tables.t_order_item.key-generator-column-name=order_item_id
sharding.jdbc.config.sharding.binding-tables=t_order,t_order_item
sharding.jdbc.config.sharding.broadcast-tables=t_config
sharding.jdbc.config.sharding.default-database-strategy.inline.sharding-column=user_id
sharding.jdbc.config.sharding.default-database-strategy.inline.algorithm-expression=master$->{user_id % 2}
sharding.jdbc.config.sharding.master-slave-rules.ds0.master-data-source-name=master0
sharding.jdbc.config.sharding.master-slave-rules.ds0.slave-data-source-names=master0slave0, master0slave1
sharding.jdbc.config.sharding.master-slave-rules.ds1.master-data-source-name=master1
sharding.jdbc.config.sharding.master-slave-rules.ds1.slave-data-source-names=master1slave0, master1slave1
數據治理
sharding.jdbc.datasource.names=ds,ds0,ds1
sharding.jdbc.datasource.ds.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.ds.driver-class-name=org.h2.Driver
sharding.jdbc.datasource.ds.url=jdbc:mysql://localhost:3306/ds
sharding.jdbc.datasource.ds.username=root
sharding.jdbc.datasource.ds.password=
sharding.jdbc.datasource.ds0.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.ds0.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.ds0.url=jdbc:mysql://localhost:3306/ds0
sharding.jdbc.datasource.ds0.username=root
sharding.jdbc.datasource.ds0.password=
sharding.jdbc.datasource.ds1.type=org.apache.commons.dbcp.BasicDataSource
sharding.jdbc.datasource.ds1.driver-class-name=com.mysql.jdbc.Driver
sharding.jdbc.datasource.ds1.url=jdbc:mysql://localhost:3306/ds1
sharding.jdbc.datasource.ds1.username=root
sharding.jdbc.datasource.ds1.password=
sharding.jdbc.config.sharding.default-data-source-name=ds
sharding.jdbc.config.sharding.default-database-strategy.inline.sharding-column=user_id
sharding.jdbc.config.sharding.default-database-strategy.inline.algorithm-expression=ds$->{user_id % 2}
sharding.jdbc.config.sharding.tables.t_order.actual-data-nodes=ds$->{0..1}.t_order$->{0..1}
sharding.jdbc.config.sharding.tables.t_order.table-strategy.inline.sharding-column=order_id
sharding.jdbc.config.sharding.tables.t_order.table-strategy.inline.algorithm-expression=t_order$->{order_id % 2}
sharding.jdbc.config.sharding.tables.t_order.key-generator-column-name=order_id
sharding.jdbc.config.sharding.tables.t_order_item.actual-data-nodes=ds$->{0..1}.t_order_item$->{0..1}
sharding.jdbc.config.sharding.tables.t_order_item.table-strategy.inline.sharding-column=order_id
sharding.jdbc.config.sharding.tables.t_order_item.table-strategy.inline.algorithm-expression=t_order_item$->{order_id % 2}
sharding.jdbc.config.sharding.tables.t_order_item.key-generator-column-name=order_item_id
sharding.jdbc.config.sharding.binding-tables=t_order,t_order_item
sharding.jdbc.config.sharding.broadcast-tables=t_config
sharding.jdbc.config.sharding.default-database-strategy.inline.sharding-column=user_id
sharding.jdbc.config.sharding.default-database-strategy.inline.algorithm-expression=master$->{user_id % 2}
sharding.jdbc.config.orchestration.name=spring_boot_ds_sharding
sharding.jdbc.config.orchestration.overwrite=true
sharding.jdbc.config.orchestration.registry.namespace=orchestration-spring-boot-sharding-test
sharding.jdbc.config.orchestration.registry.server-lists=localhost:2181
Sharding-JDBC不支持的項
DataSource接口
- 不支持timeout相關操作
Connection接口
- 不支持存儲過程,函數,遊標的操作
- 不支持執行native的SQL
- 不支持savepoint相關操作
- 不支持Schema/Catalog的操作
- 不支持自定義類型映射
Statement和PreparedStatement接口
- 不支持返回多結果集的語句(即存儲過程,非SELECT多條數據)
- 不支持國際化字符的操作
對於ResultSet接口
- 不支持對於結果集指針位置判斷
- 不支持通過非next方法改變結果指針位置
- 不支持修改結果集內容
- 不支持獲取國際化字符
- 不支持獲取Array