SpringBoot結合sharding-jdbc實現分庫分表

說明

定位爲輕量級Java框架，在Java的JDBC層提供的額外服務。 它使用客戶端直連數據庫，以jar包形式提供服務，無需額外部署和依賴，可理解爲增強版的JDBC驅動，完全兼容JDBC和各種ORM框架。

• 適用於任何基於Java的ORM框架，如：JPA, Hibernate, Mybatis, Spring JDBC Template或直接使用JDBC。
• 基於任何第三方的數據庫連接池，如：DBCP, C3P0, BoneCP, Druid, HikariCP等。
• 支持任意實現JDBC規範的數據庫。目前支持MySQL，Oracle，SQLServer和PostgreSQL。

分表配置

引入jar包

<!-- sharding-jdbc -->
<dependency>
	<groupId>org.apache.shardingsphere</groupId>
	<artifactId>sharding-jdbc-spring-boot-starter</artifactId>
	<version>4.0.0-RC2</version>
</dependency>

1、使用properties進行配置

# 分表配置
spring.shardingsphere.enabled=true
spring.shardingsphere.datasource.names=ds0,ds1

# 數據源
spring.shardingsphere.datasource.ds0.type=com.alibaba.druid.pool.DruidDataSource
spring.shardingsphere.datasource.ds0.driver-class-name=com.mysql.jdbc.Driver
spring.shardingsphere.datasource.ds0.url=jdbc:mysql://localhost:3306/ds_0?characterEncoding=utf-8
spring.shardingsphere.datasource.ds0.username=root
spring.shardingsphere.datasource.ds0.password=123456

spring.shardingsphere.datasource.ds1.type=com.alibaba.druid.pool.DruidDataSource
spring.shardingsphere.datasource.ds1.driver-class-name=com.mysql.jdbc.Driver
spring.shardingsphere.datasource.ds1.url=jdbc:mysql://localhost:3306/ds_1?characterEncoding=utf-8
spring.shardingsphere.datasource.ds1.username=root
spring.shardingsphere.datasource.ds1.password=123456

# 分表配置
spring.shardingsphere.sharding.tables.user.actual-data-nodes=ds$->{0..1}.user_$->{0..2}
spring.shardingsphere.sharding.tables.user.table-strategy.inline.sharding-column=id
spring.shardingsphere.sharding.tables.user.table-strategy.inline.algorithm-expression=user_$->{id % 3}
spring.shardingsphere.sharding.tables.user.key-generator.column=id
spring.shardingsphere.sharding.tables.user.key-generator.type=SNOWFLAKE

# 自定義分表算法
#spring.shardingsphere.sharding.tables.user.table-strategy.standard.sharding-column=created_date
#spring.shardingsphere.sharding.tables.user.table-strategy.standard.precise-algorithm-class-name=com.hsoft.vip.server.config.TableShardingAlgorithm

# 分庫配置
spring.shardingsphere.sharding.default-database-strategy.inline.sharding-column=id
spring.shardingsphere.sharding.default-database-strategy.inline.algorithm-expression=ds$->{id % 2}

# 不分庫分表的數據源指定
#spring.shardingsphere.sharding.default-data-source-name=ds0

2、使用java代碼進行配置

@Bean("datasource") // 聲明其爲Bean實例
@Primary // 在同樣的DataSource中，首先使用被標註的DataSource
public DataSource dataSource() {
	
	DataSource datasource = createDataSource();
	// 分庫設置
	Map<String, DataSource> dataSourceMap = new HashMap<>(2);
	// 添加兩個數據庫database0和database1
	// 這裏只是示例，實際業務不可能兩個datasource一樣
	dataSourceMap.put("database0", datasource);
	dataSourceMap.put("database1", datasource);
	
	// 分表設置，根據字段數據規則映射到對應的表
	TableRuleConfiguration tableRuleConfigs =new TableRuleConfiguration("t_vip_flow","database${0..1}.t_vip_flow_${201908..201910}01")  ;
	ShardingStrategyConfiguration tableShardingStrategyConfig =new StandardShardingStrategyConfiguration("created_date",new IdShardingAlgorithm(), new TableShardingAlgorithm());
	tableRuleConfigs.setTableShardingStrategyConfig(tableShardingStrategyConfig);
	// 主鍵生成策略
	KeyGeneratorConfiguration key =new KeyGeneratorConfiguration("SNOWFLAKE", "id");
	tableRuleConfigs.setKeyGeneratorConfig(key);
	ShardingRuleConfiguration shardingRuleConfiguration=new ShardingRuleConfiguration();
	shardingRuleConfiguration.setTableRuleConfigs(Arrays.asList(tableRuleConfigs));
	// 分庫策略
	ShardingStrategyConfiguration defaultDatabaseShardingStrategyConfig =new StandardShardingStrategyConfiguration("id",new DatabaseShardingAlgorithm());
	shardingRuleConfiguration.setDefaultDatabaseShardingStrategyConfig(defaultDatabaseShardingStrategyConfig);
	try {
		Properties props =new Properties();
		props.put("sql.show", true);
		DataSource dataSource = ShardingDataSourceFactory.createDataSource(dataSourceMap, shardingRuleConfiguration, props);
		return dataSource;
	} catch (SQLException e) {
		e.printStackTrace();
	}
	return datasource;
}

private DataSource createDataSource() {
	DruidDataSource datasource = new DruidDataSource();
	datasource.setUrl(url);
	datasource.setUsername(username);
	datasource.setPassword(password);
	datasource.setDriverClassName(driver);
	datasource.setInitialSize(initialSize);
	datasource.setMinIdle(minIdle);
	datasource.setMaxActive(maxActive);
	datasource.setMaxWait(maxWait);
	datasource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
	datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
	datasource.setValidationQuery(validationQuery);
	datasource.setTestWhileIdle(testWhileIdle);
	datasource.setTestOnBorrow(testOnBorrow);
	datasource.setTestOnReturn(testOnReturn);
	datasource.setPoolPreparedStatements(poolPreparedStatements);
	datasource.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize);
	// 配置removeAbandoned對性能會有一些影響，建議懷疑存在泄漏之後再打開。在上面的配置中，如果連接超過30分鐘未關閉，就會被強行回收，並且日誌記錄連接申請時的調用堆棧。
	datasource.setRemoveAbandoned(true);
	datasource.setLogAbandoned(true);
	datasource.setRemoveAbandonedTimeout(1800);// 30分鐘
	try {
		datasource.setFilters(filters);
	} catch (SQLException e) {
		logger.error("druid configuration initialization filter", e);
	}
	datasource.setConnectionProperties(connectionProperties);
	return datasource;
}

分表算法

日期精確映射

public class IdShardingAlgorithm implements PreciseShardingAlgorithm<Date> {

	@Override
	public String doSharding(Collection<String> availableTargetNames, PreciseShardingValue<Date> shardingValue) {
		Date date = shardingValue.getValue();
		String firstDay =DateOperator.formatDate(DateOperator.getFirstDayOfMonth(date),"yyyyMMdd");
		return availableTargetNames.stream().filter(p -> p.endsWith(firstDay)).findFirst().orElse(null);
	}

}

日期範圍映射

public class TableShardingAlgorithm implements RangeShardingAlgorithm<Date> {
	@Override
	public Collection<String> doSharding(Collection<String> availableTargetNames,
			RangeShardingValue<Date> shardingValue) {
		Date dateFrom = shardingValue.getValueRange().lowerEndpoint();
		Date dateEnd = shardingValue.getValueRange().upperEndpoint();
		int monthFrom = Integer.valueOf(DateOperator.formatDate(DateOperator.getFirstDayOfMonth(dateFrom),"yyyyMM"));
		int monthEnd = Integer.valueOf(DateOperator.formatDate(DateOperator.getFirstDayOfMonth(dateEnd),"yyyyMM"));
		List<String> lstMonth=new ArrayList<>();
		for(int i = monthFrom;i<=monthEnd;i++) {
			lstMonth.add(i+"01");
		}
		return availableTargetNames.stream().filter(p ->lstMonth.contains(p.substring(p.length()-8))).collect(Collectors.toList());
	}
}

分庫算法

根據id分庫（這裏只是示例，實際業務不可能根據id分庫）

public class DatabaseShardingAlgorithm implements PreciseShardingAlgorithm<Comparable<?>> {

	@Override
	public String doSharding(Collection<String> availableTargetNames, PreciseShardingValue<Comparable<?>> shardingValue) {
		Long value;
		if (shardingValue.getValue() instanceof Integer) {
			value = Long.valueOf(shardingValue.getValue().toString());
		}else {
			value =(Long)shardingValue.getValue();
		}
		long index = value % 2L;
		return availableTargetNames.stream().filter(p -> p.endsWith(index + "")).findFirst().orElse(null);
	}

}

注意事項

1. 用於分庫分表的字段，必須在查詢條件中
2. 當前僅支持三種查詢條件：=，in，between

package org.apache.shardingsphere.core.constant;
public enum ShardingOperator {
    EQUAL, BETWEEN, IN
}

3. 框架自帶兩種id生成器（SNOWFLAKE、UUID）。使用框架自動生成id時，insert語句中其他所有的字段值不能是null，否則插入時，id的賦值會錯位

@RequiredArgsConstructor
@Getter
public abstract class InsertOptimizeResultUnit {
    //...省略其他代碼...
    public final void addColumnParameter(final Object parameter) {
        parameters[getCurrentIndex(parameters)] = parameter;
    }
    
    private int getCurrentIndex(final Object[] array) {
        int count = 0;
        for (Object each : array) {
            // 這裏判斷了不爲null才計數
            if (null != each) {
                count++;
            }
        }
        return count;
    }
    //...省略其他代碼...
}

參考

• https://shardingsphere.apache.org/document/current/cn/overview/
• https://github.com/yinjihuan/sharding-jdbc