一款优秀数据库中间件的不完全解析

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/d8/d800399b0223fc49fdafc49a7ce5144c.gif","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/4e/4e375e843a31b6f08434a5327e576d21.png","alt":null,"title":"后台提供高并发系列历史文章阶段总结版，欢迎关注","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"高并发系列历史文章微信链接文档","attrs":{}}]},{"type":"numberedlist","attrs":{"start":null,"normalizeStart":1},"content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"垂直性能提升","attrs":{}}]}]}]},{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1.1. ","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?__biz=MzA4ODUzMDg5NQ==&mid=2650000954&idx=1&sn=a9ee98310e583b1712e1e64988d2a796&scene=21#wechat_redirect","title":null,"type":null},"content":[{"type":"text","text":"架构优化：集群部署，负载均衡","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1.2. ","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?__biz=MzA4ODUzMDg5NQ==&mid=2650000991&idx=1&sn=4cd73cc5aa4ccb97d9823db82737d14b&scene=21#wechat_redirect","title":null,"type":null},"content":[{"type":"text","text":"万亿流量下负载均衡的实现","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1.3. ","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?__biz=MzA4ODUzMDg5NQ==&mid=2650001031&idx=1&sn=75b0eea86788b7b59c61875745b38c4c&scene=21#wechat_redirect","title":null,"type":null},"content":[{"type":"text","text":"架构优化：消息中间件的妙用","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1.4. ","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?__biz=MzA4ODUzMDg5NQ==&mid=2650001031&idx=1&sn=75b0eea86788b7b59c61875745b38c4c&scene=21#wechat_redirect","title":null,"type":null},"content":[{"type":"text","text":"架构优化：用消息队列实现存储降级","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1.5. ","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?__biz=MzA4ODUzMDg5NQ==&mid=2650001071&idx=1&sn=fe00cfd25ae6c8595bcc2aef84ed102f&scene=21#wechat_redirect","title":null,"type":null},"content":[{"type":"text","text":"存储优化：mysql的索引原理和优化","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1.6. ","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?__biz=MzA4ODUzMDg5NQ==&mid=2650001136&idx=1&sn=2585d7dcf8b0e4328fe07eca4e7fe085&scene=21#wechat_redirect","title":null,"type":null},"content":[{"type":"text","text":"索引优化补充篇：explain索引优化实战","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1.7. ","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s?__biz=MzA4ODUzMDg5NQ==&mid=2650001108&idx=1&sn=5c246e6888438575f74147892671c2d1&scene=21#wechat_redirect","title":null,"type":null},"content":[{"type":"text","text":"存储优化：详解分库分表","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1.8. 本篇内容：详解数据库中间件","attrs":{}}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"Part1 数据库中间件有啥用","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"有一天，你去三亚玩耍，就想玩个冲浪，即时你不差钱，难道还要自己采买快艇、滑板等等装备来满足这为数不多的心血来潮么。租一个就行了嘛。这其实就是连接池的作用。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"数据库中间件可以理解为是一种具有连接池功能，但比连接池更高级的、带很多附加功能的辅助组件，不仅可以租冲浪板，还可以提供地点推荐、上保险等等各类服务。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"从网上的资料看，zdal应该算是半开源的，好像是之前开源过，但后续没有准备维护，然后就删除了，不过github被fork下来好多，随便一搜就是一片，当前，只是老的版本。目前蚂蚁内部的zdal好像已经更新到zdal5了吧，那咱可就看不到了。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/1c/1c6df9159f8c2cfc7c0ef8a14b2025e4.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"越复杂的系统，数据库中间件的作用越大。就拿zdal来说，它提供分库分表，结果集合并，sql解析，数据库failover动态切换等数据访问层统一解决方案。下面就一起来看下，其内部实现是怎么样的。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"Part2 架构剖析之高屋建瓴","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/9a/9a6ae9793a273eb2284714608c65149c.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"2.1整体概述","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如上图所示，zdal有四个重要的组成部分：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"价值体现--客户端Client包","attrs":{}},{"type":"text","text":"。对外暴露基本操作接口，用于业务层简单黑盒的操作数据源；业务只和client交互，动态切换/路由等逻辑只需要进行规则配置，相关逻辑由zdal实现。","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"核心功能--连接管理datasource包","attrs":{}},{"type":"text","text":"。最核心的能力，提供多种类型数据库的连接管理；不管功能多花哨，最终目的还是为了解决数据库连接的问题。","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"关键能力--SQL解析parser包","attrs":{}},{"type":"text","text":"。基础SQL解析能力；解析sql类型、字段名称、数据库等等，配合规则进行路由","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"扩展能力--库表路由rule包","attrs":{}},{"type":"text","text":"。根据parser解析出的字段确定逻辑库表和物理库表。","attrs":{}}]}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"2.2组件图看架构","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"组件图对整体架构和各组件及相互联系的理解可以起到很好的帮助。一个简版的组件图画了好久，还有不少错，不过大概是这么个意思，哎，基本功要丢~","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ec/ec8d214f4751d0cecf8da1d685ce6f1f.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"对照上图可以比较清晰的看到：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Client包对应用层暴露的数据源、负责监听配置动态变更的监听组件、负责加载组织各部分的配置组件、负责加载spring bean 和库表规则的配置组件；","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Client中加载了规则组件，实现逻辑表和数据库的路由规则。","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Client中的库表配置调用datasource中的数据源管理服务并构建连接池的连接池；","attrs":{}}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Client中的SqlDispatcher服务调用SQL解析组件实现SQL解析。","attrs":{}}]}]}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"Part3细节剖析之一叶知秋","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"3.1配置加载和bean初始化","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"大部分情况下，我们使用如mybatis这样的ORM框架来进行数据库操作，其实不管是ORM还是其他方式，应用层都需要对数据源进行配置。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"所以，client对外暴露了一个符合JDBC标准的datasource数据源，用来满足应用层ORM等框架配置数据源的要求--","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"ZdalDataSource","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/23/2305a5987f9c359aa83941eaaea6ee84.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如图片被压缩看不清，后台回复获取","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"//只提供了一个init方法，这也是spring启动时时，必须要调用的初始化方法，所有功能，都从这里开始\npublic class ZdalDataSource extends AbstractZdalDataSource implements DataSource{\n    public void init() {\n        try {\n            super.initZdalDataSource();\n        } catch (Exception e) {\n            CONFIG_LOGGER.error(\"...\");\n            throw new ZdalClientException(e);\n        }\n    }\n\n复制代码","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"ZdalDataSource#init()","attrs":{}},{"type":"text","text":" 方法即为配置加载的核心入口，init中负责加载spring配置，根据配置初始化数据源，并创建连接池，同时，将逻辑表和物理库的对应关系都维护起来供后续路由调用。","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"    /*父类的init方法*/\nprotected void initZdalDataSource() {\n    /*用FileSystemXmlApplicationContext方式加载配置文件中的数据源和规则，转化成zdalConfig对象*/\n    this.zdalConfig = ZdalConfigurationLoader.getInstance().getZdalConfiguration(appName,dbmode, appDsName, configPath);\n    this.dbConfigType = zdalConfig.getDataSourceConfigType();\n   this.dbType = zdalConfig.getDbType();\n   //初始化数据源\n   this.initDataSources(zdalConfig);\n   this.inited.set(true);\n    }\n}\n\n复制代码","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"从上面的类图和这里的两个入口方法大概了解到zdal配置加载的启动流程。下面我们就来详细看一下，读写分离和分库分表的规则是怎么被加载，怎么起作用的。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"3.2细说读写分离","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"读写分离配置的加载","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"首先，我们需要有数据源的相关配置，如下图：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/d2/d2c37efa05bd361a69679ee05eac8251.png","alt":"图片","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"此XML配置会在init方法被调用时，被初始化，解析成ZdalConfig类的属性，ZdalConfig类的主要成员见下面代码：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"public class ZdalConfig {\n    /** key=dsName;value=DataSourceParameter 所有物理数据源的配置项,比如用户名，密码，库名等 */\n    private Map dataSourceParameters = new ConcurrentHashMap();\n    /** 逻辑数据源和物理数据源的对应关系:key=logicDsName,value=physicDsName */\n    private Map              logicPhysicsDsNames  = new ConcurrentHashMap();\n    /** 数据源的读写规则，比如只读，或读写等配置*/\n    private Map              groupRules           = new ConcurrentHashMap();\n    /** 异常转移的数据源规则*/\n    private Map              failoverRules        = new ConcurrentHashMap();\n    //一份完整的读写分离和分库分表规则配置\n    private AppRule                          appRootRule;\n\n复制代码","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以看到，xml中的规则，被解析到xxxRules里。这里以groupRules为例，failover同理。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下一步则是通过解析得到的zdalConfig 来初始化数据源：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"protected final void initDataSources(ZdalConfig zdalConfig) {\n    //DataSourceParameter中存的是数据源参数，如用户名密码，最大最小连接数等\n    for (Entry entry : zdalConfig.getDataSourceParameters().entrySet()) {\n        try {\n           //初始化连接池\n           ZDataSource zDataSource = new ZDataSource(/*设置最大最小连接数*/createDataSourceDO(entry.getValue(),zdalConfig.getDbType(), appDsName + \".\" + entry.getKey()));\n           this.dataSourcesMap.put(entry.getKey(), zDataSource);\n        } catch (Exception e) {\n            //...\n        }\n   }\n  //其他分支略，只看最简单的分组模式\n  if (dbConfigType.isGroup()) {\n       //读写配置赋值\n       this.rwDataSourcePoolConfig = zdalConfig.getGroupRules();\n       //初始化多份读库下的负载均衡\n       this.initForLoadBalance(zdalConfig.getDbType());\n  }\n  //注册监听：为了满足动态切换\n  this.initConfigListener();\n}\n\n复制代码","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"initForLoadBalance的方法如下：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"private void initForLoadBalance(DBType dbType) {\n    Map dsSelectors = this.buildRwDbSelectors(this.rwDataSourcePoolConfig);\n    this.runtimeConfigHolder.set(new ZdalRuntime(dsSelectors));\n    this.setDbTypeForDBSelector(dbType);\n}\n\n复制代码","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以看到，首先构建出了DB选择器，然后赋值给了runtimeConfigHolder供运行时获取。而构建DB选择器的时候，其实是按读写两个维度，把所有数据源都构建了一遍，即group_r和group_w下都包含5个数据源，只不过各自的权重不一样：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"//比如按上面的配置写库只有一个，但是也会包含全数据源\n\ngroup_0_w_0 :\ngroup_0_w_1 :\ngroup_0_w_2 :\ngroup_0_w_3 :\ngroup_0_w_4 :\n\n//上述就是写相关的DBSelecter的内容。\n\n复制代码","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"读写分离怎么起作用","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"以delete为例，更新删除是要操作写库的","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":" public void delete(ZdalDataSource dataSource) {\n     String deleteSql = \"delete from test\";\n     Connection conn = null;\n     PreparedStatement pst = null;\n     try {\n        conn = dataSource.getConnection();\n        pst = conn.prepareStatement(deleteSql);\n        pst.execute();\n     } catch (Exception e) {\n            //...\n     } finally {\n           //资源关闭\n     }\n }\n\n复制代码","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"getConnection会从上文中提到的runtimeConfigHolder中获取DBSelecter，然后执行execute方法","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":" public boolean execute() throws SQLException {\n    SqlType sqlType = getSqlType(sql);\n    // SELECT相关的就选择group_r对应的DBSelecter\n   if (sqlType == SqlType.SELECT || sqlType == SqlType.SELECT_FOR_UPDATE|| sqlType == SqlType.SELECT_FROM_DUAL) {\n     //略\n    return true;\n    //update/delete相关的就选择group_w对应的DBSelecter\n  } else if (sqlType == SqlType.INSERT || sqlType == SqlType.UPDATE|| sqlType == SqlType.DELETE) {\n       if (super.dbConfigType == DataSourceConfigType.GROUP) {\n           executeUpdate0();\n       } else {\n           executeUpdate();\n      }\n      return false;\n  } \n}\n\n复制代码","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果是读取相关的，那就选_r的DBSelecter，如果是写相关的，那就选_W的DBSelecter。那么executeUpdate0中是怎么执行区分读写数据源的呢，其实就是把这一组的数据源根据权重筛选一遍。","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"// WeightRandom#select(int[], java.lang.String[])\nprivate String select(int[] areaEnds, String[] keys) {\n   //这里的areaEnds数组，是一个累加范围值数据\n   //比如三个库权重    10   9   8\n   //那么areaEnds就是  10  19  27 是对每个权重的累加，最后一个值是总和\n   int sum = areaEnds[areaEnds.length - 1];\n   //这样随机出来的数，是符合权重分布的\n   int rand = random.nextInt(sum);\n   for (int i = 0; i