mongodb 源码实现系列 - command命令处理模块源码实现一

原創

2020-11-26 20:08

(); \n31. //kind为0对应命令请求body内容，内容通过bson报错 \n32. switch (sectionKind) { \n33. //sections第一个字节是0说明是body \n34. case Section::kBody: { \n35. //默认只能有一个body \n36. uassert(40430, \"Multiple body sections in message\", !haveBody); \n37. haveBody = true; \n38. //命令请求的bson信息保存在这里 \n39. msg.body = sectionsBuf.read>(); \n40. break; \n41. } \n42. \n43. //DocSequence暂时没看明白，用到的地方很少，跳过，后续等 \n44. //该系列文章主流功能分析完成后，从头再回首分析 \n45. case Section::kDocSequence: { \n46. ...... \n47. } \n48. } \n49. } \n50. //OP_MSG必须有body内容 \n51. uassert(40587, \"OP_MSG messages must have a body\", haveBody); \n52. //body和sequence去重判断 \n53. for (const auto& docSeq : msg.sequences) { \n54. ...... \n55. } \n56. return msg; \n} ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" OpMsg类被OpMsgRequest类继承，OpMsgRequest类中核心接口就是解析出OpMsg.body中的库信息和表信息，OpMsgRequest类代码实现如下：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"cpp"},"content":[{"type":"text","text":"1.//协议解析得时候会用到，见runCommands \n2.struct OpMsgRequest : public OpMsg { \n3. ...... \n4. //构造初始化 \n5. explicit OpMsgRequest(OpMsg&& generic) : OpMsg(std::move(generic)) {} \n6. //opMsgRequestFromAnyProtocol->OpMsgRequest::parse \n7. //从message中解析出OpMsg所需成员信息 \n8. static OpMsgRequest parse(const Message& message) { \n9. //OpMsg::parse \n10. return OpMsgRequest(OpMsg::parse(message)); \n11. } \n12. //根据db body extraFields填充OpMsgRequest \n13. static OpMsgRequest fromDBAndBody(... { \n14. OpMsgRequest request; \n15. request.body = ([&] { \n16. //填充request.body \n17. ...... \n18. }()); \n19. return request; \n20. } \n21. //从body中获取db name \n22. StringData getDatabase() const { \n23. if (auto elem = body[\"$db\"]) \n24. return elem.checkAndGetStringData(); \n25. uasserted(40571, \"OP_MSG requests require a $db argument\"); \n26. } \n27. //find insert 等命令信息 body中的第一个elem就是command 名 \n28. StringData getCommandName() const { \n29. return body.firstElementFieldName(); \n30. } \n}; ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" OpMsgRequest通过OpMsg::parse(message)解析出OpMsg信息，从而获取到body内容，GetCommandName()接口和getDatabase()则分别从body中获取库DB信息、命令名信息。通过该类相关接口，命令名(find、write、update等)和DB库都获取到了。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" OpMsg模块除了OP","attrs":{}},{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"MSG相关报文解析外，还负责OP","attrs":{}},{"type":"text","text":"MSG报文组装填充，该模块接口功能大全如下表：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/f7/f730feeb5fce4ef2221fb5354d09eb7f.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"5. Mongod实例服务入口核心代码实现","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Mongod实例服务入口类ServiceEntryPointMongod继承ServiceEntryPointImpl类，mongod实例的报文解析处理、命令解析、命令执行都由该类负责处理。ServiceEntryPointMongod核心接口可以细分为：opCode解析及回调处理、命令解析及查找、命令执行三个子模块。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"5.1 opCode解析及回调处理","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" OpCode操作码解析及其回调处理由ServiceEntryPointMongod::handleRequest(...)接口实现，核心代码实现如下:","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"cpp"},"content":[{"type":"text","text":"1.//mongod服务对于客户端请求的处理 \n2.//通过状态机SSM模块的如下接口调用:ServiceStateMachine::_processMessage \n3.DbResponse ServiceEntryPointMongod::handleRequest(OperationContext* opCtx, const Message& m) { \n4. //获取opCode，3.6版本对应客户端默认使用OP_MSG \n5. NetworkOp op = m.operation(); \n6. ...... \n7. //根据message构造DbMessage \n8. DbMessage dbmsg(m); \n9. //根据操作上下文获取对应的client \n10. Client& c = *opCtx->getClient(); \n11. ...... \n12. //获取库.表信息，注意只有dbUpdate(currentOp.elapsedTimeExcludingPauses()); \n47. ...... \n48. //慢日志记录 \n49. if (shouldLogOpDebug || (shouldSample && debug.executionTimeMicros > logThresholdMs * 1000LL)) { \n50. Locker::LockerInfo lockerInfo; \n51. //OperationContext::lockState LockerImpl<>::getLockerInfo \n52. opCtx->lockState()->getLockerInfo(&lockerInfo); \n53. \n54. //OpDebug::report 记录慢日志到日志文件 \n55. log() <done(); \n28. //responseLength赋值 \n29. CurOp::get(opCtx)->debug().responseLength = response.header().dataLen(); \n30. // 返回 \n31. return DbResponse{std::move(response)}; \n} ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" RunCommands(...)接口从message中解析出OpMsg信息，然后获取该OpMsg对应的command命令信息，最后执行该命令对应的后续处理操作。主要功能说明如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"①　获取该OpCode对应replyBuilder，OP_MSG操作对应builder为OpMsgReplyBuilder。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"②　根据message解析出OpMsgRequest数据，OpMsgRequest来中包含了真正的命令请求bson信息。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"③　opCtx初始化操作。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"④　通过request.getCommandName()返回命令信息(如“find”、“update”等字符串)。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑤　通过Command::findCommand(command name)从CommandMap这个map表中查找是否支持该 command命令。如果没找到说明不支持，如果找到说明支持。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑥　调用execCommandDatabase(...)执行该命令，并获取命令的执行结果。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑦　根据command执行结果构造response并返回","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"5.3命令执行","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"cpp"},"content":[{"type":"text","text":"1.void execCommandDatabase(...) { \n2. ...... \n3. //获取dbname \n4. const auto dbname = request.getDatabase().toString(); \n5. ...... \n6. //mab表存放从bson中解析出的elem信息 \n7. StringMap topLevelFields; \n8. //body elem解析 \n9. for (auto&& element : request.body) { \n10. //获取bson中的elem信息 \n11. StringData fieldName = element.fieldNameStringData(); \n12. //如果elem信息重复，则异常处理 \n13. ...... \n14. } \n15. //如果是help命令，则给出help提示 \n16. if (Command::isHelpRequest(helpField)) { \n17. //给出help提示 \n18. Command::generateHelpResponse(opCtx, replyBuilder, *command); \n19. return; \n20. } \n21. //权限认证检查，检查该命令执行权限 \n22. uassertStatusOK(Command::checkAuthorization(command, opCtx, request)); \n23. ...... \n24. \n25. //该命令执行次数统计 db.serverStatus().metrics.commands可以获取统计信息 \n26. command->incrementCommandsExecuted(); \n27. //真正的命令执行在这里面 \n28. retval = runCommandImpl(opCtx, command, request, replyBuilder, startOperationTime); \n29. //该命令执行失败次数统计 \n30. if (!retval) { \n31. command->incrementCommandsFailed(); \n32. } \n33. ...... \n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" execCommandDatabase(...)最终调用RunCommandImpl(...)进行对应命令的真正处理，该接口核心代码实现如下：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"cpp"},"content":[{"type":"text","text":"1.bool runCommandImpl(...) { \n2. //获取命令请求内容body \n3. BSONObj cmd = request.body; \n4. //获取请求中的DB库信息 \n5. const std::string db = request.getDatabase().toString(); \n6. //ReadConcern检查 \n7. Status rcStatus = waitForReadConcern( \n8. opCtx, repl::ReadConcernArgs::get(opCtx), command->allowsAfterClusterTime(cmd)); \n9. //ReadConcern检查不通过，直接异常提示处理 \n10. if (!rcStatus.isOK()) { \n11. //异常处理 \n12. return; \n13. } \n14. if (!command->supportsWriteConcern(cmd)) { \n15. //命令不支持WriteConcern，但是对应的请求中却带有WriteConcern配置，直接报错不支持 \n16. if (commandSpecifiesWriteConcern(cmd)) { \n17. //异常处理\"Command does not support writeConcern\" \n18. ...... \n19. return result; \n20. } \n21. //调用Command::publicRun执行不同命令操作 \n22. result = command->publicRun(opCtx, request, inPlaceReplyBob); \n23. } \n24. //提取WriteConcernOptions信息 \n25. auto wcResult = extractWriteConcern(opCtx, cmd, db); \n26. //提取异常，直接异常处理 \n27. if (!wcResult.isOK()) { \n28. //异常处理 \n29. ...... \n30. return result; \n31. } \n32. ...... \n33. //执行对应的命令Command::publicRun，执行不同命令操作 \n34. result = command->publicRun(opCtx, request, inPlaceReplyBob); \n35. ...... \n36.}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" RunCommandImpl(...)接口最终调用该接口入参的command，执行 command->publicRun(...)接口，也就是命令模块的公共publicRun。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"5.4总结","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Mongod服务入口首先从message中解析出opCode操作码，3.6版本对应客户端默认操作码为OP_MSQ，解析出该操作对应OpMsgRequest信息。然后从message原始数据中解析出command命令字符串后，继续通过全局Map表种查找是否支持该命令操作，如果支持则执行该命令；如果不支持，直接异常打印，同时返回。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"6. Mongos实例服务入口核心代码实现","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" mongos服务入口核心代码实现过程和mongod服务入口代码实现流程几乎相同，mongos实例message解析、OP_MSG操作码处理、command命令查找等流程和上一章节mongod实例处理过程类似，本章节不在详细分析。Mongos实例服务入口处理调用流程如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" ServiceEntryPointMongos::handleRequest(...)->Strategy::clientCommand(...)-->runCommand(...)->execCommandClient(...)","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/51/511238a22e3d645d35c8977bc9db1185.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 最后的接口核心代码实现如下：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"cpp"},"content":[{"type":"text","text":"1.void runCommand(...) { \n2. ...... \n3. //获取请求命令name \n4. auto const commandName = request.getCommandName(); \n5. //从全局map表中查找 \n6. auto const command = Command::findCommand(commandName); \n7. //没有对应的command存在,抛异常说明不支持该命令 \n8. if (!command) { \n9. ...... \n10. return; \n11. } \n12. ...... \n13. //执行命令 \n14. execCommandClient(opCtx, command, request, builder); \n15. ...... \n16.} \n17.\n18.void execCommandClient(...) \n19.{ \n20. ...... \n21. //认证检查，是否有操作该command命令的权限，没有则异常提示 \n22. Status status = Command::checkAuthorization(c, opCtx, request); \n23. if (!status.isOK()) { \n24. Command::appendCommandStatus(result, status); \n25. return; \n26. } \n27. //该命令的执行次数自增，代理上面也是要计数的 \n28. c->incrementCommandsExecuted(); \n29. //如果需要command统计，则加1 \n30. if (c->shouldAffectCommandCounter()) { \n31. globalOpCounters.gotCommand(); \n32. } \n33. ...... \n34. //有部分命令不支持writeconcern配置，报错 \n35. bool supportsWriteConcern = c->supportsWriteConcern(request.body); \n36. //不支持writeconcern又带有该参数的请求，直接异常处理\"Command does not support writeConcern\" \n37. if (!supportsWriteConcern && !wcResult.getValue().usedDefault) { \n38. ...... \n39. return; \n40. } \n41. //执行本命令对应的公共publicRun接口，Command::publicRun \n42. ok = c->publicRun(opCtx, request, result); \n43. ...... \n44.} ","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Tips: mongos和mongod实例服务入口核心代码实现的一点小区别","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"①　Mongod实例opCode操作码解析、OpMsg解析、command查找及对应命令调用处理都由class ServiceEntryPointMongod{...}类一起完成。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"②　mongos实例则把opCode操作码解析交由class ServiceEntryPointMongos{...}类实现，OpMsg解析、command查找及对应命令调用处理放到了clase Strategy{...}类来处理。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"7.总结","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Mongodb报文解析及组装流程总结","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"①　一个完整mongodb报文由通用报文header头部+body部分组成。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"②　Body部分内容，根据报文头部的opCode来决定不同的body内容。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"③　3.6版本对应客户端请求opCode默认为OP_MSG，该操作码对应body部分由flagBits + sections + checksum组成，其中sections中存放的是真正的命令请求信息，已bson数据格式保存。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"④　Header头部和body报文体封装及解析过程由class Message {...}类实现","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑤　Body中对应command命令名、库名、表名的解析在mongodb(version<3.6)低版本协议中由class DbMessage {...}类实现","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑥　Body中对应command命令名、库名、表名的解析在mongodb(version<3.6)低版本协议中由struct OpMsgRequest{...}结构和struct OpMsg {...}类实现","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Mongos和mongod实例的服务入口处理流程大同小异，整体处理流程如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"①　从message解析出opCode操作码，根据不同操作码执行对应操作码回调。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"②　根据message解析出OpMsg request信息，mongodb报文的命令信息就存储在该body中，该body已bson格式存储。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"③　从body中解析出command命令字符串信息(如“insert”、“update”等)。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"④　从全局_commands map表中查找是否支持该命令，如果支持则执行该命令处理，如果不支持则直接报错提示。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑤　最终找到对应command命令后，执行command的功能run接口。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 图形化总结如下:","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ef/efc040eb2e9f6a9947b61bc25f6cde82.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":" 说明：","attrs":{}},{"type":"text","text":"第3章的协议解析及封装过程实际上应该算是网络处理模块范畴，本文为了分析command命令处理模块方便，把该部分实现归纳到了命令处理模块，这样方便理解。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":" Tips: ","attrs":{}},{"type":"text","text":"下期继续分享不同command命令执行细节。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"8.遗留问题","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 第1章节中的统计信息，将在command模块核心代码分析完毕后揭晓答案，《mongodb command命令处理模块源码实现二》中继续分析，敬请关注。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}