mongodb 源碼實現系列 - command命令處理模塊源碼實現一

原創

2020-11-26 20:08

(); \n31. //kind爲0對應命令請求body內容，內容通過bson報錯 \n32. switch (sectionKind) { \n33. //sections第一個字節是0說明是body \n34. case Section::kBody: { \n35. //默認只能有一個body \n36. uassert(40430, \"Multiple body sections in message\", !haveBody); \n37. haveBody = true; \n38. //命令請求的bson信息保存在這裏 \n39. msg.body = sectionsBuf.read>(); \n40. break; \n41. } \n42. \n43. //DocSequence暫時沒看明白，用到的地方很少，跳過，後續等 \n44. //該系列文章主流功能分析完成後，從頭再回首分析 \n45. case Section::kDocSequence: { \n46. ...... \n47. } \n48. } \n49. } \n50. //OP_MSG必須有body內容 \n51. uassert(40587, \"OP_MSG messages must have a body\", haveBody); \n52. //body和sequence去重判斷 \n53. for (const auto& docSeq : msg.sequences) { \n54. ...... \n55. } \n56. return msg; \n} ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" OpMsg類被OpMsgRequest類繼承，OpMsgRequest類中核心接口就是解析出OpMsg.body中的庫信息和表信息，OpMsgRequest類代碼實現如下：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"cpp"},"content":[{"type":"text","text":"1.//協議解析得時候會用到，見runCommands \n2.struct OpMsgRequest : public OpMsg { \n3. ...... \n4. //構造初始化 \n5. explicit OpMsgRequest(OpMsg&& generic) : OpMsg(std::move(generic)) {} \n6. //opMsgRequestFromAnyProtocol->OpMsgRequest::parse \n7. //從message中解析出OpMsg所需成員信息 \n8. static OpMsgRequest parse(const Message& message) { \n9. //OpMsg::parse \n10. return OpMsgRequest(OpMsg::parse(message)); \n11. } \n12. //根據db body extraFields填充OpMsgRequest \n13. static OpMsgRequest fromDBAndBody(... { \n14. OpMsgRequest request; \n15. request.body = ([&] { \n16. //填充request.body \n17. ...... \n18. }()); \n19. return request; \n20. } \n21. //從body中獲取db name \n22. StringData getDatabase() const { \n23. if (auto elem = body[\"$db\"]) \n24. return elem.checkAndGetStringData(); \n25. uasserted(40571, \"OP_MSG requests require a $db argument\"); \n26. } \n27. //find insert 等命令信息 body中的第一個elem就是command 名 \n28. StringData getCommandName() const { \n29. return body.firstElementFieldName(); \n30. } \n}; ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" OpMsgRequest通過OpMsg::parse(message)解析出OpMsg信息，從而獲取到body內容，GetCommandName()接口和getDatabase()則分別從body中獲取庫DB信息、命令名信息。通過該類相關接口，命令名(find、write、update等)和DB庫都獲取到了。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" OpMsg模塊除了OP","attrs":{}},{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"MSG相關報文解析外，還負責OP","attrs":{}},{"type":"text","text":"MSG報文組裝填充，該模塊接口功能大全如下表：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/f7/f730feeb5fce4ef2221fb5354d09eb7f.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"5. Mongod實例服務入口核心代碼實現","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Mongod實例服務入口類ServiceEntryPointMongod繼承ServiceEntryPointImpl類，mongod實例的報文解析處理、命令解析、命令執行都由該類負責處理。ServiceEntryPointMongod核心接口可以細分爲：opCode解析及回調處理、命令解析及查找、命令執行三個子模塊。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"5.1 opCode解析及回調處理","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" OpCode操作碼解析及其回調處理由ServiceEntryPointMongod::handleRequest(...)接口實現，核心代碼實現如下:","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"cpp"},"content":[{"type":"text","text":"1.//mongod服務對於客戶端請求的處理 \n2.//通過狀態機SSM模塊的如下接口調用:ServiceStateMachine::_processMessage \n3.DbResponse ServiceEntryPointMongod::handleRequest(OperationContext* opCtx, const Message& m) { \n4. //獲取opCode，3.6版本對應客戶端默認使用OP_MSG \n5. NetworkOp op = m.operation(); \n6. ...... \n7. //根據message構造DbMessage \n8. DbMessage dbmsg(m); \n9. //根據操作上下文獲取對應的client \n10. Client& c = *opCtx->getClient(); \n11. ...... \n12. //獲取庫.表信息，注意只有dbUpdate(currentOp.elapsedTimeExcludingPauses()); \n47. ...... \n48. //慢日誌記錄 \n49. if (shouldLogOpDebug || (shouldSample && debug.executionTimeMicros > logThresholdMs * 1000LL)) { \n50. Locker::LockerInfo lockerInfo; \n51. //OperationContext::lockState LockerImpl<>::getLockerInfo \n52. opCtx->lockState()->getLockerInfo(&lockerInfo); \n53. \n54. //OpDebug::report 記錄慢日誌到日誌文件 \n55. log() <done(); \n28. //responseLength賦值 \n29. CurOp::get(opCtx)->debug().responseLength = response.header().dataLen(); \n30. // 返回 \n31. return DbResponse{std::move(response)}; \n} ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" RunCommands(...)接口從message中解析出OpMsg信息，然後獲取該OpMsg對應的command命令信息，最後執行該命令對應的後續處理操作。主要功能說明如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"①　獲取該OpCode對應replyBuilder，OP_MSG操作對應builder爲OpMsgReplyBuilder。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"②　根據message解析出OpMsgRequest數據，OpMsgRequest來中包含了真正的命令請求bson信息。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"③　opCtx初始化操作。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"④　通過request.getCommandName()返回命令信息(如“find”、“update”等字符串)。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑤　通過Command::findCommand(command name)從CommandMap這個map表中查找是否支持該 command命令。如果沒找到說明不支持，如果找到說明支持。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑥　調用execCommandDatabase(...)執行該命令，並獲取命令的執行結果。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑦　根據command執行結果構造response並返回","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"5.3命令執行","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"cpp"},"content":[{"type":"text","text":"1.void execCommandDatabase(...) { \n2. ...... \n3. //獲取dbname \n4. const auto dbname = request.getDatabase().toString(); \n5. ...... \n6. //mab表存放從bson中解析出的elem信息 \n7. StringMap topLevelFields; \n8. //body elem解析 \n9. for (auto&& element : request.body) { \n10. //獲取bson中的elem信息 \n11. StringData fieldName = element.fieldNameStringData(); \n12. //如果elem信息重複，則異常處理 \n13. ...... \n14. } \n15. //如果是help命令，則給出help提示 \n16. if (Command::isHelpRequest(helpField)) { \n17. //給出help提示 \n18. Command::generateHelpResponse(opCtx, replyBuilder, *command); \n19. return; \n20. } \n21. //權限認證檢查，檢查該命令執行權限 \n22. uassertStatusOK(Command::checkAuthorization(command, opCtx, request)); \n23. ...... \n24. \n25. //該命令執行次數統計 db.serverStatus().metrics.commands可以獲取統計信息 \n26. command->incrementCommandsExecuted(); \n27. //真正的命令執行在這裏面 \n28. retval = runCommandImpl(opCtx, command, request, replyBuilder, startOperationTime); \n29. //該命令執行失敗次數統計 \n30. if (!retval) { \n31. command->incrementCommandsFailed(); \n32. } \n33. ...... \n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" execCommandDatabase(...)最終調用RunCommandImpl(...)進行對應命令的真正處理，該接口核心代碼實現如下：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"cpp"},"content":[{"type":"text","text":"1.bool runCommandImpl(...) { \n2. //獲取命令請求內容body \n3. BSONObj cmd = request.body; \n4. //獲取請求中的DB庫信息 \n5. const std::string db = request.getDatabase().toString(); \n6. //ReadConcern檢查 \n7. Status rcStatus = waitForReadConcern( \n8. opCtx, repl::ReadConcernArgs::get(opCtx), command->allowsAfterClusterTime(cmd)); \n9. //ReadConcern檢查不通過，直接異常提示處理 \n10. if (!rcStatus.isOK()) { \n11. //異常處理 \n12. return; \n13. } \n14. if (!command->supportsWriteConcern(cmd)) { \n15. //命令不支持WriteConcern，但是對應的請求中卻帶有WriteConcern配置，直接報錯不支持 \n16. if (commandSpecifiesWriteConcern(cmd)) { \n17. //異常處理\"Command does not support writeConcern\" \n18. ...... \n19. return result; \n20. } \n21. //調用Command::publicRun執行不同命令操作 \n22. result = command->publicRun(opCtx, request, inPlaceReplyBob); \n23. } \n24. //提取WriteConcernOptions信息 \n25. auto wcResult = extractWriteConcern(opCtx, cmd, db); \n26. //提取異常，直接異常處理 \n27. if (!wcResult.isOK()) { \n28. //異常處理 \n29. ...... \n30. return result; \n31. } \n32. ...... \n33. //執行對應的命令Command::publicRun，執行不同命令操作 \n34. result = command->publicRun(opCtx, request, inPlaceReplyBob); \n35. ...... \n36.}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" RunCommandImpl(...)接口最終調用該接口入參的command，執行 command->publicRun(...)接口，也就是命令模塊的公共publicRun。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"5.4總結","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Mongod服務入口首先從message中解析出opCode操作碼，3.6版本對應客戶端默認操作碼爲OP_MSQ，解析出該操作對應OpMsgRequest信息。然後從message原始數據中解析出command命令字符串後，繼續通過全局Map表種查找是否支持該命令操作，如果支持則執行該命令；如果不支持，直接異常打印，同時返回。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"6. Mongos實例服務入口核心代碼實現","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" mongos服務入口核心代碼實現過程和mongod服務入口代碼實現流程幾乎相同，mongos實例message解析、OP_MSG操作碼處理、command命令查找等流程和上一章節mongod實例處理過程類似，本章節不在詳細分析。Mongos實例服務入口處理調用流程如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" ServiceEntryPointMongos::handleRequest(...)->Strategy::clientCommand(...)-->runCommand(...)->execCommandClient(...)","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/51/511238a22e3d645d35c8977bc9db1185.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 最後的接口核心代碼實現如下：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"cpp"},"content":[{"type":"text","text":"1.void runCommand(...) { \n2. ...... \n3. //獲取請求命令name \n4. auto const commandName = request.getCommandName(); \n5. //從全局map表中查找 \n6. auto const command = Command::findCommand(commandName); \n7. //沒有對應的command存在,拋異常說明不支持該命令 \n8. if (!command) { \n9. ...... \n10. return; \n11. } \n12. ...... \n13. //執行命令 \n14. execCommandClient(opCtx, command, request, builder); \n15. ...... \n16.} \n17.\n18.void execCommandClient(...) \n19.{ \n20. ...... \n21. //認證檢查，是否有操作該command命令的權限，沒有則異常提示 \n22. Status status = Command::checkAuthorization(c, opCtx, request); \n23. if (!status.isOK()) { \n24. Command::appendCommandStatus(result, status); \n25. return; \n26. } \n27. //該命令的執行次數自增，代理上面也是要計數的 \n28. c->incrementCommandsExecuted(); \n29. //如果需要command統計，則加1 \n30. if (c->shouldAffectCommandCounter()) { \n31. globalOpCounters.gotCommand(); \n32. } \n33. ...... \n34. //有部分命令不支持writeconcern配置，報錯 \n35. bool supportsWriteConcern = c->supportsWriteConcern(request.body); \n36. //不支持writeconcern又帶有該參數的請求，直接異常處理\"Command does not support writeConcern\" \n37. if (!supportsWriteConcern && !wcResult.getValue().usedDefault) { \n38. ...... \n39. return; \n40. } \n41. //執行本命令對應的公共publicRun接口，Command::publicRun \n42. ok = c->publicRun(opCtx, request, result); \n43. ...... \n44.} ","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Tips: mongos和mongod實例服務入口核心代碼實現的一點小區別","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"①　Mongod實例opCode操作碼解析、OpMsg解析、command查找及對應命令調用處理都由class ServiceEntryPointMongod{...}類一起完成。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"②　mongos實例則把opCode操作碼解析交由class ServiceEntryPointMongos{...}類實現，OpMsg解析、command查找及對應命令調用處理放到了clase Strategy{...}類來處理。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"7.總結","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Mongodb報文解析及組裝流程總結","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"①　一個完整mongodb報文由通用報文header頭部+body部分組成。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"②　Body部分內容，根據報文頭部的opCode來決定不同的body內容。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"③　3.6版本對應客戶端請求opCode默認爲OP_MSG，該操作碼對應body部分由flagBits + sections + checksum組成，其中sections中存放的是真正的命令請求信息，已bson數據格式保存。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"④　Header頭部和body報文體封裝及解析過程由class Message {...}類實現","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑤　Body中對應command命令名、庫名、表名的解析在mongodb(version<3.6)低版本協議中由class DbMessage {...}類實現","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑥　Body中對應command命令名、庫名、表名的解析在mongodb(version<3.6)低版本協議中由struct OpMsgRequest{...}結構和struct OpMsg {...}類實現","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Mongos和mongod實例的服務入口處理流程大同小異，整體處理流程如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"①　從message解析出opCode操作碼，根據不同操作碼執行對應操作碼回調。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"②　根據message解析出OpMsg request信息，mongodb報文的命令信息就存儲在該body中，該body已bson格式存儲。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"③　從body中解析出command命令字符串信息(如“insert”、“update”等)。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"④　從全局_commands map表中查找是否支持該命令，如果支持則執行該命令處理，如果不支持則直接報錯提示。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"⑤　最終找到對應command命令後，執行command的功能run接口。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 圖形化總結如下:","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ef/efc040eb2e9f6a9947b61bc25f6cde82.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":" 說明：","attrs":{}},{"type":"text","text":"第3章的協議解析及封裝過程實際上應該算是網絡處理模塊範疇，本文爲了分析command命令處理模塊方便，把該部分實現歸納到了命令處理模塊，這樣方便理解。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":" Tips: ","attrs":{}},{"type":"text","text":"下期繼續分享不同command命令執行細節。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"8.遺留問題","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 第1章節中的統計信息，將在command模塊核心代碼分析完畢後揭曉答案，《mongodb command命令處理模塊源碼實現二》中繼續分析，敬請關注。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}