【得物技术】交易轨迹系统

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"我们不生产数据，我们只是数据的搬运工。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"背景","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"现有系统操作日志都分散在各个域的系统中，而且没有形成规范的系统。导致排查线上问题全是去日志系统各种搜，效率很低。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我们针对系统调用链路有trace串联，跨域查问题很爽。但是我们对单据纬度没有trace串联，单据流转对于我们来说是黑盒。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"开发人员对自己的功能线上情况不清楚，比如：每天卖家发货量多少？不清楚；成功率多少？不清楚；平均响应多少？不清楚；针对当前量，我们是不是需要做一些优化？其实有时候并不是不去关注，只是没有一个很直观的系统去很直观的呈现出来。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"为了解决上述问题，交易轨迹系统就应运而生了。","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"实现思路","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"怎样去埋数据？","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"1. 方案一：手动埋点（不推荐）","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"对现有系统侵入性较高，而且埋点对系统稳定性有一定的影响，不到万不得已不推荐。但是我们提供了这项功能。支持三种接入方式，feign、dubbo和rocketmq。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"2. 方案二：日志清洗（推荐）","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"怎样对现有系统不改造，利用现有的资源去寻找事件action？我们发现access.log可以完美的利用起来。但是有一些超时任务和mq触发的任务没有access.log，现阶段需要考虑其他方式接入。后续我们考虑提供公共工具，将mq和超时任务触发的动作，写入access.log并提供全套的清洗入库套件。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"3. 方案三：binlog（不推荐）","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"为什么不推荐，因为binlog拉取不到入参出参，对排查问题没有太大的作用。但是有一点优势很明显，我们可以追踪前后两次的表更。比如：修改地址（A地址->B地址）","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/eb/ebe1b69f529672c447fb0e53a4471ae2.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"heading","attrs":{"align":null,"level":4}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"怎样去数据清洗？","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"1. 功能独立","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"和业务系统隔离，独立一套系统去做数据清洗","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"2. 实时生效","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"利用groovy脚本做清洗逻辑，脚本写好后保存实时生效。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"3. 数据扩展","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"日志里面的数据不满足埋点需求，我们提供了dubbo和feign的扩展，可以去业务系统查询数据","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"4. 批量操作支持","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"批量操作会涉及到多个单号的流转，脚本默认返回List，对多个单号进行埋点，支持一次解析，多条数据落库","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"5. 主子单号支持","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"（同理批量操作）","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"6. 日志太复杂？解析费劲？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"针对日志分析，我们提供的公共的数据获取工具，直接调用方法即可","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"例子（寄售申请）：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":null},"content":[{"type":"text","text":"package com.shizhuang.duapp.trade.script.biz.deposit.js\n\nimport com.alibaba.fastjson.JSONArray\nimport com.alibaba.fastjson.JSONObject\nimport com.google.common.collect.Lists\nimport com.shizhuang.duapp.trade.cycle.api.resultdata.BizResult\nimport com.shizhuang.duapp.trade.cycle.api.script.AccessLogToBizResultBaseScript\nimport com.shizhuang.duapp.trade.cycle.api.sourcedata.AccessLogData\nimport com.shizhuang.duapp.trade.cycle.api.util.AccessLogUtils\n\n/**\n * @program: trade-cycle-center\n *\n * @author: 小猪佩奇*\n * @create: 2020-11-20 17:08\n * */\nclass ConsignApplyCreate2BizResult extends AccessLogToBizResultBaseScript {\n List parse(AccessLogData accessLogData) {\n // 响应结果\n String responseData = accessLogData.getResponseData()\n // 执行时间\n String takeTime = accessLogData.getTakeTime()\n // http 返回code码\n String httpStatus = accessLogData.getHttpStatus()\n // 获取uid\n String uid = AccessLogUtils.getUid(accessLogData)\n List applyNoList = this.analyzeApplyItemNo(responseData);\n List bizResultList = Lists.newArrayList();\n for (applyItemNo in applyNoList) {\n BizResult result = new BizResult()\n result.setOperatorId(uid)\n result.setOperationBizNo(applyItemNo)\n result.setOperationTime(accessLogData.getOperationTime())\n result.setOperationSubType(\"app寄售申请\")\n result.setOperationTrace(accessLogData.getOperationTrace())\n result.setOperationCostTime(Long.valueOf(takeTime))\n result.setOperationResult(\"200\" == httpStatus ? 0 : 1)\n //result.setOperationExtend()\n result.setOperationDetailReq(accessLogData.getPayload())\n result.setOperationDetailResp(accessLogData.getResponseData())\n bizResultList.add(result);\n }\n return bizResultList;\n }\n\n /**\n * 分析单号\n * @param responseData\n * @return\n */\n List analyzeApplyItemNo(String responseData) {\n List list = Lists.newArrayList()\n\n JSONObject jsonObject = JSONObject.parseObject(responseData)\n JSONObject data = jsonObject.getJSONObject(\"data\")\n JSONObject applyProduct = data.getJSONObject(\"applyProduct\")\n if (Objects.isNull(applyProduct)) {\n return list\n }\n\n JSONArray applyItems = applyProduct.getJSONArray(\"applyItems\")\n if (Objects.nonNull(applyItems)) {\n for (Object applyItem : applyItems) {\n JSONObject parseObject = JSONObject.parseObject(applyItem.toString())\n String applyItemNo = parseObject.getString(\"applyItemNo\");\n list.add(applyItemNo);\n }\n }\n return list\n }\n}\n","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"怎样去做数据呈现？","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"1. 单据纬度","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/2c/2c6ae1219909f1d76d45f70f7dcf47c8.png","alt":null,"title":"寄存单交易轨迹","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/71/714cf0e3d3cdeadbcb9bae513bd1df82.png","alt":null,"title":"客服工单轨迹","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"查询时段段内，单据的流转轨迹（支持跨域跨系统展示）。根据来源数据和结果数据，排查问题根源。如需查看详细日志，可以根据traceId去日志系统去查看详细trace日志。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"2. 人纬度","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/5f/5f882ef4010c2235349934568d2d600e.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"统计时间段内用户的操作行为，方便排查某个时间段内用户的操作。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"怎样自定义图形化？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"TDengine时序数据库存储的存储的数据都是带有时间戳的，且本身很好地支持时间维度的聚合，正好可以结合grafana对操作日志中的数据进行分析，展示各类业务运转的情况。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/03/03e019bbb8a8ce6b59012d813feff216.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"系统接入","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"access log方式接入","attrs":{}}]},{"type":"numberedlist","attrs":{"start":"1","normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"下载脚本项目，编写脚本代码","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"新增事件脚本类，将脚本代码配置到交易轨迹系统","attrs":{}}]}],"attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/3a/3a88cf5917aa75531014e958a8fb4e86.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"numberedlist","attrs":{"start":"3","normalizeStart":"3"},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":3,"align":null,"origin":null},"content":[{"type":"text","text":"添加路由规则，关联事件和脚本","attrs":{}}]}],"attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/e2/e2c0fec7681bfb574ab1130a63691a32.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"rocketMq方式接入","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"mq接入方式和rpc方式相同，各业务系统把数据封装好，通过硬编码的方式将数据发送到mq，轨迹系统订阅消息进行数据清洗并持久化。有时候获取事件不是很方便，通过mq的方式接入也未尝不是一种好的方式。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"扩展","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在存储上我们支持跨度一年的轨迹查询，在存储上已经做过优化，接入方可以不考虑存储问题。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在和财务系统、客服系统的对接中。我们发现还有更多的方向去扩展功能，不仅仅是局限于交易轨迹。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"财务系统需要记录和第三方交互的请求报文，排查问题的时候直接通过单号快速查询第三方报文。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"客服系统需要记录客服会话的生命周期，通过会话id可以直观的查看会话链路。并且可以通过客服id可以快速统计客服的接入量统计。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"后续会加强图形化的功能和简易数据分析的能力","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"理论上我们只提供了一种存储能力，规范了数据存储的格式。具体怎样使用，使用方可以自定义。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"最后打个广告，欢迎大家接入。","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"文/得物技术 ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Ambition","attrs":{}}]}]}