需求
文章,最開始文章詳情需要顯示點贊數量、訪問數量,以前做法是在調用查詢接口 數據庫+1 點贊時候訪問量+1
update question q set q.view_count=q.view_count+1 where id=1 類似這樣做法,其實在高併發場景不合理的,但是還好
需求改變
需要支持時間搜索 搜索某一段時間的訪問量
我的方案
參考《redis-緩存設計-統計1秒 5秒 1分鐘 訪問數量》
比如我設計1分鐘延遲,同一分鐘時間片都是redis incr 指定字段時間片,然後時間片過了在刷到數據庫,因爲不支持時分秒搜索 那麼一個文章就算一天點贊10萬次上億次一天就生成一條數據 後續都是update 不存在埋點數據過大問題
數據庫表
--文章統計指標 CREATE TABLE `question_metric_item` ( `id` BIGINT(20) NOT NULL AUTO_INCREMENT COMMENT 'ID', `question_id` BIGINT(20) NOT NULL COMMENT '文章id', `answer_count` int(11) default 0 COMMENT '評論量', `subscription_count` int(11) default 0 COMMENT '關注量', `help_count` int(11) default 0 COMMENT '有幫助', `comment_count` int(11) default 0 COMMENT '評論回答數', `no_help_count` int(11) default 0 COMMENT '無幫助', `share_count` int(11) default 0 COMMENT '分享數', `view_count` int(11) default 0 COMMENT '查看數', `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '創建時間', `provider_id` BIGINT(20) COMMENT '服務商id', PRIMARY KEY (`id`), ) ENGINE = InnoDB AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8 ROW_FORMAT = DYNAMIC COMMENT ='文章統計指標';
埋點實現
在對應地方調用對應方法埋點
/** * 新增閱讀量 * * @param questionId * @return */ public Boolean addViewCount(Long providerId, Long questionId, Date date, Long count) { if (questionId == null) { return false; } addCount(providerId, QuestionMetricItemRo_.viewCount, questionId, count, date); return true; } /** * 新增評論量 * * @param questionId * @param count * @return */ public Boolean addAnswerCount(Long providerId, Long questionId, Date date, Long count) { addCount(providerId, QuestionMetricItemRo_.answerCount, questionId, count, date); return true; } /** * 關注與取消關注 * * @param questionId * @param date * @param count * @return */ public Boolean addSubscriptionCount(Long providerId, Long questionId, Date date, Long count) { addCount(providerId, QuestionMetricItemRo_.subscriptionCount, questionId, count, date); return true; } /** * 新增回答量 * * @param questionId * @param date * @param count * @return */ public Boolean addCommentCount(Long providerId, Long questionId, Date date, Long count) { addCount(providerId, QuestionMetricItemRo_.commentCount, questionId, count, date); return true; } /** * 有幫助 * * @param questionId * @param date * @param count * @return */ public Boolean addHelpCount(Long providerId, Long questionId, Date date, Long count) { addCount(providerId, QuestionMetricItemRo_.helpCount, questionId, count, date); return true; } /** * 無幫助 * * @param questionId * @param date * @param count * @return */ public Boolean addNoHelpCount(Long providerId, Long questionId, Date date, Long count) { addCount(providerId, QuestionMetricItemRo_.noHelpCount, questionId, count, date); return true; } /** * 更新分享數 * * @param questionId * @param date * @param count * @return */ public Boolean addShareCount(Long providerId, Long questionId, Date date, Long count) { addCount(providerId, QuestionMetricItemRo_.shareCount, questionId, count, date); return true; } public void addCount(Long providerId, String field, Long questionId, Long count, Date currentDate) { if (log.isDebugEnabled()) { log.debug("field={},questionId={},count={}", field, questionId, count); } String id = formatId(currentDate, questionId); String redisIdKey = this.getRoPrimaryId(id); //添加到當日list集合 String listKey = getRoPrefix("list"); Date formatDate = formatDate(currentDate); //時間片過了則統計 Date scoreDate = Time.when(formatDate).setMinute(Time.when(formatDate).getMinute() + PREISION_MINUTE).setSecond(5).getDate(); if (!exists(redisIdKey)) { Map<String, String> values = MapUtil.newMap(QuestionMetricItemRo_.createdAt, Time.when(currentDate).toString(Time.DEFAULT_TIME_FORMATS[0]), QuestionMetricItemRo_.id, id, QuestionMetricItemRo_.questionId, String.valueOf(questionId), QuestionMetricItemRo_.providerId, String.valueOf(providerId)); this.hset(redisIdKey, values); //定時刷入緩存 this.zadd(listKey, NumberUtil.toDouble(scoreDate.getTime()), id); //設置過期時間 expire(Sets.newHashSet(redisIdKey, listKey), getRo().expireSeconds()); } //針對文章數量累加1 this.hincrBy(redisIdKey, field, count); } public Date formatDate(Date date) { int preision = 60000 * PREISION_MINUTE; //算出x分鐘內的時間片 long startDateTime = (long) (date.getTime() / preision) * preision; return new Date(startDateTime); }
定時任務同步
當然同步的時候除了同步每日的,還需要往主表的Ro進行數量增加
/** * 同步redis的統計數量到數據庫 */ @Override public void syncByRedis() throws ParseException { int offset = 0; int size = 50; //隊列待消費數量越多 則每次最多偏移200 Date currentDate = new Date(); Long waitCount = questionMetricItemRedisDao.listCount(currentDate); while (true) { Set<String> ids = questionMetricItemRedisDao.listKey(currentDate, offset, size); if (CollectionUtils.isEmpty(ids)) { log.debug("[QuestionMetricItemSync]沒有數據忽略,offset:{},count:{}", offset, waitCount); break; } EweiTLogHandler eweiTLogHandler = new EweiTLogHandler(); try { eweiTLogHandler.before(UUID.randomUUID().toString().replace("-", "")); //score到當前時間的數據信息 實現延遲效果 List<QuestionMetricItemRo> questionMetricItemRos = questionMetricItemRedisDao.listById(ids); log.info("[QuestionMetricItemSync]執行同步,offset:{},count:{},Ids:{}", offset, waitCount, JSON.toJSONString(ids)); log.info("執行同步questionMetricItems={}", JSON.toJSONString(questionMetricItemRos)); List<QuestionMetricItem> questionMetricItems = QuestionMetricItemConvert.INSTANCE.toQuestionMetricItem(questionMetricItemRos); SpringUtil.getBean(IQuestionMetricItemService.class).batchSaveAndSyncQuestion(questionMetricItems); } catch (Exception e) { //埋點失敗也不影響後續執行 log.error("[QuestionMetricItemSync]執行同步異常", e); } finally { questionMetricItemRedisDao.deleteByIdAndDelRouting(ids); eweiTLogHandler.clear(); } } }
針對實時性
我們知道統計是根據我們的時間片有一定延遲的,針對報表有一定延遲是可以接收的,但是針對詳情需要實時性
不可能sum我們分日期的統計,直接用主表又會有延遲,很簡單 通過主表的數量+當前時間片未同步的數量就好了
如以下代碼
/** * 合併當前時間片還未同步到redis的count * * @param * @return */ public void combineCount(Question question) { if (question == null) { return; } String formatId = formatId(new Date(), NumberUtil.toLong(question.getId())); QuestionMetricItemRo itemRo = findById(formatId); if (itemRo == null) { return; } Integer answerCount = NumberUtil.toInteger(question.getAnswerCount(), 0) + (NumberUtil.toInteger(question.getAnswerCount(), 0)); question.setAnswerCount(answerCount); Integer subscriptionCount = NumberUtil.toInteger(question.getSubscriptionCount(), 0) + (NumberUtil.toInteger(itemRo.getSubscriptionCount(), 0)); question.setSubscriptionCount(subscriptionCount); Integer commentCount = NumberUtil.toInteger(question.getCommentCount(), 0) + (NumberUtil.toInteger(itemRo.getCommentCount(), 0)); question.setCommentCount(commentCount); Integer helpCount = NumberUtil.toInteger(question.getVoteCount(), 0) + (NumberUtil.toInteger(itemRo.getHelpCount(), 0)); question.setVoteCount(helpCount); Integer noHelpCount = NumberUtil.toInteger(question.getNoHelpCount(), 0) + (NumberUtil.toInteger(itemRo.getNoHelpCount(), 0)); question.setNoHelpCount(noHelpCount); Integer shareCount = NumberUtil.toInteger(question.getShareCount(), 0) + (NumberUtil.toInteger(itemRo.getShareCount(), 0)); question.setShareCount(shareCount); Integer viewCount = NumberUtil.toInteger(question.getViewCount(), 0) + (NumberUtil.toInteger(itemRo.getViewCount(), 0)); question.setViewCount(viewCount); }