攜程Elasticsearch數據同步實踐

{"type":"doc","content":[{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"一、背景"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Elasticsearch是最近幾年非常熱門的分佈式搜索和數據分析引擎，攜程內部不僅使用ES實現了大規模的日誌平臺，也廣泛使用ES實現了各個業務場景的搜索、推薦等功能。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本文聚焦在業務搜索的場景分享了我們在做數據同步方面的思考和實踐，希望能對大家有所啓發。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"二、現狀調研"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"數據同步是個很麻煩的事情，在各種論壇、分享中被大家反覆討論。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們的需求大致包括全量、增量地從Hive、MySql、Soa服務、Mq等不同類型的數據源獲取數據，部分數據還需要進行一定的計算或者轉換，然後近實時地同步到ES中，以被用戶搜索到。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"爲了討論方便，假定本文的場景是文章搜索的場景："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1）索引內容爲文章，主要的信息保存在article表裏；"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"2）每個文章關聯了tag，保存在article_tag表裏；"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"3）tag表裏的tagName也需要進入ES索引，以便使用標籤名字搜索文章；"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在以前同步這樣的數據進入ES，單條文章的數據組裝僞代碼如下："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"List tagIds = articleTagDao.query(\"select tagId from article_tags where articleId=?\", articleId);\nList tags =tagDao.query(\"select id, name from tags whereid in (?)\");\nArticleInEs articleInEs = new ArticleInEs();\narticleInEs.setTagIds(tagIds);\narticleInEs.setTagNames(tags.stream().filter(tag-> tagIds.contains(tag.getId())).map(TagPojo::getName).collect(Collectors.toList()));"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}