ELK使用記錄-Elasticsearch使用記錄

Java RestHighLevelClient 操作

分組聚合帶條件

//查詢es 書籍日誌
        SearchRequest searchRequest = new SearchRequest("bookanalysis");
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
        List<QueryBuilder> filter = boolQuery.filter();
        long start = LocalDateTime.now().plusDays(-1).withHour(0).withMinute(0).withSecond(0).withNano(0).toInstant(ZoneOffset.of("+8")).toEpochMilli();
//        long start = LocalDate.now().plusDays(-1).atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli();
        long end = LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli();

        //開始時間
        Long startMilli = start;
        //結束時間
        Long endMilli = end;
        //時間區間條件
        filter.add(rangeQuery(startMilli, endMilli, "create_time"));
        //表示是閱讀書籍數據
        filter.add(QueryBuilders.matchQuery("log_type",0));

        //這裏拼接動態條件
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

        //添加條件
        searchSourceBuilder.query(boolQuery);
        //時間分桶
        DateHistogramAggregationBuilder builder = AggregationBuilders
                .dateHistogram("create_time")
                .field("create_time")
                .calendarInterval(DateHistogramInterval.HOUR).format("yyyy-MM-dd HH:mm:ss");

        //書籍桶
        TermsAggregationBuilder book_idField = AggregationBuilders.terms("book_id").field("book_id");
        TermsAggregationBuilder chapter_idField = AggregationBuilders.terms("content_id").field("content_id");
        //將子桶加入桶中
        builder.subAggregation(book_idField.subAggregation(chapter_idField));

        //閱讀章節數
//          book_idField.subAggregation(AggregationBuilders.count("content_id_count").field("content_id"));
        //閱讀UV
        chapter_idField.subAggregation(AggregationBuilders.cardinality("read_uv").field("user_id"));

        searchSourceBuilder.aggregation(builder);
        searchRequest.source(searchSourceBuilder);
        SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);


        //每日的
        List<AppStatisticsChapterUvEveryday> everydayList = new ArrayList<>();
        //累計的
        List<AppStatisticsChapterUvTotal> totalList = new ArrayList<>();


        ParsedDateHistogram terms = response.getAggregations().get("create_time");
        Iterator<? extends Histogram.Bucket> timeIter = terms.getBuckets().iterator();
        while (timeIter.hasNext()){
            Histogram.Bucket timeBucket = timeIter.next();
            Object create_time = timeBucket.getKey();
            ParsedLongTerms bookTerms = timeBucket.getAggregations().get("book_id");
            Iterator<? extends Terms.Bucket> bookIter = bookTerms.getBuckets().iterator();
            while (bookIter.hasNext()){
                Terms.Bucket bookBucket = bookIter.next();
                int book_id = bookBucket.getKeyAsNumber().intValue();
                ParsedLongTerms contentTerms = bookBucket.getAggregations().get("content_id");
                Iterator<? extends Terms.Bucket> contentIter = contentTerms.getBuckets().iterator();
                while (contentIter.hasNext()) {
                    Terms.Bucket contentBucket = contentIter.next();
                    int chapter_id = contentBucket.getKeyAsNumber().intValue();
                    Aggregations aggregations = contentBucket.getAggregations();
                    ZonedDateTime date = (ZonedDateTime) create_time;

                    AppStatisticsChapterUvEveryday chapterUvEveryday = new AppStatisticsChapterUvEveryday();
                    Date createTime = new SimpleDateFormat("yyyy-MM-dd").parse(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
                            .format(date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime()));
                    chapterUvEveryday.setStatistics_date(createTime);
                    //閱讀UV
                    ParsedCardinality read_uv = aggregations.get("read_uv");
                    chapterUvEveryday.setRead_uv((int) read_uv.getValue());
                    //書籍id
                    chapterUvEveryday.setBook_id(book_id);
                    //章節id
                    chapterUvEveryday.setChapter_id(chapter_id);
                    //章節名稱
                    AppBookChapter appBookChapter = appBookChapterService.chapterInfoById(chapter_id);
                    if (appBookChapter==null){
                        continue;
                    }
                    chapterUvEveryday.setChapter_name(appBookChapter.getTitle());
                    everydayList.add(chapterUvEveryday);
                    //查詢上次的累計內容
                    AppStatisticsChapterUvTotal one = appStatisticsChapterUvTotalService.getOne(
                            new QueryWrapper<AppStatisticsChapterUvTotal>()
                                    .eq("chapter_id", chapter_id)
                                    .orderByDesc("statistics_date")
                                    .last("limit 0,1")
                    );
                    if (one == null) {
                        one = new AppStatisticsChapterUvTotal();
                        one.setStatistics_date(createTime);
                        //閱讀UV
                        one.setRead_uv((int) read_uv.getValue());
                        //書籍id
                        one.setBook_id(book_id);
                        //章節id
                        one.setChapter_id(chapter_id);
                        //章節名稱
                        one.setChapter_name(appBookChapter.getTitle());
                    }else{
                        one.setId(null);
                        one.setStatistics_date(createTime);
                        //閱讀UV
                        one.setRead_uv(one.getRead_uv()+ (int) read_uv.getValue());
                        //書籍id
                        one.setBook_id(book_id);
                        //章節id
                        one.setChapter_id(chapter_id);
                        //章節名稱
                        one.setChapter_name(appBookChapter.getTitle());
                    }
                    totalList.add(one);
                }
            }
        }
        appStatisticsChapterUvTotalService.saveBatch(totalList);
        appStatisticsChapterUvEverydayService.saveBatch(everydayList);

獲取桶內的真正數據hits

TopHitsAggregationBuilder topHitsAggregationBuilder = AggregationBuilders.topHits("top");
build.subAggregation(topHitsAggregationBuilder);


TopHits topHits = Bucket.getAggregations().get("top");
for (SearchHit hit : topHits.getHits().getHits()) {
    hit.getSourceAsMap().get("device_id");
}


//獲取超過1w條數據 需要加上  "track_total_hits":true ，不然只能顯示出9999條
searchRequest.source(searchSourceBuilder.trackTotalHits(true)); 



//博客學習
//內容: Top Hits Aggregation
//網址: https://blog.csdn.net/ctwy291314/article/details/82773180 

普通查詢,使用查詢內容

    @Test
    public void test() throws IOException {
        SearchRequest searchRequest = new SearchRequest("useranalysis");
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
        List<QueryBuilder> filter = boolQuery.filter();
        //時間區間條件
        filter.add(rangeQuery(1611590400000L, 1611676800000L, "create_time"));
        filter.add(QueryBuilders.matchQuery("channel_id",50000011104L));
        filter.add(QueryBuilders.matchQuery("is_new",1));
        //這裏拼接動態條件
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        //添加條件
        searchSourceBuilder.query(boolQuery);

        searchRequest.source(searchSourceBuilder.trackTotalHits(true).size(20001));
        SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        HashSet<String> userIds = new HashSet<>();
        for (SearchHit hit : response.getHits().getHits()) {
            String user_id = hit.getSourceAsMap().get("user_id").toString();
            userIds.add(user_id);
        }
        System.out.println(userIds);
    }

使用模板查詢(dsl語言)

    public int test(List<Integer> userIds) throws IOException {
        long start = LocalDateTime.now().plusDays(-1).withHour(0).withMinute(0).withSecond(0).withNano(0).toInstant(ZoneOffset.of("+8")).toEpochMilli();
        long end = LocalDateTime.now().plusDays(0).withHour(0).withMinute(0).withSecond(0).withNano(0).toInstant(ZoneOffset.of("+8")).toEpochMilli();
        SearchTemplateRequest request = new SearchTemplateRequest();
        request.setRequest(new SearchRequest("bookanalysis"));
        request.setScriptType(ScriptType.INLINE);
        String script = "{\n" +
                "  \"query\": {\n" +
                "    \"bool\": {\n" +
                "      \"must\": [\n" +
                "        {\n" +
                "          \"range\": {\n" +
                "            \"create_time\": {\n" +
                "              \"gte\": \"{{start}}\",\n" +
                "              \"lte\": \"{{end}}\"\n" +
                "            }\n" +
                "          }\n" +
                "        },\n" +
                "        {\n" +
                "          \"terms\": {\n" +
                "            \"user_id\":{{user_id}}\n" +
                "          }\n" +
                "        }\n" +
                "      ]\n" +
                "    }\n" +
                "  },\n" +
                "  \"size\": 0,\n" +
                "  \"track_total_hits\": true"+
                "}";
        request.setScript(script);
        Map<String, Object> params = new HashMap<>();
        params.put("user_id", JSONObject.toJSONString(userIds));
        params.put("start", start);
        params.put("end", end);
        request.setScriptParams(params);
        SearchTemplateResponse searchTemplateResponse = restHighLevelClient.searchTemplate(request, RequestOptions.DEFAULT);
        SearchResponse response = searchTemplateResponse.getResponse();
        TotalHits totalHits = response.getHits().getTotalHits();
        return (int) totalHits.value;
    }

刪除數據

    public ReturnT<String> deleteEveryday(String param) throws IOException {
        //刪除指定的傳入的索引
        try {
            DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(param);
            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
            List<QueryBuilder> filter = boolQueryBuilder.filter();
            long start = LocalDateTime.now().plusDays(-10).withHour(23).withMinute(59).withSecond(59).withNano(0).toInstant(ZoneOffset.of("+8")).toEpochMilli();
            long end = LocalDateTime.now().plusDays(-2).withHour(0).withMinute(0).withSecond(0).withNano(0).toInstant(ZoneOffset.of("+8")).toEpochMilli();
            XxlJobLogger.log("\t 開始刪除數據:("+start+"-"+end+") 索引地址:"+param+"。\t 開始時間:"+LocalDateTime.now().toString());
            filter.add(rangeQuery(start, end, "create_time"));
            deleteByQueryRequest.setQuery(boolQueryBuilder);
            //並行 5條線程同時執行
            deleteByQueryRequest.setSlices(5);
            //批次大小 一次刪除多少
            deleteByQueryRequest.setBatchSize(5000);
            // 更新最大文檔數
//            deleteByQueryRequest.setSize(10);
            //使用滾動參數來控制“搜索上下文”存活的時間  因爲刪除數據前是先查詢到這個數據
            deleteByQueryRequest.setScroll(TimeValue.timeValueMinutes(10));
            //刷新索引
            deleteByQueryRequest.setRefresh(true);
            //超時時間 2小時
            deleteByQueryRequest.setTimeout(TimeValue.timeValueHours(2));
            BulkByScrollResponse response = restHighLevelClient.deleteByQuery(deleteByQueryRequest,RequestOptions.DEFAULT);
            XxlJobLogger.log("\t 刪除數據:"+response.getTotal()+" 條"+"\t 結束時間:"+LocalDateTime.now().toString());
                        //使用異步刪除
            /*
            Cancellable response = restHighLevelClient.deleteByQueryAsync(
                    deleteByQueryRequest,
                    RequestOptions.DEFAULT,
                    new ActionListener<BulkByScrollResponse>() {
                        @Override public void onResponse(BulkByScrollResponse response) {
//                            XxlJobLogger.log("\t 刪除數據:"+response.getTotal()+" 條"+"\t 異步結束時間:"+LocalDateTime.now().toString());
                            System.out.println("\t 刪除數據:"+response.getTotal()+" 條"+"\t 異步結束時間:"+LocalDateTime.now().toString());
                        }
                        @Override public void onFailure(Exception e) {
                            e.printStackTrace();
//                            XxlJobLogger.log("\t 異步刪除數據報錯了........"+LocalDateTime.now().toString()+"\t"+e.getMessage());
                            System.out.println("\t 異步刪除數據報錯了........"+LocalDateTime.now().toString()+"\t"+e.getMessage());
                        }
                    });
            //取消刪除
//            response.cancel();
//            Thread.sleep(2000);
             */
        } catch (Exception e) {
            e.printStackTrace();
            XxlJobLogger.log("\t 刪除數據報錯了........"+LocalDateTime.now().toString()+"\t"+e.getMessage());
            return ReturnT.FAIL;
        }
        return ReturnT.SUCCESS;