1.Es的三种客户端
a.Transport Client,基于transport的连接,使用es的9300端口
b.JestClient,基于http的连接,使用es的9200端口
3.RestClient,基于http的连接,es官方推荐,使用9200端口
2.文档来源:
基于jestClient的es聚合: https://blog.csdn.net/lvyuan1234/article/details/78655493 (此文解决了jest基本的聚合操作)
基于时间的es聚合: https://blog.csdn.net/xuyingzhong/article/details/78839744 (此文解决了不返回空bucket的问题)
es官方文档: https://www.elastic.co/guide/en/elasticsearch/reference/5.4
基于scroll的深度分页: https://www.jianshu.com/p/32f4d276d433 (需要注意的是每次传入scroll时长)
第一次查询:加入_doc进行排序
/index/type/_search?scroll=5m&size=10000 (tips:5m是指游标存在的时长,es默认size不超过10000)
"{\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"query\":{\"match_all\":{}}}"
带条件和排序查询
{"sort": [{"_doc": {"order": "asc"}}],"query": {"bool": {"must": [{"term": {"SuccessSign": 0}}]}}}
第二次查询:以后每次查询需要带入上一次查询的scroll_id
_search/scroll?scroll=5m,不需要指定index,type
"{\"scroll_id\": " + "\"" + scroll_id + "\"" + "}"
3.基于时间聚合的代码:
@Service
@Slf4j
public class ResourceServiceImpl implements ResourceService {
@Resource(name = "viidJestClient")
private JestClient jestClient;
private static final String ENTRYTIME = "EntryTime";
private static final String VIID = "viid";
private static final String VIDEOSLICE = "VideoSlice";
private static final String IMAGE = "Image";
@Override
public List<ResourceVO> summary(QueryParam queryParam) {
List<Object> rangeTime = queryParam.getTerms().stream().filter(term -> term.getColumn().equals(ENTRYTIME))
.map(Term::getValue).collect(Collectors.toList());
if (2 != rangeTime.size() || CollectionUtils.isEmpty(rangeTime)) {
throw new ParamNoExistException(ErrorConstant.PARAM_NO_EXIST, I18nConstant.DTAE_PARAM_NOT_EXIST);
}
Object beginTime = rangeTime.get(0);
Object endTime = rangeTime.get(1);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery(ENTRYTIME).
gte(beginTime).lte(endTime));
DateHistogramAggregationBuilder field = AggregationBuilders.dateHistogram("agg").field(ENTRYTIME);
field.dateHistogramInterval(DateHistogramInterval.DAY);
field.format("yyyy-MM-dd");
// 强制返回空buckets
field.minDocCount(0);
field.extendedBounds(new ExtendedBounds(parseDate(beginTime), parseDate(endTime)));
searchSourceBuilder.query(queryBuilder);
searchSourceBuilder.aggregation(field);
searchSourceBuilder.size(0);
Search videoSearch = new Search.Builder(searchSourceBuilder.toString()).addIndex(VIID).addType(VIDEOSLICE).build();
Search imageSearch = new Search.Builder(searchSourceBuilder.toString()).addIndex(VIID).addType(IMAGE).build();
SearchResult videoResult = null;
SearchResult imageResult = null;
try {
videoResult = jestClient.execute(videoSearch);
imageResult = jestClient.execute(imageSearch);
} catch (IOException e) {
throw new JestExcuteException(HttpStatusEnum.INTERNAL_SERVER_ERROR.value(), ErrorConstant.JEST_EXCUTE_ERROR);
}
List<ResourceVO> resourceVOS = Lists.newArrayList();
assert videoResult != null;
List<DateHistogramAggregation.DateHistogram> videoBuckets = videoResult.getAggregations().getDateHistogramAggregation("agg").getBuckets();
for (DateHistogramAggregation.DateHistogram videoBucket : videoBuckets) {
ResourceVO resourceVO = new ResourceVO();
resourceVO.setTime(videoBucket.getTimeAsString());
resourceVO.setVideoCount(videoBucket.getCount());
resourceVOS.add(resourceVO);
}
assert imageResult != null;
List<DateHistogramAggregation.DateHistogram> imageBulkets = imageResult.getAggregations().getDateHistogramAggregation("agg").getBuckets();
for (DateHistogramAggregation.DateHistogram imageBulket : imageBulkets) {
ResourceVO imageResourceVO = resourceVOS.stream().filter(resourceVO -> resourceVO.getTime().equals(imageBulket.getTimeAsString())).findFirst().get();
imageResourceVO.setImageCount(imageBulket.getCount());
resourceVOS.add(imageResourceVO);
}
return resourceVOS;
}