SpringBoot使用JestClient操作ElasticSearch

項目配置

項目中使用了Apollo進行配置管理

application.properties中配置如下

spring.elasticsearch.jest.uris = http://test-es.umgsai.com
spring.elasticsearch.jest.read-timeout = 3s
spring.elasticsearch.jest.connection-timeout = 2s

配置好之後就可以在項目中通過Resource註解拿到JestClient實例了

---

創建索引

創建index：POST http://localhost:8080/es/createIndex?index=main_order

@RequestMapping("/es")
@RestController
public class EsController {

    @Resource
    protected JestClient jestClient;

    @RequestMapping(value = "/createIndex", method = RequestMethod.POST)
    public Object createIndex(String index) throws IOException {
        CreateIndex createIndex = new CreateIndex.Builder(index).build();
        JestResult result = jestClient.execute(createIndex);
        if (result.isSucceeded()) {
            return Response.success(null);
        }
        return Response.fail(Code.ERROR_PARAM.getCode(), result.getErrorMessage());
    }
}

---

刪除索引

刪除index：DELETE http://localhost:8080/es/deleteIndex?index=main_order

@RequestMapping(value = "/deleteIndex", method = RequestMethod.DELETE)
public Object deleteIndex(String index) throws IOException {
    DeleteIndex deleteIndex = new DeleteIndex.Builder(index).build();
    JestResult result = jestClient.execute(deleteIndex);
    if (result.isSucceeded()) {
        return Response.success(null);
    }
    return Response.fail(Code.ERROR_PARAM.getCode(), result.getErrorMessage());
}

---

創建Mapping

PUT http://localhost:8080/es/putMapping

參數名	值
index	main_order
type	main_order
mapping	如下JSON

{
    "main_order":{
        "properties":{
            "orderId":{
                "type":"keyword"
            },
            "tags":{
                "type":"long"
            },
            "version":{
                "type":"integer"
            },
            "createTime":{
                "type":"date",
                "format":"yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
            },
            "updateTime":{
                "type":"date",
                "format":"yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
            }
        }
    }
}

@RequestMapping(value = "/putMapping", method = RequestMethod.PUT)
public Object putMapping(String index, String type, String mapping) throws IOException {
    PutMapping putMapping = new PutMapping.Builder(
            index,
            type,
            mapping
    ).build();

    JestResult result = jestClient.execute(putMapping);
    if (result.isSucceeded()) {
        return Response.success(null);
    }
    return Response.fail(Code.ERROR_PARAM.getCode(), result.getErrorMessage());
}

---

查詢Mapping

@RequestMapping(value = "/getMapping", method = RequestMethod.GET)
public Object getMapping(String index) throws IOException {

    Action getMapping = new GetMapping.Builder().addIndex(index).build();
    JestResult result = jestClient.execute(getMapping);

    if (result.isSucceeded()) {
        return Response.success(JSON.parse(result.getJsonString()));
    }
    return Response.fail(Code.ERROR_PARAM.getCode(), result.getErrorMessage());
}

ES不支持刪除mapping，官方建議先刪除一個index，然後重新創建index，重新創建mapping

 https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-delete-mapping.html
 It is no longer possible to delete the mapping for a type. Instead you should delete the index and recreate it with the new mappings.

---

異步寫入數據

public <DATA> Response<Object> insertOrUpdate(String indexName, String typeName, List<DATA> dataList, String id, JestResultHandler<JestResult> jestResultHandler) {
    Bulk.Builder bulk = new Bulk.Builder();
    for (DATA data : dataList) {
        Index index = new Index.Builder(data).
                index(indexName).
                type(typeName).
                id(id).
                build();
        bulk.addAction(index);
    }
    jestClient.executeAsync(bulk.build(), jestResultHandler);
    return Response.success(null);
}

---

同步寫入數據

    public <DATA> JestResult insertOrUpdate(String indexName, String typeName, List<DATA> dataList, String id) throws Exception {
        Bulk.Builder bulk = new Bulk.Builder();
        for (DATA data : dataList) {
            Index index = new Index.Builder(data).
                    index(indexName).
                    type(typeName).
                    id(id).
                    build();
            bulk.addAction(index);
        }
        return jestClient.execute(bulk.build());
    }

---

更新

public JestResult update(String indexName, String typeName, String id, Map<String, Object> paramMap) throws Exception {
    Map<String, Object> docMap = Maps.newHashMap();
    docMap.put("doc", paramMap);
    final Update update = new Update.Builder(docMap).id(id).build();
    final Bulk bulk = new Bulk.Builder()
            .addAction(update).defaultIndex(indexName).defaultType(typeName)
            .build();
    return jestClient.execute(bulk);
}

---

刪除

// 異步刪除
public Response<Object> deleteByKeyList(String indexName, String typeName, String deleteKeyName, List<String> deleteKeyList, JestResultHandler<JestResult> jestResultHandler) {
    BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().filter(QueryBuilders.termsQuery(deleteKeyName, deleteKeyList));
    DeleteByQuery deleteByQuery = new DeleteByQuery.Builder(new SearchSourceBuilder().query(boolQueryBuilder).toString())
            .addIndex(indexName)
            .addType(typeName)
            .refresh(true).build();
    jestClient.executeAsync(deleteByQuery, jestResultHandler);
    return Response.success(null);
}

// 同步刪除
public JestResult deleteByKeyList(String indexName, String typeName, String deleteKeyName, List<String> deleteKeyList) throws Exception {
    BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().filter(QueryBuilders.termsQuery(deleteKeyName, deleteKeyList));
    DeleteByQuery deleteByQuery = new DeleteByQuery.Builder(new SearchSourceBuilder().query(boolQueryBuilder).toString())
            .addIndex(indexName)
            .addType(typeName)
            .refresh(true).build();
    return jestClient.execute(deleteByQuery);
}

// 按文檔ID異步刪除
public Response<Object> deleteById(String indexName, String typeName, String id, JestResultHandler<JestResult> jestResultHandler) {
    Delete.Builder builder = new Delete.Builder(id);
    Delete delete = builder.index(indexName).type(typeName).build();
    jestClient.executeAsync(delete, jestResultHandler);
    return Response.success(null);
}

---

查詢數據

public <T> Response<PageResult<T>> query(BoolQueryBuilder boolQueryBuilder, String indexName, String typeName, Class<T> sourceType,
                                         String orderBy, Boolean desc, List<String> fieldList, Integer pageNum, Integer pageSize) {
    String query = "";
    Transaction transaction = Cat.newTransaction(TransactionType.ES_SEARCH_DATA, indexName);
    try {
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(boolQueryBuilder);
        if (CollectionUtils.isNotEmpty(fieldList)) {
            //指定查詢字段，不傳則查詢全部
            searchSourceBuilder.fetchSource(fieldList.toArray(new String[0]), null);
        }
        EsBuilder.pageParamBuilder(searchSourceBuilder, orderBy, desc, pageNum, pageSize);
        query = searchSourceBuilder.toString();
        int from = searchSourceBuilder.from();
        if (from > esSearchMaxFrom) {
            String errorMsg = String.format("從ES查詢數據出現錯誤，indexName=%s, typeName=%s, query=%s, msg=不允許深度分頁", indexName, typeName, query);
            log.error(errorMsg);
            orderErrorHandler.save("", OrderErrorTypeEnum.SEARCH_ES_ERROR, errorMsg);
            return Response.fail(Code.ERROR_PARAM.getCode(), "不允許深度分頁，請檢查參數");
        }
        log.info("SearchManager.query, indexName={}, typeName={}, query={}", indexName, typeName, query);
        Search search = new Search.Builder(query).addIndex(indexName).addType(typeName).build();
        SearchResult searchResult = jestClient.execute(search);
        if (!searchResult.isSucceeded()) {
            OrderSearchException orderSearchException = new OrderSearchException(ErrorCode.QUERY_ERROR);
            transaction.setStatus(orderSearchException);
            String errorMsg = String.format("從ES查詢數據出現錯誤，indexName=%s, typeName=%s, query=%s, msg=%s", indexName, typeName, query, searchResult.getErrorMessage());
            log.error(errorMsg, orderSearchException);
            orderErrorHandler.save("", OrderErrorTypeEnum.SEARCH_ES_ERROR, errorMsg);
            return Response.fail(ErrorCode.QUERY_ERROR.getCode(), searchResult.getErrorMessage());
        }
        transaction.setStatus(Transaction.SUCCESS);
        long totalCount = searchResult.getTotal();
        PageResult<T> pageResult = new PageResult<>();
        pageResult.setCount(totalCount);
        List<T> resultList = Lists.newArrayList();
        List<SearchResult.Hit<T, Void>> hits = searchResult.getHits(sourceType);
        if (CollectionUtils.isNotEmpty(hits)) {
            for (SearchResult.Hit<T, Void> hit : hits) {
                resultList.add(hit.source);
            }
        }
        pageResult.setList(resultList);
        pageResult.setEnd(CollectionUtils.size(resultList) < searchSourceBuilder.size());
        return Response.success(pageResult);
    } catch (Exception e) {
        transaction.setStatus(e);
        String errorMsg = String.format("從ES查詢數據出現異常，indexName=%s, typeName=%s, query=%s, msg=%s", indexName, typeName, query, e.getMessage());
        orderErrorHandler.save("", OrderErrorTypeEnum.SEARCH_ES_ERROR, errorMsg);
        log.error(errorMsg, e);
        return Response.fail(ErrorCode.QUERY_ERROR.getCode(), errorMsg);
    } finally {
        transaction.complete();
    }
}

---

查詢數量

public Response<Long> queryCount(BoolQueryBuilder boolQueryBuilder, String indexName, String typeName) {
    String query = "";
    Transaction transaction = Cat.newTransaction(TransactionType.ES_SEARCH_COUNT, indexName);
    try {
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(boolQueryBuilder);
        query = searchSourceBuilder.toString();
        Count count = new Count.Builder()
                .addIndex(indexName)
                .addType(typeName)
                .query(query)
                .build();
        log.info("SearchManager.queryCount, indexName={}, typeName={}, query={}", indexName, typeName, query);
        CountResult countResult = jestClient.execute(count);
        if (!countResult.isSucceeded()) {
            OrderSearchException orderSearchException = new OrderSearchException(ErrorCode.QUERY_ERROR);
            transaction.setStatus(orderSearchException);
            String errorMsg = String.format("從ES查詢數量出現錯誤，indexName=%s, typeName=%s, query=%s, msg=%s", indexName, typeName, query, countResult.getErrorMessage());
            orderErrorHandler.save("", OrderErrorTypeEnum.SEARCH_ES_ERROR, errorMsg);
            log.error(errorMsg, orderSearchException);
            return Response.fail(ErrorCode.QUERY_ERROR.getCode(), countResult.getErrorMessage());
        }
        Double resultsCount = countResult.getCount();
        transaction.setStatus(Transaction.SUCCESS);
        return Response.success(resultsCount.longValue());
    } catch (Exception e) {
        transaction.setStatus(e);
        String errorMsg = String.format("從ES查詢數量出現異常，indexName=%s, typeName=%s, query=%s, msg=%s", indexName, typeName, query, e.getMessage());
        log.error(errorMsg, e);
        orderErrorHandler.save("", OrderErrorTypeEnum.SEARCH_ES_ERROR, errorMsg);
        return Response.fail(ErrorCode.QUERY_ERROR.getCode(), errorMsg);
    } finally {
        transaction.complete();
    }
}

---

JestClient連接池

JestClient是基於HttpClient的，使用了http連接池和ES服務端進行通信。
httpclient通過maxTotal和defaultMaxPerRoute來配置連接池。
maxTotal：設置連接池的最大連接數，即整個池子的大小。初始值20
defaultMaxPerRoute：設置每一個路由的最大連接數，這裏的路由是指IP+PORT，例如連接池大小(MaxTotal)設置爲300，路由連接數設置爲200(DefaultMaxPerRoute)，對於www.a.com與www.b.com兩個路由來說，發起服務的主機連接到每個路由的最大連接數（併發數）不能超過200，兩個路由的總連接數不能超過300。初始值2.
在defaultMaxPerRoute小於maxTotal時，主要是defaultMaxPerRoute起作用，因爲在我這個例子中只有一個 ip+port

設置maxTotal和defaultMaxPerRoute的方法

@Slf4j
@Component
public class HttpSettingsCustomizer implements HttpClientConfigBuilderCustomizer {
 
    //Apollo配置
    @Value("${es.max.total.connection:100}")
    private Integer maxTotalConnection;
    @Value("${es.default.max.total.connection.per.route:35}")
    private Integer defaultMaxTotalConnectionPerRoute;
 
    @Override
    public void customize(HttpClientConfig.Builder builder) {
        log.info("set maxTotalConnection={}, defaultMaxTotalConnectionPerRoute={}", maxTotalConnection, defaultMaxTotalConnectionPerRoute);
        builder.maxTotalConnection(maxTotalConnection).defaultMaxTotalConnectionPerRoute(defaultMaxTotalConnectionPerRoute);
    }
}

參數設置過程可參考JestAutoConfiguration源碼

@Configuration
@ConditionalOnClass(JestClient.class)
@EnableConfigurationProperties(JestProperties.class)
@AutoConfigureAfter(GsonAutoConfiguration.class)
public class JestAutoConfiguration {
 
   private final JestProperties properties;
 
   private final ObjectProvider<Gson> gsonProvider;
 
   private final List<HttpClientConfigBuilderCustomizer> builderCustomizers;
 
   public JestAutoConfiguration(JestProperties properties, ObjectProvider<Gson> gson,
         ObjectProvider<List<HttpClientConfigBuilderCustomizer>> builderCustomizers) {
      this.properties = properties;
      this.gsonProvider = gson;
      this.builderCustomizers = builderCustomizers.getIfAvailable();
   }
 
   @Bean(destroyMethod = "shutdownClient")
   @ConditionalOnMissingBean
   public JestClient jestClient() {
      JestClientFactory factory = new JestClientFactory();
      factory.setHttpClientConfig(createHttpClientConfig());
      return factory.getObject();
   }
 
   protected HttpClientConfig createHttpClientConfig() {
      HttpClientConfig.Builder builder = new HttpClientConfig.Builder(
            this.properties.getUris());
      PropertyMapper map = PropertyMapper.get();
      map.from(this.properties::getUsername).whenHasText().to((username) -> builder
            .defaultCredentials(username, this.properties.getPassword()));
      Proxy proxy = this.properties.getProxy();
      map.from(proxy::getHost).whenHasText().to((host) -> {
         Assert.notNull(proxy.getPort(), "Proxy port must not be null");
         builder.proxy(new HttpHost(host, proxy.getPort()));
      });
      map.from(this.gsonProvider::getIfUnique).whenNonNull().to(builder::gson);
      map.from(this.properties::isMultiThreaded).to(builder::multiThreaded);
      map.from(this.properties::getConnectionTimeout).whenNonNull()
            .asInt(Duration::toMillis).to(builder::connTimeout);
      map.from(this.properties::getReadTimeout).whenNonNull().asInt(Duration::toMillis)
            .to(builder::readTimeout);
      customize(builder);
      return builder.build();
   }
 
   private void customize(HttpClientConfig.Builder builder) {
      if (this.builderCustomizers != null) {
         for (HttpClientConfigBuilderCustomizer customizer : this.builderCustomizers) {
            customizer.customize(builder);
         }
      }
   }
 
}

---

參考：https://www.jianshu.com/p/df5d89955eaa
https://docs.spring.io/spring-boot/docs/2.0.3.RELEASE/reference/htmlsingle/#boot-features-elasticsearch