elasticsearch7.1.1學習之整合springboot——data整合

9.30的時候maven公庫裏發佈了spring-data-elasticsearch3.2.0的正式版本，那麼主要新的特性是支持響應式編程（這個響應式編程通過異步返回的方式來提高吞吐量，可以和springwebflux一起使用，主要返回的對象有mono,flux）和升級到支持elasticsearch6.8.1，而對應的springboot版本是2.2.0。下圖是版本對應的關係

Spring Data Release Train	Spring Data Elasticsearch	Elasticsearch	Spring Boot
Moore[1]	3.2.x[1]	6.8.1 / 7.x[2]	2.2.0[1]
Lovelace	3.1.x	6.2.2 / 7.x[2]	2.1.x
Kay[3]	3.0.x[3]	5.5.0	2.0.x[3]
Ingalls[3]	2.1.x[3]	2.4.0	1.5.x[3]

話不多說，開始編寫代碼,這裏用的是springboot2.2.0版本配套的es starter

 

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.2.0</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.gdut.imis</groupId>
    <artifactId>es-data-demo</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>es-data-demo</name>
    <description>Demo project for Spring Boot</description>

    <properties>
        <java.version>1.8</java.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
        </dependency>
       <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-webflux</artifactId>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
            <exclusions>
                <exclusion>
                    <groupId>org.junit.vintage</groupId>
                    <artifactId>junit-vintage-engine</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>

    

</project>

寫配置：

由於官方建議用高版本的客戶端，所以使用restHighLevelClient

package com.gdut.imis.esdatademo.configuration;

import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.convert.converter.Converter;
import org.springframework.core.convert.support.DefaultConversionService;
import org.springframework.data.convert.ReadingConverter;
import org.springframework.data.convert.WritingConverter;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.RestClients;
import org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient;
import org.springframework.data.elasticsearch.client.reactive.ReactiveRestClients;
import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfiguration;
import org.springframework.data.elasticsearch.core.ElasticsearchEntityMapper;
import org.springframework.data.elasticsearch.core.EntityMapper;
import org.springframework.data.elasticsearch.core.convert.ElasticsearchCustomConversions;
import org.springframework.data.geo.Point;
import java.time.Duration;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

/**
 * @author lulu
 * @Date 2019/10/7 14:09
 */
@Configuration
public class EsConfiguration extends AbstractElasticsearchConfiguration {


    @Bean
    @Override
    public RestHighLevelClient elasticsearchClient() {
     //鏈接配置
        ClientConfiguration clientConfiguration = ClientConfiguration.builder().connectedTo("localhost:9200")
                .withConnectTimeout(Duration.ofSeconds(15)).withSocketTimeout(Duration.ofSeconds(15))
                //.withBasicAuth()
                .build();
        return RestClients.create(clientConfiguration).rest();
    }
    @Bean
    //reactive配置
    ReactiveElasticsearchClient client() {
        ClientConfiguration clientConfiguration = ClientConfiguration.builder()
                .connectedTo("localhost:9200")
                .build();

        return ReactiveRestClients.create(clientConfiguration);
    }
    @Bean
    @Override
    public EntityMapper entityMapper() {

        //entityMapper可以自定義怎麼把實體對象和json進行映射
        // elasticsearchMappingContext返回一個具有@Document註解的實體類集合上下文
        //DefaultConversionService裏定義一系列的轉換方式
        ElasticsearchEntityMapper entityMapper = new ElasticsearchEntityMapper(
                elasticsearchMappingContext(), new DefaultConversionService()
        );
        //conversionService用於轉換對象,如果沒有自定義的顯式配置，則有默認實現
        entityMapper.setConversions(elasticsearchCustomConversions());
        return entityMapper;
    }

    @Bean
    @Override
    //自定義的轉換
    public ElasticsearchCustomConversions elasticsearchCustomConversions() {
        return new ElasticsearchCustomConversions(
                Arrays.asList(new PointToMap(), new MapToPoint()));
    }

    @WritingConverter
    static class PointToMap implements Converter<Point, Map<String, Object>> {
        @Override
        public Map<String, Object> convert(Point p) {
            Map map = new HashMap();
            map.put("user_lat", p.getX());
            map.put("user_lon", p.getY());
            return map;
        }
    }

    @ReadingConverter
    static class MapToPoint implements Converter<Map<String, Double>, Point> {
        @Override
        public Point convert(Map<String, Double> map) {
            return new Point(map.get("user_lat"), map.get("user_lon"));
        }
    }
}

定義了一個User實體，這裏面的註解類型有

• @Id：作用在字段上面，用於標識對象，類似主鍵的作用。

• @Document：作用在類上面，表明該類是映射到數據庫的對象（實體類）。其中比較重要的屬性是：

  • indexName：用於存儲此實體的索引的名稱

  • type：映射類型。如果未設置，則使用小寫的類的簡單名稱，最好設置爲"_doc"，這樣不會有warn的日誌。

  • shards：索引的分片數。

  • replicas：索引的副本數。

  • refreshIntervall：索引的刷新間隔。用於索引創建。默認值爲“ 1s”。

  • indexStoreType：索引的索引存儲類型。用於索引創建。默認值爲“ fs”。

  • createIndex：配置是否在存儲庫引導中創建索引。默認值爲true。

  • versionType：版本管理的配置。默認值爲EXTERNAL。

• @Transient：默認情況下，所有私有字段都映射到文檔，此註釋作用的字段將不會映射到數據庫中

• @Field：在字段級別應用並定義字段的屬性，大多數屬性映射到各自的Elasticsearch映射定義：

  • name：字段名稱，將在Elasticsearch文檔中表示，如果未設置，則使用Java字段名稱。

  • type：字段類型，可以是Text，Integer，Long，Date，Float，Double，Boolean，Object，Auto，Nested，Ip，Attachment，Keyword之一。

  • format和日期類型的pattern自定義定義。

  • store：標記是否將原始字段值存儲在Elasticsearch中，默認值爲false。

  • analyzer，searchAnalyzer，normalizer用於指定自定義自定義分析和正規化。

  • copy_to：將多個文檔字段複製到的目標字段。

  • fielddata: 作用於text類型，設置爲true的字段可以對其進行聚合

• @GeoPoint：將字段標記爲geo_point數據類型。如果字段是GeoPoint類的實例，則可以省略。

package com.gdut.imis.esdatademo.entity;

import lombok.Data;
import org.springframework.data.annotation.Id;
import org.springframework.data.annotation.Transient;
import org.springframework.data.elasticsearch.annotations.*;

import java.util.List;


/**
 * @author lulu
 * @Date 2019/10/7 14:19
 */
@Document(indexName = "user_info",shards = 2,type = "_doc")
@Data
public class User {
    @Id
    private Integer userId;

    @Field(name="user_name",type = FieldType.Keyword)
    private String name;

    private Address location;

    @Field(name="user_birthday",type=FieldType.Date,format = DateFormat.date_hour_minute_second )
    private String birthDay;

    @Transient
    private List<Job> jobList;
}

package com.gdut.imis.esdatademo.repository;

import com.gdut.imis.esdatademo.entity.User;
import org.springframework.data.domain.Page;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Repository;

import java.util.List;

/**
 * @author lulu
 * @Date 2019/10/7 14:51
 */
@Repository
public interface UserRepository extends ElasticsearchRepository<User, Integer> {
    /**
     * 根據用戶名模糊查詢
     * @param name
     * @return
     */
    List<User> findAllByNameLike(String name);

    /**
     * 按用戶名模糊查詢and根據Address屬性下的city進行查詢，並且按照id排序
     * @param userName
     * @param city
     * @return
     */
    List<User> findUsersByNameLikeAndLocationCityEqualsOrderByUserId(
            String userName,
            String city
    );
}

定義好實體以後，再繼承ElasticsearchRepository就可以了，其中可以自定義一些方法名，orm框架會根據一定方法把他解析成對應的查詢語句並且執行，這裏定義方法可能會有一個點要注意，就是如果User對象有一個localtionCity屬性，他的Address屬性有一個city屬性，此時需要把LocationCity改爲Location_City纔可以查找得到,其實用法都是差不多，主要是跟據條件查詢

另外的一個job類

package com.gdut.imis.esdatademo.entity;

import lombok.Data;
import lombok.experimental.Accessors;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;

/**
 * @author lulu
 * @Date 2019/10/7 19:37
 */
@Document(indexName = "user_job",replicas = 2,shards = 1, type = "_doc",createIndex = false)
@Data
@Accessors(chain = true)
public class Job {
    private String desc;
    @Id
    private String jobName;
    private Double salary;

}

package com.gdut.imis.esdatademo.repository;

import com.gdut.imis.esdatademo.entity.Job;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.stereotype.Repository;

/**
 * @author lulu
 * @Date 2019/10/7 19:49
 */
@Repository
public interface JobRepository extends ElasticsearchRepository<Job,String> {
Page<Job> findByDescLike(String job, Pageable pageable);

}

這裏面主要是一個分頁的方法，其實用法比較普遍和簡單，也沒什麼好說的，如果想自定義查詢，也可以使用ElasticsearchRestTemplate作爲查詢工具，template的方法還是很多的，而且構造的查詢也可以相對複雜，並且支持聚合、批量操作等等。配置和上面一樣，直接注入就可以使用,下面這個則是支持reactive流式

package com.gdut.imis.esdatademo.repository;

import com.gdut.imis.esdatademo.entity.Job;
import org.springframework.data.elasticsearch.repository.ReactiveElasticsearchRepository;

/**
 * @author lulu
 * @Date 2019/10/9 13:23
 */
@Repository
public interface JobReactiveRepository extends ReactiveElasticsearchRepository<Job, String> {
}

例子

package com.gdut.imis.esdatademo.service;

import com.gdut.imis.esdatademo.entity.Job;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.index.query.*;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.ParsedCardinality;
import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats;
import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.ReactiveElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * @author lulu
 * @Date 2019/10/7 21:26
 */
@Service
public class JobService {
    @Autowired
    private ElasticsearchRestTemplate template;
    @Autowired
    private ReactiveElasticsearchTemplate reactiveTemplate;

    public Flux<Job> queryDemo(String jobName,Double from,Double to){

        /**
         * 構造布爾查詢
         */
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
        //模糊查詢名字
        QueryStringQueryBuilder name = QueryBuilders.queryStringQuery(jobName).field("jobName");
        boolQuery.must(name);
        //限定薪水範圍
        RangeQueryBuilder salary = QueryBuilders.rangeQuery("salary").lte(to).gte(from);
        boolQuery.must(salary);
        NativeSearchQuery query=new NativeSearchQuery(boolQuery);
        //定義排序
        Sort.TypedSort<Job> sort = Sort.sort(Job.class);
        Sort descending = sort.by(Job::getSalary).descending();
        //分頁
        PageRequest request=PageRequest.of(0,10,descending);
        query.setPageable(request);
        Flux<Job> jobFlux = reactiveTemplate.find(query, Job.class);
        return jobFlux;
    }

    public Map<String,Long> termDemo(){
        MatchAllQueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
        //terms聚合，會以一個個桶的形式返回
        TermsAggregationBuilder userCity = AggregationBuilders.terms("user_city").field("location.city");
        NativeSearchQuery searchQuery=new NativeSearchQuery(queryBuilder);
        searchQuery.addIndices("user_info");
        searchQuery.setSearchType(SearchType.DEFAULT);
        searchQuery.addTypes("_doc");
        searchQuery.addAggregation(userCity);
        ParsedStringTerms city = template.query(searchQuery, response -> (ParsedStringTerms) response.getAggregations().getAsMap().get("user_city"));
       Map<String,Long> map= city.getBuckets().stream().filter(e->((Bucket) e).getDocCount()>5).collect(Collectors.toMap(bucket -> bucket.getKey().toString(), Bucket::getDocCount));
       return map;
    }

    public Map<String,Aggregation>  aggDemo(){
        MatchAllQueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
        StatsAggregationBuilder agg = AggregationBuilders.stats("salary_stats").field("salary");
        CardinalityAggregationBuilder agg2 = AggregationBuilders.cardinality("user_address_code_stats").field("location.code");
        NativeSearchQuery searchQuery=new NativeSearchQuery(queryBuilder);
        searchQuery.addIndices("user_job","user_info");
        searchQuery.addTypes("_doc");
        searchQuery.addAggregation(agg);
        searchQuery.addAggregation(agg2);
        Map<String, Aggregation> query = template.query(searchQuery, response -> response.getAggregations().asMap());
        ParsedStats stats= (ParsedStats) query.get("salary_stats");
        ParsedCardinality cardinality= (ParsedCardinality) query.get("user_address_code_stats");
        return query;
    }

}

controller，其實webflux還有另一種編程方式，是handler+routing的方式開發，有興趣的可以自行了解下，因爲我對這個webflux只停留於簡單瞭解的狀態，就不獻醜了哈哈，以後有機會再補上

package com.gdut.imis.esdatademo.controller;

import com.gdut.imis.esdatademo.entity.Address;
import com.gdut.imis.esdatademo.entity.Job;
import com.gdut.imis.esdatademo.entity.User;
import com.gdut.imis.esdatademo.repository.JobReactiveRepository;
import com.gdut.imis.esdatademo.repository.JobRepository;
import com.gdut.imis.esdatademo.repository.UserRepository;
import com.gdut.imis.esdatademo.service.JobService;
import org.elasticsearch.search.aggregations.Aggregation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.geo.Point;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import reactor.core.publisher.Flux;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

/**
 * @author lulu
 * @Date 2019/10/7 14:54
 */
@RestController
public class TestController {

    @Autowired
    private UserRepository userRepository;
    @Autowired
    private JobRepository jobRepository;
    @Autowired
    private JobReactiveRepository jobReactiveRepository;
    @Autowired
    private JobService jobService;
    private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");


    @GetMapping("/getUserByName")
    public List<User> userList(@RequestParam("name") String name) {
        return userRepository.findAllByNameLike(name);
    }
    @GetMapping("/getUserByCity")
    public List<User> getUserList(@RequestParam("name")String name,
            @RequestParam("city")String city){
        return userRepository.findUsersByNameLikeAndLocationCityEqualsOrderByUserId(name,city);
    }

    @GetMapping("/getFlux")
    public Flux<Job> queryDemo(@RequestParam("jobName") String jobName,@RequestParam("from")Double from,@RequestParam("to") Double to){
       return jobService.queryDemo(jobName,from,to);
    }

    @GetMapping("/createUser")
    public Iterable<User> createUser(@RequestParam("from")Integer from,@RequestParam("to")Integer to) {
        String[] country = {"uk", "china", "japan"};
        String[] city = {"london", "gz", "tokyo"};
        String[] street = {"a", "b", "c"};
        List<User> users = IntStream.rangeClosed(from, to).mapToObj(e -> {
                    User u = new User();
                    Point point = new Point(Math.random() * 100, Math.random() * 100);
                    u.setName("No" + e);
                    u.setUserId(e);
                    Address address = new Address();
                    address.setCountry(country[e % country.length]);
                    address.setCity(city[e % city.length]);
                    address.setStreet(street[e % street.length]);
                    address.setCode(e%2);
                    address.setPoint(point);
                    u.setLocation(address);
                    u.setBirthDay(dateFormat.format(new Date()));
                    return u;
                }
        ).collect(Collectors.toList());

        return userRepository.saveAll(users);
    }
    @GetMapping("/createManyJob")
    public Iterable<Job> jobList(@RequestParam("from")Integer from,@RequestParam("to")Integer to) {
        List<Job> jobList = IntStream.rangeClosed(from, to).mapToObj(e -> {
            String res = UUID.randomUUID().toString();
            Job job = new Job().setJobName(e + "")
                    .setDesc(res).setSalary(Math.random() * 1000);
            return job;
        }).collect(Collectors.toList());
//        jobReactiveRepository.saveAll(jobList);
        return jobRepository.saveAll(jobList);
    }

    @GetMapping("/getJobAgg")
    public Map<String,Aggregation> getJobAgg(){
        return jobService.aggDemo();
    }
    @GetMapping("/getJob")
    public List<Job> getJob(@RequestParam("name") String name,
                            @RequestParam("index") Integer index,
                            @RequestParam("size") Integer size
    ) {
        Sort.TypedSort<Job> sort = Sort.sort(Job.class);
        Sort descending = sort.by(Job::getSalary).descending();
        PageRequest request = PageRequest.of(index, size, descending);
        Page<Job> byDescLike = jobRepository.findByDescLike(name, request);
        return byDescLike.getContent();
    }

}

總結：這裏面主要還是理解好es本身的json查詢語句和聚合是怎麼用的，再根據實際要求對應官方文檔進行理解使用，其實版本升級以後就是某些api用法不同了，大體還是差不多的，高級的聚合查詢我還不會，可以參考下面的鏈接，還有一個小配置，如果想看到template發送了什麼返回了什麼，可以配置日誌級別

logging:
 level:
  org.springframework.data.elasticsearch.client.WIRE: trace

參考鏈接：

template使用：https://blog.csdn.net/Topdandan/article/details/81436141

官方文檔：https://docs.spring.io/spring-data/elasticsearch/docs/3.2.0.RELEASE/reference/html/#new-features

webflux:https://www.cnblogs.com/limuma/p/9315343.html,只是把文中的map改爲和elasticsearch交互