elasticsearch 腳本排序主要用於複雜場景的綜合排序，腳本語言有 java ,painless,groovy,目前調研的爲painless。
- kibana 腳本創建mapping
DELETE ksc.metadata
PUT ksc.metadata
{
"mappings": {
"doc": {
"properties": {
"hotnum": {
"type": "long"
},
"ds": {
"type": "object",
"properties": {
"dsid": {
"type": "long"
},
"dsname": {
"type": "text",
"fielddata": true
}
}
},
"tlbtype": {
"type":"keyword"
},
"dbname": {
"type": "text",
"fielddata": true
},
"createdate": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
}
}
}
}
}
- java 代碼倒入數據
package com.hys.es;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.hys.Application;
import javafx.scene.input.DataFormat;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import org.apache.http.HttpHost;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientProperties;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.elasticsearch.annotations.DateFormat;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.stream.Collectors;
/**
* es 的測試單元
*/
@ActiveProfiles("es")
@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class,webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class EsTest {
@Autowired
ElasticsearchTemplate elasticsearchTemplate;
@Autowired
RestClientProperties restClientProperties;
RestHighLevelClient restHighLevelClient = null;
String index = "test";
Client client = null;
String type = "student";
@Before
public void inserIndexIfNotExist(){
if(!elasticsearchTemplate.indexExists(index)){
elasticsearchTemplate.createIndex(this.index);
}
client = elasticsearchTemplate.getClient();
HttpHost[] httpHosts = new HttpHost[restClientProperties.getUris().size()];
int index = 0;
restClientProperties.getUris().forEach(uri->{
httpHosts[index] = HttpHost.create(uri);
});
restHighLevelClient = new RestHighLevelClient(
RestClient.builder(
httpHosts
));
}
/**
* 插入模板
*/
@Test
public void insertTemplate(){
String setting = "{\n" +
" \"template\" : \"te*\",\n" +
" \"order\" : 1,\n" +
" \"settings\" : {\n" +
" \"number_of_shards\" : 1\n" +
" },\n" +
" \"mappings\" : {\n" +
" \"type1\" : {\n" +
" \"_source\" : { \"enabled\" : true }\n" +
" }\n" +
" }\n" +
"}";
boolean ok = elasticsearchTemplate.createIndex(this.index, setting);
System.out.println(ok);
}
@Test
public void insertTemplate2() throws IOException {
String setting = "{\n" +
" \"template\" : \"te*\",\n" +
" \"order\" : 1,\n" +
" \"settings\" : {\n" +
" \"number_of_shards\" : 1\n" +
" },\n" +
" \"mappings\" : {\n" +
" \"type1\" : {\n" +
" \"_source\" : { \"enabled\" : true }\n" +
" }\n" +
" }\n" +
"}";
CreateIndexRequest request = new CreateIndexRequest(index);//創建索引
//創建的每個索引都可以有與之關聯的特定設置。
request.settings(Settings.builder().loadFromSource(setting,XContentType.JSON)
);
//創建索引時創建文檔類型映射
request.mapping("tweet",//類型定義
" {\n" +
" \"tweet\": {\n" +
" \"properties\": {\n" +
" \"message\": {\n" +
" \"type\": \"text\"\n" +
" }\n" +
" }\n" +
" }\n" +
" }",//類型映射，需要的是一個JSON字符串
XContentType.JSON);
//爲索引設置一個別名
request.alias(
new Alias("twitter_alias")
);
//可選參數
request.timeout(TimeValue.timeValueMinutes(2));//超時,等待所有節點被確認(使用TimeValue方式)
//request.timeout("2m");//超時,等待所有節點被確認(使用字符串方式)
request.masterNodeTimeout(TimeValue.timeValueMinutes(1));//連接master節點的超時時間(使用TimeValue方式)
//request.masterNodeTimeout("1m");//連接master節點的超時時間(使用字符串方式)
request.waitForActiveShards(2);//在創建索引API返回響應之前等待的活動分片副本的數量，以int形式表示。
//request.waitForActiveShards(ActiveShardCount.DEFAULT);//在創建索引API返回響應之前等待的活動分片副本的數量，以ActiveShardCount形式表示。
//同步執行
CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(request);
//異步執行
//異步執行創建索引請求需要將CreateIndexRequest實例和ActionListener實例傳遞給異步方法：
//CreateIndexResponse的典型監聽器如下所示：
//異步方法不會阻塞並立即返回。
ActionListener<CreateIndexResponse> listener = new ActionListener<CreateIndexResponse>() {
@Override
public void onResponse(CreateIndexResponse createIndexResponse) {
//如果執行成功，則調用onResponse方法;
}
@Override
public void onFailure(Exception e) {
//如果失敗，則調用onFailure方法。
}
};
restHighLevelClient.indices().createAsync(request, listener);//要執行的CreateIndexRequest和執行完成時要使用的ActionListener
//返回的CreateIndexResponse允許檢索有關執行的操作的信息，如下所示：
boolean acknowledged = createIndexResponse.isAcknowledged();//指示是否所有節點都已確認請求
boolean shardsAcknowledged = createIndexResponse.isShardsAcknowledged();//指示是否在超時之前爲索引中的每個分片啓動了必需的分片副本數
System.out.println(acknowledged);
}
@Test
public void insertTemplate3() throws IOException {
PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("templateahys");
List<String> indexPatterns = new ArrayList<String>();
indexPatterns.add("ubi*");
putIndexTemplateRequest.patterns(indexPatterns);
Map<String, Object> settings = new HashMap<>();
settings.put("number_of_shards", 1);
putIndexTemplateRequest.settings(settings);
PutIndexTemplateResponse putIndexTemplateResponse = restHighLevelClient.indices().putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT);
System.out.println(putIndexTemplateResponse.isAcknowledged());
}
@Test
public void insertTemplate4() throws IOException {
PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("templateahys1");
putIndexTemplateRequest.source("{\n" +
" \"template\" : \"*\",\n" +
" \"order\" : 0,\n" +
" \"settings\" : {\n" +
" \"number_of_shards\" : 1\n" +
" },\n" +
" \"mappings\" : {\n" +
" \"type1\" : {\n" +
" \"_source\" : { \"enabled\" : false }\n" +
" }\n" +
" }\n" +
"}",XContentType.JSON);
PutIndexTemplateResponse putIndexTemplateResponse = restHighLevelClient.indices().putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT);
System.out.println(putIndexTemplateResponse.isAcknowledged());
}
@Test
public void validate(){
PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("templateahys1");
putIndexTemplateRequest.source("{\n" +
" \"template\" : \"*\",\n" +
" \"order\" : 0,\n" +
" \"settings1\" : {\n" +
" \"number_of_shards\" : 1\n" +
" },\n" +
" \"mappings\" : {\n" +
" \"type1\" : {\n" +
" \"_source\" : { \"enabled\" : false }\n" +
" }\n" +
" }\n" +
"}",XContentType.JSON);
System.out.println("ok");
}
//插入type
@Test
public void querySoreSingle() throws IOException {
//https://blog.csdn.net/u012270682/article/details/80165836
String index = "ksc.metadata";
String type = "doc";
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.must(QueryBuilders.multiMatchQuery("ds62","ds.dsname"));
// String idOrCode = "params.current1-doc['createdate'].date";
//根據熱度排序
String hotNumCode = "long hotscore = doc['hotnum'].value/params['hotnumSize']*10;if(hotscore < params['hotnumMinscore']){hotscore =params['hotnumMinscore'];}" +
"else if(hotscore > params['hotnumMaxscore']){hotscore =params['hotnumMaxscore'];}return hotscore;";
Map<String,Object> params = new HashMap<>();
// params.put("current1",new SimpleDateFormat("yyyy-MM-dd").format(LocalDateTime.now()));
params.put("current1",new Date());
params.put("hotnumSize",100);
params.put("hotnumMinscore",10);
params.put("hotnumMaxscore",100);
//params.put("hotnumsize",100);
Script script = new Script(ScriptType.INLINE,"painless",hotNumCode,params);
ScoreFunctionBuilder scoreFunctionBuilder = ScoreFunctionBuilders.scriptFunction(script);
FunctionScoreQueryBuilder scoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder,scoreFunctionBuilder)
.scoreMode(FunctionScoreQuery.ScoreMode.SUM).boostMode(CombineFunction.REPLACE);
NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
builder.withQuery(scoreQueryBuilder).withIndices(index).withTypes(type);
NativeSearchQuery nativeSearchQuery = builder.build();
String queryc = nativeSearchQuery.getQuery().toString();
System.out.println(queryc);
List<Map> maps = elasticsearchTemplate.queryForList(nativeSearchQuery, Map.class);
System.out.println(maps);
}
@Test
public void querySoreMult() throws IOException {
//https://blog.csdn.net/u012270682/article/details/80165836
String index = "ksc.metadata";
String type = "doc";
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.must(QueryBuilders.multiMatchQuery("ds62","ds.dsname"));
// String idOrCode = "params.current1-doc['createdate'].date";
//根據熱度排序
String hotNumCode = "long hotscore = doc['hotnum'].value/params['hotnumSize']*10;if(hotscore < params['hotnumMinscore']){hotscore =params['hotnumMinscore'];}" +
"else if(hotscore > params['hotnumMaxscore']){hotscore =params['hotnumMaxscore'];}return hotscore;";
Map<String,Object> params = new HashMap<>();
// params.put("current1",new SimpleDateFormat("yyyy-MM-dd").format(LocalDateTime.now()));
params.put("current1",new Date());
params.put("hotnumSize",100);
params.put("hotnumMinscore",10);
params.put("hotnumMaxscore",100);
Script hotScript = new Script(ScriptType.INLINE,"painless",hotNumCode,params);
ScoreFunctionBuilder hotScriptScoreFunctionBuilder = ScoreFunctionBuilders.scriptFunction(hotScript);
//時間排序
String timeCode = "Calendar c = Calendar.getInstance(); c.setTime(new Date());c.add(Calendar.DATE, - 7);long beforeNowSeven = c.getTimeInMillis();long differ = (doc['createdate'].value.millis - beforeNowSeven)/86400000;if(differ> 7){differ = 1;}return differ;";
Script timeScript = new Script(ScriptType.INLINE,"painless",timeCode,params);
ScoreFunctionBuilder timeScriptScoreFunctionBuilder = ScoreFunctionBuilders.scriptFunction(timeScript);
//類型排序
String tlbTypeCode = "if(doc['tlbtype'].value=='table'){return 2;}else{return 1;}";
Script tlbTypeScript = new Script(ScriptType.INLINE,"painless",tlbTypeCode,params);
ScoreFunctionBuilder tlbScriptScoreFunctionBuilder = ScoreFunctionBuilders.scriptFunction(tlbTypeScript);
FunctionScoreQueryBuilder.FilterFunctionBuilder[] filterFunctionBuilders = new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{
new FunctionScoreQueryBuilder.FilterFunctionBuilder(hotScriptScoreFunctionBuilder),
new FunctionScoreQueryBuilder.FilterFunctionBuilder(timeScriptScoreFunctionBuilder),
new FunctionScoreQueryBuilder.FilterFunctionBuilder(tlbScriptScoreFunctionBuilder)
};
FunctionScoreQueryBuilder scoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder,filterFunctionBuilders)
.scoreMode(FunctionScoreQuery.ScoreMode.SUM).boostMode(CombineFunction.REPLACE);
NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
builder.withQuery(scoreQueryBuilder).withIndices(index).withTypes(type);
NativeSearchQuery nativeSearchQuery = builder.build();
String queryc = nativeSearchQuery.getQuery().toString();
System.out.println(queryc);
List<Map> maps = elasticsearchTemplate.queryForList(nativeSearchQuery, Map.class);
System.out.println(maps);
}
@Test
public void batchData() throws IOException {
String index = "ksc.metadata";
String type = "doc";
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout(TimeValue.timeValueMinutes(2));
bulkRequest.timeout("2m");
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
for (int i = 0;i<1000;i++) {
int r = new Random().nextInt(50);
int d = new Random().nextInt(100);
Ds ds = new Ds();
ds.setDsid(d);
ds.setDsname("ds"+d);
LocalDateTime dateTime = LocalDateTime.of(2019, 8,getZ(10) , getZ(24) ,getZ(60) );
Doc doc = new Doc();
doc.setCreatedate(dateTime.format(formatter));
doc.setHotnum(new Random().nextInt(1000));
doc.setDbname("db"+r);
doc.setTlbtype();
doc.setDs(ds);
bulkRequest.add(new IndexRequest(index, type, UUID.randomUUID().toString())
.source(JSONObject.toJSONString(doc),XContentType.JSON));
}
BulkResponse response = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
//response.hasFailures()
System.out.println(response.status());
}
private int getZ(int seed){
int i = 0;
for(;;){
i=new Random().nextInt(seed);
if (i > 0){
return i;
}
}
}
@Test
public void batch100Minlin() throws IOException {
for (int i = 0;i<2;i++){
batchData();
}
}
@Test
public void queryScript(){
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");Calendar c = Calendar.getInstance();
//過去七天
c.setTime(new Date());
c.add(Calendar.DATE, - 7);
Date d = c.getTime();
long time = d.getTime();
//86400000
System.out.println(24*60*60*1000);
String day = format.format(d);
System.out.println("過去七天："+day);
}
@After
public void close() throws IOException {
if(client!=null){
client.close();
}
if(restHighLevelClient!=null){
restHighLevelClient.close();
}
}
@Document(indexName = "test", type = "student", createIndex = false)
public static class Student {
@Field
String id;
@Field
String name;
@Field
Long age;
@Field(format = DateFormat.year_month_day)
Date birthDay;
}
@Data
public static class Doc{
Integer hotnum;
String dbname;
//@JsonFormat (shape = JsonFormat.Shape.STRING, pattern ="yyyy-MM-dd HH:mm:ss")
//LocalDateTime createdate;
String createdate;
String tlbtype;
public void setTlbtype(){
boolean aBoolean = new Random().nextBoolean();
if(aBoolean){
tlbtype = "table";
}else {
tlbtype = "col";
}
}
Ds ds;
}
@Data
public static class Ds{
Integer dsid;
String dsname;
}
}
- kibana腳本查詢
GET ksc.metadata/_search
{
"from": 0,
"size": 50,
"query": {
"function_score": {
"query": {
"multi_match": {
"query": "ds62",
"fields": [
"ds.dsname"
]
}
},
"score_mode": "sum",
"boost_mode" : "replace",
"functions": [
{
"script_score": {
"script": {
"inline": "if(doc['tlbtype'].value=='table'){return 2;}else{return 1;}",
"params": {
},
"lang": "painless"
}
}
},
{
"script_score": {
"script": {
"inline": "Calendar c = Calendar.getInstance(); c.setTime(new Date());c.add(Calendar.DATE, - 7);long beforeNowSeven = c.getTimeInMillis();long differ = (doc['createdate'].value.millis - beforeNowSeven)/86400000;if(differ> 7){differ = 1;}return differ;",
"params": {
}
, "lang": "painless"
}
}
},
{
"script_score": {
"script": {
"inline": "long hotscore = doc['hotnum'].value/params['hotnumSize']*10;if(hotscore < params['hotnumMinscore']){hotscore =params['hotnumMinscore'];}else if(hotscore > params['hotnumMaxscore']){hotscore =params['hotnumMaxscore'];}return hotscore;",
"lang": "painless",
"params": {
"current1": "2019-08-10T10:33:53.470Z",
"hotnumSize": 100,
"hotnumMinscore": 10,
"hotnumMaxscore": 100
}
}
}
}
]
}
}
}
- java 代碼實現搜索