項目背景
需要在非Spring環境(Spark)下使用ES的增刪改查,新增此工具類方便開發。
maven依賴
<!-- 若需要對ES增刪改查 需引入ES高級客戶端 -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.3.2</version>
</dependency>
<!-- Junit測試 -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13-beta-3</version>
</dependency>
工具類源代碼
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.replication.ReplicationResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.rest.RestStatus;
import java.io.IOException;
import java.util.*;
/**
* ES連接工具包
* @author licanfeng
* @date 2019-09-26
*/
public class EsUtils {
/**
* 構造方法私有 不允許new創建實例
*/
private EsUtils(){
}
private static Logger LOGGER = Logger.getLogger(EsUtils.class);
private static volatile RestHighLevelClient restHighLevelClient = null;
/**
* ES 中index後綴 日期
*/
private static final String LOG_DATE_PATTERN = "yyyyMMdd";
/**
* 獲取ES連接
*/
public static synchronized RestHighLevelClient getClient(String esNode, Integer esPort, String esUserName, String esPassword) {
if(restHighLevelClient == null){
//密碼驗證
final CredentialsProvider credentialsProvider =
new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(esUserName, esPassword));
RestClientBuilder builder = RestClient.builder(
new HttpHost(esNode, esPort))
.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder
.setDefaultCredentialsProvider(credentialsProvider));
builder.setMaxRetryTimeoutMillis(60000);
restHighLevelClient = new RestHighLevelClient(builder);
LOGGER.info("ES連接創建成功！");
}
return restHighLevelClient;
}
/**
* 創建文檔
* @param index 索引名
* @param josonStr 文檔json字符串
* @return
* @throws Exception
*/
public static boolean createDocument(String index, String josonStr) throws Exception {
return createDocument(index, "doc", null, josonStr);
}
/**
* 創建文檔
* @param index
* @param josonStr
* @return
* @throws Exception
*/
public static boolean createDocument(String index, Map<String, Object> map) throws Exception {
return createDocument(index, "doc", null, map);
}
/**
* 創建指定ID文檔：jsonStr
* 若documentId已存在 則更新文檔
* @param index 索引名
* @param indexType 默認使用doc類型
* @param documentId documentId推薦爲空，由ES自動創建
* @param josonStr 文檔json字符串
* @return
* @throws Exception
*/
public static boolean createDocument(String index, String indexType, String documentId, String josonStr) throws Exception {
IndexRequest request = new IndexRequest(index, indexType, documentId);
request.source(josonStr, XContentType.JSON);
IndexResponse indexResponse = restHighLevelClient.index(request);
if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
|| indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
return true;
}
ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
return true;
}
if (shardInfo.getFailed() > 0) {
for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
.getFailures()) {
throw new Exception(failure.reason());
}
}
return false;
}
/**
* 創建指定ID文檔：jsonStr
* 若documentId已存在 則更新文檔
* @param index 索引名
* @param indexType 默認使用doc類型
* @param documentId documentId推薦爲空，由ES自動創建
* @param map 文檔Map
* @return
* @throws Exception
*/
public static boolean createDocument(String index, String indexType, String documentId, Map<String,Object> map) throws Exception {
IndexRequest request = new IndexRequest(index, indexType, documentId);
request.source(map);
IndexResponse indexResponse = restHighLevelClient.index(request);
if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
|| indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
return true;
}
ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
return true;
}
if (shardInfo.getFailed() > 0) {
for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
.getFailures()) {
throw new Exception(failure.reason());
}
}
return false;
}
/**
* 刪除文檔 默認doc文檔
*
* @param index 索引名
* @param documentId 文檔id
* @return 刪除成功：true 刪除失敗：false
* @throws Exception
*/
public static boolean deleteDocument(String index, String documentId) throws Exception {
return deleteDocument(index, "doc", documentId);
}
/**
* 刪除文檔
*
* @param index
* @return
* @throws Exception
*/
public static boolean deleteDocument(String index, String indexType, String documentId) throws Exception {
DeleteRequest request = new DeleteRequest(index, indexType, documentId);
DeleteResponse deleteResponse = restHighLevelClient.delete(request);
if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
return false;
}
ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo();
if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
return false;
}
if (shardInfo.getFailed() > 0) {
for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
.getFailures()) {
throw new Exception(failure.reason());
}
}
return true;
}
/**
* 查找文檔
* @param index 索引名
* @param type 類型
* @param id 文檔id
* @throws IOException
*/
public static Map<String, Object> findByIdAsMap(String index, String id) {
return findByIdAsMap(index, "doc", id);
}
/**
* 查找文檔
* @param index 索引名
* @param type 類型
* @param id 文檔id
* @throws IOException
*/
public static Map<String, Object> findByIdAsMap(String index, String type, String id) {
Map<String, Object> map = new HashMap();
//查
GetRequest getRequest = new GetRequest(index, type, id);
GetResponse getResponse = null;
try {
getResponse = restHighLevelClient.get(getRequest);
map = getResponse.getSourceAsMap();
} catch (IOException e) {
e.printStackTrace();
}
return map;
}
/**
* 驗證索引是否存在
*
* @param index
* 索引名稱
* @return
* @throws Exception
*/
public static boolean indexExists(String index) throws Exception {
GetIndexRequest request = new GetIndexRequest();
request.indices(index);
request.local(false);
request.humanReadable(true);
boolean exists = restHighLevelClient.indices().exists(request);
return exists;
}
/**
* 生成按天生成的index
* 生成格式如aion_trace_sys_20190701
* @param indexPrefix 日誌index名前綴 如aion_trace_sys_
* @param begin 開始時間
* @param end 結束時間
* @return index list
*/
public static List<String> makeDayRangeIndex(String indexPrefix, long begin, long end) {
Long oneDay = 1000 * 60 * 60 * 24L;
Long time = begin;
List<String> list = new ArrayList<>();
while (time <= end) {
Date d = new Date(time);
String dateStr = DateFormatUtils.format(d, LOG_DATE_PATTERN);
time += oneDay;
list.add(indexPrefix + dateStr);
}
return list;
}
/**
* 創建索引 不推薦手動創建，推薦使用模板創建，參見README.md
* @param index
* @param indexType
* @param properties
* 結構: {name:{type:text}} {age:{type:integer}}
* @return
* @throws Exception
*/
public static boolean createIndex(String index, String indexType, Map<String, Object> properties) throws Exception {
if (indexExists(index)) {
return true;
}
CreateIndexRequest request = new CreateIndexRequest(index);
request.settings(
Settings.builder()
.put("index.number_of_shards", 30)
.put("index.number_of_replicas", 0)
);
Map<String, Object> jsonMap = new HashMap<>();
Map<String, Object> mapping = new HashMap<>();
if (null != properties) {
mapping.put("properties", properties);
}
jsonMap.put(indexType, mapping);
request.mapping(indexType, jsonMap);
CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(
request);
boolean acknowledged = createIndexResponse.isAcknowledged();
return acknowledged;
}
/**
* 刪除索引
*
* @param index
* @return
* @throws Exception
*/
public static boolean deleteIndex(String index) throws Exception {
try {
DeleteIndexRequest request = new DeleteIndexRequest(index);
DeleteIndexResponse deleteIndexResponse = restHighLevelClient.indices().delete(request);
return deleteIndexResponse.isAcknowledged();
} catch (ElasticsearchException exception) {
if (exception.status() == RestStatus.NOT_FOUND) {
return true;
} else {
return false;
}
}
}
}
測試用例
import org.junit.Assert;
import org.junit.Test;
import org.mortbay.util.ajax.JSON;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ESTest {
/**
* 查找文檔
*/
@Test
public void findDocumentTest() {
EsUtils.getClient("10.10.10.10", 9280, "admin", "password");
Map<String, Object> map = EsUtils.findByIdAsMap("aion_log_20190829", "doc", "3ig_3WwBGY6TmRjEbkFj");
map.entrySet().forEach(x -> System.out.println("key:" + x.getKey() + " ,value:" + x.getValue()));
}
/**
* 創建更新文檔
*/
@Test
public void createDocumentByJsonTest() throws Exception {
EsUtils.getClient("10.10.10.10", 9280, "admin", "password");
Map<String,Object> map = new HashMap<>();
map.put("Name", "李燦峯");
map.put("Age", 100);
map.put("Grade", 100.0);
map.put("method", "createDocumentByJsonTest");
String string = JSON.toString(map);
System.out.println(string);
boolean result = EsUtils.createDocument("test_2019_09_23", "doc", "testID001",string);
Assert.assertTrue(result);
}
/**
* 創建更新文檔
*/
@Test
public void createDocumentByMapTest() throws Exception {
EsUtils.getClient("10.10.10.10", 9280, "admin", "password");
Map<String,Object> map = new HashMap<>();
map.put("Name", "李燦峯");
map.put("Age", 100);
map.put("Grade", 100.0);
map.put("method", "createDocumentByMapTest");
boolean result = EsUtils.createDocument("test_2019_09_23", "doc", null, map);
Assert.assertTrue(result);
}
/**
* 刪除文檔
*/
@Test
public void deleteDocumentTest() throws Exception {
EsUtils.getClient("10.10.10.10", 9280, "admin", "password");
boolean result = EsUtils.deleteDocument("test_2019_09_23", "testID001");
Assert.assertTrue(result);
}
/**
* 判斷索引是否存在
*/
@Test
public void indexExistsTest() throws Exception {
EsUtils.getClient("10.10.10.10", 9280, "admin", "password");
boolean result = EsUtils.indexExists("test_2019_09_23");
Assert.assertTrue(result);
}
/**
* 刪除索引
*/
@Test
public void deleteIndexTest() throws Exception {
EsUtils.getClient("10.10.10.10", 9280, "admin", "password");
boolean result = EsUtils.deleteIndex("test_2019_09_23");
Assert.assertTrue(result);
}
/**
* 創建索引
*/
@Test
public void createIndexTest() throws Exception {
EsUtils.getClient("10.10.10.10", 9280, "admin", "password");
boolean result = EsUtils.createIndex("test_2019_09_24", "doc",null );
Assert.assertTrue(result);
}
/**
* 索引日期
*/
@Test
public void makeDayRangeIndexTest() throws Exception {
EsUtils.getClient("10.10.10.10", 9280, "admin", "password");
//08-22 00:00 -- 09-22 00:00
List<String> result = EsUtils.makeDayRangeIndex("aion_log_",1566403200000L, 1569081600000L );
result.forEach(System.out::println);
}
}