import com.alibaba.fastjson.JSON;
import com.pojo.es.DataSourceEs;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.cluster.NodesInfo;
import io.searchbox.core.*;
import io.searchbox.indices.mapping.GetMapping;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.*;
import java.util.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
/**
* @explain 使用JestClient操作ES工具類
* 目前此工具類，支持大部分5.0+和6.0+版本的ES
* 使用前，必須先創建JestClient對象
* 【注意】只對ES的SQL查詢不支持。5.0+只支持url直接拼接SQL語句，6.0+只支持JSON參數發送SQL語句，呵呵…
* @author Song
* @date 2019/9/1
*/
@Component
public class JestClientUtil {
public Log log = LogFactory.getLog(JestClientUtil.class);
@Autowired
//常量池
private AppProperties appProperties;
@Autowired
//個人項目時需要的工具類，可刪除
EsSelectSqlUtil esSelectSqlUtil;
/**
* 【核心】創建JestClient
*/
public JestClient getJestClientIf(String ip, String user, String password){
if (StringUtils.isEmpty(user) && StringUtils.isEmpty(password)) {
return this.getJestClient(ip);
}else {
return this.getJestClient(ip, user, password);
}
}
/**
* 創建不帶密碼的JestClient
*/
private JestClient getJestClient(String ip) {
JestClientFactory factory = new JestClientFactory();
HttpClientConfig.Builder builder = new HttpClientConfig.Builder(ip);
factory.setHttpClientConfig(builder.connTimeout(60000).readTimeout(60000).multiThreaded(true).build());
return factory.getObject();
}
/**
* 創建帶密碼的JestClient
* @param ip ES的地址
* @param userName 用戶名
* @param password 密碼
*/
private JestClient getJestClient(String ip, String userName, String password){
JestClientFactory factory = new JestClientFactory();
HttpClientConfig.Builder builder = new HttpClientConfig.Builder(ip);
if (!StringUtils.isEmpty(userName) && !StringUtils.isEmpty(password)) {
builder.defaultCredentials(userName, password);
}
factory.setHttpClientConfig(builder.connTimeout(60000).readTimeout(60000).multiThreaded(true).build());
return factory.getObject();
}
/**
* 關閉JestClient
*/
private void closeClient(JestClient jestClient){
if (jestClient != null) {
try {
jestClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* ES中批量新增信息
*/
public <T> boolean insertEs(JestClient jestClient, String indexName, String typeName, List<T> list) {
//List<Object> list2 = new ArrayList<Object>();
//list2.add(new UserTest(1L, "Walker", 20, "程序猿程序員工程師"));
/*list2.add(new UserTest(2L, "李四", 24, "李四是個測試工程師"));
list2.add(new UserTest(3L, "王五", 25, "王五是個運維工程師"));
list2.add(new UserTest(4L, "張三", 25, "張三是個運維工程師"));
list2.add(new UserTest(5L, "劉六", 25, "劉流是個運維工程師"));*/
/*Map<String, Map<String, Map<String, String>>> mapOne = new HashMap<>();
Map<String, Map<String, String>> mapTwo = new HashMap<>();
Map<String, String> mapThree = new HashMap<>();
mapOne.put("testOne",mapTwo);
mapTwo.put("testTwo", mapThree);
mapThree.put("testThree", "啊哈哈哈哈");
list2.add(mapOne);*/
boolean result = false;
try {
Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(typeName);
for (T obj : list) {
Index index = new Index.Builder(obj).build();
bulk.addAction(index);
}
BulkResult br = jestClient.execute(bulk.build());
if (br.isSucceeded()) {
return true;
}else {
String errorMessage = br.getErrorMessage();
List<BulkResult.BulkResultItem> items = br.getItems();
items.forEach(x -> log.info("Insert ElasticSearch Exception========>>>>" + x.error +"|" +x.type +"|" +
x.errorReason + "|" + x.errorType +"|" + x.operation + "|" + x.version + "|" + errorMessage));
return false;
}
//result = insertBatch(jestClient,indexName, typeName,list);
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 查詢信息1
* @param content 查詢的關鍵字
* @param pageIndex 頁數
* @param pageSize 顯示行數
* @param indexName 查詢的Index名稱
* @param typeName 查詢的Type名稱
*/
public String selectEs(JestClient jestClient, String content, Integer pageIndex, Integer pageSize, String indexName, String typeName) {
try {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if (!StringUtils.isEmpty(content)) {
searchSourceBuilder.query(QueryBuilders.queryStringQuery(content));
}
//分頁設置
searchSourceBuilder.from(pageIndex).size(pageSize);
System.out.println("全文搜索查詢語句:"+searchSourceBuilder.toString());
Search search = new Search.Builder(searchSourceBuilder.toString())
.addIndex(indexName)
.addType(typeName)
.build();
JestResult jr = jestClient.execute(search);
return jr.getSourceAsString();
} catch (Exception e) {
e.printStackTrace();
return "";
}
}
/**
* 獲得當前ES的所有index信息
*/
public List<String> getIndexAll(JestClient jestClient) {
Cat cat = new Cat.IndicesBuilder().build();
String jsonString = null;
List<String> indexNameList = new ArrayList<>();
try {
jsonString = jestClient.execute(cat).getJsonString();
if (StringUtils.isEmpty(jsonString)) {
return null;
}
System.out.println("當前ES中的所有Index："+ jsonString);
List<Map<String,Object>> list = (List)JSON.parse(jsonString);
list.forEach(x -> {
System.out.println(x+"====>>>>");
indexNameList.add(String.valueOf(x.get("index")));
});
return indexNameList;
} catch (Exception e) {
//e.printStackTrace();
return null;
}
}
/**
* 獲取當前Index下的所有Type信息
*/
public List<String> getTypeAll(JestClient jestClient,String indexName) throws Exception {
GetMapping getMapping = new GetMapping.Builder().addIndex(indexName).build();
JestResult jr =jestClient.execute(getMapping);
List<String> typeList = new ArrayList<>();
Map<String, Map<String,Map<String, Object>>> indexMap = (Map)JSON.parse(jr.getJsonString());
Set<Map.Entry<String, Map<String,Map<String, Object>>>> indexSet = indexMap.entrySet();
indexSet.forEach(indexKey -> {
Map<String, Map<String, Object>> mappingMap = indexKey.getValue();
Set<Map.Entry<String, Map<String, Object>>> mappingSet = mappingMap.entrySet();
mappingSet.forEach(mappingKey -> {
Map<String, Object> typeMap = mappingKey.getValue();
Set<Map.Entry<String, Object>> typeSet = typeMap.entrySet();
typeSet.forEach(typeKey -> {
typeList.add(typeKey.getKey());
});
});
});
return typeList;
//return jr.getJsonString();
}
/**
* 根據Index和Type查詢出的信息，獲取其中的所有Field信息
* @param str selectEs()方法返回的JSON信息
*/
public List<String> getFieldAll(String str){
Map<String, Object> map = (Map)JSON.parse(str);
if (map == null || map.isEmpty()) {
return list;
}
Set<Map.Entry<String, Object>> entries = map.entrySet();
entries.forEach(x -> {
AtomicReference<String> fieldOne = new AtomicReference<>(null);
fieldOne.set(x.getKey());
if (ifMap(x.getValue())) {
//核心方法
forGetMap(x.getValue(),x.getKey());
Map<String, Object> objectMap = (Map)x.getValue();
Set<Map.Entry<String, Object>> entries1 = objectMap.entrySet();
entries1.forEach(objectKey -> {
fieldOne.set(fieldOne + "." + objectKey.getKey());
});
}else {
list.add(x.getKey());
}
});
return list;
}
/**
* 【核心】使用SQL語句查詢ES信息方法
* @param sql 用戶輸入的SQL語句【注意，SQL語句中的空格需要全部替換成%20，不然HTTP不認識空格】
* @param dataSourceEs ES在數據庫中的信息
*/
public List<Map<String, Object>> runEsSql(String sql, DataSourceEs dataSourceEs) throws IOException {
if (dataSourceEs == null || StringUtils.isEmpty(sql)) {
return null;
}
//替換傳來SQL中的空格爲http請求能識別的
String overSql = sql.replaceAll(" ", "%20");
String uri = "http://" + dataSourceEs.getEsIp() + ":" + dataSourceEs.getEsPort() + "/_sql?sql=" + overSql;
//5.0+只支持url拼接，6.0+只支持JSON，mlgb都給你
Map<String, String> map = new LinkedHashMap<>(1);
//JSON不需要替換
map.put("sql", sql);
String string = JSON.toJSONString(map);
System.out.println("---->"+string);
PrintWriter out = null;
//try {
URL url = new URL(uri);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoOutput(true); // 設置該連接是可以輸出的
connection.setRequestMethod("GET"); // 設置請求方式
connection.setRequestProperty("content-type", "application/json");
out = new PrintWriter(connection.getOutputStream());
// 發送請求參數
out.write(string);
// flush輸出流的緩衝
out.flush();
connection.connect();
BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"));
String line = null;
StringBuilder result = new StringBuilder();
while ((line = br.readLine()) != null) { // 讀取數據
result.append(line).append("\n");
}
connection.disconnect();
br.close();
String resultOver = result.toString();
//ES返回結果爲固定，拿到有用的信息
Map<String, Object> mapOne = (Map) JSON.parse(resultOver);
Map<String, Object> hits = (Map)mapOne.get("hits");
return (List)hits.get("hits");
/*} catch (Exception e) {
log.info("請求數據平臺接口錯誤=====>>>>>>"+ e.getMessage());
return null;
}*/
}
/**
* 查看ElasticSearch版本節點信息
*/
public JestResult getNodesInfo(JestClient jestClient){
NodesInfo nodesInfo = new NodesInfo.Builder().build();
JestResult result = null ;
try {
result = jestClient.execute(nodesInfo);
} catch (IOException e) {
e.printStackTrace();
}
return result;
}
/**
* 請求數據平臺接口，獲取日誌集信息
* @param typeName 從ES獲取到的Type名稱
*/
public String getZorkData(String typeName){
String uri = appProperties.getZorkDataUrl() + typeName;
try {
URL url = new URL(uri);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoOutput(true); // 設置該連接是可以輸出的
connection.setRequestMethod("POST"); // 設置請求方式
connection.connect();
BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"));
String line = null;
StringBuilder result = new StringBuilder();
while ((line = br.readLine()) != null) { // 讀取數據
result.append(line).append("\n");
}
connection.disconnect();
br.close();
return result.toString();
} catch (Exception e) {
log.info("請求數據平臺接口錯誤=====>>>>>>"+ e.getMessage());
return null;
}
}
/**
* 判斷是否可以強轉Map的方法
*/
public boolean ifMap(Object object){
if (object == null) {
return false;
}
try {
//如果可以強轉MAP，那就返回可以
Map<String, Object> objectMap = (Map)object;
return true;
}catch (Exception e){
//否則就是強制類型轉換異常，代表不可以轉MAP，直接拿Value
return false;
}
}
public List<String> list = new ArrayList<>();
/**
* 由於無法確定使用該平臺的操作人員，之前在ElasticSearch中錄入的數據結構、字段是什麼樣的
* 但有一點確定的是，ES返回的信息必然是JSON，所以使用無限循環去“探索”其結構，
* 無法強轉就證明，一步到胃了。
* @param object this的Value
* @param objectStr this的上一個父級名稱
* @return 計數器
*/
public int forGetMap(Object object, String objectStr) {
AtomicInteger integer = new AtomicInteger();
for (;;){
try {
Map<String, Object> objectMap = (Map)object;
Set<Map.Entry<String, Object>> entries = objectMap.entrySet();
entries.forEach(x -> {
//應該在這裏進行獲取字段信息
list.add((objectStr + "." + x.getKey()));
integer.set(this.forGetMap(x.getValue(), (objectStr + "." + x.getKey())));
});
//如果可以強轉MAP，統計當前字段可以強轉幾次Map
return integer.get();
}catch (Exception e){
//否則就是強制類型轉換異常，代表不可以轉MAP
break;
}
}
return integer.get();
}
}
所需依賴：
<!-- ElasticSearch依賴 -->
<dependency>
<groupId>io.searchbox</groupId>
<artifactId>jest</artifactId>
<version>6.3.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch -->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<!-- 這裏的版本是你的ES版本 -->
<version>5.4.2</version>
</dependency>
<!-- 添加字段需要的JAR -->
<!-- https://mvnrepository.com/artifact/org.elasticsearch.client/transport -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<!-- 這裏的版本是你的ES版本 -->
<version>5.4.2</version>
</dependency>
項目中需要直接對ES進行操作，獲取其內部信息等操作。所以通過各種辦法蒐集，整合了一份ES的工具類，希望可以幫到大家。
我建議各位先一個一個方法copy後，稍微琢磨一下，註釋都有。
注意：執行工具類中的所有方法，都先需要創建JestClient後，才能操作ES。