import com.alibaba.fastjson.JSON;
import com.pojo.es.DataSourceEs;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.cluster.NodesInfo;
import io.searchbox.core.*;
import io.searchbox.indices.mapping.GetMapping;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.*;
import java.util.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
/**
* @explain 使用JestClient操作ES工具类
* 目前此工具类，支持大部分5.0+和6.0+版本的ES
* 使用前，必须先创建JestClient对象
* 【注意】只对ES的SQL查询不支持。5.0+只支持url直接拼接SQL语句，6.0+只支持JSON参数发送SQL语句，呵呵…
* @author Song
* @date 2019/9/1
*/
@Component
public class JestClientUtil {
public Log log = LogFactory.getLog(JestClientUtil.class);
@Autowired
//常量池
private AppProperties appProperties;
@Autowired
//个人项目时需要的工具类，可删除
EsSelectSqlUtil esSelectSqlUtil;
/**
* 【核心】创建JestClient
*/
public JestClient getJestClientIf(String ip, String user, String password){
if (StringUtils.isEmpty(user) && StringUtils.isEmpty(password)) {
return this.getJestClient(ip);
}else {
return this.getJestClient(ip, user, password);
}
}
/**
* 创建不带密码的JestClient
*/
private JestClient getJestClient(String ip) {
JestClientFactory factory = new JestClientFactory();
HttpClientConfig.Builder builder = new HttpClientConfig.Builder(ip);
factory.setHttpClientConfig(builder.connTimeout(60000).readTimeout(60000).multiThreaded(true).build());
return factory.getObject();
}
/**
* 创建带密码的JestClient
* @param ip ES的地址
* @param userName 用户名
* @param password 密码
*/
private JestClient getJestClient(String ip, String userName, String password){
JestClientFactory factory = new JestClientFactory();
HttpClientConfig.Builder builder = new HttpClientConfig.Builder(ip);
if (!StringUtils.isEmpty(userName) && !StringUtils.isEmpty(password)) {
builder.defaultCredentials(userName, password);
}
factory.setHttpClientConfig(builder.connTimeout(60000).readTimeout(60000).multiThreaded(true).build());
return factory.getObject();
}
/**
* 关闭JestClient
*/
private void closeClient(JestClient jestClient){
if (jestClient != null) {
try {
jestClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* ES中批量新增信息
*/
public <T> boolean insertEs(JestClient jestClient, String indexName, String typeName, List<T> list) {
//List<Object> list2 = new ArrayList<Object>();
//list2.add(new UserTest(1L, "Walker", 20, "程序猿程序员工程师"));
/*list2.add(new UserTest(2L, "李四", 24, "李四是个测试工程师"));
list2.add(new UserTest(3L, "王五", 25, "王五是个运维工程师"));
list2.add(new UserTest(4L, "张三", 25, "张三是个运维工程师"));
list2.add(new UserTest(5L, "刘六", 25, "刘流是个运维工程师"));*/
/*Map<String, Map<String, Map<String, String>>> mapOne = new HashMap<>();
Map<String, Map<String, String>> mapTwo = new HashMap<>();
Map<String, String> mapThree = new HashMap<>();
mapOne.put("testOne",mapTwo);
mapTwo.put("testTwo", mapThree);
mapThree.put("testThree", "啊哈哈哈哈");
list2.add(mapOne);*/
boolean result = false;
try {
Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(typeName);
for (T obj : list) {
Index index = new Index.Builder(obj).build();
bulk.addAction(index);
}
BulkResult br = jestClient.execute(bulk.build());
if (br.isSucceeded()) {
return true;
}else {
String errorMessage = br.getErrorMessage();
List<BulkResult.BulkResultItem> items = br.getItems();
items.forEach(x -> log.info("Insert ElasticSearch Exception========>>>>" + x.error +"|" +x.type +"|" +
x.errorReason + "|" + x.errorType +"|" + x.operation + "|" + x.version + "|" + errorMessage));
return false;
}
//result = insertBatch(jestClient,indexName, typeName,list);
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 查询信息1
* @param content 查询的关键字
* @param pageIndex 页数
* @param pageSize 显示行数
* @param indexName 查询的Index名称
* @param typeName 查询的Type名称
*/
public String selectEs(JestClient jestClient, String content, Integer pageIndex, Integer pageSize, String indexName, String typeName) {
try {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if (!StringUtils.isEmpty(content)) {
searchSourceBuilder.query(QueryBuilders.queryStringQuery(content));
}
//分页设置
searchSourceBuilder.from(pageIndex).size(pageSize);
System.out.println("全文搜索查询语句:"+searchSourceBuilder.toString());
Search search = new Search.Builder(searchSourceBuilder.toString())
.addIndex(indexName)
.addType(typeName)
.build();
JestResult jr = jestClient.execute(search);
return jr.getSourceAsString();
} catch (Exception e) {
e.printStackTrace();
return "";
}
}
/**
* 获得当前ES的所有index信息
*/
public List<String> getIndexAll(JestClient jestClient) {
Cat cat = new Cat.IndicesBuilder().build();
String jsonString = null;
List<String> indexNameList = new ArrayList<>();
try {
jsonString = jestClient.execute(cat).getJsonString();
if (StringUtils.isEmpty(jsonString)) {
return null;
}
System.out.println("当前ES中的所有Index："+ jsonString);
List<Map<String,Object>> list = (List)JSON.parse(jsonString);
list.forEach(x -> {
System.out.println(x+"====>>>>");
indexNameList.add(String.valueOf(x.get("index")));
});
return indexNameList;
} catch (Exception e) {
//e.printStackTrace();
return null;
}
}
/**
* 获取当前Index下的所有Type信息
*/
public List<String> getTypeAll(JestClient jestClient,String indexName) throws Exception {
GetMapping getMapping = new GetMapping.Builder().addIndex(indexName).build();
JestResult jr =jestClient.execute(getMapping);
List<String> typeList = new ArrayList<>();
Map<String, Map<String,Map<String, Object>>> indexMap = (Map)JSON.parse(jr.getJsonString());
Set<Map.Entry<String, Map<String,Map<String, Object>>>> indexSet = indexMap.entrySet();
indexSet.forEach(indexKey -> {
Map<String, Map<String, Object>> mappingMap = indexKey.getValue();
Set<Map.Entry<String, Map<String, Object>>> mappingSet = mappingMap.entrySet();
mappingSet.forEach(mappingKey -> {
Map<String, Object> typeMap = mappingKey.getValue();
Set<Map.Entry<String, Object>> typeSet = typeMap.entrySet();
typeSet.forEach(typeKey -> {
typeList.add(typeKey.getKey());
});
});
});
return typeList;
//return jr.getJsonString();
}
/**
* 根据Index和Type查询出的信息，获取其中的所有Field信息
* @param str selectEs()方法返回的JSON信息
*/
public List<String> getFieldAll(String str){
Map<String, Object> map = (Map)JSON.parse(str);
if (map == null || map.isEmpty()) {
return list;
}
Set<Map.Entry<String, Object>> entries = map.entrySet();
entries.forEach(x -> {
AtomicReference<String> fieldOne = new AtomicReference<>(null);
fieldOne.set(x.getKey());
if (ifMap(x.getValue())) {
//核心方法
forGetMap(x.getValue(),x.getKey());
Map<String, Object> objectMap = (Map)x.getValue();
Set<Map.Entry<String, Object>> entries1 = objectMap.entrySet();
entries1.forEach(objectKey -> {
fieldOne.set(fieldOne + "." + objectKey.getKey());
});
}else {
list.add(x.getKey());
}
});
return list;
}
/**
* 【核心】使用SQL语句查询ES信息方法
* @param sql 用户输入的SQL语句【注意，SQL语句中的空格需要全部替换成%20，不然HTTP不认识空格】
* @param dataSourceEs ES在数据库中的信息
*/
public List<Map<String, Object>> runEsSql(String sql, DataSourceEs dataSourceEs) throws IOException {
if (dataSourceEs == null || StringUtils.isEmpty(sql)) {
return null;
}
//替换传来SQL中的空格为http请求能识别的
String overSql = sql.replaceAll(" ", "%20");
String uri = "http://" + dataSourceEs.getEsIp() + ":" + dataSourceEs.getEsPort() + "/_sql?sql=" + overSql;
//5.0+只支持url拼接，6.0+只支持JSON，mlgb都给你
Map<String, String> map = new LinkedHashMap<>(1);
//JSON不需要替换
map.put("sql", sql);
String string = JSON.toJSONString(map);
System.out.println("---->"+string);
PrintWriter out = null;
//try {
URL url = new URL(uri);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoOutput(true); // 设置该连接是可以输出的
connection.setRequestMethod("GET"); // 设置请求方式
connection.setRequestProperty("content-type", "application/json");
out = new PrintWriter(connection.getOutputStream());
// 发送请求参数
out.write(string);
// flush输出流的缓冲
out.flush();
connection.connect();
BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"));
String line = null;
StringBuilder result = new StringBuilder();
while ((line = br.readLine()) != null) { // 读取数据
result.append(line).append("\n");
}
connection.disconnect();
br.close();
String resultOver = result.toString();
//ES返回结果为固定，拿到有用的信息
Map<String, Object> mapOne = (Map) JSON.parse(resultOver);
Map<String, Object> hits = (Map)mapOne.get("hits");
return (List)hits.get("hits");
/*} catch (Exception e) {
log.info("请求数据平台接口错误=====>>>>>>"+ e.getMessage());
return null;
}*/
}
/**
* 查看ElasticSearch版本节点信息
*/
public JestResult getNodesInfo(JestClient jestClient){
NodesInfo nodesInfo = new NodesInfo.Builder().build();
JestResult result = null ;
try {
result = jestClient.execute(nodesInfo);
} catch (IOException e) {
e.printStackTrace();
}
return result;
}
/**
* 请求数据平台接口，获取日志集信息
* @param typeName 从ES获取到的Type名称
*/
public String getZorkData(String typeName){
String uri = appProperties.getZorkDataUrl() + typeName;
try {
URL url = new URL(uri);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoOutput(true); // 设置该连接是可以输出的
connection.setRequestMethod("POST"); // 设置请求方式
connection.connect();
BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"));
String line = null;
StringBuilder result = new StringBuilder();
while ((line = br.readLine()) != null) { // 读取数据
result.append(line).append("\n");
}
connection.disconnect();
br.close();
return result.toString();
} catch (Exception e) {
log.info("请求数据平台接口错误=====>>>>>>"+ e.getMessage());
return null;
}
}
/**
* 判断是否可以强转Map的方法
*/
public boolean ifMap(Object object){
if (object == null) {
return false;
}
try {
//如果可以强转MAP，那就返回可以
Map<String, Object> objectMap = (Map)object;
return true;
}catch (Exception e){
//否则就是强制类型转换异常，代表不可以转MAP，直接拿Value
return false;
}
}
public List<String> list = new ArrayList<>();
/**
* 由于无法确定使用该平台的操作人员，之前在ElasticSearch中录入的数据结构、字段是什么样的
* 但有一点确定的是，ES返回的信息必然是JSON，所以使用无限循环去“探索”其结构，
* 无法强转就证明，一步到胃了。
* @param object this的Value
* @param objectStr this的上一个父级名称
* @return 计数器
*/
public int forGetMap(Object object, String objectStr) {
AtomicInteger integer = new AtomicInteger();
for (;;){
try {
Map<String, Object> objectMap = (Map)object;
Set<Map.Entry<String, Object>> entries = objectMap.entrySet();
entries.forEach(x -> {
//应该在这里进行获取字段信息
list.add((objectStr + "." + x.getKey()));
integer.set(this.forGetMap(x.getValue(), (objectStr + "." + x.getKey())));
});
//如果可以强转MAP，统计当前字段可以强转几次Map
return integer.get();
}catch (Exception e){
//否则就是强制类型转换异常，代表不可以转MAP
break;
}
}
return integer.get();
}
}
所需依赖：
<!-- ElasticSearch依赖 -->
<dependency>
<groupId>io.searchbox</groupId>
<artifactId>jest</artifactId>
<version>6.3.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch -->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<!-- 这里的版本是你的ES版本 -->
<version>5.4.2</version>
</dependency>
<!-- 添加字段需要的JAR -->
<!-- https://mvnrepository.com/artifact/org.elasticsearch.client/transport -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<!-- 这里的版本是你的ES版本 -->
<version>5.4.2</version>
</dependency>
项目中需要直接对ES进行操作，获取其内部信息等操作。所以通过各种办法搜集，整合了一份ES的工具类，希望可以帮到大家。
我建议各位先一个一个方法copy后，稍微琢磨一下，注释都有。
注意：执行工具类中的所有方法，都先需要创建JestClient后，才能操作ES。