<dependencies>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.0.0</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.0.0</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>7.0.0</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>transport-netty4-client</artifactId>
<version>7.0.0</version>
</dependency>
</dependencies>
import org.elasticsearch.action.search.SearchResponse;
import java.util.List;
import java.util.Map;
/**
* @Author: gh
* @Description: elasticsearch查詢數據(分頁)、添加數據、修改數據、刪除數據、展示所有庫表、統計每張表的記錄總條數;
* es的使用率,已使用的存儲空間大小。按任意字段進行模糊查詢。獲取所有字段的名稱。
*/
public interface EsDao {
/**
* 分頁查詢
* @param index 索引名稱
* @param pageSize 頁的大小
* @param pageNum 第幾頁
* @return
*/
public SearchResponse getAllRowsBySearchAfter(String index, Integer pageSize, Integer pageNum);
public SearchResponse getAllRowsByFromSize(String index, Integer pageSize, Integer pageNum);
public List<Map<String, Object>> getAllRowsByScroll(String index);
/**
* 統計每個index下的document總數。
* count API的命令：GET /index_name/type_name/_count
* @param indexes 查詢一個或多個index
* @return
*/
public long totalCount(String...indexes);
/**
* 查詢某條document是否存在。
* @param index 索引名稱
* @param id _id字段對應的主鍵值。
* @return true存在，false不存在。
*/
public boolean docIsExist(String index,String id);
/**
* 新增一條document。
* @param index 索引名稱
* @param kvs 各個字段名稱（key）和對應的值（value）
* @return 200/201表示新增成功；其他如400表示新增失敗。
*/
public int insertDoc(String index,Map<String, Object> kvs);
/**
* 刪除一條document。
* @param index 索引名稱
* @param id _id字段對應的主鍵值。
* @return 200/201表示刪除成功；其他如400表示刪除失敗。
*/
public int deleteDoc(String index,String id);
/**
* 更新一條document
* @param index 索引名稱
* @param id _id字段對應的主鍵值。
* @param kvs 各個字段名稱（key）和對應的值（value）
* @return 200/201表示更新成功；其他如400表示更新失敗。
*/
public int updateDoc(String index,String id,Map<String, Object> kvs);
/**
* es使用率 = cluster store(size_in_bytes) / (所有節點的磁盤可用空間 + size_in_bytes)
* size_in_bytes: "_all":{"total":{"store":{"size_in_bytes": ***}}}
* 某個節點的磁盤可用空間：（節點狀態）"fs":{"total":{"available_in_bytes":***}}
* @return
*/
public double usedRate();
/**
* 獲取某個cluster下所有的index名稱列表。
* 命令：_cluster/state/routing_table
* @param clusterName 集羣名稱
* @return
*/
public Map<String, List<String>> getClusterIndexes(String clusterName);
/**
* 獲取某個index下所有的type名稱列表。
* _type字段從elasticsearch 6.0.0版本開始過時，7.0版本後不再使用。
* @return
*/
public Map<String,List<String>> getIndexTypes(String clusterName);
/**
* @return 全部已使用存儲空間的大小
*/
public double storeSizeOfMB();
/**
* @param index 需要查詢的索引名稱
* @return （某個索引下）已使用存儲空間的大小
*/
public double storeSizeOfMB(String index);
/**
* 暫行：1.統計某個集羣（相當於數據庫）下所有index（相當於表）的數量。默認集羣時elasticsearch。
* 擱置：2.統計某個index（相當於數據庫）下所有type（相當於表）的數量。
* @return 表的數量
*/
public int countTables();
/**
* 獲取某個集羣（相當於數據庫）下所有的indice（相當於表）的名稱列表。
* @return
*/
public List<String> getTablenamesOfDB();
/**
* 按任意字段進行模糊查詢
* @param indexName 索引名
* @param fieldName 字段名
* @param fieldValue 字段模糊值
*/
public SearchResponse queryByRandomField(String indexName,String fieldName,String fieldValue,
int pageSize,int pageNum);
/**
* @param indexName 索引名
* @param fieldName 字段名
* @param fieldValue 字段模糊值
* @return 模糊查詢時返回的記錄總數
*/
public long totalCountOfFuzzyQuery(String indexName,String fieldName,String fieldValue);
/**
* 獲取 某個索引的所有字段名
* @param indexName 索引的名稱。例如：table1
* @return map<k,v> k:字段名，v:空字符串
*/
public Map<String,Object> getColumnNames(String indexName);
}
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.genius.pojo.pg.dto.DataBaseDTO;
import com.genius.util.StringUtil;
import com.genius.util.common.FileSizeUtil;
import com.genius.util.common.SizeUnitEnum;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.*;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.client.indices.*;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.*;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.locationtech.jts.geom.Coordinate;
import org.springframework.util.StringUtils;
import java.io.*;
import java.sql.*;
import java.util.*;
import java.util.stream.Collectors;
/**
* @Description: 使用es JDBC 方式進行 查詢數據(分頁)、添加數據、修改數據、刪除數據、展示所有庫表。
* 【注意】
* 1.Elasticsearch JDBC方式不提供連接池機制；可以使用第三方連接池的機制。
* 2.使用Elasticsearch x-pack-sql-jdbc的包會報錯：
* java.sql.SQLInvalidAuthorizationSpecException:
* current license is non-compliant for [jdbc]
* 使用JDBC客戶端，elasticsearch需要升級到白金級：https://www.elastic.co/cn/subscriptions
*/
public class EsDaoImpl implements EsDao{
RestHighLevelClient restClient = null;
DataBaseDTO dataBaseDTO = null;
public EsDaoImpl(DataBaseDTO dbd) {
try {
dataBaseDTO = dbd;
this.restClient = connectToES();
//initClient();
} catch (Exception e) {
e.printStackTrace();
}
}
public RestHighLevelClient getRestClient() {
return restClient;
}
public void setRestClient(RestHighLevelClient restClient) {
this.restClient = restClient;
}
/**
* 關閉連接
*/
public void close(){
try{
if(this.restClient != null){
restClient.close();
}
}catch(IOException e){
e.printStackTrace();
}
}
public boolean connected(){
try {
if(getRestClient() != null && getRestClient().ping(RequestOptions.DEFAULT)){
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
return false;
}
/**
* JDBC連接elasticsearch。http port=9200
* jdbc:es://[[http|https]://][host[:port]]/[prefix]<[?[option=value]&]
* e.g. jdbc:es://http://server:3456/?timezone=UTC&page.size=250
* 安全通信：SSL
* @return
* @throws SQLException
*/
/*private Connection connectToES_abandon() throws SQLException {
if(conn == null){
if(dataBaseDTO != null){
String url = "jdbc:es://"+dataBaseDTO.getIp()+":9200";
System.out.println(url);
EsDataSource ds = new EsDataSource();
ds.setUrl(url);
Properties props = new Properties();
*//*props.put("user", "test_admin");
props.put("password", "x-pack-test-password");*//*
ds.setProperties(props);
return ds.getConnection();
//return DriverManager.getConnection(url, props);
}
}
return conn;
}*/
/**
* 連接 elasticsearch 並獲取 index信息
* @return
* @throws SQLException
*/
private RestHighLevelClient connectToES() {
try {
String esIP = dataBaseDTO.getIp();
//http port = 9200
String esPort = dataBaseDTO.getHost();
//索引。
//String index = dataBaseDTO.getDbName();
//TODO: 用戶名、密碼做校驗。目前cluster是默認的。
return new RestHighLevelClient(
RestClient.builder(new HttpHost(esIP, Integer.valueOf(esPort), "http")));
//flag = indexIsExist(index);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
@Override
public SearchResponse getAllRowsByScrollScan(String index, Integer pageSize, Integer pageNum) {
return getAllRowsByScrollScan(null,index,pageSize,pageNum);
}
private SearchResponse getAllRowsByScrollScan(SearchSourceBuilder sourceBuilder,String index,
Integer pageSize,Integer pageNum) {
if(pageSize==null || pageSize<1){
pageSize = 10;
}
if(pageNum==null || pageNum<1){
pageNum = 1;
}
SearchRequest search = new SearchRequest(index);
SearchResponse resp = null;
if(sourceBuilder == null){
sourceBuilder = new SearchSourceBuilder();
}
sourceBuilder.size(pageSize);
sourceBuilder.sort("_id", SortOrder.ASC);
search.source(sourceBuilder)
.searchType(SearchType.QUERY_THEN_FETCH)
.scroll(TimeValue.timeValueSeconds(60));
try {
resp = restClient.search(search, RequestOptions.DEFAULT);
SearchHit[] hits1 = resp.getHits().getHits();
//查詢結果太少，直接返回
if(hits1 == null || hits1.length < pageSize){
return resp;
}
if(pageNum > 1){
String scrollId = resp.getScrollId();
for(int i=1;i<pageNum;i++){
//利用scroll id繼續查詢
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(TimeValue.timeValueSeconds(60));
resp = restClient.scroll(scrollRequest, RequestOptions.DEFAULT);
SearchHit[] hits2 = resp.getHits().getHits();
//查詢結果太少，直接返回
if(hits2 == null || hits2.length < pageSize){
break;
}
scrollId = resp.getScrollId();
}
//清除滾屏
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
restClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
}
} catch (IOException e) {
e.printStackTrace();
}
return resp;
}
@Override
public List<Map<String, Object>> getAllRowsByScroll(String indexName, String column, String value) {
List<Map<String, Object>> collect = new ArrayList<>();
final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(60));
SearchResponse resp = null;
SearchRequest search = new SearchRequest(indexName);
search.scroll(scroll);
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
try {
setQuerySearchBuilder(sourceBuilder,indexName,column,value);
sourceBuilder.size(100);
sourceBuilder.sort("_id", SortOrder.ASC);
search.source(sourceBuilder)
.searchType(SearchType.QUERY_THEN_FETCH)
.scroll(TimeValue.timeValueSeconds(60));
resp = restClient.search(search, RequestOptions.DEFAULT);
assert resp != null;
String scrollId;
int count = 0;
do {
count++;
System.out.println("=================="+count);
collect.addAll(Arrays.stream(resp.getHits().getHits()).map(m->{
Map<String, Object> oneRowData = m.getSourceAsMap(); //sourceAsMap 可能爲null
if(oneRowData != null){
oneRowData.put("_id", m.getId());
//oneRowData.put("_type", hit.getType());
}
return oneRowData;
}).collect(Collectors.toList()));
scrollId = resp.getScrollId();
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
resp = restClient.scroll(scrollRequest, RequestOptions.DEFAULT);
} while (resp.getHits().getHits().length != 0);
//清除滾屏
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
//也可以選擇setScrollIds()將多個scrollId一起使用
clearScrollRequest.addScrollId(scrollId);
restClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
if (collect.size() == 0 || collect == null) {
return null;
}
} catch (IOException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
return collect;
}
@Override
public List<Map<String, Object>> getAllRowsByScroll(String index) {
List<Map<String, Object>> collect = new ArrayList<>();
final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(60));
SearchResponse resp = null;
SearchRequest search = new SearchRequest(index);
search.scroll(scroll);
try {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
sourceBuilder.size(100);
sourceBuilder.sort("_id", SortOrder.ASC);
search.source(sourceBuilder);
resp = restClient.search(search, RequestOptions.DEFAULT);
assert resp != null;
String scrollId;
int count = 0;
do {
Arrays.stream(resp.getHits().getHits()).forEach(hit->{
Map<String,Object> map=hit.getSourceAsMap();
map.put("_id",hit.getId());
collect.add(map);
});
scrollId = resp.getScrollId();
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
resp = restClient.scroll(scrollRequest, RequestOptions.DEFAULT);
} while (resp.getHits().getHits().length != 0);
//清除滾屏
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
//也可以選擇setScrollIds()將多個scrollId一起使用
clearScrollRequest.addScrollId(scrollId);
restClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
if (collect.size() == 0 || collect == null) {
return null;
}
} catch (IOException e) {
e.printStackTrace();
}
return collect;
}
@Override
public SearchResponse getAllRows(String index) {
//如果不判斷索引是否存在，可能會報錯：IndexNotFoundException: no such index
SearchResponse resp = null;
SearchRequest search = new SearchRequest(index);
try {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
search.source(sourceBuilder);
resp = restClient.search(search, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
return resp;
}
@Override
public SearchResponse getAllRowsByFromSize(String index, Integer pageSize, Integer pageNum) {
if(pageSize==null || pageSize<1){
pageSize = 10;
}
if(pageNum==null || pageNum<1){
pageNum = 1;
}
//如果不判斷索引是否存在，可能會報錯：IndexNotFoundException: no such index
SearchResponse resp = null;
SearchRequest search = new SearchRequest(index);
try {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//分頁查詢
sourceBuilder.from(pageSize*pageNum-pageSize);
sourceBuilder.size(pageSize);
sourceBuilder.sort("_id", SortOrder.ASC);
search.source(sourceBuilder);
resp = restClient.search(search, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
//當使用 TransportClient 時的API.
/*if(indexIsExist(index)) {
resp = client.prepareSearch(index)
.setQuery(QueryBuilders.matchAllQuery())
.setFrom(pageSize*pageNum-pageSize).setSize(pageSize)
//如果不排序，每次返回的結果是亂序的
.addSort("_id", SortOrder.ASC)
//.setPostFilter(QueryBuilders.rangeQuery("doc.offset").from(7000).to(10000))
.get();
}*/
return resp;
}
@Override
public SearchResponse getAllRowsBySearchAfter(String index, Integer pageSize, Integer pageNum) {
if(pageSize==null || pageSize<1){
pageSize = 10;
}
if(pageNum==null || pageNum<1){
pageNum = 1;
}
//============NOTE: API方式一---使用high level API
SearchResponse resp = null;
Object[] sortValues = null;
int counter = 0;
try {
//TODO:問題-pageNum大時，速度非常慢！
do{
//計數：當前是第幾頁
counter += 1;
SearchRequest search = new SearchRequest(index);
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//分頁查詢
sourceBuilder.size(pageSize);
sourceBuilder.sort("_id", SortOrder.ASC);
//第一次搜索時，沒有search_after參數。
if(sortValues != null){
sourceBuilder.searchAfter(sortValues);
}
search.source(sourceBuilder);
resp = restClient.search(search, RequestOptions.DEFAULT);
SearchHits hits = resp.getHits();
//當搜索沒有結果時，hitSize = 0
int hitSize= hits.getHits().length;
if(hitSize == 0){
break;
}
SearchHit hit = hits.getHits()[hitSize - 1];
sortValues = hit.getSortValues();
}while(counter < pageNum);
} catch (IOException e) {
e.printStackTrace();
}
return resp;
//============NOTE: API方式二---使用low level API
/*Object tiebreaker = null; //查詢的最後一個結果的ID
String jsonParam = null;
try {
do{
if(tiebreaker == null){
jsonParam = "{\"size\": "+pageSize+
",\"sort\": [{\"_id\": \"desc\"}]}";
}else{
jsonParam = "{\"size\": "+pageSize+"," +
"\"search_after\":"+tiebreaker+","+
"\"sort\": [{\"_id\": \"desc\"}]}";
}
//search_after請求
Request req = new Request("get", index+"/_search");
req.setJsonEntity(jsonParam);
RestClient client = restClient.getLowLevelClient();
Response resp = client.performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
JSONArray jsonArray = jo.getJSONObject("hits").getJSONArray("hits");
int dataSize = jsonArray.size();
if(dataSize > 0){
*//* XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, jsonArray.toJSONString());*//*
//返回的分頁結果集
Object lastResult = jsonArray.get(jsonArray.size() - 1);
if(lastResult instanceof JSONObject){
tiebreaker = ((JSONObject) lastResult).getJSONArray("sort");
}
}else{
break;
}
}else{
break;
}
}while(true);
} catch (IOException e) {
e.printStackTrace();
}
return null;*/
}
public Map<String,Object> extractResponse(SearchResponse resp){
Map<String,Object> rowDatas = new HashMap<>();
List<Map<String,Object>> data = new ArrayList<>();
Set<String> fields = new HashSet<>();
if(resp != null){
SearchHits hits = resp.getHits();
Iterator<SearchHit> ite = hits.iterator();
while(ite.hasNext()){
SearchHit hit = ite.next();
//sourceAsMap 可能爲null
Map<String, Object> oneRowData = hit.getSourceAsMap();
if(oneRowData != null){
oneRowData.put("_id", hit.getId());
//oneRowData.put("_type", hit.getType());
//[NOTE:]前端需要顯示幾何對象的經緯度數值字符串
//[NOTE:]有routing路由的，需要加入路由
}
fields.addAll(oneRowData.keySet());
data.add(oneRowData);
}
}
rowDatas.put("data", data);
rowDatas.put("fields", fields);
rowDatas.put("pk", "_id");
return rowDatas;
}
private long getMaxresult(String index){
//ClusterGetSettingsRequest cgsr = new ClusterGetSettingsRequest();
long maxResult = 10000;
try {
Request req = new Request("get", index+"/_settings");
RestClient client = restClient.getLowLevelClient();
Response resp = client.performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
//查詢設置
JSONObject settingObj = jo.getJSONObject(index)
.getJSONObject("settings")
.getJSONObject("index");
String value = settingObj.getString("max_result_window");
if(value == null){
return maxResult; //默認大小10000
}
maxResult = Long.valueOf(value);
}
} catch (IOException e) {
e.printStackTrace();
}
return maxResult;
}
/**
* 判斷索引是否存在
* @param index 索引名稱
* @return boolean
*/
private boolean isIndexExist(String index){
try{
if(!StringUtils.isEmpty(index)){
GetIndexRequest gir = new GetIndexRequest(index);
return restClient.indices().exists(gir, RequestOptions.DEFAULT);
}
}catch(Exception e){
e.printStackTrace();
}
//當使用 TransportClient 時的API.
//BasicConfigurator.configure();
/*AdminClient admin = client.admin();
IndicesAdminClient indices = admin.indices();
IndicesExistsRequestBuilder ierb = indices.prepareExists(index);
ActionFuture<IndicesExistsResponse> future = ierb.execute();
IndicesExistsResponse resp = future.actionGet();
if(resp.isExists()){
System.out.format("index [%s] is exist...\n", index);
return true;
}else {
System.out.format("index [%s] is NOT exist...\n", index);
return false;
}*/
return false;
}
//插入數據時，需要注意數據類型是否匹配。
@Override
public int insertDoc(String index,Map<String, Object> jsonMap) {
try {
if(jsonMap != null){
//將幾何類型的轉爲特定對象
Map<String, Object> columns = getColumnNames(index);
Set<String> keys = jsonMap.keySet();
for (String key : keys) {
EsFieldType eft = EsFieldType.GEO_POINT;
if(eft.getType().equals(columns.get(key))){
//如果是幾何類型，就轉換
Object transferedField = eft.getTransferedField(jsonMap.get(key).toString());
if(transferedField==null){
return 0;
}
jsonMap.put(key, transferedField);
}
}
//不設置id時，id會自動生成。
IndexRequest indexRequest = new IndexRequest(index).source(jsonMap);
//opType must be 'create' or 'index'.
// optype=index時，如果某ID對應的document已經存在，它將被替換。
//NOTE: 要麼需要判斷該id是否存在，已存在則不新增；要麼讓id自動生成。
indexRequest.opType(DocWriteRequest.OpType.INDEX);
//indexRequest.timeout(TimeValue.timeValueSeconds(1));
//強制刷新，消除延遲
indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
/*
*幾何類型的字段，可能報錯：需要按幾何對象格式去存儲。
*Elasticsearch exception [type=mapper_parsing_exception,
* reason=failed to parse field [location] of type [geo_point]]]
*/
IndexResponse indexResponse = restClient.index(indexRequest, RequestOptions.DEFAULT);
int result = indexResponse.status().getStatus();
if(result==RestStatus.OK.getStatus() || result == RestStatus.CREATED.getStatus()){
return 1; //新增成功
}
}
} catch (Exception e) {
e.printStackTrace();
}
return 0;
}
@Override
public int deleteDoc(String index, String id) {
try {
DeleteRequest delRequest = new DeleteRequest(index,id);
// 如果存在routing,需要指定routing
String routing = getRouting(index, id);
if(routing != null){
delRequest.routing(routing);
}
//強制刷新，消除延遲
delRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
DeleteResponse delResp = restClient.delete(delRequest, RequestOptions.DEFAULT);
int result = delResp.status().getStatus();
if(result==RestStatus.OK.getStatus() || result == RestStatus.CREATED.getStatus()){
return 1; //刪除成功
}
} catch (IOException e) {
e.printStackTrace();
}
return 0;//刪除失敗
}
@Override
public int updateDoc(String index, String id, Map<String, Object> kvs) {
try {
//將幾何類型的轉爲特定對象
Map<String, Object> columns = getColumnNames(index);
Set<String> keys = kvs.keySet();
for (String key : keys) {
EsFieldType eft = EsFieldType.GEO_POINT;
if(eft.getType().equals(columns.get(key))){
//如果是幾何類型，就轉換
Object transferedField = eft.getTransferedField(kvs.get(key).toString());
if(transferedField==null){
return 0;
}
kvs.put(key, transferedField);
}
}
UpdateRequest updateRequest = new UpdateRequest(index, id);
/*
* 報錯：ElasticsearchStatusException[Elasticsearch exception [type=document_missing_exception
* 如果存在routing,需要指定routing
*/
String routing = getRouting(index, id);
if(routing != null){
updateRequest.routing(routing);
}
updateRequest.doc(kvs);
//NOTE:查詢document，如果不存在，就不更新。
UpdateResponse updateResp = restClient.update(updateRequest, RequestOptions.DEFAULT);
//強制刷新，消除延遲
updateResp.setForcedRefresh(true);
int result = updateResp.status().getStatus();
if(result==RestStatus.OK.getStatus() || result == RestStatus.CREATED.getStatus()){
return 1; //更新成功
}
} catch (Exception e) {
e.printStackTrace();
}
return 0;
}
@Override
public boolean docIsExist(String index, String id) {
boolean flag = false;
try {
GetRequest getRequest = new GetRequest(index, id);
GetResponse getResp = restClient.get(getRequest, RequestOptions.DEFAULT);
flag = getResp.isExists();
} catch (IOException e) {
e.printStackTrace();
}finally {
return flag;
}
}
@Override
public long totalCount(String...indexes) {
long countNum = 0;
try {
//直接統計總數，沒有設置search條件。
CountRequest countRequest = new CountRequest(indexes);
CountResponse countResp = restClient.count(countRequest, RequestOptions.DEFAULT);
countNum = countResp.getCount();
} catch (IOException e) {
e.printStackTrace();
}finally {
return countNum;
}
}
@Override
public double usedRate() {
//es使用率 = cluster store(size_in_bytes) / (所有節點的磁盤可用空間 + size_in_bytes)
double rate = 0.0;
try {
Request req = new Request("get", "_cluster/stats");
RestClient client = restClient.getLowLevelClient();
Response resp = client.performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
//查詢整個集羣的已用的存儲空間大小（包括所有的index）
long totalIndexSizes = jo.getJSONObject("indices")
.getJSONObject("store")
.getLongValue("size_in_bytes");
//查詢各個節點上的磁盤可用空間總和
long totalAvailableFSSizes = jo.getJSONObject("nodes")
.getJSONObject("fs")
.getLongValue("available_in_bytes");
System.out.println(totalIndexSizes+"==============="+totalAvailableFSSizes);
rate = (double)totalIndexSizes / (totalAvailableFSSizes + totalIndexSizes);
}
} catch (IOException e) {
e.printStackTrace();
}finally {
return Double.parseDouble(String.format("%.2f",rate*100));
}
}
@Override
public double storeSizeOfDB(SizeUnitEnum unit) {
return storeSizeOfDB(null,unit);
}
@Override
public double storeSizeOfDB(String index,SizeUnitEnum unit) {
try {
Request req = new Request("get", "_stats");
RestClient client = restClient.getLowLevelClient();
Response resp = client.performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
if(StringUtils.isEmpty(index)){
//查詢整個集羣的已用的存儲空間大小（包括所有的index）
long bytes = jo.getJSONObject("_all")
.getJSONObject("total")
.getJSONObject("store")
.getLongValue("size_in_bytes");
return FileSizeUtil.valueOf((double)bytes, unit);
}else{
//查詢某個索引下已用的存儲空間大小
if(isIndexExist(index)){
long bytes = jo.getJSONObject("indices")
.getJSONObject(index)
.getJSONObject("total")
.getJSONObject("store")
.getLongValue("size_in_bytes");
return FileSizeUtil.valueOf((double)bytes, unit);
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
return 0.0;
}
@Override
public double storeSizeOfTbl(String[] indices, SizeUnitEnum unit) {
try {
if(indices != null ){
Request req = new Request("get", "_stats");
RestClient client = restClient.getLowLevelClient();
Response resp = client.performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
JSONObject indiceJO = jo.getJSONObject("indices");
//查詢某個索引下已用的存儲空間大小
long bytes = 0L;
for (String index : indices) {
//判斷索引是否存在
//if(isIndexExist(index)){ }
if(indiceJO.get(index) != null){
bytes += indiceJO
.getJSONObject(index)
.getJSONObject("total")
.getJSONObject("store")
.getLongValue("size_in_bytes");
}
}
return FileSizeUtil.valueOf((double)bytes, unit);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return 0.0;
}
@Override
public int countTables() {
//默認的集羣：elasticsearch
return countIndices("elasticsearch");
}
private int countIndices(String clusterName){
//ClusterStatsRequest req = new ClusterStatsRequest();
//Request req = new Request("get", "_cluster/stats");
return getTablenamesOfDB().size();
}
@Override
public List<String> getTablenamesOfDB() {
List<String> nameList = new ArrayList<>();
List<String> filteredNameList = new ArrayList<>();
try {
Request req = new Request("get", "_stats");
Response resp = restClient.getLowLevelClient().performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
//獲取所有的indices
JSONObject indices = jo.getJSONObject("indices");
nameList.addAll(indices.keySet());
}
//過濾掉自動生成的index
for (String idx : nameList) {
if(!idx.startsWith(".")){
filteredNameList.add(idx);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return filteredNameList;
}
@Override
public SearchResponse queryByRandomField(String indexName, String fieldName, String fieldValue,
int pageSize,int pageNum) {
SearchResponse resp = null;
try {
/* SearchRequest search = new SearchRequest(indexName);
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//分頁查詢
sourceBuilder.from(pageSize*pageNum-pageSize);
sourceBuilder.size(pageSize);
sourceBuilder.sort("_id", SortOrder.ASC);
setQuerySearchBuilder(sourceBuilder,indexName,fieldName,fieldValue);
search.source(sourceBuilder);
resp = restClient.search(search, RequestOptions.DEFAULT);*/
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
setQuerySearchBuilder(sourceBuilder,indexName,fieldName,fieldValue);
resp = getAllRowsByScrollScan(sourceBuilder, indexName, pageSize, pageNum);
}catch(Exception e){
e.printStackTrace();
}
return resp;
}
private void setQuerySearchBuilder(SearchSourceBuilder sourceBuilder,
String indexName,
String fieldName, String fieldValue) throws Exception{
String[]numeric = {"long","integer","short","byte","double","float","half_float","scaled_float"};
List<String> numericTypes = Arrays.asList(numeric);
//獲取字段類型
String fieldType = getFieldType(indexName, fieldName);
if("text".equals(fieldType) || "keyword".equals(fieldType)){
//以*開頭的wildcardQuery非常慢，不建議使用
//模糊查詢：fuzzy，wildcard: 只針對text，keyword類型字段。
//text類型：可以使用fuzzy模糊查詢。keyword類型，使用fuzzy查詢失效。
sourceBuilder.query(QueryBuilders.wildcardQuery(fieldName,"*" + fieldValue + "*"));
//sourceBuilder.query(QueryBuilders.fuzzyQuery(fieldName, fieldValue).fuzziness(Fuzziness.AUTO));
//sourceBuilder.query(QueryBuilders.matchQuery(fieldName, fieldValue).fuzziness(Fuzziness.AUTO));
}else if(numericTypes.contains(fieldType)){
if(StringUtil.isNumeric(fieldValue)){
sourceBuilder.query(QueryBuilders.rangeQuery(fieldName).gte(fieldValue));
}
}else if("geo_point".equals(fieldType)){
//Geo fields do not support exact searching, use dedicated geo queries instead
if(fieldValue != null){
String[] locations = fieldValue.split(",");
if(locations.length == 4){
double top = StringUtil.isNumeric(locations[0].trim())?Double.valueOf(locations[0].trim()):90;
double left = StringUtil.isNumeric(locations[1].trim())?Double.valueOf(locations[1].trim()): -180;
double bottom = StringUtil.isNumeric(locations[2].trim())?Double.valueOf(locations[2].trim()) :-90;
double right = StringUtil.isNumeric(locations[3].trim())?Double.valueOf(locations[3].trim()): 180;
sourceBuilder.query(QueryBuilders.geoBoundingBoxQuery(fieldName)
.setCorners(top, left, bottom, right));
}
}
}else if("geo_shape".equals(fieldType)){
//Geo fields do not support exact searching, use dedicated geo queries instead
if(fieldValue != null){
String[] locations = fieldValue.split(",");
if(locations.length == 4){
double top = StringUtil.isNumeric(locations[0].trim())?Double.valueOf(locations[0].trim()):90;
double left = StringUtil.isNumeric(locations[1].trim())?Double.valueOf(locations[1].trim()): -180;
double bottom = StringUtil.isNumeric(locations[2].trim())?Double.valueOf(locations[2].trim()) :-90;
double right = StringUtil.isNumeric(locations[3].trim())?Double.valueOf(locations[3].trim()): 180;
List<Coordinate> coordinates = new CoordinatesBuilder().coordinate(left, top)
.coordinate(right, bottom).build();
GeometryCollectionBuilder gcb = new GeometryCollectionBuilder();
gcb.coordinates(coordinates);
sourceBuilder.query(QueryBuilders.geoWithinQuery(fieldName, gcb));
}
}
}else{
sourceBuilder.query(QueryBuilders.matchQuery(fieldName, fieldValue));
}
}
@Override
public long totalCountOfFuzzyQuery(String indexName, String fieldName, String fieldValue) {
long counter = 0;
try {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//模糊查詢：fuzzy，wildcard
setQuerySearchBuilder(sourceBuilder,indexName,fieldName,fieldValue);
//直接統計總數，設置search條件。
CountRequest countRequest = new CountRequest(new String[]{indexName},sourceBuilder);
CountResponse countResponse = restClient.count(countRequest, RequestOptions.DEFAULT);
counter = countResponse.getCount();
} catch (Exception e) {
e.printStackTrace();
}
return counter;
}
//獲取字段類型
private String getFieldType(String indice,String fieldName)throws IOException{
Map<String, MappingMetaData> mappings = getMappingInfo(indice);
Map<String, Object> source = mappings.get(indice).getSourceAsMap();
Object properties = source.get("properties");
if(properties instanceof LinkedHashMap){
LinkedHashMap map = (LinkedHashMap)properties;
Object field = map.get(fieldName);
if(field instanceof LinkedHashMap){
LinkedHashMap fieldMap = (LinkedHashMap)field;
String type = fieldMap.get("type").toString();
return type;
}
}
return null;
}
/**
* 獲取mapping信息
* @param indice
* @return
* @throws IOException
*/
public Map<String, MappingMetaData> getMappingInfo(String indice) throws IOException{
GetMappingsRequest gmr = new GetMappingsRequest();
gmr.indices(indice);
GetMappingsResponse resp = restClient.indices()
.getMapping(gmr, RequestOptions.DEFAULT);
Map<String, MappingMetaData> mappings = resp.mappings();
return mappings;
}
@Override
public Map<String, Object> getColumnNames(String indexName) {
Map<String, Object> columnNames = new HashMap<>();
GetMappingsRequest mappingsRequest = new GetMappingsRequest().indices(indexName);
try {
GetMappingsResponse mappingsResponse = restClient.indices()
.getMapping(mappingsRequest, RequestOptions.DEFAULT);
Map<String, MappingMetaData> mappings = mappingsResponse.mappings();
if(mappings != null){
MappingMetaData metaData = mappings.get(indexName);
if(metaData != null){
Map<String, Object> sourceAsMap = metaData.getSourceAsMap();//properties
if(sourceAsMap != null){
Collection<Object> collection = sourceAsMap.values();//Object = map
Map<String,Object> tmp = new HashMap<>();
Iterator<Object> ite = collection.iterator();
while (ite.hasNext()){
tmp.putAll((Map<String,Object>)ite.next());
}
Set<String> fields = tmp.keySet();
//提取字段名和類型
for (String field : fields) {
Map<String,Object> fieldMap = (Map<String,Object>)tmp.get(field);
columnNames.put(field, fieldMap.get("type"));
}
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
return columnNames;
}
/**
* 根據索引ID查詢對應document，並返回routing內容
* @param index
* @param id
*/
private String getRouting(String index, String id){
SearchResponse resp = queryByRandomField(index, "_id", id, 1, 1);
if(resp != null){
SearchHits hits = resp.getHits();
Iterator<SearchHit> ite = hits.iterator();
while(ite.hasNext()){
SearchHit hit = ite.next();
DocumentField df = hit.field("_routing");
if(df != null){
List<Object> values = df.getValues();
if(values != null){
String valStr = values.toString();
//將routing轉成字符串返回
return valStr.substring(1, valStr.length()-1);
}
}
}
}
return null;
}
@Override
public Map<String, List<String>> getClusterIndexes(String clusterName) {
return null;
}
@Override
public Map<String, List<String>> getIndexTypes(String clusterName) {
return null;
}
}
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
import java.util.regex.Pattern;
/**
* @Description: elasticsearch字段類型：
* https://www.elastic.co/guide/en/elasticsearch/reference/6.0/mapping-types.html
* 該枚舉類用於轉換elasticsearch特殊字段的數據值
*/
public enum EsFieldType {
/**
* 幾何點類型，接收經緯度數據對。
* geo_point有四種形式：
* #對象 "location": {"lat": 41.12,"lon": -71.34}
* #字符串 "location": "41.12,-71.34"
* #geohash "location": "drm3btev3e86"
* #數組 "location": [ -71.34, 41.12 ]
*/
GEO_POINT("geo_point"){
@Override
public Object fieldValueTransfer(String fieldValue) {
String rex1 = "\\{(\\s)*\"lat\"(\\s)*:(\\s)*(\\-|\\+)?\\d+(\\.\\d+)?(\\s)*,"+
"(\\s)*\"lon\"(\\s)*:(\\s)*(\\-|\\+)?\\d+(\\.\\d+)?(\\s)*\\}";
String rex2 = "(\\-|\\+)?\\d+(\\.\\d+)?(\\s)*,(\\s)*(\\-|\\+)?\\d+(\\.\\d+)?";
String rex3 = "^[a-z0-9]+$";
String rex4 = "^\\[(\\s)*(\\-|\\+)?\\d+(\\.\\d+)?(\\s)*,(\\s)*(\\-|\\+)?\\d+(\\.\\d+)?(\\s)*\\]$";
if(match(rex1,fieldValue)){
//json object
return parseJsonToGeopoint(fieldValue);
}else if(match(rex4,fieldValue)){
//array
return JSON.parseArray(fieldValue);
}else if(match(rex2,fieldValue)){
//string
return fieldValue;
}else if(match(rex3,fieldValue)){
//geohash,不能有大寫的英文字母
return GeoPoint.fromGeohash(fieldValue);
}else{
return null;
}
}
private GeoPoint parseJsonToGeopoint(String jsonStr){
//{"lat": 41.12,"lon": -71.34}
JSONObject jo = JSON.parseObject(jsonStr);
//直接返回JSONObject也可以，但經緯度前後順序會顛倒。
return new GeoPoint(jo.getDoubleValue("lat"), jo.getDoubleValue("lon"));
}
};
EsFieldType(String type) {
this.type = type;
}
public Object getTransferedField(String fieldValue){
return fieldValueTransfer(fieldValue);
}
public boolean match(String rex,String input){
Pattern p = Pattern.compile(rex);
return p.matcher(input).matches();
}
private String type;
public String getType() {
return type;
}
protected abstract Object fieldValueTransfer(String fieldValue);
}
