package com.ncs.dao;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.Requests;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.FuzzyQueryBuilder;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.PrefixQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.index.query.SpanFirstQueryBuilder;
import org.elasticsearch.index.query.WildcardQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.google.common.collect.Maps;
import com.ncs.Conf;
import com.ncs.entity.ESEntity;
/**
* 操作Elasticserach
* @author syl
*
* @date 2017年4月7日 下午5:59:23
*/
public class ElasticSearchDAO {
private static final Logger logger = LoggerFactory
.getLogger(ElasticSearchDAO.class);
// 聲明靜態配置
private static TransportClient client = null;
private volatile static BulkRequestBuilder prepareBulk;
// 初始化靜態配置
static {
/*// es 5.X的連接方式
String EsHosts = Conf.get("es.hosts");
Settings settings = Settings.builder()
.put("cluster.name", Conf.get("es.cluster.name"))// 設置集羣名稱
.put("tclient.transport.sniff", true).build();// 自動嗅探整個集羣的狀態，把集羣中其它機器的ip地址加到客戶端中
// 獲取客戶端
client = new PreBuiltTransportClient(setting);
String[] nodes = EsHosts.split(",");
for (String node : nodes) {
if (node.length() > 0) {// 跳過爲空的node（當開頭、結尾有逗號或多個連續逗號時會出現空node）
String[] hostPort = node.split(":");
try {
client.addTransportAddress(new InetSocketTransportAddress(
InetAddress.getByName(hostPort[0]), Integer.parseInt(hostPort[1])));
} catch (Exception e) {
e.printStackTrace();
}
}
} */
// es 2.4.4的連接方式
String EsHosts = Conf.get("es.hosts");
Settings settings = Settings.settingsBuilder()
.put("cluster.name", Conf.get("es.cluster.name"))// 設置集羣名稱
.put("tclient.transport.sniff", true).build();// 自動嗅探整個集羣的狀態，把集羣中其它機器的ip地址加到客戶端中
// 獲取客戶端
client = TransportClient.builder().settings(settings).build();
String[] nodes = EsHosts.split(",");
for (String node : nodes) {
if (node.length() > 0) {// 跳過爲空的node（當開頭、結尾有逗號或多個連續逗號時會出現空node）
String[] hostPort = node.split(":");
try {
client.addTransportAddress(new InetSocketTransportAddress(
InetAddress.getByName(hostPort[0]), Integer.parseInt(hostPort[1])));
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
/**
* 關閉客戶端，釋放資源
*/
public static void closeClient() {
client.close();
logger.info("client closed!");
}
/**
* 創建索引名稱
*
* @param indexname
* 索引名稱
*/
public static void createIndexName(String indexname) {
client.admin().indices().prepareCreate(indexname).execute().actionGet();
logger.info("create index " + indexname + " success!");
}
/**
* 創建mapping(feid("indexAnalyzer","ik")該字段分詞IK索引；
* feid("searchAnalyzer","ik")該字段分詞ik查詢；具體分詞插件請看IK分詞插件說明)
*
* @param indexname
* 索引名稱；
* @param mappingType
* 索引類型
* @throws Exception
*/
public static void createMapping(String indexname, String mappingType)
throws Exception {
new XContentFactory();
XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
// .startObject("_ttl").field("enable", false).endObject()
.startObject("properties")
.startObject("domain").field("type", "string")
.field("store", "yes").field("index", "not_analyzed")
.endObject()
.startObject("rowKeys").field("type", "string")
.field("store", "yes").field("index", "not_analyzed")
.endObject()
// .startObject("clazz").field("type", "string")
// .field("store", "yes").field("index", "not_analyzed")
// .endObject()
// .startObject("type").field("type", "string")
// .field("store", "yes").field("index", "not_analyzed")
// .endObject()
// .startObject("rdata").field("type", "string")
// .field("store", "yes").field("index", "not_analyzed")
// .endObject()
// .startObject("ispId")
// .field("type", "integer").field("store", "yes")
// .field("index", "not_analyzed")
// .endObject()
// .startObject("version")
// .field("type", "long").field("store", "yes")
// .field("index", "not_analyzed")
// .endObject()
.endObject().endObject();
PutMappingRequest mapping = Requests.putMappingRequest(indexname)
.type(mappingType).source(builder);
client.admin().indices().putMapping(mapping).actionGet();
logger.info("create mapping " + mappingType + " success!");
}
/**
* 建立索引,並插入數據
*
* @param indexName
* 爲索引庫名，一個es集羣中可以有多個索引庫。 名稱必須爲小寫
* @param indexType
* Type爲索引類型，是用來區分同索引庫下不同類型的數據的，一個索引庫下可以有多個索引類型。
* @param jsondata
* json格式的數據集合
* @return
*/
public static void createIndexResponse(String indexname, String type,
List<String> jsondata) {
// 創建索引庫 需要注意的是.setRefresh(true)這裏一定要設置,否則第一次建立索引查找不到數據
IndexRequestBuilder requestBuilder = client.prepareIndex(indexname,
type).setRefresh(true);
for (int i = 0; i < jsondata.size(); i++) {
IndexResponse actionGet = requestBuilder.setId(i + 1 + "").setSource(jsondata.get(i))
.execute().actionGet();
logger.info("response.getVersion():" + actionGet.getVersion());
}
}
/**
* 建立索引,並插入一條數據
*
* @param indexname
* 爲索引庫名
* @param type
* 索引類型
* @param jsondata
* json格式的數據
* @return
*/
public static IndexRequestBuilder createIndexResponse(String indexname,
String type, String jsondata, String id) {
IndexRequestBuilder response = client.prepareIndex(indexname, type)
// 必須爲對象單獨指定ID
.setId(id).setSource(jsondata);
// 多次index這個版本號會變
// logger.info("response.getVersion():" + response.getVersion());
return response;
}
/**
* 批量處理的準備
* @param indexname // 索引名
* @param type // 類型名
* @param jsondata // json字符串
* @param id // id
*/
public static void creatIndexResponseBulk(String indexname,
String type, String jsondata, String id) throws Exception{
prepareBulk = client.prepareBulk();
prepareBulk = prepareBulk.add(ElasticSearchDAO.createIndexResponse(indexname, type, jsondata, id));
}
/**
* 執行批量插入，並初始化prepareBulk
*/
public static void execute() throws Exception {
// 每1000條提交一次
prepareBulk.execute().actionGet();
}
/**
* 根據索引庫名、類型名、id,刪除對應的數據
*
* @param indexname
* 索引庫名
* @param type
* 類型名
* @param id
* 每一條數據的唯一標識
*/
public static DeleteResponse deleteResponse(String indexname, String type,
String id) {
DeleteResponse response = client.prepareDelete(indexname, type, id)
.execute().actionGet();
logger.info(response.getId() + " delete success!");
return response;
}
/**
* 根據索引名、類型名、id、列名、值，更新或添加一列到一條數據
*
* @param indexname
* 索引名
* @param type
* 類型名
* @param id
* @param cloumn
* 列名
* @param value
* 值
* @return
*/
public static UpdateResponse updataResponse(String indexname, String type,
String id, String cloumn, String value) {
Map<String, Object> params = Maps.newHashMap();
params.put(cloumn, value);
UpdateResponse response = client.prepareUpdate(indexname, type, id)
// .setScript("ctx._source." + cloumn + "=" + cloumn,
// ScriptType.INLINE).setScriptParams(params)
.execute().actionGet();
logger.info("updata success!");
return response;
}
/**
* 根據索引庫名、類型名、id,獲取對應的數據
*
* @param indexname
* 索引庫名
* @param type
* 類型名
* @param id
* 每一條數據的唯一標識
*/
public static GetResponse getResponse(String indexname, String type,
String id) {
GetResponse response = client.prepareGet(indexname, type, id).execute()
.actionGet();
logger.info("response.getId():" + response.getId() + "response.getSourceAsString():" + response.getSourceAsString());
return response;
}
/**
* 執行搜索 搜索ESEntity
*
* @param queryBuilder
* 搜索條件
* @param indexname
* 索引庫名
* @param type
* 類型名
* @return
*/
public static List<ESEntity> searcherESEntitys(
QueryBuilder queryBuilder, String indexname, String type) {
List<ESEntity> list = new ArrayList<ESEntity>();
// 執行搜索
SearchResponse searchResponse = client.prepareSearch(indexname)
.setTypes(type).setQuery(queryBuilder).execute().actionGet();
// 獲取搜索結果
SearchHits hits = searchResponse.getHits();
System.out.println("查詢到記錄數=" + hits.getTotalHits());
SearchHit[] searchHists = hits.getHits();
String domain;
String rowKeys;
// 遍歷將搜索結果封裝成Record對象，添加到list集合中
if (searchHists.length > 0) {
for (SearchHit hit : searchHists) {
domain = (String) hit.getSource().get("domain");
rowKeys = (String) hit.getSource().get("rowKeys");
list.add(new ESEntity(domain, rowKeys));
}
}
return list;
}
/**
* 將ESEntity對象集合保存到ES中
* @param esEntities 對象集合
* @param indexname 索引名稱
* @param type 索引類型
*/
public static void intoEs(List<ESEntity> esEntities,
String indexname, String type) {
String receiveString;
try {
// 將集合對象遍歷存入es中
for (ESEntity esEntity : esEntities) {
// 將對象轉換爲json字符串
receiveString = JSON.toJSONString(esEntity);
// 批量插入es之前的準備
ElasticSearchDAO.creatIndexResponseBulk(indexname, type, receiveString, esEntity.getDomain());
}
// 批量插入es中
ElasticSearchDAO.execute();
logger.info(esEntities.size() + " row eSEntity has inesrt into es!");
} catch (Exception e) {
logger.error(" inesrt into es error!" + e.toString());
e.printStackTrace();
}
}
@SuppressWarnings({ "unused" })
public static void main(String[] args) {
String indexname = "zone";
String type = "records";
// 獲得需要插入的數據
// List<String> jsondata = Zone.getInitJsonData();
// 創建index並插入數據，（只是插入數據時需要）
// ElasticSearchJavaAPI.createIndexResponse(indexname, type, jsondata);
// 查詢條件
// //////////////////////////////////////////////////////////////////////////////////////
// 按照其中某一項進行查詢
QueryBuilder queryBuilder = QueryBuilders.termQuery("owner", "sex2");
// content爲field,test爲查詢內容.
// 其中must表示必須滿足,mustNot表示必須不滿足,should表示可有可無
// 如果bool查詢語句中不存在must,則必須至少有一個should查詢,同時可以通過minimum_should_match參數來設置至少需要滿足的should個數.
QueryBuilder qb = QueryBuilders
.boolQuery()
.must(QueryBuilders.termQuery("rdata", "ns1.101domain.com"))
// .must(QueryBuilders.termQuery("rdata", "ns1.sdc.org.cn"))
.mustNot(
QueryBuilders
.termQuery("rdata", "nsgbr.comlaude.co.uk"))
.should(QueryBuilders.termQuery("rdata", "ns1.sdc.org.cn"));
// 查詢漢字時需要將漢字拆分開，因爲elasticsearch 裏默認的IK分詞器是會將每一箇中文都進行了分詞的切割
QueryBuilder queryBuilder3 = QueryBuilders.termQuery("owner", "洞");
QueryBuilder queryBuilder4 = QueryBuilders.termQuery("owner", "頭");
BoolQueryBuilder bool = QueryBuilders.boolQuery();
bool.must(queryBuilder3);
bool.must(queryBuilder4);
// 根據前綴查詢
PrefixQueryBuilder prefixQuery = QueryBuilders.prefixQuery("domain", "163");
// 通配符查詢
WildcardQueryBuilder wildcardQuery = QueryBuilders.wildcardQuery(
"rdata", "?ns"); // ? *
// 根據提供的字符串作爲前綴進行查詢
FuzzyQueryBuilder fuzzyQuery = QueryBuilders.fuzzyQuery("name", "sex");
// 範圍查詢
RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("rdata")
.from(0).to(10);
// 查詢所有
MatchAllQueryBuilder matchAllQuery = QueryBuilders.matchAllQuery();
// 根據id查詢
QueryBuilder queryBuilder2 = QueryBuilders.boolQuery().must(
QueryBuilders.termQuery("id", 1));
// 只匹配在Field開頭出現的, 數字表示從開頭起的幾個單詞內查詢, 則此查詢意思是:ns中的開頭3個單詞內,nsgbr是否能查詢到
SpanFirstQueryBuilder spanFirstQuery = QueryBuilders.spanFirstQuery(
QueryBuilders.spanTermQuery("rdata", "nsgbr"), // Query
3 // Max End position
);
// String查詢
QueryStringQueryBuilder queryString = QueryBuilders.queryStringQuery("sex");
// 文本查詢
// MatchQueryBuilder textPhrase = QueryBuilders.textPhrase("", null);
// /////////////////////////////////////////////////////////////////////////////
// List<DNResourceRecord> result = ElasticSearchDAO.searcherDNResourceRecords(prefixQuery, indexname,
// type);
// for (int i = 0; i < result.size(); i++) {
// DNResourceRecord record = result.get(i);
// System.out.println("(" + record.getId() + ") 主機名:" + record.getOwner()+ "\t域名:" + record.getRdata());
// }
//
// GetResponse response = ElasticSearchDAO.getResponse(indexname, type, "2");
// List<DNRegisterInfo> registerInfos = ElasticSearchDAO.searcherDNRegisterInfos(prefixQuery, indexname, type);
// for (DNRegisterInfo dnRegisterInfo : registerInfos) {
// System.out.println("主機名：" + dnRegisterInfo.getName() + " 註冊商姓名:"
// + dnRegisterInfo.getRegistrantName() + " " + dnRegisterInfo.getUpdatedDate());
// }
// List<ESEntity> list = ElasticSearchDAO.searcherESEntitys(prefixQuery, indexname, type);
// for (ESEntity esEntity : list) {
// System.out.println(esEntity);
// }
// 創建索引和mapping
ElasticSearchDAO.createIndexName("zone");
try {
ElasticSearchDAO.createMapping("zone", "records");
} catch (Exception e) {
e.printStackTrace();
}
// System.out.println("********************************");
// ElasticSearchDAO.deleteResponse(indexname, type, "sex4.tld.");
// ElasticSearchDAO.updataResponse(indexname, type, "sex4.tld.", "id", "sex4.tld._info");
ElasticSearchDAO.closeClient();
}
}
Java操作elasticsearch的工具類
發表評論
所有評論
還沒有人評論，想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章