ElasticSearch6.x 基於SpringBoot 實現ElasticSearch的文檔批處理管理

SpringBoot 功能封裝涉及ElasticSearch的文檔批處理管理方法約定如下：

public void createBlukProcessDocument(String index, String type, List<Map<String, Object>> context):基於BulkProcessor 實現批處理

public BulkResponse createBulkDocument(String index, String type, List<Map<String, Object>> context)：實現批處理

在上一篇文中說到：ElasticSearch6.x 基於SpringBoot 實現ElasticSearch連接功能封裝,將約定的方法填充到ElasticSearchIndexUtil.java 工具類中。

/**
 * ela index 工具類
 * 
 * @author zzg
 *
 */
@Component
public class ElasticSearchIndexUtil {
	// 引入 Ela 連接實列化對象
	@Autowired
	private TransportClient client;


   /**
	 * 功能描述：批量導入文檔
	 * 
	 * @param index
	 *            索引名
	 * @param type
	 *            索引類型
	 * @param context
	 *            文檔內容
	 */
	public void createBlukProcessDocument(String index, String type, List<Map<String, Object>> context) {
		if (context != null && context.size() > 0) {
			BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {

				@Override
				public void beforeBulk(long executionId, BulkRequest request) {
					// TODO Auto-generated method stub

				}

				@Override
				public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
					// TODO Auto-generated method stub

				}

				@Override
				public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
					// TODO Auto-generated method stub

				}
			}).setBulkActions(10) // 每次10000個請求
					.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)) // 拆成5MB一塊
					.setFlushInterval(TimeValue.timeValueSeconds(5)) // 無論請求數量多少，每5秒鐘請求一次
					.setConcurrentRequests(1)// 設置併發請求的數量。值爲0意味着只允許執行一個請求。值爲1意味着允許1併發請求
					.setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3))
					// 設置自定義重複請求機制，最開始等待100毫秒，之後成倍增加，重試3次，當一次或者多次重複請求失敗後因爲計算資源不夠拋出EsRejectedExecutionException
					// 異常，可以通過BackoffPolicy.noBackoff()方法關閉重試機制
					.build();

			// 線程批處理
			context.stream().forEach(item -> {
				String id = (String) item.get("id");
				IndexRequest indexRequest = new IndexRequest(index, type, id);
				try {
					XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
					item.forEach((k, v) -> {
						if (!k.equalsIgnoreCase("id")) {
							try {
								builder.field(k, v);
							} catch (Exception e) {
								// TODO Auto-generated catch block
								e.printStackTrace();
							}
						}
					});
					builder.endObject();
					indexRequest.source(builder);
					bulkProcessor.add(indexRequest);
				} catch (Exception e) {
					e.printStackTrace();
				}
			});
			bulkProcessor.flush();
			// 關閉bulkProcessor
			bulkProcessor.close();
		}
	}

/**
	 * 功能描述：批量導入文檔
	 * 
	 * @param index
	 *            索引名
	 * @param type
	 *            索引類型
	 * @param context
	 *            文檔內容
	 */
	public BulkResponse createBulkDocument(String index, String type, List<Map<String, Object>> context) {
		BulkRequestBuilder requestBuilder = client.prepareBulk();

		context.stream().forEach(item -> {
			String id = (String) item.get("id");
			IndexRequestBuilder indexResponse = client.prepareIndex(index, type, id);

			try {
				XContentBuilder builder = XContentFactory.jsonBuilder().startObject();

				item.forEach((k, v) -> {
					if (!k.equalsIgnoreCase("id")) {
						try {
							builder.field(k, v);

						} catch (Exception e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						}

					}
				});
				builder.endObject();
				indexResponse.setSource(builder);
			} catch (Exception e) {
				e.printStackTrace();
			}
			requestBuilder.add(indexResponse);
		});
		return requestBuilder.get();
	}
}

編寫測試工具類Test.java ，測試相關封裝的功能代碼：

@RunWith(SpringRunner.class)
@SpringBootTest
// 由於是Web項目，Junit需要模擬ServletContext，因此我們需要給我們的測試類加上@WebAppConfiguration。
@WebAppConfiguration
public class Test {
	@Autowired
	private ElasticSearchIndexUtil util;

@org.junit.Test
	public void createBulkDocument() {
		List<Map<String, Object>> container = new ArrayList<Map<String, Object>>();

		Map<String, Object> obj1 = new HashMap<String, Object>();
		obj1.put("id", "5");
		obj1.put("postDate", "2019-07-18");
		obj1.put("message", "java 從入門到精通");
		obj1.put("user", "王剛");

		Map<String, Object> obj2 = new HashMap<String, Object>();
		obj2.put("id", "6");
		obj2.put("postDate", "2019-07-18");
		obj2.put("message", "python 從入門到精通");
		obj2.put("user", "光明科技");

		container.add(obj1);
		container.add(obj2);

		BulkResponse response = util.createBulkDocument("book", "library", container);
		if (response.hasFailures()) {
			System.out.println("error");
		}
	}

	@org.junit.Test
	public void createBlukProcessDocument() {
		List<Map<String, Object>> container = new ArrayList<Map<String, Object>>();

		Map<String, Object> obj1 = new HashMap<String, Object>();
		obj1.put("id", "9");
		obj1.put("postDate", "2019-07-18");
		obj1.put("message", "java 從入門到精通");
		obj1.put("user", "王剛");

		Map<String, Object> obj2 = new HashMap<String, Object>();
		obj2.put("id", "10");
		obj2.put("postDate", "2019-07-18");
		obj2.put("message", "python 從入門到精通");
		obj2.put("user", "光明科技");

		container.add(obj1);
		container.add(obj2);

		util.createBlukProcessDocument("book", "library", container);
	}


}