EsRejectedExecutionExceptionindices:data/read/mget[shard]異常處理與線程池設置

報錯

• elasticsearch查詢失敗
• java日誌：

org.frameworkset.elasticsearch.ElasticSearchException: java.net.SocketTimeoutException: Read timed out
	at org.frameworkset.elasticsearch.client.ElasticSearchRestClient._executeHttp(ElasticSearchRestClient.java:636)
	at org.frameworkset.elasticsearch.client.ElasticSearchRestClient.executeHttp(ElasticSearchRestClient.java:493)
	at org.frameworkset.elasticsearch.client.ElasticSearchRestClient.executeHttp(ElasticSearchRestClient.java:661)
	at org.frameworkset.elasticsearch.client.ElasticSearchRestClient.executeHttp(ElasticSearchRestClient.java:467)
	at org.frameworkset.elasticsearch.client.RestClientUtil.getDocument(RestClientUtil.java:982)
	at org.frameworkset.elasticsearch.client.RestClientUtil.getDocument(RestClientUtil.java:1087)
	at cn.lonsun.es.base.BbossUtil.getDocument(BbossUtil.java:225)
	at cn.lonsun.searchEngine.impl.IndexEsServiceImpl.getOneById(IndexEsServiceImpl.java:593)
	at cn.lonsun.searchEngine.impl.SearchEsServiceImpl.scrollSearchForPage(SearchEsServiceImpl.java:283)

• es日誌

[2020-06-09T11:06:24,198][DEBUG][o.e.a.g.TransportShardMultiGetAction] [node-1-master-8-9200] null: failed to execute [org.elasticsearch.action.get.MultiGetShardRequest@11e6ef27]
org.elasticsearch.transport.RemoteTransportException: [node-1-master-8-9200][192.168.4.8:9300][indices:data/read/mget[shard][s]]
Caused by: org.elasticsearch.common.util.concurrent.EsRejectedExecutionException: rejected execution of org.elasticsearch.transport.TransportService$7@198d4842 on EsThreadPoolExecutor[get, queue capacity = 1000, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@171dd8cf[Running, pool size = 8, active threads = 8, queued tasks = 1000, completed tasks = 9786245]]
	at org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:50) ~[elasticsearch-5.5.2.jar:5.5.2]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830) ~[?:1.8.0_201]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379) ~[?:1.8.0_201]
	at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.doExecute(EsThreadPoolExecutor.java:94) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.execute(EsThreadPoolExecutor.java:89) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService.sendLocalRequest(TransportService.java:640) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService.access$000(TransportService.java:74) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService$3.sendRequest(TransportService.java:137) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:586) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:519) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:494) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.perform(TransportSingleShardAction.java:233) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.onFailure(TransportSingleShardAction.java:196) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.access$1300(TransportSingleShardAction.java:123) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction$2.handleException(TransportSingleShardAction.java:252) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1067) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TcpTransport.lambda$handleException$16(TcpTransport.java:1467) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:110) [elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:1465) [elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:1457) [elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1401) [elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) [transport-netty4-5.5.2.jar:5.5.2]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_201]

原因

• 高併發請求，線程池不夠用，es拒絕服務
• 參考：EsRejectedExecutionException排錯與線程池類型
• 官方文檔：https://www.elastic.co/guide/en/elasticsearch/reference/5.5/modules-threadpool.html

解決

• 重啓。是的，最簡單粗暴，最快捷的處理方式，就是重啓。如果是集羣部署，可以挨個重啓，可能在重啓到某一個的時候就會發現服務可用了，繼續把所有節點重啓一遍。
• 更好配置的服務器，更多的集羣節點。我們生產用的es服務器，配置是四核八線程32G內存，集羣節點也就3個。是比較低的配置，和比較簡陋的集羣。但數據量較大，搜索併發較高時，es服務撐不住了，導致連接池爆了。
• 目前報錯是使用Rest client接口，使用的是HTTP API請求，連接池爆了報錯的，後續會改成Transport client連接，是tcp連接，壓測下是否有問題。
• 未完待續