1. 集群环境
ambari-version：2.7.5
HDP-version：3.0
2.问题描述
hadoop-yarn的启动之后，运行一段时间，莫名其妙的出现新的任务无法提交上去，查看yarn的状态之后，发现yarn的状态都是正常的，并且所有的资源都是充足的，但是提交任务之后就会一直处于accept状态
3.问题表现
4.问题排查
4.1 首先查看了日志，发现了有报错信息：
2023-02-02 15:56:57,934 INFO capacity.LeafQueue (LeafQueue.java:activateApplications(911)) - Application application_1673397736150_0694 from user: hdfs activated in queue: default
2023-02-02 15:56:57,937 INFO capacity.LeafQueue (LeafQueue.java:addApplicationAttempt(941)) - Application added - appId: application_1673397736150_0694 user: hdfs, leaf-queue: default #user-pending-applications: 0 #user-active-applications: 1 #queue-pending-applications: 0 #queue-active-applications: 1
2023-02-02 15:56:57,938 INFO capacity.CapacityScheduler (CapacityScheduler.java:addApplicationAttempt(1036)) - Added Application Attempt appattempt_1673397736150_0694_000001 to scheduler from user hdfs in queue default
2023-02-02 15:56:57,938 INFO attempt.RMAppAttemptImpl (RMAppAttemptImpl.java:handle(925)) - appattempt_1673397736150_0694_000001 State change from SUBMITTED to SCHEDULED on event = ATTEMPT_ADDED
2023-02-02 15:57:28,011 ERROR webapp.Dispatcher (Dispatcher.java:service(171)) - error handling URI: /cluster
org.eclipse.jetty.io.RuntimeIOException: org.eclipse.jetty.io.EofException
at org.eclipse.jetty.server.ResponseWriter.isOpen(ResponseWriter.java:133)
at org.eclipse.jetty.server.ResponseWriter.println(ResponseWriter.java:314)
at org.apache.hadoop.yarn.webapp.hamlet2.HamletImpl$EImp._p(HamletImpl.java:110)
at org.apache.hadoop.yarn.webapp.hamlet2.Hamlet$SCRIPT.__(Hamlet.java:457)
at org.apache.hadoop.yarn.server.resourcemanager.webapp.RMAppsBlock.renderData(RMAppsBlock.java:199)
at org.apache.hadoop.yarn.server.webapp.AppsBlock.render(AppsBlock.java:145)
at org.apache.hadoop.yarn.webapp.view.HtmlBlock.render(HtmlBlock.java:69)
at org.apache.hadoop.yarn.webapp.view.HtmlBlock.renderPartial(HtmlBlock.java:79)
at org.apache.hadoop.yarn.webapp.View.render(View.java:243)
at org.apache.hadoop.yarn.webapp.view.HtmlBlock$Block.subView(HtmlBlock.java:43)
at org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.__(Hamlet.java:30354)
at org.apache.hadoop.yarn.server.resourcemanager.webapp.AppsBlockWithMetrics.render(AppsBlockWithMetrics.java:29)
at org.apache.hadoop.yarn.webapp.view.HtmlBlock.render(HtmlBlock.java:69)
at org.apache.hadoop.yarn.webapp.view.HtmlBlock.renderPartial(HtmlBlock.java:79)
at org.apache.hadoop.yarn.webapp.View.render(View.java:243)
at org.apache.hadoop.yarn.webapp.view.HtmlPage$Page.subView(HtmlPage.java:49)
at org.apache.hadoop.yarn.webapp.hamlet2.HamletImpl$EImp._v(HamletImpl.java:117)
at org.apache.hadoop.yarn.webapp.hamlet2.Hamlet$TD.__(Hamlet.java:848)
at org.apache.hadoop.yarn.webapp.view.TwoColumnLayout.render(TwoColumnLayout.java:71)
at org.apache.hadoop.yarn.webapp.view.HtmlPage.render(HtmlPage.java:82)
at org.apache.hadoop.yarn.webapp.Dispatcher.render(Dispatcher.java:206)
at org.apache.hadoop.yarn.webapp.Dispatcher.service(Dispatcher.java:165)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:287)
at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:277)
at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:182)
at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85)
at com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:941)
at com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:875)
at org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppFilter.doFilter(RMWebAppFilter.java:178)
at com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:829)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:119)
at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:133)
at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:130)
at com.google.inject.servlet.GuiceFilter$Context.call(GuiceFilter.java:203)
at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:130)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.hadoop.security.http.XFrameOptionsFilter.doFilter(XFrameOptionsFilter.java:57)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:644)
at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:592)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.hadoop.security.http.CrossOriginFilter.doFilter(CrossOriginFilter.java:98)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.hadoop.http.HttpServer2$QuotingInputFilter.doFilter(HttpServer2.java:1619)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.hadoop.http.NoCacheFilter.doFilter(NoCacheFilter.java:45)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:539)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:333)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.eclipse.jetty.io.EofException
at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:199)
at org.eclipse.jetty.io.WriteFlusher.flush(WriteFlusher.java:420)
at org.eclipse.jetty.io.WriteFlusher.write(WriteFlusher.java:313)
at org.eclipse.jetty.io.AbstractEndPoint.write(AbstractEndPoint.java:147)
at org.eclipse.jetty.server.HttpConnection$SendCallback.process(HttpConnection.java:745)
at org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:241)
at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:224)
at org.eclipse.jetty.server.HttpConnection.send(HttpConnection.java:519)
at org.eclipse.jetty.server.HttpChannel.sendResponse(HttpChannel.java:753)
at org.eclipse.jetty.server.HttpChannel.write(HttpChannel.java:804)
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:235)
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:219)
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:470)
at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:167)
at org.eclipse.jetty.server.Utf8HttpWriter.write(Utf8HttpWriter.java:183)
at org.eclipse.jetty.server.HttpWriter.write(HttpWriter.java:71)
at org.eclipse.jetty.server.HttpWriter.write(HttpWriter.java:65)
at org.eclipse.jetty.server.ResponseWriter.write(ResponseWriter.java:231)
at org.eclipse.jetty.server.ResponseWriter.write(ResponseWriter.java:248)
at org.eclipse.jetty.server.ResponseWriter.print(ResponseWriter.java:298)
at org.apache.hadoop.yarn.webapp.hamlet2.HamletImpl$EImp._p(HamletImpl.java:107)
... 70 more
Caused by: java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:51)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:177)
... 90 more
2023-02-02 15:57:36,903 INFO zookeeper.ReadOnlyZKClient (ReadOnlyZKClient.java:run(315)) - 0x0769d513 no activities for 60000 ms, close active connection. Will reconnect next time when there are new requests
从上面的报错日志中可以看出，报错信息跟yarn貌似没有半毛钱的关系，所以即系排查，由于是因为任务提交不上去，也就是说任务无法正常申请到资源，所以故而联想到可能会是resourceManager的内存是不是有什么问题，所以获取了resourceManager的日志以及jvm的对内存信息
4.2 resourceManager的日志排查以及堆内存信息的排查
4.2.1 查看日志
从日志中并未查出有用的信息，然后接着查看了jvm堆信息
4.2.2 查看进程堆信息
[yarn@node3 ~]$ jmap -heap 29691
Attaching to process ID 29691, please wait...
Debugger attached successfully.
Server compiler detected.
JVM version is 25.161-b12
using thread-local object allocation.
Parallel GC with 8 thread(s)
Heap Configuration:
MinHeapFreeRatio = 0
MaxHeapFreeRatio = 100
MaxHeapSize = 2147483648 (2048.0MB)
NewSize = 175112192 (167.0MB)
MaxNewSize = 715653120 (682.5MB)
OldSize = 351272960 (335.0MB)
NewRatio = 2
SurvivorRatio = 8
MetaspaceSize = 21807104 (20.796875MB)
CompressedClassSpaceSize = 1073741824 (1024.0MB)
MaxMetaspaceSize = 17592186044415 MB
G1HeapRegionSize = 0 (0.0MB)
Heap Usage:
PS Young Generation
Eden Space:
capacity = 49807360 (47.5MB)
used = 23949536 (22.840057373046875MB)
free = 25857824 (24.659942626953125MB)
48.08433131167763% used
From Space:
capacity = 524288 (0.5MB)
used = 425984 (0.40625MB)
free = 98304 (0.09375MB)
81.25% used
To Space:
capacity = 524288 (0.5MB)
used = 0 (0.0MB)
free = 524288 (0.5MB)
0.0% used
PS Old Generation
capacity = 284164096 (271.0MB)
used = 173603056 (165.56077575683594MB)
free = 110561040 (105.43922424316406MB)
61.09253717964426% used
27626 interned Strings occupying 2802968 bytes.
通过jvm的堆信息查看，内存使用情况也是正常的，这就奇怪了，于是接着往下排查。
4.3 引发的思考
既然堆内存的使用情况也都正常，那会是什么原因呢，难道是yarn的资源调度的问题，导致无法申请资源？带着这个问题，找了一下yarn的源码中关于资源计算这方面的信息，果然是大有所收获。
5.源码中寻找问题
5.1 追踪源码
找到源码中对应的类：【ResourceCalculator.class】，在这个class有两个实现,分别是：
- DefaultResourceCalculator.class
- DominantResourceCalculator.class
来分别看一下两个类中对于资源计算的不同方式：
1. DefaultResourceCalculator.class
...
@Private
@Unstable
public class DefaultResourceCalculator extends ResourceCalculator {
private static final Log LOG =
LogFactory.getLog(DefaultResourceCalculator.class);
@Override
public int compare(Resource unused, Resource lhs, Resource rhs,
boolean singleType) {
// Only consider memory 这里只考虑内存
return Long.compare(lhs.getMemorySize(), rhs.getMemorySize());
}
@Override
public long computeAvailableContainers(Resource available, Resource required) {
// Only consider memory
return available.getMemorySize() / required.getMemorySize();
}
...
@Override
public long computeAvailableContainers(Resource available, Resource required) {
// Only consider memory
return available.getMemorySize() / required.getMemorySize();
}
...
2. DominantResourceCalculator.class
...
/** 这里会比较两种资源，内存和vcore
* Compare two resources - if the value for every resource type for the lhs
* is greater than that of the rhs, return 1. If the value for every resource
* type in the lhs is less than the rhs, return -1. Otherwise, return 0
*
* @param lhs resource to be compared
* @param rhs resource to be compared
* @return 0, 1, or -1
*/
private int compare(Resource lhs, Resource rhs) {
boolean lhsGreater = false;
boolean rhsGreater = false;
int ret = 0;
int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
ResourceInformation lhsResourceInformation = lhs
.getResourceInformation(i);
ResourceInformation rhsResourceInformation = rhs
.getResourceInformation(i);
int diff = lhsResourceInformation.compareTo(rhsResourceInformation);
if (diff >= 1) {
lhsGreater = true;
} else if (diff <= -1) {
rhsGreater = true;
}
}
if (lhsGreater && rhsGreater) {
ret = 0;
} else if (lhsGreater) {
ret = 1;
} else if (rhsGreater) {
ret = -1;
}
return ret;
}
...
@Override
public long computeAvailableContainers(Resource available,
Resource required) {
long min = Long.MAX_VALUE;
int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
ResourceInformation availableResource = available
.getResourceInformation(i);
ResourceInformation requiredResource = required.getResourceInformation(i);
if (requiredResource.getValue() != 0) {
long tmp = availableResource.getValue() / requiredResource.getValue();
min = min < tmp ? min : tmp;
}
}
return min > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) min;
}
...
从上面的代码中可以看出，对于默认的资源计算方式，只计算内存的资源，不会去考虑cpu的数量，但是对于第二种资源计算方式来说的话，它的内存计算是结合了内存以及cpu一起来计算的。
到此，我们找到了问题的原因,因为在ambari安装之后，默认的情况下是使用的第一种资源调度方式，这个时候需要将该资源调度方式进行修改：
修改前：
修改完成之后，在此重启yarn，问题得以解决
6. 思考
日常集群的维护的过程中，真的是会出现各种各样的意想不到的问题，当我们遇到问题的时候，
- 一定要冷静的先思考，找准方向，然后在朝着我们的方向顺藤摸瓜的去找问题的答案。
- 源码的重要性。我们在寻找很多方案之后都探寻无果的情况下，这个时候我们不妨可以从源码中着手，问题一定在其中。
- 对于问题我们需要报有一种追根问题的心态，就是一定要包某个问题吃透，了解这个问题为什么会产生，原因是什么，然后才能从根本上解决问题