執行一個hive sql時報了一個下面的錯誤,從錯誤堆棧上來看,是在SQL編譯、解析、優化過程中出的錯,還沒有提交到YARN上執行。
從Utilities.getPartitionDesc這句來看是優化的時候,在讀取分區信息時出現了內存溢出。所以想到是不是掃描的表,分區是不是太多了。
果然SQL中的一個表有36萬個分區,這種分區太多的情況應該避免的。
Exception in thread "main" java.lang.OutOfMemoryError: Java heap space
at java.util.Arrays.copyOf(Arrays.java:3332)
at java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.java:137)
at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:121)
at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:421)
at java.lang.StringBuilder.append(StringBuilder.java:136)
at org.apache.hadoop.hive.metastore.MetaStoreUtils.getDDLFromFieldSchema(MetaStoreUtils.java:871)
at org.apache.hadoop.hive.metastore.MetaStoreUtils.getSchema(MetaStoreUtils.java:1088)
at org.apache.hadoop.hive.metastore.MetaStoreUtils.getTableMetadata(MetaStoreUtils.java:881)
at org.apache.hadoop.hive.ql.metadata.Table.getMetadata(Table.java:236)
at org.apache.hadoop.hive.ql.exec.Utilities.getTableDesc(Utilities.java:1351)
at org.apache.hadoop.hive.ql.plan.PartitionDesc.<init>(PartitionDesc.java:86)
at org.apache.hadoop.hive.ql.exec.Utilities.getPartitionDesc(Utilities.java:1368)
at org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils.setMapWork(GenMapRedUtils.java:684)
at org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils.setTaskPlan(GenMapRedUtils.java:463)
at org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils.setTaskPlan(GenMapRedUtils.java:441)
at org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils.initPlan(GenMapRedUtils.java:170)
at org.apache.hadoop.hive.ql.optimizer.GenMRRedSink1.process(GenMRRedSink1.java:81)
at org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(DefaultRuleDispatcher.java:90)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(DefaultGraphWalker.java:95)
at org.apache.hadoop.hive.ql.parse.GenMapRedWalker.walk(GenMapRedWalker.java:54)
at org.apache.hadoop.hive.ql.parse.GenMapRedWalker.walk(GenMapRedWalker.java:65)
at org.apache.hadoop.hive.ql.parse.GenMapRedWalker.walk(GenMapRedWalker.java:65)
at org.apache.hadoop.hive.ql.parse.GenMapRedWalker.walk(GenMapRedWalker.java:65)
at org.apache.hadoop.hive.ql.parse.GenMapRedWalker.walk(GenMapRedWalker.java:65)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.startWalking(DefaultGraphWalker.java:110)
at org.apache.hadoop.hive.ql.parse.MapReduceCompiler.generateTaskTree(MapReduceCompiler.java:322)
at org.apache.hadoop.hive.ql.parse.TaskCompiler.compile(TaskCompiler.java:205)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:10524)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:216)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:237)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:465)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:321)
jobId:58690e4949e84814965f0787b3b8ac0b執行hive sql失敗
java.lang.InterruptedException: 執行失敗:執行hive sql失敗
at com.yixin.bdp.execute.executor.JobExecutor.getCmd(JobExecutor.java:165)
at com.yixin.bdp.execute.executor.JobExecutor.run(JobExecutor.java:46)
at com.yixin.bdp.app.App.main(App.java:14)