You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "Rajesh Balamohan (Jira)" <ji...@apache.org> on 2020/02/12 08:01:00 UTC
[jira] [Updated] (HIVE-22879) Optimise jar file loading in
CalcitePlanner
[ https://issues.apache.org/jira/browse/HIVE-22879?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Rajesh Balamohan updated HIVE-22879:
------------------------------------
Description:
{{CalcitePlanner}} internally uses {{org.codehaus.janino.UnitCompiler (calcite dependency)}} and this appears to load the jars in every thread. Need to check if this can be avoided.
Here is an example.
{noformat}
at java.util.zip.ZipFile.getEntry(Native Method)
at java.util.zip.ZipFile.getEntry(ZipFile.java:310)
- locked <0x00000005c1af21c0> (a java.util.jar.JarFile)
at java.util.jar.JarFile.getEntry(JarFile.java:240)
at java.util.jar.JarFile.getJarEntry(JarFile.java:223)
at sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:1005)
at sun.misc.URLClassPath.getResource(URLClassPath.java:212)
at java.net.URLClassLoader$1.run(URLClassLoader.java:365)
at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- locked <0x00000005caa3be88> (a java.lang.Object)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at org.codehaus.janino.ClassLoaderIClassLoader.findIClass(ClassLoaderIClassLoader.java:89)
at org.codehaus.janino.IClassLoader.loadIClass(IClassLoader.java:312)
- locked <0x0000000686136868> (a org.codehaus.janino.ClassLoaderIClassLoader)
at org.codehaus.janino.UnitCompiler.findTypeByName(UnitCompiler.java:8556)
at org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8478)
at org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8471)
at org.codehaus.janino.UnitCompiler.reclassify(UnitCompiler.java:8331)
at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6855)
at org.codehaus.janino.UnitCompiler.access$14200(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6497)
at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6494)
at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:4224)
at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6494)
at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6490)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:4148)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469)
at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116)
at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9026)
at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:7106)
at org.codehaus.janino.UnitCompiler.access$15800(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6517)
at org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6490)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469)
at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116)
at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469)
at org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9237)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9123)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9025)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5062)
at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4423)
at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4396)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5622)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5592)
at org.codehaus.janino.UnitCompiler.access$9700(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4434)
at org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4396)
at org.codehaus.janino.Java$NewInitializedArray.accept(Java.java:5373)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2649)
at org.codehaus.janino.UnitCompiler.access$2800(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1504)
at org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1487)
at org.codehaus.janino.Java$ReturnStatement.accept(Java.java:3563)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3388)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1357)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1330)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:822)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:432)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:411)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:406)
at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1414)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:406)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:378)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:237)
at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:465)
at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313)
at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235)
at org.apache.calcite.rex.RexExecutable.compile(RexExecutable.java:60)
at org.apache.calcite.rex.RexExecutable.<init>(RexExecutable.java:49)
at org.apache.calcite.rex.RexExecutorImpl.reduce(RexExecutorImpl.java:127)
at org.apache.calcite.rex.RexSimplify.simplifyCast(RexSimplify.java:1810)
at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:270)
at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473)
at org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:321)
at org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:313)
at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:288)
at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473)
at org.apache.calcite.rex.RexSimplify.simplifyAnds(RexSimplify.java:463)
at org.apache.calcite.rex.RexUtil.simplifyAnds(RexUtil.java:1748)
at org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:759)
at org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:749)
at org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSubQueryRemoveRule.onMatch(HiveSubQueryRemoveRule.java:115)
at org.apache.calcite.plan.AbstractRelOptPlanner.fireRule(AbstractRelOptPlanner.java:319)
at org.apache.calcite.plan.hep.HepPlanner.applyRule(HepPlanner.java:560)
at org.apache.calcite.plan.hep.HepPlanner.applyRules(HepPlanner.java:419)
at org.apache.calcite.plan.hep.HepPlanner.executeInstruction(HepPlanner.java:256)
at org.apache.calcite.plan.hep.HepInstruction$RuleInstance.execute(HepInstruction.java:127)
at org.apache.calcite.plan.hep.HepPlanner.executeProgram(HepPlanner.java:215)
at org.apache.calcite.plan.hep.HepPlanner.findBestExp(HepPlanner.java:202)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2518)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2484)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.hepPlan(CalcitePlanner.java:2459)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1814)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1731)
at org.apache.calcite.tools.Frameworks.lambda$withPlanner$0(Frameworks.java:130)
at org.apache.calcite.tools.Frameworks$$Lambda$162/385313421.apply(Unknown Source)
at org.apache.calcite.prepare.CalcitePrepareImpl.perform(CalcitePrepareImpl.java:915)
at org.apache.calcite.tools.Frameworks.withPrepare(Frameworks.java:179)
at org.apache.calcite.tools.Frameworks.withPlanner(Frameworks.java:125)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.logicalPlan(CalcitePlanner.java:1492)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:468)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:12488)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:358)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283)
at org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:171)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283)
at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:219)
at org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:103)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:215)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:828)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:774)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:768)
at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:125)
at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:203)
at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:325)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1688)
at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:345)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
{noformat}
was:
{{CalcitePlanner }} internally uses {{org.codehaus.janino.UnitCompiler (calcite dependency)}} and this appears to load the jars in every thread. Need to check if this can be avoided.
Here is an example.
{noformat}
at java.util.zip.ZipFile.getEntry(Native Method)
at java.util.zip.ZipFile.getEntry(ZipFile.java:310)
- locked <0x00000005c1af21c0> (a java.util.jar.JarFile)
at java.util.jar.JarFile.getEntry(JarFile.java:240)
at java.util.jar.JarFile.getJarEntry(JarFile.java:223)
at sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:1005)
at sun.misc.URLClassPath.getResource(URLClassPath.java:212)
at java.net.URLClassLoader$1.run(URLClassLoader.java:365)
at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- locked <0x00000005caa3be88> (a java.lang.Object)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at org.codehaus.janino.ClassLoaderIClassLoader.findIClass(ClassLoaderIClassLoader.java:89)
at org.codehaus.janino.IClassLoader.loadIClass(IClassLoader.java:312)
- locked <0x0000000686136868> (a org.codehaus.janino.ClassLoaderIClassLoader)
at org.codehaus.janino.UnitCompiler.findTypeByName(UnitCompiler.java:8556)
at org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8478)
at org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8471)
at org.codehaus.janino.UnitCompiler.reclassify(UnitCompiler.java:8331)
at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6855)
at org.codehaus.janino.UnitCompiler.access$14200(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6497)
at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6494)
at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:4224)
at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6494)
at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6490)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:4148)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469)
at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116)
at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9026)
at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:7106)
at org.codehaus.janino.UnitCompiler.access$15800(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6517)
at org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6490)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469)
at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116)
at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469)
at org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9237)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9123)
at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9025)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5062)
at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4423)
at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4396)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5622)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5592)
at org.codehaus.janino.UnitCompiler.access$9700(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4434)
at org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4396)
at org.codehaus.janino.Java$NewInitializedArray.accept(Java.java:5373)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2649)
at org.codehaus.janino.UnitCompiler.access$2800(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1504)
at org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1487)
at org.codehaus.janino.Java$ReturnStatement.accept(Java.java:3563)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3388)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1357)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1330)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:822)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:432)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:411)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:406)
at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1414)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:406)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:378)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:237)
at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:465)
at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313)
at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235)
at org.apache.calcite.rex.RexExecutable.compile(RexExecutable.java:60)
at org.apache.calcite.rex.RexExecutable.<init>(RexExecutable.java:49)
at org.apache.calcite.rex.RexExecutorImpl.reduce(RexExecutorImpl.java:127)
at org.apache.calcite.rex.RexSimplify.simplifyCast(RexSimplify.java:1810)
at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:270)
at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473)
at org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:321)
at org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:313)
at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:288)
at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473)
at org.apache.calcite.rex.RexSimplify.simplifyAnds(RexSimplify.java:463)
at org.apache.calcite.rex.RexUtil.simplifyAnds(RexUtil.java:1748)
at org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:759)
at org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:749)
at org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSubQueryRemoveRule.onMatch(HiveSubQueryRemoveRule.java:115)
at org.apache.calcite.plan.AbstractRelOptPlanner.fireRule(AbstractRelOptPlanner.java:319)
at org.apache.calcite.plan.hep.HepPlanner.applyRule(HepPlanner.java:560)
at org.apache.calcite.plan.hep.HepPlanner.applyRules(HepPlanner.java:419)
at org.apache.calcite.plan.hep.HepPlanner.executeInstruction(HepPlanner.java:256)
at org.apache.calcite.plan.hep.HepInstruction$RuleInstance.execute(HepInstruction.java:127)
at org.apache.calcite.plan.hep.HepPlanner.executeProgram(HepPlanner.java:215)
at org.apache.calcite.plan.hep.HepPlanner.findBestExp(HepPlanner.java:202)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2518)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2484)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.hepPlan(CalcitePlanner.java:2459)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1814)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1731)
at org.apache.calcite.tools.Frameworks.lambda$withPlanner$0(Frameworks.java:130)
at org.apache.calcite.tools.Frameworks$$Lambda$162/385313421.apply(Unknown Source)
at org.apache.calcite.prepare.CalcitePrepareImpl.perform(CalcitePrepareImpl.java:915)
at org.apache.calcite.tools.Frameworks.withPrepare(Frameworks.java:179)
at org.apache.calcite.tools.Frameworks.withPlanner(Frameworks.java:125)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.logicalPlan(CalcitePlanner.java:1492)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:468)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:12488)
at org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:358)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283)
at org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:171)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283)
at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:219)
at org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:103)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:215)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:828)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:774)
at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:768)
at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:125)
at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:203)
at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:325)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1688)
at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:345)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
{noformat}
> Optimise jar file loading in CalcitePlanner
> -------------------------------------------
>
> Key: HIVE-22879
> URL: https://issues.apache.org/jira/browse/HIVE-22879
> Project: Hive
> Issue Type: Improvement
> Components: CBO
> Reporter: Rajesh Balamohan
> Priority: Major
>
> {{CalcitePlanner}} internally uses {{org.codehaus.janino.UnitCompiler (calcite dependency)}} and this appears to load the jars in every thread. Need to check if this can be avoided.
> Here is an example.
> {noformat}
> at java.util.zip.ZipFile.getEntry(Native Method)
> at java.util.zip.ZipFile.getEntry(ZipFile.java:310)
> - locked <0x00000005c1af21c0> (a java.util.jar.JarFile)
> at java.util.jar.JarFile.getEntry(JarFile.java:240)
> at java.util.jar.JarFile.getJarEntry(JarFile.java:223)
> at sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:1005)
> at sun.misc.URLClassPath.getResource(URLClassPath.java:212)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:365)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
> - locked <0x00000005caa3be88> (a java.lang.Object)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
> at java.lang.Class.forName0(Native Method)
> at java.lang.Class.forName(Class.java:348)
> at org.codehaus.janino.ClassLoaderIClassLoader.findIClass(ClassLoaderIClassLoader.java:89)
> at org.codehaus.janino.IClassLoader.loadIClass(IClassLoader.java:312)
> - locked <0x0000000686136868> (a org.codehaus.janino.ClassLoaderIClassLoader)
> at org.codehaus.janino.UnitCompiler.findTypeByName(UnitCompiler.java:8556)
> at org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8478)
> at org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8471)
> at org.codehaus.janino.UnitCompiler.reclassify(UnitCompiler.java:8331)
> at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6855)
> at org.codehaus.janino.UnitCompiler.access$14200(UnitCompiler.java:215)
> at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6497)
> at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6494)
> at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:4224)
> at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6494)
> at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6490)
> at org.codehaus.janino.Java$Lvalue.accept(Java.java:4148)
> at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
> at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469)
> at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116)
> at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9026)
> at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:7106)
> at org.codehaus.janino.UnitCompiler.access$15800(UnitCompiler.java:215)
> at org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6517)
> at org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6490)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073)
> at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490)
> at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469)
> at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116)
> at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469)
> at org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9237)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9123)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9025)
> at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5062)
> at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:215)
> at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4423)
> at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4396)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073)
> at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396)
> at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662)
> at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5622)
> at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5592)
> at org.codehaus.janino.UnitCompiler.access$9700(UnitCompiler.java:215)
> at org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4434)
> at org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4396)
> at org.codehaus.janino.Java$NewInitializedArray.accept(Java.java:5373)
> at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396)
> at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2649)
> at org.codehaus.janino.UnitCompiler.access$2800(UnitCompiler.java:215)
> at org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1504)
> at org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1487)
> at org.codehaus.janino.Java$ReturnStatement.accept(Java.java:3563)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
> at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3388)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1357)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1330)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:822)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:432)
> at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:215)
> at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:411)
> at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:406)
> at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1414)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:406)
> at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:378)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:237)
> at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:465)
> at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313)
> at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235)
> at org.apache.calcite.rex.RexExecutable.compile(RexExecutable.java:60)
> at org.apache.calcite.rex.RexExecutable.<init>(RexExecutable.java:49)
> at org.apache.calcite.rex.RexExecutorImpl.reduce(RexExecutorImpl.java:127)
> at org.apache.calcite.rex.RexSimplify.simplifyCast(RexSimplify.java:1810)
> at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:270)
> at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473)
> at org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:321)
> at org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:313)
> at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:288)
> at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473)
> at org.apache.calcite.rex.RexSimplify.simplifyAnds(RexSimplify.java:463)
> at org.apache.calcite.rex.RexUtil.simplifyAnds(RexUtil.java:1748)
> at org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:759)
> at org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:749)
> at org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSubQueryRemoveRule.onMatch(HiveSubQueryRemoveRule.java:115)
> at org.apache.calcite.plan.AbstractRelOptPlanner.fireRule(AbstractRelOptPlanner.java:319)
> at org.apache.calcite.plan.hep.HepPlanner.applyRule(HepPlanner.java:560)
> at org.apache.calcite.plan.hep.HepPlanner.applyRules(HepPlanner.java:419)
> at org.apache.calcite.plan.hep.HepPlanner.executeInstruction(HepPlanner.java:256)
> at org.apache.calcite.plan.hep.HepInstruction$RuleInstance.execute(HepInstruction.java:127)
> at org.apache.calcite.plan.hep.HepPlanner.executeProgram(HepPlanner.java:215)
> at org.apache.calcite.plan.hep.HepPlanner.findBestExp(HepPlanner.java:202)
> at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2518)
> at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2484)
> at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.hepPlan(CalcitePlanner.java:2459)
> at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1814)
> at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1731)
> at org.apache.calcite.tools.Frameworks.lambda$withPlanner$0(Frameworks.java:130)
> at org.apache.calcite.tools.Frameworks$$Lambda$162/385313421.apply(Unknown Source)
> at org.apache.calcite.prepare.CalcitePrepareImpl.perform(CalcitePrepareImpl.java:915)
> at org.apache.calcite.tools.Frameworks.withPrepare(Frameworks.java:179)
> at org.apache.calcite.tools.Frameworks.withPlanner(Frameworks.java:125)
> at org.apache.hadoop.hive.ql.parse.CalcitePlanner.logicalPlan(CalcitePlanner.java:1492)
> at org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:468)
> at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:12488)
> at org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:358)
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283)
> at org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:171)
> at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283)
> at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:219)
> at org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:103)
> at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:215)
> at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:828)
> at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:774)
> at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:768)
> at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:125)
> at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:203)
> at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:325)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1688)
> at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:345)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)