You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user-zh@flink.apache.org by stgztsw <st...@126.com> on 2021/01/27 10:10:37 UTC
sql编译代码时超过64kb导致编译失败
sql编译代码时超过64kb导致编译失败,请问有啥办法绕过这个问题吗？尝试过把sql逻辑拆开成多个view,
但是sql逻辑优化的时候还是会合并到一起，无法绕过
Caused by: org.codehaus.janino.InternalCompilerException: Code of method
"map(Ljava/lang/Object;)Ljava/lang/Object;" of class
"ExpressionReducer$3674" grows beyond 64 KB
at org.codehaus.janino.CodeContext.makeSpace(CodeContext.java:1009)
at org.codehaus.janino.CodeContext.write(CodeContext.java:901)
at org.codehaus.janino.UnitCompiler.writeShort(UnitCompiler.java:12195)
at org.codehaus.janino.UnitCompiler.pushConstant(UnitCompiler.java:10660)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5553)
at org.codehaus.janino.UnitCompiler.access$9300(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$16.visitIntegerLiteral(UnitCompiler.java:4423)
at
org.codehaus.janino.UnitCompiler$16.visitIntegerLiteral(UnitCompiler.java:4394)
at org.codehaus.janino.Java$IntegerLiteral.accept(Java.java:5442)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4394)
at org.codehaus.janino.UnitCompiler.fakeCompile(UnitCompiler.java:3719)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5569)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5165)
at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4421)
at
org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4394)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5062)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4394)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5575)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3781)
at org.codehaus.janino.UnitCompiler.access$5900(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$13.visitMethodInvocation(UnitCompiler.java:3760)
at
org.codehaus.janino.UnitCompiler$13.visitMethodInvocation(UnitCompiler.java:3732)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5062)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3732)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2360)
at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1494)
at
org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1487)
at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2871)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1553)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:215)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1493)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1487)
at org.codehaus.janino.Java$Block.accept(Java.java:2776)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2465)
at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1495)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1487)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2947)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3388)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1357)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1330)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:822)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:432)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:215)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:411)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:406)
at
org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1414)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:406)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:378)
... 51 more
--
Sent from: http://apache-flink.147419.n8.nabble.com/
Re: sql编译代码时超过64kb导致编译失败
Posted by stgztsw <st...@126.com>.
这个限制不仅仅在janino这边，我们这边已经尝试去除了janino这边的64kb的限制，但是逻辑运行到jdk这边也出现了同样的限制。目前spark是通过spark.conf.set("spark.sql.codegen.wholeStage",false)来避免类似的问题，所以我觉得最好的方法是可以通过配置来限制一些SQL逻辑计划的优化，避免不同的view的逻辑合并到一起，这样就能通过将拆分sql来将逻辑拆分到不同的function中，从而避免一个类函数太大。
Caused by: java.lang.ClassFormatError: Invalid method Code length 123732 in
class file ExpressionReducer$3674
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
at
org.codehaus.janino.ByteArrayClassLoader.findClass(ByteArrayClassLoader.java:77)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at
org.apache.flink.table.runtime.generated.CompileUtils.doCompile(CompileUtils.java:86)
at
org.apache.flink.table.runtime.generated.CompileUtils.lambda$compile$1(CompileUtils.java:66)
at
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4742)
at
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
at
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2319)
at
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2282)
at
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2197)
... 38 more
--
Sent from: http://apache-flink.147419.n8.nabble.com/
Re: sql编译代码时超过64kb导致编译失败
Posted by Sebastian Liu <li...@gmail.com>.
这个问题需要更改CodeGen部分的代码，你可以把上述ExpressionReducer的问题补充到如下issue中，
我可以帮助一起fix. https://issues.apache.org/jira/browse/FLINK-20898
另外，辛苦也补充一个测试sql示例。
stgztsw <st...@126.com> 于2021年1月27日周三 下午6:10写道：
> sql编译代码时超过64kb导致编译失败,请问有啥办法绕过这个问题吗？尝试过把sql逻辑拆开成多个view,
> 但是sql逻辑优化的时候还是会合并到一起，无法绕过
>
> Caused by: org.codehaus.janino.InternalCompilerException: Code of method
> "map(Ljava/lang/Object;)Ljava/lang/Object;" of class
> "ExpressionReducer$3674" grows beyond 64 KB
> at org.codehaus.janino.CodeContext.makeSpace(CodeContext.java:1009)
> at org.codehaus.janino.CodeContext.write(CodeContext.java:901)
> at
> org.codehaus.janino.UnitCompiler.writeShort(UnitCompiler.java:12195)
> at
> org.codehaus.janino.UnitCompiler.pushConstant(UnitCompiler.java:10660)
> at
> org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5553)
> at
> org.codehaus.janino.UnitCompiler.access$9300(UnitCompiler.java:215)
> at
>
> org.codehaus.janino.UnitCompiler$16.visitIntegerLiteral(UnitCompiler.java:4423)
> at
>
> org.codehaus.janino.UnitCompiler$16.visitIntegerLiteral(UnitCompiler.java:4394)
> at org.codehaus.janino.Java$IntegerLiteral.accept(Java.java:5442)
> at
> org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4394)
> at
> org.codehaus.janino.UnitCompiler.fakeCompile(UnitCompiler.java:3719)
> at
> org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5569)
> at
> org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5165)
> at
> org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:215)
> at
>
> org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4421)
> at
>
> org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4394)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5062)
> at
> org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4394)
> at
> org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5575)
> at
> org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3781)
> at
> org.codehaus.janino.UnitCompiler.access$5900(UnitCompiler.java:215)
> at
>
> org.codehaus.janino.UnitCompiler$13.visitMethodInvocation(UnitCompiler.java:3760)
> at
>
> org.codehaus.janino.UnitCompiler$13.visitMethodInvocation(UnitCompiler.java:3732)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5062)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3732)
> at
> org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2360)
> at
> org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:215)
> at
>
> org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1494)
> at
>
> org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1487)
> at
> org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2871)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
> at
> org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567)
> at
> org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1553)
> at
> org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:215)
> at
> org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1493)
> at
> org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1487)
> at org.codehaus.janino.Java$Block.accept(Java.java:2776)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
> at
> org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2465)
> at
> org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:215)
> at
> org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1495)
> at
> org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1487)
> at org.codehaus.janino.Java$IfStatement.accept(Java.java:2947)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487)
> at
> org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3388)
> at
>
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1357)
> at
>
> org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1330)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:822)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:432)
> at
> org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:215)
> at
>
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:411)
> at
>
> org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:406)
> at
>
> org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1414)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:406)
> at
> org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:378)
> ... 51 more
>
>
>
> --
> Sent from: http://apache-flink.147419.n8.nabble.com/
>
--
*With kind regards
------------------------------------------------------------
Sebastian Liu 刘洋
Institute of Computing Technology, Chinese Academy of Science
Mobile\WeChat: +86—15201613655
E-mail: liuyang0704@gmail.com <li...@gmail.com>
QQ: 3239559*