You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Eli (JIRA)" <ji...@apache.org> on 2018/12/06 10:51:00 UTC
[jira] [Commented] (SPARK-18147) Broken Spark SQL Codegen
[ https://issues.apache.org/jira/browse/SPARK-18147?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16711268#comment-16711268 ]
Eli commented on SPARK-18147:
-----------------------------
I hava a similar problem at version 2.3.0 . but it work well in 2.0.1 and 2.1.0 ..
{code:java}
2018-12-06 18:45:45,945 ERROR (org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator:91) - failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 250, Column 21: Expression "project_isNull22" is not an rvalue
org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 250, Column 21: Expression "project_isNull22" is not an rvalue
at org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11821)
at org.codehaus.janino.UnitCompiler.toRvalueOrCompileException(UnitCompiler.java:7170)
at org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5332)
at org.codehaus.janino.UnitCompiler.access$9400(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$13$1.visitAmbiguousName(UnitCompiler.java:5287)
at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:4053)
at org.codehaus.janino.UnitCompiler$13.visitLvalue(UnitCompiler.java:5284)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:3977)
at org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:5280)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2391)
at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1474)
at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1466)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2926)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1532)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1472)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1466)
at org.codehaus.janino.Java$Block.accept(Java.java:2756)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2444)
at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1474)
at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1466)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2926)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1532)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1472)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1466)
at org.codehaus.janino.Java$Block.accept(Java.java:2756)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1532)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1472)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1466)
at org.codehaus.janino.Java$Block.accept(Java.java:2756)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2455)
at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1474)
at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1466)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2926)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1532)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1472)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1466)
at org.codehaus.janino.Java$Block.accept(Java.java:2756)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1571)
at org.codehaus.janino.UnitCompiler.access$2600(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitDoStatement(UnitCompiler.java:1481)
at org.codehaus.janino.UnitCompiler$6.visitDoStatement(UnitCompiler.java:1466)
at org.codehaus.janino.Java$DoStatement.accept(Java.java:3304)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1532)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1472)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1466)
at org.codehaus.janino.Java$Block.accept(Java.java:2756)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1634)
at org.codehaus.janino.UnitCompiler.access$2000(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitForStatement(UnitCompiler.java:1475)
at org.codehaus.janino.UnitCompiler$6.visitForStatement(UnitCompiler.java:1466)
at org.codehaus.janino.Java$ForStatement.accept(Java.java:2973)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1532)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1472)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1466)
at org.codehaus.janino.Java$Block.accept(Java.java:2756)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1821)
at org.codehaus.janino.UnitCompiler.access$2200(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$6.visitWhileStatement(UnitCompiler.java:1477)
at org.codehaus.janino.UnitCompiler$6.visitWhileStatement(UnitCompiler.java:1466)
at org.codehaus.janino.Java$WhileStatement.accept(Java.java:3031)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3075)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1336)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1309)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:799)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:958)
at org.codehaus.janino.UnitCompiler.access$700(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:393)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:385)
at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1286)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:385)
at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1285)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:825)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:411)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:212)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:390)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:385)
at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1405)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:385)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:357)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:234)
at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:446)
at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313)
at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:204)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:80)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1421)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1497)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1494)
at org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:1369)
at org.apache.spark.sql.execution.WholeStageCodegenExec.liftedTree1$1(WholeStageCodegenExec.scala:579)
at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:578)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)
at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:247)
at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:337)
at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:38)
at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:3272)
at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2484)
at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2484)
at org.apache.spark.sql.Dataset$$anonfun$52.apply(Dataset.scala:3253)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:77)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3252)
at org.apache.spark.sql.Dataset.head(Dataset.scala:2484)
at org.apache.spark.sql.Dataset.take(Dataset.scala:2698)
at org.apache.spark.sql.Dataset.showString(Dataset.scala:254)
at org.apache.spark.sql.Dataset.show(Dataset.scala:725)
at org.apache.spark.sql.Dataset.show(Dataset.scala:702)
at com.netentsec.uba.sparksql.SparkSqlRunner.main(SparkSqlRunner.java:211)
2018-12-06 18:45:45,950 INFO (org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator:54) -
/* 001 */ public Object generate(Object[] references) {
/* 002 */ return new GeneratedIteratorForCodegenStage3(references);
/* 003 */ }
/* 004 */
/* 005 */ final class GeneratedIteratorForCodegenStage3 extends org.apache.spark.sql.execution.BufferedRowIterator {
/* 006 */ private Object[] references;
/* 007 */ private scala.collection.Iterator[] inputs;
/* 008 */ private long scan_scanTime;
/* 009 */ private int scan_batchIdx;
/* 010 */ private org.apache.spark.sql.execution.joins.UnsafeHashedRelation bhj_relation;
/* 011 */ private boolean locallimit_stopEarly;
/* 012 */ private int locallimit_count;
/* 013 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder[] scan_mutableStateArray4 = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder[7];
/* 014 */ private org.apache.spark.sql.execution.vectorized.OnHeapColumnVector[] scan_mutableStateArray2 = new org.apache.spark.sql.execution.vectorized.OnHeapColumnVector[3];
/* 015 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] scan_mutableStateArray5 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[7];
/* 016 */ private UnsafeRow[] scan_mutableStateArray3 = new UnsafeRow[7];
/* 017 */ private org.apache.spark.sql.vectorized.ColumnarBatch[] scan_mutableStateArray1 = new org.apache.spark.sql.vectorized.ColumnarBatch[1];
/* 018 */ private scala.collection.Iterator[] scan_mutableStateArray = new scala.collection.Iterator[1];
/* 019 */
/* 020 */ public GeneratedIteratorForCodegenStage3(Object[] references) {
/* 021 */ this.references = references;
/* 022 */ }
/* 023 */
/* 024 */ public void init(int index, scala.collection.Iterator[] inputs) {
/* 025 */ partitionIndex = index;
/* 026 */ this.inputs = inputs;
/* 027 */ wholestagecodegen_init_0();
/* 028 */ wholestagecodegen_init_1();
/* 029 */ wholestagecodegen_init_2();
/* 030 */
/* 031 */ }
/* 032 */
/* 033 */ private void wholestagecodegen_init_0() {
/* 034 */ scan_mutableStateArray[0] = inputs[0];
/* 035 */
/* 036 */ scan_mutableStateArray3[0] = new UnsafeRow(3);
/* 037 */ scan_mutableStateArray4[0] = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(scan_mutableStateArray3[0], 0);
/* 038 */ scan_mutableStateArray5[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(scan_mutableStateArray4[0], 3);
/* 039 */ scan_mutableStateArray3[1] = new UnsafeRow(3);
/* 040 */ scan_mutableStateArray4[1] = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(scan_mutableStateArray3[1], 0);
/* 041 */ scan_mutableStateArray5[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(scan_mutableStateArray4[1], 3);
/* 042 */ scan_mutableStateArray3[2] = new UnsafeRow(5);
/* 043 */ scan_mutableStateArray4[2] = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(scan_mutableStateArray3[2], 32);
/* 044 */ scan_mutableStateArray5[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(scan_mutableStateArray4[2], 5);
/* 045 */
/* 046 */ bhj_relation = ((org.apache.spark.sql.execution.joins.UnsafeHashedRelation) ((org.apache.spark.broadcast.TorrentBroadcast) references[5] /* broadcast */).value()).asReadOnlyCopy();
/* 047 */ incPeakExecutionMemory(bhj_relation.estimatedSize());
/* 048 */
/* 049 */ org.apache.spark.TaskContext$.MODULE$.get().addTaskCompletionListener(new org.apache.spark.util.TaskCompletionListener() {
/* 050 */ @Override
/* 051 */ public void onTaskCompletion(org.apache.spark.TaskContext context) {
/* 052 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[6] /* avgHashProbe */).set(bhj_relation.getAverageProbesPerLookup());
/* 053 */ }
/* 054 */ });
/* 055 */
/* 056 */ }
/* 057 */
/* 058 */ private void scan_nextBatch() throws java.io.IOException {
/* 059 */ long getBatchStart = System.nanoTime();
/* 060 */ if (scan_mutableStateArray[0].hasNext()) {
/* 061 */ scan_mutableStateArray1[0] = (org.apache.spark.sql.vectorized.ColumnarBatch)scan_mutableStateArray[0].next();
/* 062 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(scan_mutableStateArray1[0].numRows());
/* 063 */ scan_batchIdx = 0;
/* 064 */ scan_mutableStateArray2[0] = (org.apache.spark.sql.execution.vectorized.OnHeapColumnVector) scan_mutableStateArray1[0].column(0);
/* 065 */ scan_mutableStateArray2[1] = (org.apache.spark.sql.execution.vectorized.OnHeapColumnVector) scan_mutableStateArray1[0].column(1);
/* 066 */ scan_mutableStateArray2[2] = (org.apache.spark.sql.execution.vectorized.OnHeapColumnVector) scan_mutableStateArray1[0].column(2);
/* 067 */
/* 068 */ }
/* 069 */ scan_scanTime += System.nanoTime() - getBatchStart;
/* 070 */ }
/* 071 */
/* 072 */ private void wholestagecodegen_init_2() {
/* 073 */ scan_mutableStateArray5[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(scan_mutableStateArray4[6], 6);
/* 074 */
/* 075 */ }
/* 076 */
/* 077 */ private void wholestagecodegen_init_1() {
/* 078 */ scan_mutableStateArray3[3] = new UnsafeRow(2);
/* 079 */ scan_mutableStateArray4[3] = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(scan_mutableStateArray3[3], 32);
/* 080 */ scan_mutableStateArray5[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(scan_mutableStateArray4[3], 2);
/* 081 */ scan_mutableStateArray3[4] = new UnsafeRow(8);
/* 082 */ scan_mutableStateArray4[4] = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(scan_mutableStateArray3[4], 64);
/* 083 */ scan_mutableStateArray5[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(scan_mutableStateArray4[4], 8);
/* 084 */ scan_mutableStateArray3[5] = new UnsafeRow(6);
/* 085 */ scan_mutableStateArray4[5] = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(scan_mutableStateArray3[5], 192);
/* 086 */ scan_mutableStateArray5[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(scan_mutableStateArray4[5], 6);
/* 087 */ scan_mutableStateArray3[6] = new UnsafeRow(6);
/* 088 */ scan_mutableStateArray4[6] = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(scan_mutableStateArray3[6], 192);
/* 089 */
/* 090 */ }
/* 091 */
/* 092 */ protected void processNext() throws java.io.IOException {
/* 093 */ if (scan_mutableStateArray1[0] == null) {
/* 094 */ scan_nextBatch();
/* 095 */ }
/* 096 */ while (scan_mutableStateArray1[0] != null) {
/* 097 */ int scan_numRows = scan_mutableStateArray1[0].numRows();
/* 098 */ int scan_localEnd = scan_numRows - scan_batchIdx;
/* 099 */ for (int scan_localIdx = 0; scan_localIdx < scan_localEnd; scan_localIdx++) {
/* 100 */ int scan_rowIdx = scan_batchIdx + scan_localIdx;
/* 101 */ do {
/* 102 */ boolean scan_isNull = scan_mutableStateArray2[0].isNullAt(scan_rowIdx);
/* 103 */ long scan_value = scan_isNull ? -1L : (scan_mutableStateArray2[0].getLong(scan_rowIdx));
/* 104 */
/* 105 */ boolean filter_isNull2 = true;
/* 106 */ long filter_value2 = -1L;
/* 107 */
/* 108 */ boolean filter_isNull4 = false;
/* 109 */ double filter_value4 = -1.0;
/* 110 */ if (false || 600.0D == 0) {
/* 111 */ filter_isNull4 = true;
/* 112 */ } else {
/* 113 */ boolean filter_isNull6 = scan_isNull;
/* 114 */ long filter_value6 = -1L;
/* 115 */ if (!filter_isNull6) {
/* 116 */ filter_value6 = scan_value / 1000000L;
/* 117 */ }
/* 118 */ boolean filter_isNull5 = filter_isNull6;
/* 119 */ double filter_value5 = -1.0;
/* 120 */ if (!filter_isNull6) {
/* 121 */ filter_value5 = (double) filter_value6;
/* 122 */ }
/* 123 */ if (filter_isNull5) {
/* 124 */ filter_isNull4 = true;
/* 125 */ } else {
/* 126 */ filter_value4 = (double)(filter_value5 / 600.0D);
/* 127 */ }
/* 128 */ }
/* 129 */ boolean filter_isNull3 = filter_isNull4;
/* 130 */ long filter_value3 = -1L;
/* 131 */
/* 132 */ if (!filter_isNull4) {
/* 133 */ filter_value3 = (long)(java.lang.Math.floor(filter_value4));
/* 134 */ }
/* 135 */ if (!filter_isNull3) {
/* 136 */ filter_isNull2 = false; // resultCode could change nullability.
/* 137 */ filter_value2 = filter_value3 * 600L;
/* 138 */
/* 139 */ }
/* 140 */ boolean filter_isNull1 = filter_isNull2;
/* 141 */ UTF8String filter_value1 = null;
/* 142 */ if (!filter_isNull1) {
/* 143 */ try {
/* 144 */ filter_value1 = UTF8String.fromString(((java.text.DateFormat) references[3] /* formatter */).format(
/* 145 */ new java.util.Date(filter_value2 * 1000L)));
/* 146 */ } catch (java.lang.IllegalArgumentException e) {
/* 147 */ filter_isNull1 = true;
/* 148 */ }
/* 149 */ }
/* 150 */ if (!(!(filter_isNull1))) continue;
/* 151 */
/* 152 */ boolean scan_isNull2 = scan_mutableStateArray2[2].isNullAt(scan_rowIdx);
/* 153 */ long scan_value2 = scan_isNull2 ? -1L : (scan_mutableStateArray2[2].getLong(scan_rowIdx));
/* 154 */
/* 155 */ if (!(!(scan_isNull2))) continue;
/* 156 */
/* 157 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[2] /* numOutputRows */).add(1);
/* 158 */
/* 159 */ boolean project_isNull6 = true;
/* 160 */ long project_value6 = -1L;
/* 161 */
/* 162 */ boolean project_isNull8 = false;
/* 163 */ double project_value8 = -1.0;
/* 164 */ if (false || 600.0D == 0) {
/* 165 */ project_isNull8 = true;
/* 166 */ } else {
/* 167 */ boolean project_isNull10 = scan_isNull;
/* 168 */ long project_value10 = -1L;
/* 169 */ if (!project_isNull10) {
/* 170 */ project_value10 = scan_value / 1000000L;
/* 171 */ }
/* 172 */ boolean project_isNull9 = project_isNull10;
/* 173 */ double project_value9 = -1.0;
/* 174 */ if (!project_isNull10) {
/* 175 */ project_value9 = (double) project_value10;
/* 176 */ }
/* 177 */ if (project_isNull9) {
/* 178 */ project_isNull8 = true;
/* 179 */ } else {
/* 180 */ project_value8 = (double)(project_value9 / 600.0D);
/* 181 */ }
/* 182 */ }
/* 183 */ boolean project_isNull7 = project_isNull8;
/* 184 */ long project_value7 = -1L;
/* 185 */
/* 186 */ if (!project_isNull8) {
/* 187 */ project_value7 = (long)(java.lang.Math.floor(project_value8));
/* 188 */ }
/* 189 */ if (!project_isNull7) {
/* 190 */ project_isNull6 = false; // resultCode could change nullability.
/* 191 */ project_value6 = project_value7 * 600L;
/* 192 */
/* 193 */ }
/* 194 */ boolean project_isNull5 = project_isNull6;
/* 195 */ UTF8String project_value5 = null;
/* 196 */ if (!project_isNull5) {
/* 197 */ try {
/* 198 */ project_value5 = UTF8String.fromString(((java.text.DateFormat) references[4] /* formatter */).format(
/* 199 */ new java.util.Date(project_value6 * 1000L)));
/* 200 */ } catch (java.lang.IllegalArgumentException e) {
/* 201 */ project_isNull5 = true;
/* 202 */ }
/* 203 */ }
/* 204 */
/* 205 */ // generate join key for stream side
/* 206 */
/* 207 */ scan_mutableStateArray4[3].reset();
/* 208 */
/* 209 */ scan_mutableStateArray5[3].zeroOutNullBytes();
/* 210 */
/* 211 */ scan_mutableStateArray5[3].write(0, scan_value2);
/* 212 */
/* 213 */ if (project_isNull5) {
/* 214 */ scan_mutableStateArray5[3].setNullAt(1);
/* 215 */ } else {
/* 216 */ scan_mutableStateArray5[3].write(1, project_value5);
/* 217 */ }
/* 218 */ scan_mutableStateArray3[3].setTotalSize(scan_mutableStateArray4[3].totalSize());
/* 219 */
/* 220 */ // find matches from HashedRelation
/* 221 */ UnsafeRow bhj_matched = scan_mutableStateArray3[3].anyNull() ? null: (UnsafeRow)bhj_relation.getValue(scan_mutableStateArray3[3]);
/* 222 */ if (bhj_matched != null) {
/* 223 */ {
/* 224 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[7] /* numOutputRows */).add(1);
/* 225 */
/* 226 */ if (locallimit_count < 21) {
/* 227 */ locallimit_count += 1;
/* 228 */
/* 229 */ boolean project_isNull19 = false;
/* 230 */ UTF8String project_value19 = null;
/* 231 */ if (!false) {
/* 232 */ project_value19 = UTF8String.fromString(String.valueOf(scan_value2));
/* 233 */ }
/* 234 */ scan_mutableStateArray4[6].reset();
/* 235 */
/* 236 */ scan_mutableStateArray5[6].zeroOutNullBytes();
/* 237 */
/* 238 */ if (project_isNull19) {
/* 239 */ scan_mutableStateArray5[6].setNullAt(0);
/* 240 */ } else {
/* 241 */ scan_mutableStateArray5[6].write(0, project_value19);
/* 242 */ }
/* 243 */
/* 244 */ if (project_isNull5) {
/* 245 */ scan_mutableStateArray5[6].setNullAt(1);
/* 246 */ } else {
/* 247 */ scan_mutableStateArray5[6].write(1, project_value5);
/* 248 */ }
/* 249 */
/* 250 */ if (project_isNull22) {
/* 251 */ scan_mutableStateArray5[6].setNullAt(2);
/* 252 */ } else {
/* 253 */ scan_mutableStateArray5[6].write(2, project_value22);
/* 254 */ }
/* 255 */
/* 256 */ if (project_isNull24) {
/* 257 */ scan_mutableStateArray5[6].setNullAt(3);
/* 258 */ } else {
/* 259 */ scan_mutableStateArray5[6].write(3, project_value24);
/* 260 */ }
/* 261 */
/* 262 */ if (project_isNull26) {
/* 263 */ scan_mutableStateArray5[6].setNullAt(4);
/* 264 */ } else {
/* 265 */ scan_mutableStateArray5[6].write(4, project_value26);
/* 266 */ }
/* 267 */
/* 268 */ if (project_isNull28) {
/* 269 */ scan_mutableStateArray5[6].setNullAt(5);
/* 270 */ } else {
/* 271 */ scan_mutableStateArray5[6].write(5, project_value28);
/* 272 */ }
/* 273 */ scan_mutableStateArray3[6].setTotalSize(scan_mutableStateArray4[6].totalSize());
/* 274 */ append(scan_mutableStateArray3[6]);
/* 275 */
/* 276 */ } else {
/* 277 */ locallimit_stopEarly = true;
/* 278 */ }
/* 279 */
/* 280 */ }
/* 281 */ }
/* 282 */
/* 283 */ } while(false);
/* 284 */ if (shouldStop()) { scan_batchIdx = scan_rowIdx + 1; return; }
/* 285 */ }
/* 286 */ scan_batchIdx = scan_numRows;
/* 287 */ scan_mutableStateArray1[0] = null;
/* 288 */ scan_nextBatch();
/* 289 */ }
/* 290 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[1] /* scanTime */).add(scan_scanTime / (1000 * 1000));
/* 291 */ scan_scanTime = 0;
/* 292 */ }
/* 293 */
/* 294 */ @Override
/* 295 */ protected boolean stopEarly() {
/* 296 */ return locallimit_stopEarly;
/* 297 */ }
/* 298 */
/* 299 */ }
{code}
> Broken Spark SQL Codegen
> ------------------------
>
> Key: SPARK-18147
> URL: https://issues.apache.org/jira/browse/SPARK-18147
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 2.0.1
> Reporter: koert kuipers
> Assignee: Liang-Chi Hsieh
> Priority: Critical
> Fix For: 2.1.0
>
>
> this is me on purpose trying to break spark sql codegen to uncover potential issues, by creating arbitrately complex data structures using primitives, strings, basic collections (map, seq, option), tuples, and case classes.
> first example: nested case classes
> code:
> {noformat}
> class ComplexResultAgg[B: TypeTag, C: TypeTag](val zero: B, result: C) extends Aggregator[Row, B, C] {
> override def reduce(b: B, input: Row): B = b
> override def merge(b1: B, b2: B): B = b1
> override def finish(reduction: B): C = result
> override def bufferEncoder: Encoder[B] = ExpressionEncoder[B]()
> override def outputEncoder: Encoder[C] = ExpressionEncoder[C]()
> }
> case class Struct2(d: Double = 0.0, s1: Seq[Double] = Seq.empty, s2: Seq[Long] = Seq.empty)
> case class Struct3(a: Struct2 = Struct2(), b: Struct2 = Struct2())
> val df1 = Seq(("a", "aa"), ("a", "aa"), ("b", "b"), ("b", null)).toDF("x", "y").groupBy("x").agg(
> new ComplexResultAgg("boo", Struct3()).toColumn
> )
> df1.printSchema
> df1.show
> {noformat}
> the result is:
> {noformat}
> [info] Cause: java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 33, Column 12: Expression "isNull1" is not an rvalue
> [info] /* 001 */ public java.lang.Object generate(Object[] references) {
> [info] /* 002 */ return new SpecificMutableProjection(references);
> [info] /* 003 */ }
> [info] /* 004 */
> [info] /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection {
> [info] /* 006 */
> [info] /* 007 */ private Object[] references;
> [info] /* 008 */ private MutableRow mutableRow;
> [info] /* 009 */ private Object[] values;
> [info] /* 010 */ private java.lang.String errMsg;
> [info] /* 011 */ private Object[] values1;
> [info] /* 012 */ private java.lang.String errMsg1;
> [info] /* 013 */ private boolean[] argIsNulls;
> [info] /* 014 */ private scala.collection.Seq argValue;
> [info] /* 015 */ private java.lang.String errMsg2;
> [info] /* 016 */ private boolean[] argIsNulls1;
> [info] /* 017 */ private scala.collection.Seq argValue1;
> [info] /* 018 */ private java.lang.String errMsg3;
> [info] /* 019 */ private java.lang.String errMsg4;
> [info] /* 020 */ private Object[] values2;
> [info] /* 021 */ private java.lang.String errMsg5;
> [info] /* 022 */ private boolean[] argIsNulls2;
> [info] /* 023 */ private scala.collection.Seq argValue2;
> [info] /* 024 */ private java.lang.String errMsg6;
> [info] /* 025 */ private boolean[] argIsNulls3;
> [info] /* 026 */ private scala.collection.Seq argValue3;
> [info] /* 027 */ private java.lang.String errMsg7;
> [info] /* 028 */ private boolean isNull_0;
> [info] /* 029 */ private InternalRow value_0;
> [info] /* 030 */
> [info] /* 031 */ private void apply_1(InternalRow i) {
> [info] /* 032 */
> [info] /* 033 */ if (isNull1) {
> [info] /* 034 */ throw new RuntimeException(errMsg3);
> [info] /* 035 */ }
> [info] /* 036 */
> [info] /* 037 */ boolean isNull24 = false;
> [info] /* 038 */ final com.tresata.spark.sql.Struct2 value24 = isNull24 ? null : (com.tresata.spark.sql.Struct2) value1.a();
> [info] /* 039 */ isNull24 = value24 == null;
> [info] /* 040 */
> [info] /* 041 */ boolean isNull23 = isNull24;
> [info] /* 042 */ final scala.collection.Seq value23 = isNull23 ? null : (scala.collection.Seq) value24.s2();
> [info] /* 043 */ isNull23 = value23 == null;
> [info] /* 044 */ argIsNulls1[0] = isNull23;
> [info] /* 045 */ argValue1 = value23;
> [info] /* 046 */
> [info] /* 047 */
> [info] /* 048 */
> [info] /* 049 */ boolean isNull22 = false;
> [info] /* 050 */ for (int idx = 0; idx < 1; idx++) {
> [info] /* 051 */ if (argIsNulls1[idx]) { isNull22 = true; break; }
> [info] /* 052 */ }
> [info] /* 053 */
> [info] /* 054 */ final ArrayData value22 = isNull22 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
> [info] /* 055 */ if (isNull22) {
> [info] /* 056 */ values1[2] = null;
> [info] /* 057 */ } else {
> [info] /* 058 */ values1[2] = value22;
> [info] /* 059 */ }
> [info] /* 060 */ }
> [info] /* 061 */
> [info] /* 062 */
> [info] /* 063 */ private void apply1_1(InternalRow i) {
> [info] /* 064 */
> [info] /* 065 */ if (isNull1) {
> [info] /* 066 */ throw new RuntimeException(errMsg7);
> [info] /* 067 */ }
> [info] /* 068 */
> [info] /* 069 */ boolean isNull41 = false;
> [info] /* 070 */ final com.tresata.spark.sql.Struct2 value41 = isNull41 ? null : (com.tresata.spark.sql.Struct2) value1.b();
> [info] /* 071 */ isNull41 = value41 == null;
> [info] /* 072 */
> [info] /* 073 */ boolean isNull40 = isNull41;
> [info] /* 074 */ final scala.collection.Seq value40 = isNull40 ? null : (scala.collection.Seq) value41.s2();
> [info] /* 075 */ isNull40 = value40 == null;
> [info] /* 076 */ argIsNulls3[0] = isNull40;
> [info] /* 077 */ argValue3 = value40;
> [info] /* 078 */
> [info] /* 079 */
> [info] /* 080 */
> [info] /* 081 */ boolean isNull39 = false;
> [info] /* 082 */ for (int idx = 0; idx < 1; idx++) {
> [info] /* 083 */ if (argIsNulls3[idx]) { isNull39 = true; break; }
> [info] /* 084 */ }
> [info] /* 085 */
> [info] /* 086 */ final ArrayData value39 = isNull39 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue3);
> [info] /* 087 */ if (isNull39) {
> [info] /* 088 */ values2[2] = null;
> [info] /* 089 */ } else {
> [info] /* 090 */ values2[2] = value39;
> [info] /* 091 */ }
> [info] /* 092 */ }
> [info] /* 093 */
> [info] /* 094 */
> [info] /* 095 */ private void apply_0(InternalRow i) {
> [info] /* 096 */
> [info] /* 097 */ if (isNull1) {
> [info] /* 098 */ throw new RuntimeException(errMsg1);
> [info] /* 099 */ }
> [info] /* 100 */
> [info] /* 101 */ boolean isNull16 = false;
> [info] /* 102 */ final com.tresata.spark.sql.Struct2 value16 = isNull16 ? null : (com.tresata.spark.sql.Struct2) value1.a();
> [info] /* 103 */ isNull16 = value16 == null;
> [info] /* 104 */
> [info] /* 105 */ boolean isNull15 = isNull16;
> [info] /* 106 */ final double value15 = isNull15 ? -1.0 : value16.d();
> [info] /* 107 */ if (isNull15) {
> [info] /* 108 */ values1[0] = null;
> [info] /* 109 */ } else {
> [info] /* 110 */ values1[0] = value15;
> [info] /* 111 */ }
> [info] /* 112 */ if (isNull1) {
> [info] /* 113 */ throw new RuntimeException(errMsg2);
> [info] /* 114 */ }
> [info] /* 115 */
> [info] /* 116 */ boolean isNull20 = false;
> [info] /* 117 */ final com.tresata.spark.sql.Struct2 value20 = isNull20 ? null : (com.tresata.spark.sql.Struct2) value1.a();
> [info] /* 118 */ isNull20 = value20 == null;
> [info] /* 119 */
> [info] /* 120 */ boolean isNull19 = isNull20;
> [info] /* 121 */ final scala.collection.Seq value19 = isNull19 ? null : (scala.collection.Seq) value20.s1();
> [info] /* 122 */ isNull19 = value19 == null;
> [info] /* 123 */ argIsNulls[0] = isNull19;
> [info] /* 124 */ argValue = value19;
> [info] /* 125 */
> [info] /* 126 */
> [info] /* 127 */
> [info] /* 128 */ boolean isNull18 = false;
> [info] /* 129 */ for (int idx = 0; idx < 1; idx++) {
> [info] /* 130 */ if (argIsNulls[idx]) { isNull18 = true; break; }
> [info] /* 131 */ }
> [info] /* 132 */
> [info] /* 133 */ final ArrayData value18 = isNull18 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue);
> [info] /* 134 */ if (isNull18) {
> [info] /* 135 */ values1[1] = null;
> [info] /* 136 */ } else {
> [info] /* 137 */ values1[1] = value18;
> [info] /* 138 */ }
> [info] /* 139 */ }
> [info] /* 140 */
> [info] /* 141 */
> [info] /* 142 */ private void apply1_0(InternalRow i) {
> [info] /* 143 */
> [info] /* 144 */ if (isNull1) {
> [info] /* 145 */ throw new RuntimeException(errMsg5);
> [info] /* 146 */ }
> [info] /* 147 */
> [info] /* 148 */ boolean isNull33 = false;
> [info] /* 149 */ final com.tresata.spark.sql.Struct2 value33 = isNull33 ? null : (com.tresata.spark.sql.Struct2) value1.b();
> [info] /* 150 */ isNull33 = value33 == null;
> [info] /* 151 */
> [info] /* 152 */ boolean isNull32 = isNull33;
> [info] /* 153 */ final double value32 = isNull32 ? -1.0 : value33.d();
> [info] /* 154 */ if (isNull32) {
> [info] /* 155 */ values2[0] = null;
> [info] /* 156 */ } else {
> [info] /* 157 */ values2[0] = value32;
> [info] /* 158 */ }
> [info] /* 159 */ if (isNull1) {
> [info] /* 160 */ throw new RuntimeException(errMsg6);
> [info] /* 161 */ }
> [info] /* 162 */
> [info] /* 163 */ boolean isNull37 = false;
> [info] /* 164 */ final com.tresata.spark.sql.Struct2 value37 = isNull37 ? null : (com.tresata.spark.sql.Struct2) value1.b();
> [info] /* 165 */ isNull37 = value37 == null;
> [info] /* 166 */
> [info] /* 167 */ boolean isNull36 = isNull37;
> [info] /* 168 */ final scala.collection.Seq value36 = isNull36 ? null : (scala.collection.Seq) value37.s1();
> [info] /* 169 */ isNull36 = value36 == null;
> [info] /* 170 */ argIsNulls2[0] = isNull36;
> [info] /* 171 */ argValue2 = value36;
> [info] /* 172 */
> [info] /* 173 */
> [info] /* 174 */
> [info] /* 175 */ boolean isNull35 = false;
> [info] /* 176 */ for (int idx = 0; idx < 1; idx++) {
> [info] /* 177 */ if (argIsNulls2[idx]) { isNull35 = true; break; }
> [info] /* 178 */ }
> [info] /* 179 */
> [info] /* 180 */ final ArrayData value35 = isNull35 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue2);
> [info] /* 181 */ if (isNull35) {
> [info] /* 182 */ values2[1] = null;
> [info] /* 183 */ } else {
> [info] /* 184 */ values2[1] = value35;
> [info] /* 185 */ }
> [info] /* 186 */ }
> [info] /* 187 */
> [info] /* 188 */
> [info] /* 189 */ public SpecificMutableProjection(Object[] references) {
> [info] /* 190 */ this.references = references;
> [info] /* 191 */ mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericMutableRow(1);
> [info] /* 192 */ this.values = null;
> [info] /* 193 */ this.errMsg = (java.lang.String) references[1];
> [info] /* 194 */ this.values1 = null;
> [info] /* 195 */ this.errMsg1 = (java.lang.String) references[2];
> [info] /* 196 */ argIsNulls = new boolean[1];
> [info] /* 197 */
> [info] /* 198 */ this.errMsg2 = (java.lang.String) references[3];
> [info] /* 199 */ argIsNulls1 = new boolean[1];
> [info] /* 200 */
> [info] /* 201 */ this.errMsg3 = (java.lang.String) references[4];
> [info] /* 202 */ this.errMsg4 = (java.lang.String) references[5];
> [info] /* 203 */ this.values2 = null;
> [info] /* 204 */ this.errMsg5 = (java.lang.String) references[6];
> [info] /* 205 */ argIsNulls2 = new boolean[1];
> [info] /* 206 */
> [info] /* 207 */ this.errMsg6 = (java.lang.String) references[7];
> [info] /* 208 */ argIsNulls3 = new boolean[1];
> [info] /* 209 */
> [info] /* 210 */ this.errMsg7 = (java.lang.String) references[8];
> [info] /* 211 */ this.isNull_0 = true;
> [info] /* 212 */ this.value_0 = null;
> [info] /* 213 */ }
> [info] /* 214 */
> [info] /* 215 */ public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection target(MutableRow row) {
> [info] /* 216 */ mutableRow = row;
> [info] /* 217 */ return this;
> [info] /* 218 */ }
> [info] /* 219 */
> [info] /* 220 */ /* Provide immutable access to the last projected row. */
> [info] /* 221 */ public InternalRow currentValue() {
> [info] /* 222 */ return (InternalRow) mutableRow;
> [info] /* 223 */ }
> [info] /* 224 */
> [info] /* 225 */ public java.lang.Object apply(java.lang.Object _i) {
> [info] /* 226 */ InternalRow i = (InternalRow) _i;
> [info] /* 227 */
> [info] /* 228 */
> [info] /* 229 */
> [info] /* 230 */ Object obj = ((Expression) references[0]).eval(null);
> [info] /* 231 */ org.apache.spark.sql.expressions.Aggregator value2 = (org.apache.spark.sql.expressions.Aggregator) obj;
> [info] /* 232 */
> [info] /* 233 */ boolean isNull4 = i.isNullAt(0);
> [info] /* 234 */ UTF8String value4 = isNull4 ? null : (i.getUTF8String(0));
> [info] /* 235 */
> [info] /* 236 */ boolean isNull3 = isNull4;
> [info] /* 237 */ final java.lang.String value3 = isNull3 ? null : (java.lang.String) value4.toString();
> [info] /* 238 */ isNull3 = value3 == null;
> [info] /* 239 */ boolean isNull1 = false || isNull3;
> [info] /* 240 */ final com.tresata.spark.sql.Struct3 value1 = isNull1 ? null : (com.tresata.spark.sql.Struct3) value2.finish(value3);
> [info] /* 241 */ isNull1 = value1 == null;
> [info] /* 242 */
> [info] /* 243 */ boolean isNull5 = false;
> [info] /* 244 */ InternalRow value5 = null;
> [info] /* 245 */ if (!false && isNull1) {
> [info] /* 246 */
> [info] /* 247 */ final InternalRow value7 = null;
> [info] /* 248 */ isNull5 = true;
> [info] /* 249 */ value5 = value7;
> [info] /* 250 */ } else {
> [info] /* 251 */
> [info] /* 252 */ boolean isNull8 = false;
> [info] /* 253 */ this.values = new Object[2];
> [info] /* 254 */ if (isNull1) {
> [info] /* 255 */ throw new RuntimeException(errMsg);
> [info] /* 256 */ }
> [info] /* 257 */
> [info] /* 258 */ boolean isNull11 = false;
> [info] /* 259 */ final com.tresata.spark.sql.Struct2 value11 = isNull11 ? null : (com.tresata.spark.sql.Struct2) value1.a();
> [info] /* 260 */ isNull11 = value11 == null;
> [info] /* 261 */ boolean isNull9 = false;
> [info] /* 262 */ InternalRow value9 = null;
> [info] /* 263 */ if (!false && isNull11) {
> [info] /* 264 */
> [info] /* 265 */ final InternalRow value13 = null;
> [info] /* 266 */ isNull9 = true;
> [info] /* 267 */ value9 = value13;
> [info] /* 268 */ } else {
> [info] /* 269 */
> [info] /* 270 */ boolean isNull14 = false;
> [info] /* 271 */ values1 = new Object[3];apply_0(i);
> [info] /* 272 */ apply_1(i);
> [info] /* 273 */ final InternalRow value14 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values1);
> [info] /* 274 */ this.values1 = null;
> [info] /* 275 */ isNull9 = isNull14;
> [info] /* 276 */ value9 = value14;
> [info] /* 277 */ }
> [info] /* 278 */ if (isNull9) {
> [info] /* 279 */ values[0] = null;
> [info] /* 280 */ } else {
> [info] /* 281 */ values[0] = value9;
> [info] /* 282 */ }
> [info] /* 283 */ if (isNull1) {
> [info] /* 284 */ throw new RuntimeException(errMsg4);
> [info] /* 285 */ }
> [info] /* 286 */
> [info] /* 287 */ boolean isNull28 = false;
> [info] /* 288 */ final com.tresata.spark.sql.Struct2 value28 = isNull28 ? null : (com.tresata.spark.sql.Struct2) value1.b();
> [info] /* 289 */ isNull28 = value28 == null;
> [info] /* 290 */ boolean isNull26 = false;
> [info] /* 291 */ InternalRow value26 = null;
> [info] /* 292 */ if (!false && isNull28) {
> [info] /* 293 */
> [info] /* 294 */ final InternalRow value30 = null;
> [info] /* 295 */ isNull26 = true;
> [info] /* 296 */ value26 = value30;
> [info] /* 297 */ } else {
> [info] /* 298 */
> [info] /* 299 */ boolean isNull31 = false;
> [info] /* 300 */ values2 = new Object[3];apply1_0(i);
> [info] /* 301 */ apply1_1(i);
> [info] /* 302 */ final InternalRow value31 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values2);
> [info] /* 303 */ this.values2 = null;
> [info] /* 304 */ isNull26 = isNull31;
> [info] /* 305 */ value26 = value31;
> [info] /* 306 */ }
> [info] /* 307 */ if (isNull26) {
> [info] /* 308 */ values[1] = null;
> [info] /* 309 */ } else {
> [info] /* 310 */ values[1] = value26;
> [info] /* 311 */ }
> [info] /* 312 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
> [info] /* 313 */ this.values = null;
> [info] /* 314 */ isNull5 = isNull8;
> [info] /* 315 */ value5 = value8;
> [info] /* 316 */ }
> [info] /* 317 */ this.isNull_0 = isNull5;
> [info] /* 318 */ this.value_0 = value5;
> [info] /* 319 */
> [info] /* 320 */ // copy all the results into MutableRow
> [info] /* 321 */
> [info] /* 322 */ if (!this.isNull_0) {
> [info] /* 323 */ mutableRow.update(0, this.value_0);
> [info] /* 324 */ } else {
> [info] /* 325 */ mutableRow.setNullAt(0);
> [info] /* 326 */ }
> [info] /* 327 */
> [info] /* 328 */ return mutableRow;
> [info] /* 329 */ }
> [info] /* 330 */ }
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org