You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Apache Spark (JIRA)" <ji...@apache.org> on 2017/10/03 04:51:00 UTC
[jira] [Assigned] (SPARK-19984) ERROR codegen.CodeGenerator: failed
to compile: org.codehaus.commons.compiler.CompileException: File
'generated.java'
[ https://issues.apache.org/jira/browse/SPARK-19984?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Apache Spark reassigned SPARK-19984:
------------------------------------
Assignee: Apache Spark
> ERROR codegen.CodeGenerator: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java'
> ---------------------------------------------------------------------------------------------------------------------
>
> Key: SPARK-19984
> URL: https://issues.apache.org/jira/browse/SPARK-19984
> Project: Spark
> Issue Type: Bug
> Components: Optimizer
> Affects Versions: 2.1.0
> Reporter: Andrey Yakovenko
> Assignee: Apache Spark
> Attachments: after_adding_count.txt, before_adding_count.txt
>
>
> I had this error few time on my local hadoop 2.7.3+Spark2.1.0 environment. This is not permanent error, next time i run it could disappear. Unfortunately i don't know how to reproduce the issue. As you can see from the log my logic is pretty complicated.
> Here is a part of log i've got (container_1489514660953_0015_01_000001)
> {code}
> 17/03/16 11:07:04 ERROR codegen.CodeGenerator: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 151, Column 29: A method named "compare" is not declared in any enclosing class nor any supertype, nor through a static import
> /* 001 */ public Object generate(Object[] references) {
> /* 002 */ return new GeneratedIterator(references);
> /* 003 */ }
> /* 004 */
> /* 005 */ final class GeneratedIterator extends org.apache.spark.sql.execution.BufferedRowIterator {
> /* 006 */ private Object[] references;
> /* 007 */ private scala.collection.Iterator[] inputs;
> /* 008 */ private boolean agg_initAgg;
> /* 009 */ private boolean agg_bufIsNull;
> /* 010 */ private long agg_bufValue;
> /* 011 */ private boolean agg_initAgg1;
> /* 012 */ private boolean agg_bufIsNull1;
> /* 013 */ private long agg_bufValue1;
> /* 014 */ private scala.collection.Iterator smj_leftInput;
> /* 015 */ private scala.collection.Iterator smj_rightInput;
> /* 016 */ private InternalRow smj_leftRow;
> /* 017 */ private InternalRow smj_rightRow;
> /* 018 */ private UTF8String smj_value2;
> /* 019 */ private java.util.ArrayList smj_matches;
> /* 020 */ private UTF8String smj_value3;
> /* 021 */ private UTF8String smj_value4;
> /* 022 */ private org.apache.spark.sql.execution.metric.SQLMetric smj_numOutputRows;
> /* 023 */ private UnsafeRow smj_result;
> /* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder smj_holder;
> /* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter smj_rowWriter;
> /* 026 */ private org.apache.spark.sql.execution.metric.SQLMetric agg_numOutputRows;
> /* 027 */ private org.apache.spark.sql.execution.metric.SQLMetric agg_aggTime;
> /* 028 */ private UnsafeRow agg_result;
> /* 029 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder agg_holder;
> /* 030 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter agg_rowWriter;
> /* 031 */ private org.apache.spark.sql.execution.metric.SQLMetric agg_numOutputRows1;
> /* 032 */ private org.apache.spark.sql.execution.metric.SQLMetric agg_aggTime1;
> /* 033 */ private UnsafeRow agg_result1;
> /* 034 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder agg_holder1;
> /* 035 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter agg_rowWriter1;
> /* 036 */
> /* 037 */ public GeneratedIterator(Object[] references) {
> /* 038 */ this.references = references;
> /* 039 */ }
> /* 040 */
> /* 041 */ public void init(int index, scala.collection.Iterator[] inputs) {
> /* 042 */ partitionIndex = index;
> /* 043 */ this.inputs = inputs;
> /* 044 */ wholestagecodegen_init_0();
> /* 045 */ wholestagecodegen_init_1();
> /* 046 */
> /* 047 */ }
> /* 048 */
> /* 049 */ private void wholestagecodegen_init_0() {
> /* 050 */ agg_initAgg = false;
> /* 051 */
> /* 052 */ agg_initAgg1 = false;
> /* 053 */
> /* 054 */ smj_leftInput = inputs[0];
> /* 055 */ smj_rightInput = inputs[1];
> /* 056 */
> /* 057 */ smj_rightRow = null;
> /* 058 */
> /* 059 */ smj_matches = new java.util.ArrayList();
> /* 060 */
> /* 061 */ this.smj_numOutputRows = (org.apache.spark.sql.execution.metric.SQLMetric) references[0];
> /* 062 */ smj_result = new UnsafeRow(2);
> /* 063 */ this.smj_holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(smj_result, 64);
> /* 064 */ this.smj_rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(smj_holder, 2);
> /* 065 */ this.agg_numOutputRows = (org.apache.spark.sql.execution.metric.SQLMetric) references[1];
> /* 066 */ this.agg_aggTime = (org.apache.spark.sql.execution.metric.SQLMetric) references[2];
> /* 067 */ agg_result = new UnsafeRow(1);
> /* 068 */ this.agg_holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(agg_result, 0);
> /* 069 */ this.agg_rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(agg_holder, 1);
> /* 070 */ this.agg_numOutputRows1 = (org.apache.spark.sql.execution.metric.SQLMetric) references[3];
> /* 071 */ this.agg_aggTime1 = (org.apache.spark.sql.execution.metric.SQLMetric) references[4];
> /* 072 */
> /* 073 */ }
> /* 074 */
> /* 075 */ private void agg_doAggregateWithoutKey() throws java.io.IOException {
> /* 076 */ // initialize aggregation buffer
> /* 077 */ agg_bufIsNull = false;
> /* 078 */ agg_bufValue = 0L;
> /* 079 */
> /* 080 */ while (!agg_initAgg1) {
> /* 081 */ agg_initAgg1 = true;
> /* 082 */ long agg_beforeAgg = System.nanoTime();
> /* 083 */ agg_doAggregateWithoutKey1();
> /* 084 */ agg_aggTime.add((System.nanoTime() - agg_beforeAgg) / 1000000);
> /* 085 */
> /* 086 */ // output the result
> /* 087 */
> /* 088 */ agg_numOutputRows.add(1);
> /* 089 */ // do aggregate
> /* 090 */ // common sub-expressions
> /* 091 */
> /* 092 */ // evaluate aggregate function
> /* 093 */ boolean agg_isNull8 = false;
> /* 094 */
> /* 095 */ long agg_value8 = -1L;
> /* 096 */ agg_value8 = agg_bufValue + agg_bufValue1;
> /* 097 */ // update aggregation buffer
> /* 098 */ agg_bufIsNull = false;
> /* 099 */ agg_bufValue = agg_value8;
> /* 100 */ }
> /* 101 */
> /* 102 */ }
> /* 103 */
> /* 104 */ private void agg_doAggregateWithoutKey1() throws java.io.IOException {
> /* 105 */ // initialize aggregation buffer
> /* 106 */ agg_bufIsNull1 = false;
> /* 107 */ agg_bufValue1 = 0L;
> /* 108 */
> /* 109 */ while (findNextInnerJoinRows(smj_leftInput, smj_rightInput)) {
> /* 110 */ int smj_size = smj_matches.size();
> /* 111 */ smj_value4 = smj_leftRow.getUTF8String(0);
> /* 112 */ for (int smj_i = 0; smj_i < smj_size; smj_i ++) {
> /* 113 */ InternalRow smj_rightRow1 = (InternalRow) smj_matches.get(smj_i);
> /* 114 */
> /* 115 */ smj_numOutputRows.add(1);
> /* 116 */
> /* 117 */ // do aggregate
> /* 118 */ // common sub-expressions
> /* 119 */
> /* 120 */ // evaluate aggregate function
> /* 121 */ boolean agg_isNull4 = false;
> /* 122 */
> /* 123 */ long agg_value4 = -1L;
> /* 124 */ agg_value4 = agg_bufValue1 + 1L;
> /* 125 */ // update aggregation buffer
> /* 126 */ agg_bufIsNull1 = false;
> /* 127 */ agg_bufValue1 = agg_value4;
> /* 128 */
> /* 129 */ }
> /* 130 */ if (shouldStop()) return;
> /* 131 */ }
> /* 132 */
> /* 133 */ }
> /* 134 */
> /* 135 */ private boolean findNextInnerJoinRows(
> /* 136 */ scala.collection.Iterator leftIter,
> /* 137 */ scala.collection.Iterator rightIter) {
> /* 138 */ smj_leftRow = null;
> /* 139 */ int comp = 0;
> /* 140 */ while (smj_leftRow == null) {
> /* 141 */ if (!leftIter.hasNext()) return false;
> /* 142 */ smj_leftRow = (InternalRow) leftIter.next();
> /* 143 */
> /* 144 */ if (agg_bufIsNull) {
> /* 145 */ smj_leftRow = null;
> /* 146 */ continue;
> /* 147 */ }
> /* 148 */ if (!smj_matches.isEmpty()) {
> /* 149 */ comp = 0;
> /* 150 */ if (comp == 0) {
> /* 151 */ comp = agg_bufValue.compare(smj_value3);
> /* 152 */ }
> /* 153 */
> /* 154 */ if (comp == 0) {
> /* 155 */ return true;
> /* 156 */ }
> /* 157 */ smj_matches.clear();
> /* 158 */ }
> /* 159 */
> /* 160 */ do {
> /* 161 */ if (smj_rightRow == null) {
> /* 162 */ if (!rightIter.hasNext()) {
> /* 163 */ smj_value3 = agg_bufValue.clone();
> /* 164 */ return !smj_matches.isEmpty();
> /* 165 */ }
> /* 166 */ smj_rightRow = (InternalRow) rightIter.next();
> /* 167 */
> /* 168 */ if (agg_bufIsNull) {
> /* 169 */ smj_rightRow = null;
> /* 170 */ continue;
> /* 171 */ }
> /* 172 */ smj_value2 = agg_bufValue.clone();
> /* 173 */ }
> /* 174 */
> /* 175 */ comp = 0;
> /* 176 */ if (comp == 0) {
> /* 177 */ comp = agg_bufValue.compare(smj_value2);
> /* 178 */ }
> /* 179 */
> /* 180 */ if (comp > 0) {
> /* 181 */ smj_rightRow = null;
> /* 182 */ } else if (comp < 0) {
> /* 183 */ if (!smj_matches.isEmpty()) {
> /* 184 */ smj_value3 = agg_bufValue.clone();
> /* 185 */ return true;
> /* 186 */ }
> /* 187 */ smj_leftRow = null;
> /* 188 */ } else {
> /* 189 */ smj_matches.add(smj_rightRow.copy());
> /* 190 */ smj_rightRow = null;;
> /* 191 */ }
> /* 192 */ } while (smj_leftRow != null);
> /* 193 */ }
> /* 194 */ return false; // unreachable
> /* 195 */ }
> /* 196 */
> /* 197 */ private void wholestagecodegen_init_1() {
> /* 198 */ agg_result1 = new UnsafeRow(1);
> /* 199 */ this.agg_holder1 = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(agg_result1, 0);
> /* 200 */ this.agg_rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(agg_holder1, 1);
> /* 201 */
> /* 202 */ }
> /* 203 */
> /* 204 */ protected void processNext() throws java.io.IOException {
> /* 205 */ while (!agg_initAgg) {
> /* 206 */ agg_initAgg = true;
> /* 207 */ long agg_beforeAgg1 = System.nanoTime();
> /* 208 */ agg_doAggregateWithoutKey();
> /* 209 */ agg_aggTime1.add((System.nanoTime() - agg_beforeAgg1) / 1000000);
> /* 210 */
> /* 211 */ // output the result
> /* 212 */
> /* 213 */ agg_numOutputRows1.add(1);
> /* 214 */ agg_rowWriter1.zeroOutNullBytes();
> /* 215 */
> /* 216 */ if (agg_bufIsNull) {
> /* 217 */ agg_rowWriter1.setNullAt(0);
> /* 218 */ } else {
> /* 219 */ agg_rowWriter1.write(0, agg_bufValue);
> /* 220 */ }
> /* 221 */ append(agg_result1.copy());
> /* 222 */ }
> /* 223 */ }
> /* 224 */ }
> org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 151, Column 29: A method named "compare" is not declared in any enclosing class nor any supertype, nor through a static import
> at org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
> at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8130)
> at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4421)
> at org.codehaus.janino.UnitCompiler.access$7500(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:3774)
> at org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:3762)
> at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:4328)
> at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3762)
> at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4933)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3189)
> at org.codehaus.janino.UnitCompiler.access$5100(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3143)
> at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3139)
> at org.codehaus.janino.Java$Assignment.accept(Java.java:3847)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3139)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2112)
> at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1377)
> at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1370)
> at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2558)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
> at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
> at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
> at org.codehaus.janino.Java$Block.accept(Java.java:2471)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2228)
> at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1378)
> at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1370)
> at org.codehaus.janino.Java$IfStatement.accept(Java.java:2621)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
> at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
> at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
> at org.codehaus.janino.Java$Block.accept(Java.java:2471)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2228)
> at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1378)
> at org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1370)
> at org.codehaus.janino.Java$IfStatement.accept(Java.java:2621)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
> at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
> at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
> at org.codehaus.janino.Java$Block.accept(Java.java:2471)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1725)
> at org.codehaus.janino.UnitCompiler.access$2100(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$6.visitWhileStatement(UnitCompiler.java:1381)
> at org.codehaus.janino.UnitCompiler$6.visitWhileStatement(UnitCompiler.java:1370)
> at org.codehaus.janino.Java$WhileStatement.accept(Java.java:2708)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894)
> at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377)
> at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369)
> at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
> at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
> at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
> at org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
> at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
> at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
> at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311)
> at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:935)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:998)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:995)
> at org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
> at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
> at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
> at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
> at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
> at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
> at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:890)
> at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:357)
> at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114)
> at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114)
> at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:135)
> at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:132)
> at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113)
> at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:225)
> at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:272)
> at org.apache.spark.sql.Dataset$$anonfun$collectAsList$1$$anonfun$apply$11.apply(Dataset.scala:2364)
> at org.apache.spark.sql.Dataset$$anonfun$collectAsList$1$$anonfun$apply$11.apply(Dataset.scala:2363)
> at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)
> at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2765)
> at org.apache.spark.sql.Dataset$$anonfun$collectAsList$1.apply(Dataset.scala:2363)
> at org.apache.spark.sql.Dataset$$anonfun$collectAsList$1.apply(Dataset.scala:2362)
> at org.apache.spark.sql.Dataset.withCallback(Dataset.scala:2778)
> at org.apache.spark.sql.Dataset.collectAsList(Dataset.scala:2362)
> at com.modeln.revvy.calcengine.spark.CMnSparkDataFrameWrapper.repartition(CMnSparkDataFrameWrapper.java:170)
> at com.modeln.revvy.calcengine.spark.handler.CMnJoinOpHandler.process(CMnJoinOpHandler.java:233)
> at com.modeln.revvy.calcengine.spark.handler.CMnJoinOpHandler.process(CMnJoinOpHandler.java:20)
> at com.modeln.revvy.calcengine.spark.handler.CMnOpPipeHandler.process(CMnOpPipeHandler.java:45)
> at com.modeln.revvy.calcengine.spark.handler.CMnOpPipeHandler.process(CMnOpPipeHandler.java:18)
> at com.modeln.revvy.calcengine.spark.handler.CMnSubAssemblyPipeHandler.process(CMnSubAssemblyPipeHandler.java:57)
> at com.modeln.revvy.calcengine.spark.handler.CMnSubAssemblyPipeHandler.process(CMnSubAssemblyPipeHandler.java:16)
> at com.modeln.revvy.calcengine.spark.CMnSparkModelGenerator.generateAndRunSparkModel(CMnSparkModelGenerator.java:173)
> at com.modeln.revvy.calcengine.spark.CMnCalcEngineDriver.main(CMnCalcEngineDriver.java:51)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:637)
> 17/03/16 11:07:04 WARN execution.WholeStageCodegenExec: Whole-stage codegen disabled for this plan:
> *HashAggregate(keys=[], functions=[count(1)], output=[count#4523L])
> +- *HashAggregate(keys=[], functions=[partial_count(1)], output=[count#4714L])
> +- *Project
> +- *SortMergeJoin [SaleId#2724], [right-SaleId#4133], Inner
> :- *Filter isnotnull(SaleId#2724)
> : +- InMemoryTableScan [SaleId#2724], [isnotnull(SaleId#2724)]
> : +- InMemoryRelation [running_total#3481L, invoicePricePerUnit__c#2721, enrollment_account_id#2722, currency__c#2723, SaleId#2724, distributorCustomer__c#2725, projected_tier_attained#3610, purchasingCustomer__c#2726, partNumber__c#2727, saleSubType__c#2728, step2#3520, step3#3541, step1#3500, totalInvoicedAmount__c#2729, overriden_tier_attained#3586, invoiceDate__c#2730, quantity__c#2731L, endCustomer__c#2732, tx_group_count#2733L, quoteId__c#2734, loadDate__c#2735, calc_tier_attained#3563], true, 10000, StorageLevel(disk, memory, deserialized, 1 replicas)
> : +- *Project [running_total#3481L, invoicePricePerUnit__c#2721, enrollment_account_id#2722, currency__c#2723, SaleId#2724, distributorCustomer__c#2725, CASE WHEN (CASE WHEN (quantity__c#2731L > 0) THEN cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(38,1)) ELSE CASE WHEN (((running_total#3481L - quantity__c#2731L) > 0) && ((running_total#3481L - quantity__c#2731L) > 250000)) THEN CASE WHEN (running_total#3481L <= 250000) THEN cast(CheckOverflow((250000.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(38,1)) ELSE cast(cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(38,0)) as decimal(38,1)) END ELSE 0.0 END END > 0.0) THEN 3 WHEN (CASE WHEN (quantity__c#2731L > 0) THEN CASE WHEN (running_total#3481L > 250000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) < 250000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) >= 50000) THEN CheckOverflow((cast(CheckOverflow((250000.0 - cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(23,1)) + cast(cast(quantity__c#2731L as decimal(22,1)) as decimal(23,1))), DecimalType(23,1)) ELSE 200000.0 END ELSE 0.0 END ELSE cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(23,1)) END ELSE cast(CASE WHEN (running_total#3481L > 250000) THEN 0.0 ELSE CASE WHEN (((running_total#3481L - quantity__c#2731L) > 0) && ((running_total#3481L - quantity__c#2731L) > 50000)) THEN CASE WHEN (running_total#3481L <= 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) > 250000) THEN -200000.0 ELSE CheckOverflow((50000.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) END ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 250000) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) ELSE cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(22,1)) END END ELSE 0.0 END END as decimal(23,1)) END > 0.0) THEN 2 WHEN (CASE WHEN (quantity__c#2731L > 0) THEN CASE WHEN (running_total#3481L > 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) < 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) >= 0) THEN CheckOverflow((cast(CheckOverflow((50000.0 - cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(23,1)) + cast(cast(quantity__c#2731L as decimal(22,1)) as decimal(23,1))), DecimalType(23,1)) ELSE 50000.0 END ELSE 0.0 END ELSE cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(23,1)) END ELSE cast(CASE WHEN (running_total#3481L > 50000) THEN 0.0 ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 0) THEN CASE WHEN (running_total#3481L <= 0) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) > 50000) THEN -50000.0 ELSE CheckOverflow((0.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) END ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 50000) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) ELSE cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(22,1)) END END ELSE 0.0 END END as decimal(23,1)) END > 0.0) THEN 1 ELSE -1 END AS projected_tier_attained#3610, purchasingCustomer__c#2726, partNumber__c#2727, saleSubType__c#2728, CASE WHEN (quantity__c#2731L > 0) THEN CASE WHEN (running_total#3481L > 250000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) < 250000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) >= 50000) THEN CheckOverflow((cast(CheckOverflow((250000.0 - cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(23,1)) + cast(cast(quantity__c#2731L as decimal(22,1)) as decimal(23,1))), DecimalType(23,1)) ELSE 200000.0 END ELSE 0.0 END ELSE cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(23,1)) END ELSE cast(CASE WHEN (running_total#3481L > 250000) THEN 0.0 ELSE CASE WHEN (((running_total#3481L - quantity__c#2731L) > 0) && ((running_total#3481L - quantity__c#2731L) > 50000)) THEN CASE WHEN (running_total#3481L <= 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) > 250000) THEN -200000.0 ELSE CheckOverflow((50000.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) END ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 250000) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) ELSE cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(22,1)) END END ELSE 0.0 END END as decimal(23,1)) END AS step2#3520, CASE WHEN (quantity__c#2731L > 0) THEN cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(38,1)) ELSE CASE WHEN (((running_total#3481L - quantity__c#2731L) > 0) && ((running_total#3481L - quantity__c#2731L) > 250000)) THEN CASE WHEN (running_total#3481L <= 250000) THEN cast(CheckOverflow((250000.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(38,1)) ELSE cast(cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(38,0)) as decimal(38,1)) END ELSE 0.0 END END AS step3#3541, CASE WHEN (quantity__c#2731L > 0) THEN CASE WHEN (running_total#3481L > 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) < 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) >= 0) THEN CheckOverflow((cast(CheckOverflow((50000.0 - cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(23,1)) + cast(cast(quantity__c#2731L as decimal(22,1)) as decimal(23,1))), DecimalType(23,1)) ELSE 50000.0 END ELSE 0.0 END ELSE cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(23,1)) END ELSE cast(CASE WHEN (running_total#3481L > 50000) THEN 0.0 ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 0) THEN CASE WHEN (running_total#3481L <= 0) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) > 50000) THEN -50000.0 ELSE CheckOverflow((0.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) END ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 50000) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) ELSE cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(22,1)) END END ELSE 0.0 END END as decimal(23,1)) END AS step1#3500, totalInvoicedAmount__c#2729, CASE WHEN (CASE WHEN (quantity__c#2731L > 0) THEN cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(38,1)) ELSE CASE WHEN (((running_total#3481L - quantity__c#2731L) > 0) && ((running_total#3481L - quantity__c#2731L) > 250000)) THEN CASE WHEN (running_total#3481L <= 250000) THEN cast(CheckOverflow((250000.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(38,1)) ELSE cast(cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(38,0)) as decimal(38,1)) END ELSE 0.0 END END > 0.0) THEN 3 WHEN (CASE WHEN (quantity__c#2731L > 0) THEN CASE WHEN (running_total#3481L > 250000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) < 250000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) >= 50000) THEN CheckOverflow((cast(CheckOverflow((250000.0 - cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(23,1)) + cast(cast(quantity__c#2731L as decimal(22,1)) as decimal(23,1))), DecimalType(23,1)) ELSE 200000.0 END ELSE 0.0 END ELSE cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(23,1)) END ELSE cast(CASE WHEN (running_total#3481L > 250000) THEN 0.0 ELSE CASE WHEN (((running_total#3481L - quantity__c#2731L) > 0) && ((running_total#3481L - quantity__c#2731L) > 50000)) THEN CASE WHEN (running_total#3481L <= 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) > 250000) THEN -200000.0 ELSE CheckOverflow((50000.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) END ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 250000) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) ELSE cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(22,1)) END END ELSE 0.0 END END as decimal(23,1)) END > 0.0) THEN 2 WHEN (CASE WHEN (quantity__c#2731L > 0) THEN CASE WHEN (running_total#3481L > 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) < 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) >= 0) THEN CheckOverflow((cast(CheckOverflow((50000.0 - cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(23,1)) + cast(cast(quantity__c#2731L as decimal(22,1)) as decimal(23,1))), DecimalType(23,1)) ELSE 50000.0 END ELSE 0.0 END ELSE cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(23,1)) END ELSE cast(CASE WHEN (running_total#3481L > 50000) THEN 0.0 ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 0) THEN CASE WHEN (running_total#3481L <= 0) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) > 50000) THEN -50000.0 ELSE CheckOverflow((0.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) END ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 50000) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) ELSE cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(22,1)) END END ELSE 0.0 END END as decimal(23,1)) END > 0.0) THEN 1 ELSE -1 END AS overriden_tier_attained#3586, invoiceDate__c#2730, quantity__c#2731L, endCustomer__c#2732, tx_group_count#2733L, quoteId__c#2734, loadDate__c#2735, CASE WHEN (CASE WHEN (quantity__c#2731L > 0) THEN cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(38,1)) ELSE CASE WHEN (((running_total#3481L - quantity__c#2731L) > 0) && ((running_total#3481L - quantity__c#2731L) > 250000)) THEN CASE WHEN (running_total#3481L <= 250000) THEN cast(CheckOverflow((250000.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(38,1)) ELSE cast(cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(38,0)) as decimal(38,1)) END ELSE 0.0 END END > 0.0) THEN 3 WHEN (CASE WHEN (quantity__c#2731L > 0) THEN CASE WHEN (running_total#3481L > 250000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) < 250000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) >= 50000) THEN CheckOverflow((cast(CheckOverflow((250000.0 - cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(23,1)) + cast(cast(quantity__c#2731L as decimal(22,1)) as decimal(23,1))), DecimalType(23,1)) ELSE 200000.0 END ELSE 0.0 END ELSE cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(23,1)) END ELSE cast(CASE WHEN (running_total#3481L > 250000) THEN 0.0 ELSE CASE WHEN (((running_total#3481L - quantity__c#2731L) > 0) && ((running_total#3481L - quantity__c#2731L) > 50000)) THEN CASE WHEN (running_total#3481L <= 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) > 250000) THEN -200000.0 ELSE CheckOverflow((50000.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) END ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 250000) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 250000.0), DecimalType(22,1)) ELSE cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(22,1)) END END ELSE 0.0 END END as decimal(23,1)) END > 0.0) THEN 2 WHEN (CASE WHEN (quantity__c#2731L > 0) THEN CASE WHEN (running_total#3481L > 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) < 50000) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) >= 0) THEN CheckOverflow((cast(CheckOverflow((50000.0 - cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) as decimal(23,1)) + cast(cast(quantity__c#2731L as decimal(22,1)) as decimal(23,1))), DecimalType(23,1)) ELSE 50000.0 END ELSE 0.0 END ELSE cast(CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) >= cast(quantity__c#2731L as decimal(22,1))) THEN cast(quantity__c#2731L as decimal(22,1)) ELSE CASE WHEN (CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) > 0.0) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 0.0), DecimalType(22,1)) ELSE 0.0 END END as decimal(23,1)) END ELSE cast(CASE WHEN (running_total#3481L > 50000) THEN 0.0 ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 0) THEN CASE WHEN (running_total#3481L <= 0) THEN CASE WHEN ((running_total#3481L - quantity__c#2731L) > 50000) THEN -50000.0 ELSE CheckOverflow((0.0 - cast(cast((running_total#3481L - quantity__c#2731L) as decimal(20,0)) as decimal(22,1))), DecimalType(22,1)) END ELSE CASE WHEN ((running_total#3481L - quantity__c#2731L) > 50000) THEN CheckOverflow((cast(cast(running_total#3481L as decimal(20,0)) as decimal(22,1)) - 50000.0), DecimalType(22,1)) ELSE cast((running_total#3481L - (running_total#3481L - quantity__c#2731L)) as decimal(22,1)) END END ELSE 0.0 END END as decimal(23,1)) END > 0.0) THEN 1 ELSE -1 END AS calc_tier_attained#3563]
> : +- Window [sum(measure_expr#3462L) windowspecdefinition(SaleId#2724 ASC NULLS FIRST, ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running_total#3481L], [SaleId#2724 ASC NULLS FIRST]
> : +- *Sort [SaleId#2724 ASC NULLS FIRST], false, 0
> : +- *Project [invoicePricePerUnit__c#2721, enrollment_account_id#2722, currency__c#2723, SaleId#2724, distributorCustomer__c#2725, purchasingCustomer__c#2726, partNumber__c#2727, saleSubType__c#2728, totalInvoicedAmount__c#2729, invoiceDate__c#2730, quantity__c#2731L, endCustomer__c#2732, tx_group_count#2733L, quoteId__c#2734, loadDate__c#2735, quantity__c#2731L AS measure_expr#3462L]
> : +- InMemoryTableScan [distributorCustomer__c#2725, endCustomer__c#2732, SaleId#2724, quoteId__c#2734, partNumber__c#2727, currency__c#2723, enrollment_account_id#2722, invoicePricePerUnit__c#2721, invoiceDate__c#2730, totalInvoicedAmount__c#2729, tx_group_count#2733L, loadDate__c#2735, saleSubType__c#2728, quantity__c#2731L, purchasingCustomer__c#2726]
> : +- InMemoryRelation [invoicePricePerUnit__c#2721, enrollment_account_id#2722, currency__c#2723, SaleId#2724, distributorCustomer__c#2725, purchasingCustomer__c#2726, partNumber__c#2727, saleSubType__c#2728, totalInvoicedAmount__c#2729, invoiceDate__c#2730, quantity__c#2731L, endCustomer__c#2732, tx_group_count#2733L, quoteId__c#2734, loadDate__c#2735], true, 10000, StorageLevel(disk, memory, deserialized, 1 replicas)
> : +- Coalesce 1
> : +- Scan ExistingRDD[invoicePricePerUnit__c#2721,enrollment_account_id#2722,currency__c#2723,SaleId#2724,distributorCustomer__c#2725,purchasingCustomer__c#2726,partNumber__c#2727,saleSubType__c#2728,totalInvoicedAmount__c#2729,invoiceDate__c#2730,quantity__c#2731L,endCustomer__c#2732,tx_group_count#2733L,quoteId__c#2734,loadDate__c#2735]
> +- *Sort [right-SaleId#4133 ASC NULLS FIRST], false, 0
> +- *Project [SaleId#2068 AS right-SaleId#4133]
> +- *Filter isnotnull(SaleId#2068)
> +- InMemoryTableScan [SaleId#2068], [isnotnull(SaleId#2068)]
> +- InMemoryRelation [invoicePricePerUnit__c#2065, enrollment_account_id#2066, currency__c#2067, SaleId#2068, distributorCustomer__c#2069, purchasingCustomer__c#2070, partNumber__c#2071, saleSubType__c#2072, totalInvoicedAmount__c#2073, invoiceDate__c#2074, quantity__c#2075L, endCustomer__c#2076, tx_group_count#2077L, quoteId__c#2078, loadDate__c#2079], true, 10000, StorageLevel(disk, memory, deserialized, 1 replicas)
> +- Coalesce 1
> +- Scan ExistingRDD[invoicePricePerUnit__c#2065,enrollment_account_id#2066,currency__c#2067,SaleId#2068,distributorCustomer__c#2069,purchasingCustomer__c#2070,partNumber__c#2071,saleSubType__c#2072,totalInvoicedAmount__c#2073,invoiceDate__c#2074,quantity__c#2075L,endCustomer__c#2076,tx_group_count#2077L,quoteId__c#2078,loadDate__c#2079]
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org