You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "indraneel rao (JIRA)" <ji...@apache.org> on 2017/07/12 16:28:00 UTC
[jira] [Updated] (SPARK-21391) Cannot convert a Seq of Map whose
value type is again a seq, into a dataset
[ https://issues.apache.org/jira/browse/SPARK-21391?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
indraneel rao updated SPARK-21391:
----------------------------------
Description:
There is an error while trying to create a dataset from a sequence of Maps, whose values have any kind of collections. Even when they are wrapped in a case class.
Eg : The following piece of code throws an error:
{code:java}
case class Values(values: Option[Seq[Double]])
case class ItemProperties(properties:Map[String,Values])
val l1 = List(ItemProperties(
Map(
"A1" -> Values(Some(Seq(1.0,2.0))),
"B1" -> Values(Some(Seq(44.0,55.0)))
)
),
ItemProperties(
Map(
"A2" -> Values(Some(Seq(123.0,25.0))),
"B2" -> Values(Some(Seq(445.0,35.0)))
)
)
)
l1.toDS().show()
{code}
Heres the error:
17/07/12 21:49:31 ERROR CodeGenerator: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private boolean resultIsNull;
/* 009 */ private java.lang.String argValue;
/* 010 */ private Object[] values;
/* 011 */ private boolean resultIsNull1;
/* 012 */ private scala.collection.Seq argValue1;
/* 013 */ private boolean isNull12;
/* 014 */ private boolean value12;
/* 015 */ private boolean isNull13;
/* 016 */ private InternalRow value13;
/* 017 */ private boolean isNull14;
/* 018 */ private InternalRow value14;
/* 019 */ private UnsafeRow result;
/* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
/* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
/* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
/* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
/* 026 */
/* 027 */ public SpecificUnsafeProjection(Object[] references) {
/* 028 */ this.references = references;
/* 029 */
/* 030 */
/* 031 */ this.values = null;
/* 032 */
/* 033 */
/* 034 */ isNull12 = false;
/* 035 */ value12 = false;
/* 036 */ isNull13 = false;
/* 037 */ value13 = null;
/* 038 */ isNull14 = false;
/* 039 */ value14 = null;
/* 040 */ result = new UnsafeRow(1);
/* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
/* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 047 */
/* 048 */ }
/* 049 */
/* 050 */ public void initialize(int partitionIndex) {
/* 051 */
/* 052 */ }
/* 053 */
/* 054 */
/* 055 */ private void evalIfTrueExpr(InternalRow i) {
/* 056 */ final InternalRow value7 = null;
/* 057 */ isNull13 = true;
/* 058 */ value13 = value7;
/* 059 */ }
/* 060 */
/* 061 */
/* 062 */ private void evalIfCondExpr(InternalRow i) {
/* 063 */
/* 064 */ isNull12 = false;
/* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
/* 066 */ }
/* 067 */
/* 068 */
/* 069 */ private void evalIfFalseExpr(InternalRow i) {
/* 070 */ values = new Object[1];
/* 071 */ resultIsNull1 = false;
/* 072 */ if (!resultIsNull1) {
/* 073 */
/* 074 */ boolean isNull11 = true;
/* 075 */ scala.Option value11 = null;
/* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
/* 077 */
/* 078 */ isNull11 = false;
/* 079 */ if (!isNull11) {
/* 080 */
/* 081 */ Object funcResult1 = null;
/* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
/* 083 */ if (funcResult1 == null) {
/* 084 */ isNull11 = true;
/* 085 */ } else {
/* 086 */ value11 = (scala.Option) funcResult1;
/* 087 */ }
/* 088 */
/* 089 */ }
/* 090 */ isNull11 = value11 == null;
/* 091 */ }
/* 092 */
/* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
/* 094 */ scala.collection.Seq value10 = isNull10 ?
/* 095 */ null : (scala.collection.Seq) value11.get();
/* 096 */ resultIsNull1 = isNull10;
/* 097 */ argValue1 = value10;
/* 098 */ }
/* 099 */
/* 100 */
/* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
/* 102 */ if (resultIsNull1) {
/* 103 */ values[0] = null;
/* 104 */ } else {
/* 105 */ values[0] = value9;
/* 106 */ }
/* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
/* 108 */ this.values = null;
/* 109 */ isNull14 = false;
/* 110 */ value14 = value8;
/* 111 */ }
/* 112 */
/* 113 */
/* 114 */ // Scala.Function1 need this
/* 115 */ public java.lang.Object apply(java.lang.Object row) {
/* 116 */ return apply((InternalRow) row);
/* 117 */ }
/* 118 */
/* 119 */ public UnsafeRow apply(InternalRow i) {
/* 120 */ holder.reset();
/* 121 */
/* 122 */ rowWriter.zeroOutNullBytes();
/* 123 */
/* 124 */
/* 125 */ boolean isNull3 = i.isNullAt(0);
/* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
/* 127 */
/* 128 */ if (isNull3) {
/* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
/* 130 */ }
/* 131 */ boolean isNull1 = true;
/* 132 */ scala.collection.immutable.Map value1 = null;
/* 133 */ if (!false) {
/* 134 */
/* 135 */ isNull1 = false;
/* 136 */ if (!isNull1) {
/* 137 */
/* 138 */ Object funcResult = null;
/* 139 */ funcResult = value3.properties();
/* 140 */ if (funcResult == null) {
/* 141 */ isNull1 = true;
/* 142 */ } else {
/* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
/* 144 */ }
/* 145 */
/* 146 */ }
/* 147 */ isNull1 = value1 == null;
/* 148 */ }
/* 149 */ MapData value = null;
/* 150 */ if (!isNull1) {
/* 151 */ final int length = value1.size();
/* 152 */ final Object[] convertedKeys = new Object[length];
/* 153 */ final Object[] convertedValues = new Object[length];
/* 154 */ int index = 0;
/* 155 */ final scala.collection.Iterator entries = value1.iterator();
/* 156 */ while(entries.hasNext()) {
/* 157 */
/* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
/* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
/* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
/* 161 */
/* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
/* 163 */
/* 164 */
/* 165 */ resultIsNull = false;
/* 166 */ if (!resultIsNull) {
/* 167 */
/* 168 */ resultIsNull = false;
/* 169 */ argValue = ExternalMapToCatalyst_key0;
/* 170 */ }
/* 171 */
/* 172 */ boolean isNull4 = resultIsNull;
/* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
/* 174 */ isNull4 = value4 == null;
/* 175 */ if (isNull4) {
/* 176 */ throw new RuntimeException("Cannot use null as map key!");
/* 177 */ } else {
/* 178 */ convertedKeys[index] = (UTF8String) value4;
/* 179 */ }
/* 180 */
/* 181 */
/* 182 */ evalIfCondExpr(i);
/* 183 */ boolean isNull5 = false;
/* 184 */ InternalRow value5 = null;
/* 185 */ if (!isNull12 && value12) {
/* 186 */ evalIfTrueExpr(i);
/* 187 */ isNull5 = isNull13;
/* 188 */ value5 = value13;
/* 189 */ } else {
/* 190 */ evalIfFalseExpr(i);
/* 191 */ isNull5 = isNull14;
/* 192 */ value5 = value14;
/* 193 */ }
/* 194 */ if (isNull5) {
/* 195 */ convertedValues[index] = null;
/* 196 */ } else {
/* 197 */ convertedValues[index] = (InternalRow) value5;
/* 198 */ }
/* 199 */
/* 200 */ index++;
/* 201 */ }
/* 202 */
/* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
/* 204 */ }
/* 205 */ if (isNull1) {
/* 206 */ rowWriter.setNullAt(0);
/* 207 */ } else {
/* 208 */ // Remember the current cursor so that we can calculate how many bytes are
/* 209 */ // written later.
/* 210 */ final int tmpCursor = holder.cursor;
/* 211 */
/* 212 */ if (value instanceof UnsafeMapData) {
/* 213 */
/* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
/* 215 */ // grow the global buffer before writing data.
/* 216 */ holder.grow(sizeInBytes);
/* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
/* 218 */ holder.cursor += sizeInBytes;
/* 219 */
/* 220 */ } else {
/* 221 */ final ArrayData keys = value.keyArray();
/* 222 */ final ArrayData values1 = value.valueArray();
/* 223 */
/* 224 */ // preserve 8 bytes to write the key array numBytes later.
/* 225 */ holder.grow(8);
/* 226 */ holder.cursor += 8;
/* 227 */
/* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
/* 229 */ final int tmpCursor1 = holder.cursor;
/* 230 */
/* 231 */
/* 232 */ if (keys instanceof UnsafeArrayData) {
/* 233 */
/* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
/* 235 */ // grow the global buffer before writing data.
/* 236 */ holder.grow(sizeInBytes1);
/* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
/* 238 */ holder.cursor += sizeInBytes1;
/* 239 */
/* 240 */ } else {
/* 241 */ final int numElements = keys.numElements();
/* 242 */ arrayWriter.initialize(holder, numElements, 8);
/* 243 */
/* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
/* 245 */ if (keys.isNullAt(index1)) {
/* 246 */ arrayWriter.setNull(index1);
/* 247 */ } else {
/* 248 */ final UTF8String element = keys.getUTF8String(index1);
/* 249 */ arrayWriter.write(index1, element);
/* 250 */ }
/* 251 */ }
/* 252 */ }
/* 253 */
/* 254 */ // Write the numBytes of key array into the first 8 bytes.
/* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
/* 256 */
/* 257 */
/* 258 */ if (values1 instanceof UnsafeArrayData) {
/* 259 */
/* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
/* 261 */ // grow the global buffer before writing data.
/* 262 */ holder.grow(sizeInBytes4);
/* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
/* 264 */ holder.cursor += sizeInBytes4;
/* 265 */
/* 266 */ } else {
/* 267 */ final int numElements1 = values1.numElements();
/* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
/* 269 */
/* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
/* 271 */ if (values1.isNullAt(index2)) {
/* 272 */ arrayWriter1.setNull(index2);
/* 273 */ } else {
/* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
/* 275 */
/* 276 */ final int tmpCursor3 = holder.cursor;
/* 277 */
/* 278 */ if (element1 instanceof UnsafeRow) {
/* 279 */
/* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
/* 281 */ // grow the global buffer before writing data.
/* 282 */ holder.grow(sizeInBytes2);
/* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
/* 284 */ holder.cursor += sizeInBytes2;
/* 285 */
/* 286 */ } else {
/* 287 */ rowWriter1.reset();
/* 288 */
/* 289 */ final ArrayData fieldName = element1.getArray(0);
/* 290 */ if (element1.isNullAt(0)) {
/* 291 */ rowWriter1.setNullAt(0);
/* 292 */ } else {
/* 293 */ // Remember the current cursor so that we can calculate how many bytes are
/* 294 */ // written later.
/* 295 */ final int tmpCursor4 = holder.cursor;
/* 296 */
/* 297 */ if (fieldName instanceof UnsafeArrayData) {
/* 298 */
/* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
/* 300 */ // grow the global buffer before writing data.
/* 301 */ holder.grow(sizeInBytes3);
/* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
/* 303 */ holder.cursor += sizeInBytes3;
/* 304 */
/* 305 */ } else {
/* 306 */ final int numElements2 = fieldName.numElements();
/* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
/* 308 */
/* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
/* 310 */ if (fieldName.isNullAt(index3)) {
/* 311 */ arrayWriter2.setNullDouble(index3);
/* 312 */ } else {
/* 313 */ final double element2 = fieldName.getDouble(index3);
/* 314 */ arrayWriter2.write(index3, element2);
/* 315 */ }
/* 316 */ }
/* 317 */ }
/* 318 */
/* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
/* 320 */ }
/* 321 */ }
/* 322 */
/* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
/* 324 */
/* 325 */ }
/* 326 */ }
/* 327 */ }
/* 328 */
/* 329 */ }
/* 330 */
/* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
/* 332 */ }
/* 333 */ result.setTotalSize(holder.totalSize());
/* 334 */ return result;
/* 335 */ }
/* 336 */ }
org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
at org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
at org.codehaus.janino.UnitCompiler.toRvalueOrCompileException(UnitCompiler.java:6639)
at org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5001)
at org.codehaus.janino.UnitCompiler.access$10500(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$13.visitAmbiguousName(UnitCompiler.java:4984)
at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:3633)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:3563)
at org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4956)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4925)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3189)
at org.codehaus.janino.UnitCompiler.access$5100(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3143)
at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3139)
at org.codehaus.janino.Java$Assignment.accept(Java.java:3847)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3139)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2112)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1377)
at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2558)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894)
at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369)
at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
at org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311)
at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:935)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:998)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:995)
at org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:890)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:405)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:359)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:874)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection$lzycompute(ExpressionEncoder.scala:266)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection(ExpressionEncoder.scala:266)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:290)
at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
at scala.collection.immutable.List.foreach(List.scala:383)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:245)
at scala.collection.immutable.List.map(List.scala:286)
at org.apache.spark.sql.SparkSession.createDataset(SparkSession.scala:429)
at org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:389)
at org.apache.spark.sql.SQLImplicits.localSeqToDatasetHolder(SQLImplicits.scala:173)
at local.TestApp$.main(TestApp.scala:63)
at local.TestApp.main(TestApp.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
Exception in thread "main" java.lang.RuntimeException: Error while encoding: java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private boolean resultIsNull;
/* 009 */ private java.lang.String argValue;
/* 010 */ private Object[] values;
/* 011 */ private boolean resultIsNull1;
/* 012 */ private scala.collection.Seq argValue1;
/* 013 */ private boolean isNull12;
/* 014 */ private boolean value12;
/* 015 */ private boolean isNull13;
/* 016 */ private InternalRow value13;
/* 017 */ private boolean isNull14;
/* 018 */ private InternalRow value14;
/* 019 */ private UnsafeRow result;
/* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
/* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
/* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
/* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
/* 026 */
/* 027 */ public SpecificUnsafeProjection(Object[] references) {
/* 028 */ this.references = references;
/* 029 */
/* 030 */
/* 031 */ this.values = null;
/* 032 */
/* 033 */
/* 034 */ isNull12 = false;
/* 035 */ value12 = false;
/* 036 */ isNull13 = false;
/* 037 */ value13 = null;
/* 038 */ isNull14 = false;
/* 039 */ value14 = null;
/* 040 */ result = new UnsafeRow(1);
/* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
/* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 047 */
/* 048 */ }
/* 049 */
/* 050 */ public void initialize(int partitionIndex) {
/* 051 */
/* 052 */ }
/* 053 */
/* 054 */
/* 055 */ private void evalIfTrueExpr(InternalRow i) {
/* 056 */ final InternalRow value7 = null;
/* 057 */ isNull13 = true;
/* 058 */ value13 = value7;
/* 059 */ }
/* 060 */
/* 061 */
/* 062 */ private void evalIfCondExpr(InternalRow i) {
/* 063 */
/* 064 */ isNull12 = false;
/* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
/* 066 */ }
/* 067 */
/* 068 */
/* 069 */ private void evalIfFalseExpr(InternalRow i) {
/* 070 */ values = new Object[1];
/* 071 */ resultIsNull1 = false;
/* 072 */ if (!resultIsNull1) {
/* 073 */
/* 074 */ boolean isNull11 = true;
/* 075 */ scala.Option value11 = null;
/* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
/* 077 */
/* 078 */ isNull11 = false;
/* 079 */ if (!isNull11) {
/* 080 */
/* 081 */ Object funcResult1 = null;
/* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
/* 083 */ if (funcResult1 == null) {
/* 084 */ isNull11 = true;
/* 085 */ } else {
/* 086 */ value11 = (scala.Option) funcResult1;
/* 087 */ }
/* 088 */
/* 089 */ }
/* 090 */ isNull11 = value11 == null;
/* 091 */ }
/* 092 */
/* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
/* 094 */ scala.collection.Seq value10 = isNull10 ?
/* 095 */ null : (scala.collection.Seq) value11.get();
/* 096 */ resultIsNull1 = isNull10;
/* 097 */ argValue1 = value10;
/* 098 */ }
/* 099 */
/* 100 */
/* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
/* 102 */ if (resultIsNull1) {
/* 103 */ values[0] = null;
/* 104 */ } else {
/* 105 */ values[0] = value9;
/* 106 */ }
/* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
/* 108 */ this.values = null;
/* 109 */ isNull14 = false;
/* 110 */ value14 = value8;
/* 111 */ }
/* 112 */
/* 113 */
/* 114 */ // Scala.Function1 need this
/* 115 */ public java.lang.Object apply(java.lang.Object row) {
/* 116 */ return apply((InternalRow) row);
/* 117 */ }
/* 118 */
/* 119 */ public UnsafeRow apply(InternalRow i) {
/* 120 */ holder.reset();
/* 121 */
/* 122 */ rowWriter.zeroOutNullBytes();
/* 123 */
/* 124 */
/* 125 */ boolean isNull3 = i.isNullAt(0);
/* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
/* 127 */
/* 128 */ if (isNull3) {
/* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
/* 130 */ }
/* 131 */ boolean isNull1 = true;
/* 132 */ scala.collection.immutable.Map value1 = null;
/* 133 */ if (!false) {
/* 134 */
/* 135 */ isNull1 = false;
/* 136 */ if (!isNull1) {
/* 137 */
/* 138 */ Object funcResult = null;
/* 139 */ funcResult = value3.properties();
/* 140 */ if (funcResult == null) {
/* 141 */ isNull1 = true;
/* 142 */ } else {
/* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
/* 144 */ }
/* 145 */
/* 146 */ }
/* 147 */ isNull1 = value1 == null;
/* 148 */ }
/* 149 */ MapData value = null;
/* 150 */ if (!isNull1) {
/* 151 */ final int length = value1.size();
/* 152 */ final Object[] convertedKeys = new Object[length];
/* 153 */ final Object[] convertedValues = new Object[length];
/* 154 */ int index = 0;
/* 155 */ final scala.collection.Iterator entries = value1.iterator();
/* 156 */ while(entries.hasNext()) {
/* 157 */
/* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
/* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
/* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
/* 161 */
/* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
/* 163 */
/* 164 */
/* 165 */ resultIsNull = false;
/* 166 */ if (!resultIsNull) {
/* 167 */
/* 168 */ resultIsNull = false;
/* 169 */ argValue = ExternalMapToCatalyst_key0;
/* 170 */ }
/* 171 */
/* 172 */ boolean isNull4 = resultIsNull;
/* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
/* 174 */ isNull4 = value4 == null;
/* 175 */ if (isNull4) {
/* 176 */ throw new RuntimeException("Cannot use null as map key!");
/* 177 */ } else {
/* 178 */ convertedKeys[index] = (UTF8String) value4;
/* 179 */ }
/* 180 */
/* 181 */
/* 182 */ evalIfCondExpr(i);
/* 183 */ boolean isNull5 = false;
/* 184 */ InternalRow value5 = null;
/* 185 */ if (!isNull12 && value12) {
/* 186 */ evalIfTrueExpr(i);
/* 187 */ isNull5 = isNull13;
/* 188 */ value5 = value13;
/* 189 */ } else {
/* 190 */ evalIfFalseExpr(i);
/* 191 */ isNull5 = isNull14;
/* 192 */ value5 = value14;
/* 193 */ }
/* 194 */ if (isNull5) {
/* 195 */ convertedValues[index] = null;
/* 196 */ } else {
/* 197 */ convertedValues[index] = (InternalRow) value5;
/* 198 */ }
/* 199 */
/* 200 */ index++;
/* 201 */ }
/* 202 */
/* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
/* 204 */ }
/* 205 */ if (isNull1) {
/* 206 */ rowWriter.setNullAt(0);
/* 207 */ } else {
/* 208 */ // Remember the current cursor so that we can calculate how many bytes are
/* 209 */ // written later.
/* 210 */ final int tmpCursor = holder.cursor;
/* 211 */
/* 212 */ if (value instanceof UnsafeMapData) {
/* 213 */
/* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
/* 215 */ // grow the global buffer before writing data.
/* 216 */ holder.grow(sizeInBytes);
/* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
/* 218 */ holder.cursor += sizeInBytes;
/* 219 */
/* 220 */ } else {
/* 221 */ final ArrayData keys = value.keyArray();
/* 222 */ final ArrayData values1 = value.valueArray();
/* 223 */
/* 224 */ // preserve 8 bytes to write the key array numBytes later.
/* 225 */ holder.grow(8);
/* 226 */ holder.cursor += 8;
/* 227 */
/* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
/* 229 */ final int tmpCursor1 = holder.cursor;
/* 230 */
/* 231 */
/* 232 */ if (keys instanceof UnsafeArrayData) {
/* 233 */
/* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
/* 235 */ // grow the global buffer before writing data.
/* 236 */ holder.grow(sizeInBytes1);
/* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
/* 238 */ holder.cursor += sizeInBytes1;
/* 239 */
/* 240 */ } else {
/* 241 */ final int numElements = keys.numElements();
/* 242 */ arrayWriter.initialize(holder, numElements, 8);
/* 243 */
/* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
/* 245 */ if (keys.isNullAt(index1)) {
/* 246 */ arrayWriter.setNull(index1);
/* 247 */ } else {
/* 248 */ final UTF8String element = keys.getUTF8String(index1);
/* 249 */ arrayWriter.write(index1, element);
/* 250 */ }
/* 251 */ }
/* 252 */ }
/* 253 */
/* 254 */ // Write the numBytes of key array into the first 8 bytes.
/* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
/* 256 */
/* 257 */
/* 258 */ if (values1 instanceof UnsafeArrayData) {
/* 259 */
/* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
/* 261 */ // grow the global buffer before writing data.
/* 262 */ holder.grow(sizeInBytes4);
/* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
/* 264 */ holder.cursor += sizeInBytes4;
/* 265 */
/* 266 */ } else {
/* 267 */ final int numElements1 = values1.numElements();
/* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
/* 269 */
/* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
/* 271 */ if (values1.isNullAt(index2)) {
/* 272 */ arrayWriter1.setNull(index2);
/* 273 */ } else {
/* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
/* 275 */
/* 276 */ final int tmpCursor3 = holder.cursor;
/* 277 */
/* 278 */ if (element1 instanceof UnsafeRow) {
/* 279 */
/* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
/* 281 */ // grow the global buffer before writing data.
/* 282 */ holder.grow(sizeInBytes2);
/* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
/* 284 */ holder.cursor += sizeInBytes2;
/* 285 */
/* 286 */ } else {
/* 287 */ rowWriter1.reset();
/* 288 */
/* 289 */ final ArrayData fieldName = element1.getArray(0);
/* 290 */ if (element1.isNullAt(0)) {
/* 291 */ rowWriter1.setNullAt(0);
/* 292 */ } else {
/* 293 */ // Remember the current cursor so that we can calculate how many bytes are
/* 294 */ // written later.
/* 295 */ final int tmpCursor4 = holder.cursor;
/* 296 */
/* 297 */ if (fieldName instanceof UnsafeArrayData) {
/* 298 */
/* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
/* 300 */ // grow the global buffer before writing data.
/* 301 */ holder.grow(sizeInBytes3);
/* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
/* 303 */ holder.cursor += sizeInBytes3;
/* 304 */
/* 305 */ } else {
/* 306 */ final int numElements2 = fieldName.numElements();
/* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
/* 308 */
/* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
/* 310 */ if (fieldName.isNullAt(index3)) {
/* 311 */ arrayWriter2.setNullDouble(index3);
/* 312 */ } else {
/* 313 */ final double element2 = fieldName.getDouble(index3);
/* 314 */ arrayWriter2.write(index3, element2);
/* 315 */ }
/* 316 */ }
/* 317 */ }
/* 318 */
/* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
/* 320 */ }
/* 321 */ }
/* 322 */
/* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
/* 324 */
/* 325 */ }
/* 326 */ }
/* 327 */ }
/* 328 */
/* 329 */ }
/* 330 */
/* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
/* 332 */ }
/* 333 */ result.setTotalSize(holder.totalSize());
/* 334 */ return result;
/* 335 */ }
/* 336 */ }
externalmaptocatalyst(ExternalMapToCatalyst_key0, ObjectType(class java.lang.String), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key0, false, ObjectType(class java.lang.String)), true), ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values), if (isnull(lambdavariable(ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values)))) null else named_struct(values, newInstance(class org.apache.spark.sql.catalyst.util.GenericArrayData)), assertnotnull(input[0, local.ItemProperties, true], top level Product input object).properties) AS properties#0
+- externalmaptocatalyst(ExternalMapToCatalyst_key0, ObjectType(class java.lang.String), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key0, false, ObjectType(class java.lang.String)), true), ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values), if (isnull(lambdavariable(ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values)))) null else named_struct(values, newInstance(class org.apache.spark.sql.catalyst.util.GenericArrayData)), assertnotnull(input[0, local.ItemProperties, true], top level Product input object).properties)
+- assertnotnull(input[0, local.ItemProperties, true], top level Product input object).properties
+- assertnotnull(input[0, local.ItemProperties, true], top level Product input object)
+- input[0, local.ItemProperties, true]
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:293)
at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
at scala.collection.immutable.List.foreach(List.scala:383)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:245)
at scala.collection.immutable.List.map(List.scala:286)
at org.apache.spark.sql.SparkSession.createDataset(SparkSession.scala:429)
at org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:389)
at org.apache.spark.sql.SQLImplicits.localSeqToDatasetHolder(SQLImplicits.scala:173)
at local.TestApp$.main(TestApp.scala:63)
at local.TestApp.main(TestApp.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private boolean resultIsNull;
/* 009 */ private java.lang.String argValue;
/* 010 */ private Object[] values;
/* 011 */ private boolean resultIsNull1;
/* 012 */ private scala.collection.Seq argValue1;
/* 013 */ private boolean isNull12;
/* 014 */ private boolean value12;
/* 015 */ private boolean isNull13;
/* 016 */ private InternalRow value13;
/* 017 */ private boolean isNull14;
/* 018 */ private InternalRow value14;
/* 019 */ private UnsafeRow result;
/* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
/* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
/* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
/* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
/* 026 */
/* 027 */ public SpecificUnsafeProjection(Object[] references) {
/* 028 */ this.references = references;
/* 029 */
/* 030 */
/* 031 */ this.values = null;
/* 032 */
/* 033 */
/* 034 */ isNull12 = false;
/* 035 */ value12 = false;
/* 036 */ isNull13 = false;
/* 037 */ value13 = null;
/* 038 */ isNull14 = false;
/* 039 */ value14 = null;
/* 040 */ result = new UnsafeRow(1);
/* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
/* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 047 */
/* 048 */ }
/* 049 */
/* 050 */ public void initialize(int partitionIndex) {
/* 051 */
/* 052 */ }
/* 053 */
/* 054 */
/* 055 */ private void evalIfTrueExpr(InternalRow i) {
/* 056 */ final InternalRow value7 = null;
/* 057 */ isNull13 = true;
/* 058 */ value13 = value7;
/* 059 */ }
/* 060 */
/* 061 */
/* 062 */ private void evalIfCondExpr(InternalRow i) {
/* 063 */
/* 064 */ isNull12 = false;
/* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
/* 066 */ }
/* 067 */
/* 068 */
/* 069 */ private void evalIfFalseExpr(InternalRow i) {
/* 070 */ values = new Object[1];
/* 071 */ resultIsNull1 = false;
/* 072 */ if (!resultIsNull1) {
/* 073 */
/* 074 */ boolean isNull11 = true;
/* 075 */ scala.Option value11 = null;
/* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
/* 077 */
/* 078 */ isNull11 = false;
/* 079 */ if (!isNull11) {
/* 080 */
/* 081 */ Object funcResult1 = null;
/* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
/* 083 */ if (funcResult1 == null) {
/* 084 */ isNull11 = true;
/* 085 */ } else {
/* 086 */ value11 = (scala.Option) funcResult1;
/* 087 */ }
/* 088 */
/* 089 */ }
/* 090 */ isNull11 = value11 == null;
/* 091 */ }
/* 092 */
/* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
/* 094 */ scala.collection.Seq value10 = isNull10 ?
/* 095 */ null : (scala.collection.Seq) value11.get();
/* 096 */ resultIsNull1 = isNull10;
/* 097 */ argValue1 = value10;
/* 098 */ }
/* 099 */
/* 100 */
/* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
/* 102 */ if (resultIsNull1) {
/* 103 */ values[0] = null;
/* 104 */ } else {
/* 105 */ values[0] = value9;
/* 106 */ }
/* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
/* 108 */ this.values = null;
/* 109 */ isNull14 = false;
/* 110 */ value14 = value8;
/* 111 */ }
/* 112 */
/* 113 */
/* 114 */ // Scala.Function1 need this
/* 115 */ public java.lang.Object apply(java.lang.Object row) {
/* 116 */ return apply((InternalRow) row);
/* 117 */ }
/* 118 */
/* 119 */ public UnsafeRow apply(InternalRow i) {
/* 120 */ holder.reset();
/* 121 */
/* 122 */ rowWriter.zeroOutNullBytes();
/* 123 */
/* 124 */
/* 125 */ boolean isNull3 = i.isNullAt(0);
/* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
/* 127 */
/* 128 */ if (isNull3) {
/* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
/* 130 */ }
/* 131 */ boolean isNull1 = true;
/* 132 */ scala.collection.immutable.Map value1 = null;
/* 133 */ if (!false) {
/* 134 */
/* 135 */ isNull1 = false;
/* 136 */ if (!isNull1) {
/* 137 */
/* 138 */ Object funcResult = null;
/* 139 */ funcResult = value3.properties();
/* 140 */ if (funcResult == null) {
/* 141 */ isNull1 = true;
/* 142 */ } else {
/* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
/* 144 */ }
/* 145 */
/* 146 */ }
/* 147 */ isNull1 = value1 == null;
/* 148 */ }
/* 149 */ MapData value = null;
/* 150 */ if (!isNull1) {
/* 151 */ final int length = value1.size();
/* 152 */ final Object[] convertedKeys = new Object[length];
/* 153 */ final Object[] convertedValues = new Object[length];
/* 154 */ int index = 0;
/* 155 */ final scala.collection.Iterator entries = value1.iterator();
/* 156 */ while(entries.hasNext()) {
/* 157 */
/* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
/* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
/* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
/* 161 */
/* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
/* 163 */
/* 164 */
/* 165 */ resultIsNull = false;
/* 166 */ if (!resultIsNull) {
/* 167 */
/* 168 */ resultIsNull = false;
/* 169 */ argValue = ExternalMapToCatalyst_key0;
/* 170 */ }
/* 171 */
/* 172 */ boolean isNull4 = resultIsNull;
/* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
/* 174 */ isNull4 = value4 == null;
/* 175 */ if (isNull4) {
/* 176 */ throw new RuntimeException("Cannot use null as map key!");
/* 177 */ } else {
/* 178 */ convertedKeys[index] = (UTF8String) value4;
/* 179 */ }
/* 180 */
/* 181 */
/* 182 */ evalIfCondExpr(i);
/* 183 */ boolean isNull5 = false;
/* 184 */ InternalRow value5 = null;
/* 185 */ if (!isNull12 && value12) {
/* 186 */ evalIfTrueExpr(i);
/* 187 */ isNull5 = isNull13;
/* 188 */ value5 = value13;
/* 189 */ } else {
/* 190 */ evalIfFalseExpr(i);
/* 191 */ isNull5 = isNull14;
/* 192 */ value5 = value14;
/* 193 */ }
/* 194 */ if (isNull5) {
/* 195 */ convertedValues[index] = null;
/* 196 */ } else {
/* 197 */ convertedValues[index] = (InternalRow) value5;
/* 198 */ }
/* 199 */
/* 200 */ index++;
/* 201 */ }
/* 202 */
/* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
/* 204 */ }
/* 205 */ if (isNull1) {
/* 206 */ rowWriter.setNullAt(0);
/* 207 */ } else {
/* 208 */ // Remember the current cursor so that we can calculate how many bytes are
/* 209 */ // written later.
/* 210 */ final int tmpCursor = holder.cursor;
/* 211 */
/* 212 */ if (value instanceof UnsafeMapData) {
/* 213 */
/* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
/* 215 */ // grow the global buffer before writing data.
/* 216 */ holder.grow(sizeInBytes);
/* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
/* 218 */ holder.cursor += sizeInBytes;
/* 219 */
/* 220 */ } else {
/* 221 */ final ArrayData keys = value.keyArray();
/* 222 */ final ArrayData values1 = value.valueArray();
/* 223 */
/* 224 */ // preserve 8 bytes to write the key array numBytes later.
/* 225 */ holder.grow(8);
/* 226 */ holder.cursor += 8;
/* 227 */
/* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
/* 229 */ final int tmpCursor1 = holder.cursor;
/* 230 */
/* 231 */
/* 232 */ if (keys instanceof UnsafeArrayData) {
/* 233 */
/* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
/* 235 */ // grow the global buffer before writing data.
/* 236 */ holder.grow(sizeInBytes1);
/* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
/* 238 */ holder.cursor += sizeInBytes1;
/* 239 */
/* 240 */ } else {
/* 241 */ final int numElements = keys.numElements();
/* 242 */ arrayWriter.initialize(holder, numElements, 8);
/* 243 */
/* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
/* 245 */ if (keys.isNullAt(index1)) {
/* 246 */ arrayWriter.setNull(index1);
/* 247 */ } else {
/* 248 */ final UTF8String element = keys.getUTF8String(index1);
/* 249 */ arrayWriter.write(index1, element);
/* 250 */ }
/* 251 */ }
/* 252 */ }
/* 253 */
/* 254 */ // Write the numBytes of key array into the first 8 bytes.
/* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
/* 256 */
/* 257 */
/* 258 */ if (values1 instanceof UnsafeArrayData) {
/* 259 */
/* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
/* 261 */ // grow the global buffer before writing data.
/* 262 */ holder.grow(sizeInBytes4);
/* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
/* 264 */ holder.cursor += sizeInBytes4;
/* 265 */
/* 266 */ } else {
/* 267 */ final int numElements1 = values1.numElements();
/* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
/* 269 */
/* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
/* 271 */ if (values1.isNullAt(index2)) {
/* 272 */ arrayWriter1.setNull(index2);
/* 273 */ } else {
/* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
/* 275 */
/* 276 */ final int tmpCursor3 = holder.cursor;
/* 277 */
/* 278 */ if (element1 instanceof UnsafeRow) {
/* 279 */
/* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
/* 281 */ // grow the global buffer before writing data.
/* 282 */ holder.grow(sizeInBytes2);
/* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
/* 284 */ holder.cursor += sizeInBytes2;
/* 285 */
/* 286 */ } else {
/* 287 */ rowWriter1.reset();
/* 288 */
/* 289 */ final ArrayData fieldName = element1.getArray(0);
/* 290 */ if (element1.isNullAt(0)) {
/* 291 */ rowWriter1.setNullAt(0);
/* 292 */ } else {
/* 293 */ // Remember the current cursor so that we can calculate how many bytes are
/* 294 */ // written later.
/* 295 */ final int tmpCursor4 = holder.cursor;
/* 296 */
/* 297 */ if (fieldName instanceof UnsafeArrayData) {
/* 298 */
/* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
/* 300 */ // grow the global buffer before writing data.
/* 301 */ holder.grow(sizeInBytes3);
/* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
/* 303 */ holder.cursor += sizeInBytes3;
/* 304 */
/* 305 */ } else {
/* 306 */ final int numElements2 = fieldName.numElements();
/* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
/* 308 */
/* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
/* 310 */ if (fieldName.isNullAt(index3)) {
/* 311 */ arrayWriter2.setNullDouble(index3);
/* 312 */ } else {
/* 313 */ final double element2 = fieldName.getDouble(index3);
/* 314 */ arrayWriter2.write(index3, element2);
/* 315 */ }
/* 316 */ }
/* 317 */ }
/* 318 */
/* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
/* 320 */ }
/* 321 */ }
/* 322 */
/* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
/* 324 */
/* 325 */ }
/* 326 */ }
/* 327 */ }
/* 328 */
/* 329 */ }
/* 330 */
/* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
/* 332 */ }
/* 333 */ result.setTotalSize(holder.totalSize());
/* 334 */ return result;
/* 335 */ }
/* 336 */ }
at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306)
at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293)
at org.spark_project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
at org.spark_project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135)
at org.spark_project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410)
at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380)
at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:890)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:405)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:359)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:874)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection$lzycompute(ExpressionEncoder.scala:266)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection(ExpressionEncoder.scala:266)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:290)
... 17 more
Caused by: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private boolean resultIsNull;
/* 009 */ private java.lang.String argValue;
/* 010 */ private Object[] values;
/* 011 */ private boolean resultIsNull1;
/* 012 */ private scala.collection.Seq argValue1;
/* 013 */ private boolean isNull12;
/* 014 */ private boolean value12;
/* 015 */ private boolean isNull13;
/* 016 */ private InternalRow value13;
/* 017 */ private boolean isNull14;
/* 018 */ private InternalRow value14;
/* 019 */ private UnsafeRow result;
/* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
/* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
/* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
/* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
/* 026 */
/* 027 */ public SpecificUnsafeProjection(Object[] references) {
/* 028 */ this.references = references;
/* 029 */
/* 030 */
/* 031 */ this.values = null;
/* 032 */
/* 033 */
/* 034 */ isNull12 = false;
/* 035 */ value12 = false;
/* 036 */ isNull13 = false;
/* 037 */ value13 = null;
/* 038 */ isNull14 = false;
/* 039 */ value14 = null;
/* 040 */ result = new UnsafeRow(1);
/* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
/* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 047 */
/* 048 */ }
/* 049 */
/* 050 */ public void initialize(int partitionIndex) {
/* 051 */
/* 052 */ }
/* 053 */
/* 054 */
/* 055 */ private void evalIfTrueExpr(InternalRow i) {
/* 056 */ final InternalRow value7 = null;
/* 057 */ isNull13 = true;
/* 058 */ value13 = value7;
/* 059 */ }
/* 060 */
/* 061 */
/* 062 */ private void evalIfCondExpr(InternalRow i) {
/* 063 */
/* 064 */ isNull12 = false;
/* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
/* 066 */ }
/* 067 */
/* 068 */
/* 069 */ private void evalIfFalseExpr(InternalRow i) {
/* 070 */ values = new Object[1];
/* 071 */ resultIsNull1 = false;
/* 072 */ if (!resultIsNull1) {
/* 073 */
/* 074 */ boolean isNull11 = true;
/* 075 */ scala.Option value11 = null;
/* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
/* 077 */
/* 078 */ isNull11 = false;
/* 079 */ if (!isNull11) {
/* 080 */
/* 081 */ Object funcResult1 = null;
/* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
/* 083 */ if (funcResult1 == null) {
/* 084 */ isNull11 = true;
/* 085 */ } else {
/* 086 */ value11 = (scala.Option) funcResult1;
/* 087 */ }
/* 088 */
/* 089 */ }
/* 090 */ isNull11 = value11 == null;
/* 091 */ }
/* 092 */
/* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
/* 094 */ scala.collection.Seq value10 = isNull10 ?
/* 095 */ null : (scala.collection.Seq) value11.get();
/* 096 */ resultIsNull1 = isNull10;
/* 097 */ argValue1 = value10;
/* 098 */ }
/* 099 */
/* 100 */
/* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
/* 102 */ if (resultIsNull1) {
/* 103 */ values[0] = null;
/* 104 */ } else {
/* 105 */ values[0] = value9;
/* 106 */ }
/* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
/* 108 */ this.values = null;
/* 109 */ isNull14 = false;
/* 110 */ value14 = value8;
/* 111 */ }
/* 112 */
/* 113 */
/* 114 */ // Scala.Function1 need this
/* 115 */ public java.lang.Object apply(java.lang.Object row) {
/* 116 */ return apply((InternalRow) row);
/* 117 */ }
/* 118 */
/* 119 */ public UnsafeRow apply(InternalRow i) {
/* 120 */ holder.reset();
/* 121 */
/* 122 */ rowWriter.zeroOutNullBytes();
/* 123 */
/* 124 */
/* 125 */ boolean isNull3 = i.isNullAt(0);
/* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
/* 127 */
/* 128 */ if (isNull3) {
/* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
/* 130 */ }
/* 131 */ boolean isNull1 = true;
/* 132 */ scala.collection.immutable.Map value1 = null;
/* 133 */ if (!false) {
/* 134 */
/* 135 */ isNull1 = false;
/* 136 */ if (!isNull1) {
/* 137 */
/* 138 */ Object funcResult = null;
/* 139 */ funcResult = value3.properties();
/* 140 */ if (funcResult == null) {
/* 141 */ isNull1 = true;
/* 142 */ } else {
/* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
/* 144 */ }
/* 145 */
/* 146 */ }
/* 147 */ isNull1 = value1 == null;
/* 148 */ }
/* 149 */ MapData value = null;
/* 150 */ if (!isNull1) {
/* 151 */ final int length = value1.size();
/* 152 */ final Object[] convertedKeys = new Object[length];
/* 153 */ final Object[] convertedValues = new Object[length];
/* 154 */ int index = 0;
/* 155 */ final scala.collection.Iterator entries = value1.iterator();
/* 156 */ while(entries.hasNext()) {
/* 157 */
/* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
/* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
/* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
/* 161 */
/* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
/* 163 */
/* 164 */
/* 165 */ resultIsNull = false;
/* 166 */ if (!resultIsNull) {
/* 167 */
/* 168 */ resultIsNull = false;
/* 169 */ argValue = ExternalMapToCatalyst_key0;
/* 170 */ }
/* 171 */
/* 172 */ boolean isNull4 = resultIsNull;
/* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
/* 174 */ isNull4 = value4 == null;
/* 175 */ if (isNull4) {
/* 176 */ throw new RuntimeException("Cannot use null as map key!");
/* 177 */ } else {
/* 178 */ convertedKeys[index] = (UTF8String) value4;
/* 179 */ }
/* 180 */
/* 181 */
/* 182 */ evalIfCondExpr(i);
/* 183 */ boolean isNull5 = false;
/* 184 */ InternalRow value5 = null;
/* 185 */ if (!isNull12 && value12) {
/* 186 */ evalIfTrueExpr(i);
/* 187 */ isNull5 = isNull13;
/* 188 */ value5 = value13;
/* 189 */ } else {
/* 190 */ evalIfFalseExpr(i);
/* 191 */ isNull5 = isNull14;
/* 192 */ value5 = value14;
/* 193 */ }
/* 194 */ if (isNull5) {
/* 195 */ convertedValues[index] = null;
/* 196 */ } else {
/* 197 */ convertedValues[index] = (InternalRow) value5;
/* 198 */ }
/* 199 */
/* 200 */ index++;
/* 201 */ }
/* 202 */
/* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
/* 204 */ }
/* 205 */ if (isNull1) {
/* 206 */ rowWriter.setNullAt(0);
/* 207 */ } else {
/* 208 */ // Remember the current cursor so that we can calculate how many bytes are
/* 209 */ // written later.
/* 210 */ final int tmpCursor = holder.cursor;
/* 211 */
/* 212 */ if (value instanceof UnsafeMapData) {
/* 213 */
/* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
/* 215 */ // grow the global buffer before writing data.
/* 216 */ holder.grow(sizeInBytes);
/* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
/* 218 */ holder.cursor += sizeInBytes;
/* 219 */
/* 220 */ } else {
/* 221 */ final ArrayData keys = value.keyArray();
/* 222 */ final ArrayData values1 = value.valueArray();
/* 223 */
/* 224 */ // preserve 8 bytes to write the key array numBytes later.
/* 225 */ holder.grow(8);
/* 226 */ holder.cursor += 8;
/* 227 */
/* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
/* 229 */ final int tmpCursor1 = holder.cursor;
/* 230 */
/* 231 */
/* 232 */ if (keys instanceof UnsafeArrayData) {
/* 233 */
/* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
/* 235 */ // grow the global buffer before writing data.
/* 236 */ holder.grow(sizeInBytes1);
/* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
/* 238 */ holder.cursor += sizeInBytes1;
/* 239 */
/* 240 */ } else {
/* 241 */ final int numElements = keys.numElements();
/* 242 */ arrayWriter.initialize(holder, numElements, 8);
/* 243 */
/* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
/* 245 */ if (keys.isNullAt(index1)) {
/* 246 */ arrayWriter.setNull(index1);
/* 247 */ } else {
/* 248 */ final UTF8String element = keys.getUTF8String(index1);
/* 249 */ arrayWriter.write(index1, element);
/* 250 */ }
/* 251 */ }
/* 252 */ }
/* 253 */
/* 254 */ // Write the numBytes of key array into the first 8 bytes.
/* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
/* 256 */
/* 257 */
/* 258 */ if (values1 instanceof UnsafeArrayData) {
/* 259 */
/* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
/* 261 */ // grow the global buffer before writing data.
/* 262 */ holder.grow(sizeInBytes4);
/* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
/* 264 */ holder.cursor += sizeInBytes4;
/* 265 */
/* 266 */ } else {
/* 267 */ final int numElements1 = values1.numElements();
/* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
/* 269 */
/* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
/* 271 */ if (values1.isNullAt(index2)) {
/* 272 */ arrayWriter1.setNull(index2);
/* 273 */ } else {
/* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
/* 275 */
/* 276 */ final int tmpCursor3 = holder.cursor;
/* 277 */
/* 278 */ if (element1 instanceof UnsafeRow) {
/* 279 */
/* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
/* 281 */ // grow the global buffer before writing data.
/* 282 */ holder.grow(sizeInBytes2);
/* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
/* 284 */ holder.cursor += sizeInBytes2;
/* 285 */
/* 286 */ } else {
/* 287 */ rowWriter1.reset();
/* 288 */
/* 289 */ final ArrayData fieldName = element1.getArray(0);
/* 290 */ if (element1.isNullAt(0)) {
/* 291 */ rowWriter1.setNullAt(0);
/* 292 */ } else {
/* 293 */ // Remember the current cursor so that we can calculate how many bytes are
/* 294 */ // written later.
/* 295 */ final int tmpCursor4 = holder.cursor;
/* 296 */
/* 297 */ if (fieldName instanceof UnsafeArrayData) {
/* 298 */
/* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
/* 300 */ // grow the global buffer before writing data.
/* 301 */ holder.grow(sizeInBytes3);
/* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
/* 303 */ holder.cursor += sizeInBytes3;
/* 304 */
/* 305 */ } else {
/* 306 */ final int numElements2 = fieldName.numElements();
/* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
/* 308 */
/* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
/* 310 */ if (fieldName.isNullAt(index3)) {
/* 311 */ arrayWriter2.setNullDouble(index3);
/* 312 */ } else {
/* 313 */ final double element2 = fieldName.getDouble(index3);
/* 314 */ arrayWriter2.write(index3, element2);
/* 315 */ }
/* 316 */ }
/* 317 */ }
/* 318 */
/* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
/* 320 */ }
/* 321 */ }
/* 322 */
/* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
/* 324 */
/* 325 */ }
/* 326 */ }
/* 327 */ }
/* 328 */
/* 329 */ }
/* 330 */
/* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
/* 332 */ }
/* 333 */ result.setTotalSize(holder.totalSize());
/* 334 */ return result;
/* 335 */ }
/* 336 */ }
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:941)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:998)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:995)
at org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
... 30 more
Caused by: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
at org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
at org.codehaus.janino.UnitCompiler.toRvalueOrCompileException(UnitCompiler.java:6639)
at org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5001)
at org.codehaus.janino.UnitCompiler.access$10500(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$13.visitAmbiguousName(UnitCompiler.java:4984)
at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:3633)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:3563)
at org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4956)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4925)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3189)
at org.codehaus.janino.UnitCompiler.access$5100(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3143)
at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3139)
at org.codehaus.janino.Java$Assignment.accept(Java.java:3847)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3139)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2112)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1377)
at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2558)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894)
at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369)
at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
at org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311)
at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:935)
... 34 more
17/07/12 21:49:31 INFO SparkContext: Invoking stop() from shutdown hook
17/07/12 21:49:31 INFO SparkUI: Stopped Spark web UI at http://192.168.0.101:4040
17/07/12 21:49:31 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
17/07/12 21:49:31 INFO MemoryStore: MemoryStore cleared
17/07/12 21:49:31 INFO BlockManager: BlockManager stopped
17/07/12 21:49:31 INFO BlockManagerMaster: BlockManagerMaster stopped
17/07/12 21:49:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
17/07/12 21:49:31 INFO SparkContext: Successfully stopped SparkContext
17/07/12 21:49:31 INFO ShutdownHookManager: Shutdown hook called
17/07/12 21:49:31 INFO ShutdownHookManager: Deleting directory /private/var/folders/fw/lg6m2xps12bfwd7x0hj3rpzm0000gn/T/spark-886b188a-a3c3-460e-96a0-b7a4d1aa09de
Process finished with exit code 1
was:
There is an error while trying to create a dataset from a sequence of Maps, whose values have any kind of collections. Even when they are wrapped in a case class.
Eg : The following piece of code throws an error:
{code:java}
case class Values(values: Option[Seq[Double]])
case class ItemProperties(properties:Map[String,Values])
case class A(values :Set[Double])
val values3 = Set(1.0,2.0,3.0)
spark.createDataset(Seq(values3)).show()
val l1 = List(ItemProperties(
Map(
"A1" -> Values(Some(Seq(1.0,2.0))),
"B1" -> Values(Some(Seq(44.0,55.0)))
)
),
ItemProperties(
Map(
"A2" -> Values(Some(Seq(123.0,25.0))),
"B2" -> Values(Some(Seq(445.0,35.0)))
)
)
)
l1.toDS().show()
{code}
Heres the error:
17/07/12 21:49:31 ERROR CodeGenerator: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private boolean resultIsNull;
/* 009 */ private java.lang.String argValue;
/* 010 */ private Object[] values;
/* 011 */ private boolean resultIsNull1;
/* 012 */ private scala.collection.Seq argValue1;
/* 013 */ private boolean isNull12;
/* 014 */ private boolean value12;
/* 015 */ private boolean isNull13;
/* 016 */ private InternalRow value13;
/* 017 */ private boolean isNull14;
/* 018 */ private InternalRow value14;
/* 019 */ private UnsafeRow result;
/* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
/* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
/* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
/* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
/* 026 */
/* 027 */ public SpecificUnsafeProjection(Object[] references) {
/* 028 */ this.references = references;
/* 029 */
/* 030 */
/* 031 */ this.values = null;
/* 032 */
/* 033 */
/* 034 */ isNull12 = false;
/* 035 */ value12 = false;
/* 036 */ isNull13 = false;
/* 037 */ value13 = null;
/* 038 */ isNull14 = false;
/* 039 */ value14 = null;
/* 040 */ result = new UnsafeRow(1);
/* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
/* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 047 */
/* 048 */ }
/* 049 */
/* 050 */ public void initialize(int partitionIndex) {
/* 051 */
/* 052 */ }
/* 053 */
/* 054 */
/* 055 */ private void evalIfTrueExpr(InternalRow i) {
/* 056 */ final InternalRow value7 = null;
/* 057 */ isNull13 = true;
/* 058 */ value13 = value7;
/* 059 */ }
/* 060 */
/* 061 */
/* 062 */ private void evalIfCondExpr(InternalRow i) {
/* 063 */
/* 064 */ isNull12 = false;
/* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
/* 066 */ }
/* 067 */
/* 068 */
/* 069 */ private void evalIfFalseExpr(InternalRow i) {
/* 070 */ values = new Object[1];
/* 071 */ resultIsNull1 = false;
/* 072 */ if (!resultIsNull1) {
/* 073 */
/* 074 */ boolean isNull11 = true;
/* 075 */ scala.Option value11 = null;
/* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
/* 077 */
/* 078 */ isNull11 = false;
/* 079 */ if (!isNull11) {
/* 080 */
/* 081 */ Object funcResult1 = null;
/* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
/* 083 */ if (funcResult1 == null) {
/* 084 */ isNull11 = true;
/* 085 */ } else {
/* 086 */ value11 = (scala.Option) funcResult1;
/* 087 */ }
/* 088 */
/* 089 */ }
/* 090 */ isNull11 = value11 == null;
/* 091 */ }
/* 092 */
/* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
/* 094 */ scala.collection.Seq value10 = isNull10 ?
/* 095 */ null : (scala.collection.Seq) value11.get();
/* 096 */ resultIsNull1 = isNull10;
/* 097 */ argValue1 = value10;
/* 098 */ }
/* 099 */
/* 100 */
/* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
/* 102 */ if (resultIsNull1) {
/* 103 */ values[0] = null;
/* 104 */ } else {
/* 105 */ values[0] = value9;
/* 106 */ }
/* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
/* 108 */ this.values = null;
/* 109 */ isNull14 = false;
/* 110 */ value14 = value8;
/* 111 */ }
/* 112 */
/* 113 */
/* 114 */ // Scala.Function1 need this
/* 115 */ public java.lang.Object apply(java.lang.Object row) {
/* 116 */ return apply((InternalRow) row);
/* 117 */ }
/* 118 */
/* 119 */ public UnsafeRow apply(InternalRow i) {
/* 120 */ holder.reset();
/* 121 */
/* 122 */ rowWriter.zeroOutNullBytes();
/* 123 */
/* 124 */
/* 125 */ boolean isNull3 = i.isNullAt(0);
/* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
/* 127 */
/* 128 */ if (isNull3) {
/* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
/* 130 */ }
/* 131 */ boolean isNull1 = true;
/* 132 */ scala.collection.immutable.Map value1 = null;
/* 133 */ if (!false) {
/* 134 */
/* 135 */ isNull1 = false;
/* 136 */ if (!isNull1) {
/* 137 */
/* 138 */ Object funcResult = null;
/* 139 */ funcResult = value3.properties();
/* 140 */ if (funcResult == null) {
/* 141 */ isNull1 = true;
/* 142 */ } else {
/* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
/* 144 */ }
/* 145 */
/* 146 */ }
/* 147 */ isNull1 = value1 == null;
/* 148 */ }
/* 149 */ MapData value = null;
/* 150 */ if (!isNull1) {
/* 151 */ final int length = value1.size();
/* 152 */ final Object[] convertedKeys = new Object[length];
/* 153 */ final Object[] convertedValues = new Object[length];
/* 154 */ int index = 0;
/* 155 */ final scala.collection.Iterator entries = value1.iterator();
/* 156 */ while(entries.hasNext()) {
/* 157 */
/* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
/* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
/* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
/* 161 */
/* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
/* 163 */
/* 164 */
/* 165 */ resultIsNull = false;
/* 166 */ if (!resultIsNull) {
/* 167 */
/* 168 */ resultIsNull = false;
/* 169 */ argValue = ExternalMapToCatalyst_key0;
/* 170 */ }
/* 171 */
/* 172 */ boolean isNull4 = resultIsNull;
/* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
/* 174 */ isNull4 = value4 == null;
/* 175 */ if (isNull4) {
/* 176 */ throw new RuntimeException("Cannot use null as map key!");
/* 177 */ } else {
/* 178 */ convertedKeys[index] = (UTF8String) value4;
/* 179 */ }
/* 180 */
/* 181 */
/* 182 */ evalIfCondExpr(i);
/* 183 */ boolean isNull5 = false;
/* 184 */ InternalRow value5 = null;
/* 185 */ if (!isNull12 && value12) {
/* 186 */ evalIfTrueExpr(i);
/* 187 */ isNull5 = isNull13;
/* 188 */ value5 = value13;
/* 189 */ } else {
/* 190 */ evalIfFalseExpr(i);
/* 191 */ isNull5 = isNull14;
/* 192 */ value5 = value14;
/* 193 */ }
/* 194 */ if (isNull5) {
/* 195 */ convertedValues[index] = null;
/* 196 */ } else {
/* 197 */ convertedValues[index] = (InternalRow) value5;
/* 198 */ }
/* 199 */
/* 200 */ index++;
/* 201 */ }
/* 202 */
/* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
/* 204 */ }
/* 205 */ if (isNull1) {
/* 206 */ rowWriter.setNullAt(0);
/* 207 */ } else {
/* 208 */ // Remember the current cursor so that we can calculate how many bytes are
/* 209 */ // written later.
/* 210 */ final int tmpCursor = holder.cursor;
/* 211 */
/* 212 */ if (value instanceof UnsafeMapData) {
/* 213 */
/* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
/* 215 */ // grow the global buffer before writing data.
/* 216 */ holder.grow(sizeInBytes);
/* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
/* 218 */ holder.cursor += sizeInBytes;
/* 219 */
/* 220 */ } else {
/* 221 */ final ArrayData keys = value.keyArray();
/* 222 */ final ArrayData values1 = value.valueArray();
/* 223 */
/* 224 */ // preserve 8 bytes to write the key array numBytes later.
/* 225 */ holder.grow(8);
/* 226 */ holder.cursor += 8;
/* 227 */
/* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
/* 229 */ final int tmpCursor1 = holder.cursor;
/* 230 */
/* 231 */
/* 232 */ if (keys instanceof UnsafeArrayData) {
/* 233 */
/* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
/* 235 */ // grow the global buffer before writing data.
/* 236 */ holder.grow(sizeInBytes1);
/* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
/* 238 */ holder.cursor += sizeInBytes1;
/* 239 */
/* 240 */ } else {
/* 241 */ final int numElements = keys.numElements();
/* 242 */ arrayWriter.initialize(holder, numElements, 8);
/* 243 */
/* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
/* 245 */ if (keys.isNullAt(index1)) {
/* 246 */ arrayWriter.setNull(index1);
/* 247 */ } else {
/* 248 */ final UTF8String element = keys.getUTF8String(index1);
/* 249 */ arrayWriter.write(index1, element);
/* 250 */ }
/* 251 */ }
/* 252 */ }
/* 253 */
/* 254 */ // Write the numBytes of key array into the first 8 bytes.
/* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
/* 256 */
/* 257 */
/* 258 */ if (values1 instanceof UnsafeArrayData) {
/* 259 */
/* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
/* 261 */ // grow the global buffer before writing data.
/* 262 */ holder.grow(sizeInBytes4);
/* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
/* 264 */ holder.cursor += sizeInBytes4;
/* 265 */
/* 266 */ } else {
/* 267 */ final int numElements1 = values1.numElements();
/* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
/* 269 */
/* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
/* 271 */ if (values1.isNullAt(index2)) {
/* 272 */ arrayWriter1.setNull(index2);
/* 273 */ } else {
/* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
/* 275 */
/* 276 */ final int tmpCursor3 = holder.cursor;
/* 277 */
/* 278 */ if (element1 instanceof UnsafeRow) {
/* 279 */
/* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
/* 281 */ // grow the global buffer before writing data.
/* 282 */ holder.grow(sizeInBytes2);
/* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
/* 284 */ holder.cursor += sizeInBytes2;
/* 285 */
/* 286 */ } else {
/* 287 */ rowWriter1.reset();
/* 288 */
/* 289 */ final ArrayData fieldName = element1.getArray(0);
/* 290 */ if (element1.isNullAt(0)) {
/* 291 */ rowWriter1.setNullAt(0);
/* 292 */ } else {
/* 293 */ // Remember the current cursor so that we can calculate how many bytes are
/* 294 */ // written later.
/* 295 */ final int tmpCursor4 = holder.cursor;
/* 296 */
/* 297 */ if (fieldName instanceof UnsafeArrayData) {
/* 298 */
/* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
/* 300 */ // grow the global buffer before writing data.
/* 301 */ holder.grow(sizeInBytes3);
/* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
/* 303 */ holder.cursor += sizeInBytes3;
/* 304 */
/* 305 */ } else {
/* 306 */ final int numElements2 = fieldName.numElements();
/* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
/* 308 */
/* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
/* 310 */ if (fieldName.isNullAt(index3)) {
/* 311 */ arrayWriter2.setNullDouble(index3);
/* 312 */ } else {
/* 313 */ final double element2 = fieldName.getDouble(index3);
/* 314 */ arrayWriter2.write(index3, element2);
/* 315 */ }
/* 316 */ }
/* 317 */ }
/* 318 */
/* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
/* 320 */ }
/* 321 */ }
/* 322 */
/* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
/* 324 */
/* 325 */ }
/* 326 */ }
/* 327 */ }
/* 328 */
/* 329 */ }
/* 330 */
/* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
/* 332 */ }
/* 333 */ result.setTotalSize(holder.totalSize());
/* 334 */ return result;
/* 335 */ }
/* 336 */ }
org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
at org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
at org.codehaus.janino.UnitCompiler.toRvalueOrCompileException(UnitCompiler.java:6639)
at org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5001)
at org.codehaus.janino.UnitCompiler.access$10500(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$13.visitAmbiguousName(UnitCompiler.java:4984)
at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:3633)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:3563)
at org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4956)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4925)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3189)
at org.codehaus.janino.UnitCompiler.access$5100(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3143)
at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3139)
at org.codehaus.janino.Java$Assignment.accept(Java.java:3847)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3139)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2112)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1377)
at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2558)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894)
at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369)
at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
at org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311)
at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:935)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:998)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:995)
at org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:890)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:405)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:359)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:874)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection$lzycompute(ExpressionEncoder.scala:266)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection(ExpressionEncoder.scala:266)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:290)
at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
at scala.collection.immutable.List.foreach(List.scala:383)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:245)
at scala.collection.immutable.List.map(List.scala:286)
at org.apache.spark.sql.SparkSession.createDataset(SparkSession.scala:429)
at org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:389)
at org.apache.spark.sql.SQLImplicits.localSeqToDatasetHolder(SQLImplicits.scala:173)
at local.TestApp$.main(TestApp.scala:63)
at local.TestApp.main(TestApp.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
Exception in thread "main" java.lang.RuntimeException: Error while encoding: java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private boolean resultIsNull;
/* 009 */ private java.lang.String argValue;
/* 010 */ private Object[] values;
/* 011 */ private boolean resultIsNull1;
/* 012 */ private scala.collection.Seq argValue1;
/* 013 */ private boolean isNull12;
/* 014 */ private boolean value12;
/* 015 */ private boolean isNull13;
/* 016 */ private InternalRow value13;
/* 017 */ private boolean isNull14;
/* 018 */ private InternalRow value14;
/* 019 */ private UnsafeRow result;
/* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
/* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
/* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
/* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
/* 026 */
/* 027 */ public SpecificUnsafeProjection(Object[] references) {
/* 028 */ this.references = references;
/* 029 */
/* 030 */
/* 031 */ this.values = null;
/* 032 */
/* 033 */
/* 034 */ isNull12 = false;
/* 035 */ value12 = false;
/* 036 */ isNull13 = false;
/* 037 */ value13 = null;
/* 038 */ isNull14 = false;
/* 039 */ value14 = null;
/* 040 */ result = new UnsafeRow(1);
/* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
/* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 047 */
/* 048 */ }
/* 049 */
/* 050 */ public void initialize(int partitionIndex) {
/* 051 */
/* 052 */ }
/* 053 */
/* 054 */
/* 055 */ private void evalIfTrueExpr(InternalRow i) {
/* 056 */ final InternalRow value7 = null;
/* 057 */ isNull13 = true;
/* 058 */ value13 = value7;
/* 059 */ }
/* 060 */
/* 061 */
/* 062 */ private void evalIfCondExpr(InternalRow i) {
/* 063 */
/* 064 */ isNull12 = false;
/* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
/* 066 */ }
/* 067 */
/* 068 */
/* 069 */ private void evalIfFalseExpr(InternalRow i) {
/* 070 */ values = new Object[1];
/* 071 */ resultIsNull1 = false;
/* 072 */ if (!resultIsNull1) {
/* 073 */
/* 074 */ boolean isNull11 = true;
/* 075 */ scala.Option value11 = null;
/* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
/* 077 */
/* 078 */ isNull11 = false;
/* 079 */ if (!isNull11) {
/* 080 */
/* 081 */ Object funcResult1 = null;
/* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
/* 083 */ if (funcResult1 == null) {
/* 084 */ isNull11 = true;
/* 085 */ } else {
/* 086 */ value11 = (scala.Option) funcResult1;
/* 087 */ }
/* 088 */
/* 089 */ }
/* 090 */ isNull11 = value11 == null;
/* 091 */ }
/* 092 */
/* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
/* 094 */ scala.collection.Seq value10 = isNull10 ?
/* 095 */ null : (scala.collection.Seq) value11.get();
/* 096 */ resultIsNull1 = isNull10;
/* 097 */ argValue1 = value10;
/* 098 */ }
/* 099 */
/* 100 */
/* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
/* 102 */ if (resultIsNull1) {
/* 103 */ values[0] = null;
/* 104 */ } else {
/* 105 */ values[0] = value9;
/* 106 */ }
/* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
/* 108 */ this.values = null;
/* 109 */ isNull14 = false;
/* 110 */ value14 = value8;
/* 111 */ }
/* 112 */
/* 113 */
/* 114 */ // Scala.Function1 need this
/* 115 */ public java.lang.Object apply(java.lang.Object row) {
/* 116 */ return apply((InternalRow) row);
/* 117 */ }
/* 118 */
/* 119 */ public UnsafeRow apply(InternalRow i) {
/* 120 */ holder.reset();
/* 121 */
/* 122 */ rowWriter.zeroOutNullBytes();
/* 123 */
/* 124 */
/* 125 */ boolean isNull3 = i.isNullAt(0);
/* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
/* 127 */
/* 128 */ if (isNull3) {
/* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
/* 130 */ }
/* 131 */ boolean isNull1 = true;
/* 132 */ scala.collection.immutable.Map value1 = null;
/* 133 */ if (!false) {
/* 134 */
/* 135 */ isNull1 = false;
/* 136 */ if (!isNull1) {
/* 137 */
/* 138 */ Object funcResult = null;
/* 139 */ funcResult = value3.properties();
/* 140 */ if (funcResult == null) {
/* 141 */ isNull1 = true;
/* 142 */ } else {
/* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
/* 144 */ }
/* 145 */
/* 146 */ }
/* 147 */ isNull1 = value1 == null;
/* 148 */ }
/* 149 */ MapData value = null;
/* 150 */ if (!isNull1) {
/* 151 */ final int length = value1.size();
/* 152 */ final Object[] convertedKeys = new Object[length];
/* 153 */ final Object[] convertedValues = new Object[length];
/* 154 */ int index = 0;
/* 155 */ final scala.collection.Iterator entries = value1.iterator();
/* 156 */ while(entries.hasNext()) {
/* 157 */
/* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
/* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
/* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
/* 161 */
/* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
/* 163 */
/* 164 */
/* 165 */ resultIsNull = false;
/* 166 */ if (!resultIsNull) {
/* 167 */
/* 168 */ resultIsNull = false;
/* 169 */ argValue = ExternalMapToCatalyst_key0;
/* 170 */ }
/* 171 */
/* 172 */ boolean isNull4 = resultIsNull;
/* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
/* 174 */ isNull4 = value4 == null;
/* 175 */ if (isNull4) {
/* 176 */ throw new RuntimeException("Cannot use null as map key!");
/* 177 */ } else {
/* 178 */ convertedKeys[index] = (UTF8String) value4;
/* 179 */ }
/* 180 */
/* 181 */
/* 182 */ evalIfCondExpr(i);
/* 183 */ boolean isNull5 = false;
/* 184 */ InternalRow value5 = null;
/* 185 */ if (!isNull12 && value12) {
/* 186 */ evalIfTrueExpr(i);
/* 187 */ isNull5 = isNull13;
/* 188 */ value5 = value13;
/* 189 */ } else {
/* 190 */ evalIfFalseExpr(i);
/* 191 */ isNull5 = isNull14;
/* 192 */ value5 = value14;
/* 193 */ }
/* 194 */ if (isNull5) {
/* 195 */ convertedValues[index] = null;
/* 196 */ } else {
/* 197 */ convertedValues[index] = (InternalRow) value5;
/* 198 */ }
/* 199 */
/* 200 */ index++;
/* 201 */ }
/* 202 */
/* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
/* 204 */ }
/* 205 */ if (isNull1) {
/* 206 */ rowWriter.setNullAt(0);
/* 207 */ } else {
/* 208 */ // Remember the current cursor so that we can calculate how many bytes are
/* 209 */ // written later.
/* 210 */ final int tmpCursor = holder.cursor;
/* 211 */
/* 212 */ if (value instanceof UnsafeMapData) {
/* 213 */
/* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
/* 215 */ // grow the global buffer before writing data.
/* 216 */ holder.grow(sizeInBytes);
/* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
/* 218 */ holder.cursor += sizeInBytes;
/* 219 */
/* 220 */ } else {
/* 221 */ final ArrayData keys = value.keyArray();
/* 222 */ final ArrayData values1 = value.valueArray();
/* 223 */
/* 224 */ // preserve 8 bytes to write the key array numBytes later.
/* 225 */ holder.grow(8);
/* 226 */ holder.cursor += 8;
/* 227 */
/* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
/* 229 */ final int tmpCursor1 = holder.cursor;
/* 230 */
/* 231 */
/* 232 */ if (keys instanceof UnsafeArrayData) {
/* 233 */
/* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
/* 235 */ // grow the global buffer before writing data.
/* 236 */ holder.grow(sizeInBytes1);
/* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
/* 238 */ holder.cursor += sizeInBytes1;
/* 239 */
/* 240 */ } else {
/* 241 */ final int numElements = keys.numElements();
/* 242 */ arrayWriter.initialize(holder, numElements, 8);
/* 243 */
/* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
/* 245 */ if (keys.isNullAt(index1)) {
/* 246 */ arrayWriter.setNull(index1);
/* 247 */ } else {
/* 248 */ final UTF8String element = keys.getUTF8String(index1);
/* 249 */ arrayWriter.write(index1, element);
/* 250 */ }
/* 251 */ }
/* 252 */ }
/* 253 */
/* 254 */ // Write the numBytes of key array into the first 8 bytes.
/* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
/* 256 */
/* 257 */
/* 258 */ if (values1 instanceof UnsafeArrayData) {
/* 259 */
/* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
/* 261 */ // grow the global buffer before writing data.
/* 262 */ holder.grow(sizeInBytes4);
/* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
/* 264 */ holder.cursor += sizeInBytes4;
/* 265 */
/* 266 */ } else {
/* 267 */ final int numElements1 = values1.numElements();
/* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
/* 269 */
/* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
/* 271 */ if (values1.isNullAt(index2)) {
/* 272 */ arrayWriter1.setNull(index2);
/* 273 */ } else {
/* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
/* 275 */
/* 276 */ final int tmpCursor3 = holder.cursor;
/* 277 */
/* 278 */ if (element1 instanceof UnsafeRow) {
/* 279 */
/* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
/* 281 */ // grow the global buffer before writing data.
/* 282 */ holder.grow(sizeInBytes2);
/* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
/* 284 */ holder.cursor += sizeInBytes2;
/* 285 */
/* 286 */ } else {
/* 287 */ rowWriter1.reset();
/* 288 */
/* 289 */ final ArrayData fieldName = element1.getArray(0);
/* 290 */ if (element1.isNullAt(0)) {
/* 291 */ rowWriter1.setNullAt(0);
/* 292 */ } else {
/* 293 */ // Remember the current cursor so that we can calculate how many bytes are
/* 294 */ // written later.
/* 295 */ final int tmpCursor4 = holder.cursor;
/* 296 */
/* 297 */ if (fieldName instanceof UnsafeArrayData) {
/* 298 */
/* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
/* 300 */ // grow the global buffer before writing data.
/* 301 */ holder.grow(sizeInBytes3);
/* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
/* 303 */ holder.cursor += sizeInBytes3;
/* 304 */
/* 305 */ } else {
/* 306 */ final int numElements2 = fieldName.numElements();
/* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
/* 308 */
/* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
/* 310 */ if (fieldName.isNullAt(index3)) {
/* 311 */ arrayWriter2.setNullDouble(index3);
/* 312 */ } else {
/* 313 */ final double element2 = fieldName.getDouble(index3);
/* 314 */ arrayWriter2.write(index3, element2);
/* 315 */ }
/* 316 */ }
/* 317 */ }
/* 318 */
/* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
/* 320 */ }
/* 321 */ }
/* 322 */
/* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
/* 324 */
/* 325 */ }
/* 326 */ }
/* 327 */ }
/* 328 */
/* 329 */ }
/* 330 */
/* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
/* 332 */ }
/* 333 */ result.setTotalSize(holder.totalSize());
/* 334 */ return result;
/* 335 */ }
/* 336 */ }
externalmaptocatalyst(ExternalMapToCatalyst_key0, ObjectType(class java.lang.String), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key0, false, ObjectType(class java.lang.String)), true), ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values), if (isnull(lambdavariable(ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values)))) null else named_struct(values, newInstance(class org.apache.spark.sql.catalyst.util.GenericArrayData)), assertnotnull(input[0, local.ItemProperties, true], top level Product input object).properties) AS properties#0
+- externalmaptocatalyst(ExternalMapToCatalyst_key0, ObjectType(class java.lang.String), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key0, false, ObjectType(class java.lang.String)), true), ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values), if (isnull(lambdavariable(ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values)))) null else named_struct(values, newInstance(class org.apache.spark.sql.catalyst.util.GenericArrayData)), assertnotnull(input[0, local.ItemProperties, true], top level Product input object).properties)
+- assertnotnull(input[0, local.ItemProperties, true], top level Product input object).properties
+- assertnotnull(input[0, local.ItemProperties, true], top level Product input object)
+- input[0, local.ItemProperties, true]
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:293)
at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
at scala.collection.immutable.List.foreach(List.scala:383)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:245)
at scala.collection.immutable.List.map(List.scala:286)
at org.apache.spark.sql.SparkSession.createDataset(SparkSession.scala:429)
at org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:389)
at org.apache.spark.sql.SQLImplicits.localSeqToDatasetHolder(SQLImplicits.scala:173)
at local.TestApp$.main(TestApp.scala:63)
at local.TestApp.main(TestApp.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private boolean resultIsNull;
/* 009 */ private java.lang.String argValue;
/* 010 */ private Object[] values;
/* 011 */ private boolean resultIsNull1;
/* 012 */ private scala.collection.Seq argValue1;
/* 013 */ private boolean isNull12;
/* 014 */ private boolean value12;
/* 015 */ private boolean isNull13;
/* 016 */ private InternalRow value13;
/* 017 */ private boolean isNull14;
/* 018 */ private InternalRow value14;
/* 019 */ private UnsafeRow result;
/* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
/* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
/* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
/* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
/* 026 */
/* 027 */ public SpecificUnsafeProjection(Object[] references) {
/* 028 */ this.references = references;
/* 029 */
/* 030 */
/* 031 */ this.values = null;
/* 032 */
/* 033 */
/* 034 */ isNull12 = false;
/* 035 */ value12 = false;
/* 036 */ isNull13 = false;
/* 037 */ value13 = null;
/* 038 */ isNull14 = false;
/* 039 */ value14 = null;
/* 040 */ result = new UnsafeRow(1);
/* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
/* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 047 */
/* 048 */ }
/* 049 */
/* 050 */ public void initialize(int partitionIndex) {
/* 051 */
/* 052 */ }
/* 053 */
/* 054 */
/* 055 */ private void evalIfTrueExpr(InternalRow i) {
/* 056 */ final InternalRow value7 = null;
/* 057 */ isNull13 = true;
/* 058 */ value13 = value7;
/* 059 */ }
/* 060 */
/* 061 */
/* 062 */ private void evalIfCondExpr(InternalRow i) {
/* 063 */
/* 064 */ isNull12 = false;
/* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
/* 066 */ }
/* 067 */
/* 068 */
/* 069 */ private void evalIfFalseExpr(InternalRow i) {
/* 070 */ values = new Object[1];
/* 071 */ resultIsNull1 = false;
/* 072 */ if (!resultIsNull1) {
/* 073 */
/* 074 */ boolean isNull11 = true;
/* 075 */ scala.Option value11 = null;
/* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
/* 077 */
/* 078 */ isNull11 = false;
/* 079 */ if (!isNull11) {
/* 080 */
/* 081 */ Object funcResult1 = null;
/* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
/* 083 */ if (funcResult1 == null) {
/* 084 */ isNull11 = true;
/* 085 */ } else {
/* 086 */ value11 = (scala.Option) funcResult1;
/* 087 */ }
/* 088 */
/* 089 */ }
/* 090 */ isNull11 = value11 == null;
/* 091 */ }
/* 092 */
/* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
/* 094 */ scala.collection.Seq value10 = isNull10 ?
/* 095 */ null : (scala.collection.Seq) value11.get();
/* 096 */ resultIsNull1 = isNull10;
/* 097 */ argValue1 = value10;
/* 098 */ }
/* 099 */
/* 100 */
/* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
/* 102 */ if (resultIsNull1) {
/* 103 */ values[0] = null;
/* 104 */ } else {
/* 105 */ values[0] = value9;
/* 106 */ }
/* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
/* 108 */ this.values = null;
/* 109 */ isNull14 = false;
/* 110 */ value14 = value8;
/* 111 */ }
/* 112 */
/* 113 */
/* 114 */ // Scala.Function1 need this
/* 115 */ public java.lang.Object apply(java.lang.Object row) {
/* 116 */ return apply((InternalRow) row);
/* 117 */ }
/* 118 */
/* 119 */ public UnsafeRow apply(InternalRow i) {
/* 120 */ holder.reset();
/* 121 */
/* 122 */ rowWriter.zeroOutNullBytes();
/* 123 */
/* 124 */
/* 125 */ boolean isNull3 = i.isNullAt(0);
/* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
/* 127 */
/* 128 */ if (isNull3) {
/* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
/* 130 */ }
/* 131 */ boolean isNull1 = true;
/* 132 */ scala.collection.immutable.Map value1 = null;
/* 133 */ if (!false) {
/* 134 */
/* 135 */ isNull1 = false;
/* 136 */ if (!isNull1) {
/* 137 */
/* 138 */ Object funcResult = null;
/* 139 */ funcResult = value3.properties();
/* 140 */ if (funcResult == null) {
/* 141 */ isNull1 = true;
/* 142 */ } else {
/* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
/* 144 */ }
/* 145 */
/* 146 */ }
/* 147 */ isNull1 = value1 == null;
/* 148 */ }
/* 149 */ MapData value = null;
/* 150 */ if (!isNull1) {
/* 151 */ final int length = value1.size();
/* 152 */ final Object[] convertedKeys = new Object[length];
/* 153 */ final Object[] convertedValues = new Object[length];
/* 154 */ int index = 0;
/* 155 */ final scala.collection.Iterator entries = value1.iterator();
/* 156 */ while(entries.hasNext()) {
/* 157 */
/* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
/* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
/* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
/* 161 */
/* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
/* 163 */
/* 164 */
/* 165 */ resultIsNull = false;
/* 166 */ if (!resultIsNull) {
/* 167 */
/* 168 */ resultIsNull = false;
/* 169 */ argValue = ExternalMapToCatalyst_key0;
/* 170 */ }
/* 171 */
/* 172 */ boolean isNull4 = resultIsNull;
/* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
/* 174 */ isNull4 = value4 == null;
/* 175 */ if (isNull4) {
/* 176 */ throw new RuntimeException("Cannot use null as map key!");
/* 177 */ } else {
/* 178 */ convertedKeys[index] = (UTF8String) value4;
/* 179 */ }
/* 180 */
/* 181 */
/* 182 */ evalIfCondExpr(i);
/* 183 */ boolean isNull5 = false;
/* 184 */ InternalRow value5 = null;
/* 185 */ if (!isNull12 && value12) {
/* 186 */ evalIfTrueExpr(i);
/* 187 */ isNull5 = isNull13;
/* 188 */ value5 = value13;
/* 189 */ } else {
/* 190 */ evalIfFalseExpr(i);
/* 191 */ isNull5 = isNull14;
/* 192 */ value5 = value14;
/* 193 */ }
/* 194 */ if (isNull5) {
/* 195 */ convertedValues[index] = null;
/* 196 */ } else {
/* 197 */ convertedValues[index] = (InternalRow) value5;
/* 198 */ }
/* 199 */
/* 200 */ index++;
/* 201 */ }
/* 202 */
/* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
/* 204 */ }
/* 205 */ if (isNull1) {
/* 206 */ rowWriter.setNullAt(0);
/* 207 */ } else {
/* 208 */ // Remember the current cursor so that we can calculate how many bytes are
/* 209 */ // written later.
/* 210 */ final int tmpCursor = holder.cursor;
/* 211 */
/* 212 */ if (value instanceof UnsafeMapData) {
/* 213 */
/* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
/* 215 */ // grow the global buffer before writing data.
/* 216 */ holder.grow(sizeInBytes);
/* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
/* 218 */ holder.cursor += sizeInBytes;
/* 219 */
/* 220 */ } else {
/* 221 */ final ArrayData keys = value.keyArray();
/* 222 */ final ArrayData values1 = value.valueArray();
/* 223 */
/* 224 */ // preserve 8 bytes to write the key array numBytes later.
/* 225 */ holder.grow(8);
/* 226 */ holder.cursor += 8;
/* 227 */
/* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
/* 229 */ final int tmpCursor1 = holder.cursor;
/* 230 */
/* 231 */
/* 232 */ if (keys instanceof UnsafeArrayData) {
/* 233 */
/* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
/* 235 */ // grow the global buffer before writing data.
/* 236 */ holder.grow(sizeInBytes1);
/* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
/* 238 */ holder.cursor += sizeInBytes1;
/* 239 */
/* 240 */ } else {
/* 241 */ final int numElements = keys.numElements();
/* 242 */ arrayWriter.initialize(holder, numElements, 8);
/* 243 */
/* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
/* 245 */ if (keys.isNullAt(index1)) {
/* 246 */ arrayWriter.setNull(index1);
/* 247 */ } else {
/* 248 */ final UTF8String element = keys.getUTF8String(index1);
/* 249 */ arrayWriter.write(index1, element);
/* 250 */ }
/* 251 */ }
/* 252 */ }
/* 253 */
/* 254 */ // Write the numBytes of key array into the first 8 bytes.
/* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
/* 256 */
/* 257 */
/* 258 */ if (values1 instanceof UnsafeArrayData) {
/* 259 */
/* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
/* 261 */ // grow the global buffer before writing data.
/* 262 */ holder.grow(sizeInBytes4);
/* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
/* 264 */ holder.cursor += sizeInBytes4;
/* 265 */
/* 266 */ } else {
/* 267 */ final int numElements1 = values1.numElements();
/* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
/* 269 */
/* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
/* 271 */ if (values1.isNullAt(index2)) {
/* 272 */ arrayWriter1.setNull(index2);
/* 273 */ } else {
/* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
/* 275 */
/* 276 */ final int tmpCursor3 = holder.cursor;
/* 277 */
/* 278 */ if (element1 instanceof UnsafeRow) {
/* 279 */
/* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
/* 281 */ // grow the global buffer before writing data.
/* 282 */ holder.grow(sizeInBytes2);
/* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
/* 284 */ holder.cursor += sizeInBytes2;
/* 285 */
/* 286 */ } else {
/* 287 */ rowWriter1.reset();
/* 288 */
/* 289 */ final ArrayData fieldName = element1.getArray(0);
/* 290 */ if (element1.isNullAt(0)) {
/* 291 */ rowWriter1.setNullAt(0);
/* 292 */ } else {
/* 293 */ // Remember the current cursor so that we can calculate how many bytes are
/* 294 */ // written later.
/* 295 */ final int tmpCursor4 = holder.cursor;
/* 296 */
/* 297 */ if (fieldName instanceof UnsafeArrayData) {
/* 298 */
/* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
/* 300 */ // grow the global buffer before writing data.
/* 301 */ holder.grow(sizeInBytes3);
/* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
/* 303 */ holder.cursor += sizeInBytes3;
/* 304 */
/* 305 */ } else {
/* 306 */ final int numElements2 = fieldName.numElements();
/* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
/* 308 */
/* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
/* 310 */ if (fieldName.isNullAt(index3)) {
/* 311 */ arrayWriter2.setNullDouble(index3);
/* 312 */ } else {
/* 313 */ final double element2 = fieldName.getDouble(index3);
/* 314 */ arrayWriter2.write(index3, element2);
/* 315 */ }
/* 316 */ }
/* 317 */ }
/* 318 */
/* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
/* 320 */ }
/* 321 */ }
/* 322 */
/* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
/* 324 */
/* 325 */ }
/* 326 */ }
/* 327 */ }
/* 328 */
/* 329 */ }
/* 330 */
/* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
/* 332 */ }
/* 333 */ result.setTotalSize(holder.totalSize());
/* 334 */ return result;
/* 335 */ }
/* 336 */ }
at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306)
at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293)
at org.spark_project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
at org.spark_project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135)
at org.spark_project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410)
at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380)
at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:890)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:405)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:359)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:874)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection$lzycompute(ExpressionEncoder.scala:266)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection(ExpressionEncoder.scala:266)
at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:290)
... 17 more
Caused by: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private boolean resultIsNull;
/* 009 */ private java.lang.String argValue;
/* 010 */ private Object[] values;
/* 011 */ private boolean resultIsNull1;
/* 012 */ private scala.collection.Seq argValue1;
/* 013 */ private boolean isNull12;
/* 014 */ private boolean value12;
/* 015 */ private boolean isNull13;
/* 016 */ private InternalRow value13;
/* 017 */ private boolean isNull14;
/* 018 */ private InternalRow value14;
/* 019 */ private UnsafeRow result;
/* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
/* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
/* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
/* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
/* 026 */
/* 027 */ public SpecificUnsafeProjection(Object[] references) {
/* 028 */ this.references = references;
/* 029 */
/* 030 */
/* 031 */ this.values = null;
/* 032 */
/* 033 */
/* 034 */ isNull12 = false;
/* 035 */ value12 = false;
/* 036 */ isNull13 = false;
/* 037 */ value13 = null;
/* 038 */ isNull14 = false;
/* 039 */ value14 = null;
/* 040 */ result = new UnsafeRow(1);
/* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
/* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
/* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
/* 047 */
/* 048 */ }
/* 049 */
/* 050 */ public void initialize(int partitionIndex) {
/* 051 */
/* 052 */ }
/* 053 */
/* 054 */
/* 055 */ private void evalIfTrueExpr(InternalRow i) {
/* 056 */ final InternalRow value7 = null;
/* 057 */ isNull13 = true;
/* 058 */ value13 = value7;
/* 059 */ }
/* 060 */
/* 061 */
/* 062 */ private void evalIfCondExpr(InternalRow i) {
/* 063 */
/* 064 */ isNull12 = false;
/* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
/* 066 */ }
/* 067 */
/* 068 */
/* 069 */ private void evalIfFalseExpr(InternalRow i) {
/* 070 */ values = new Object[1];
/* 071 */ resultIsNull1 = false;
/* 072 */ if (!resultIsNull1) {
/* 073 */
/* 074 */ boolean isNull11 = true;
/* 075 */ scala.Option value11 = null;
/* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
/* 077 */
/* 078 */ isNull11 = false;
/* 079 */ if (!isNull11) {
/* 080 */
/* 081 */ Object funcResult1 = null;
/* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
/* 083 */ if (funcResult1 == null) {
/* 084 */ isNull11 = true;
/* 085 */ } else {
/* 086 */ value11 = (scala.Option) funcResult1;
/* 087 */ }
/* 088 */
/* 089 */ }
/* 090 */ isNull11 = value11 == null;
/* 091 */ }
/* 092 */
/* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
/* 094 */ scala.collection.Seq value10 = isNull10 ?
/* 095 */ null : (scala.collection.Seq) value11.get();
/* 096 */ resultIsNull1 = isNull10;
/* 097 */ argValue1 = value10;
/* 098 */ }
/* 099 */
/* 100 */
/* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
/* 102 */ if (resultIsNull1) {
/* 103 */ values[0] = null;
/* 104 */ } else {
/* 105 */ values[0] = value9;
/* 106 */ }
/* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
/* 108 */ this.values = null;
/* 109 */ isNull14 = false;
/* 110 */ value14 = value8;
/* 111 */ }
/* 112 */
/* 113 */
/* 114 */ // Scala.Function1 need this
/* 115 */ public java.lang.Object apply(java.lang.Object row) {
/* 116 */ return apply((InternalRow) row);
/* 117 */ }
/* 118 */
/* 119 */ public UnsafeRow apply(InternalRow i) {
/* 120 */ holder.reset();
/* 121 */
/* 122 */ rowWriter.zeroOutNullBytes();
/* 123 */
/* 124 */
/* 125 */ boolean isNull3 = i.isNullAt(0);
/* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
/* 127 */
/* 128 */ if (isNull3) {
/* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
/* 130 */ }
/* 131 */ boolean isNull1 = true;
/* 132 */ scala.collection.immutable.Map value1 = null;
/* 133 */ if (!false) {
/* 134 */
/* 135 */ isNull1 = false;
/* 136 */ if (!isNull1) {
/* 137 */
/* 138 */ Object funcResult = null;
/* 139 */ funcResult = value3.properties();
/* 140 */ if (funcResult == null) {
/* 141 */ isNull1 = true;
/* 142 */ } else {
/* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
/* 144 */ }
/* 145 */
/* 146 */ }
/* 147 */ isNull1 = value1 == null;
/* 148 */ }
/* 149 */ MapData value = null;
/* 150 */ if (!isNull1) {
/* 151 */ final int length = value1.size();
/* 152 */ final Object[] convertedKeys = new Object[length];
/* 153 */ final Object[] convertedValues = new Object[length];
/* 154 */ int index = 0;
/* 155 */ final scala.collection.Iterator entries = value1.iterator();
/* 156 */ while(entries.hasNext()) {
/* 157 */
/* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
/* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
/* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
/* 161 */
/* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
/* 163 */
/* 164 */
/* 165 */ resultIsNull = false;
/* 166 */ if (!resultIsNull) {
/* 167 */
/* 168 */ resultIsNull = false;
/* 169 */ argValue = ExternalMapToCatalyst_key0;
/* 170 */ }
/* 171 */
/* 172 */ boolean isNull4 = resultIsNull;
/* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
/* 174 */ isNull4 = value4 == null;
/* 175 */ if (isNull4) {
/* 176 */ throw new RuntimeException("Cannot use null as map key!");
/* 177 */ } else {
/* 178 */ convertedKeys[index] = (UTF8String) value4;
/* 179 */ }
/* 180 */
/* 181 */
/* 182 */ evalIfCondExpr(i);
/* 183 */ boolean isNull5 = false;
/* 184 */ InternalRow value5 = null;
/* 185 */ if (!isNull12 && value12) {
/* 186 */ evalIfTrueExpr(i);
/* 187 */ isNull5 = isNull13;
/* 188 */ value5 = value13;
/* 189 */ } else {
/* 190 */ evalIfFalseExpr(i);
/* 191 */ isNull5 = isNull14;
/* 192 */ value5 = value14;
/* 193 */ }
/* 194 */ if (isNull5) {
/* 195 */ convertedValues[index] = null;
/* 196 */ } else {
/* 197 */ convertedValues[index] = (InternalRow) value5;
/* 198 */ }
/* 199 */
/* 200 */ index++;
/* 201 */ }
/* 202 */
/* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
/* 204 */ }
/* 205 */ if (isNull1) {
/* 206 */ rowWriter.setNullAt(0);
/* 207 */ } else {
/* 208 */ // Remember the current cursor so that we can calculate how many bytes are
/* 209 */ // written later.
/* 210 */ final int tmpCursor = holder.cursor;
/* 211 */
/* 212 */ if (value instanceof UnsafeMapData) {
/* 213 */
/* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
/* 215 */ // grow the global buffer before writing data.
/* 216 */ holder.grow(sizeInBytes);
/* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
/* 218 */ holder.cursor += sizeInBytes;
/* 219 */
/* 220 */ } else {
/* 221 */ final ArrayData keys = value.keyArray();
/* 222 */ final ArrayData values1 = value.valueArray();
/* 223 */
/* 224 */ // preserve 8 bytes to write the key array numBytes later.
/* 225 */ holder.grow(8);
/* 226 */ holder.cursor += 8;
/* 227 */
/* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
/* 229 */ final int tmpCursor1 = holder.cursor;
/* 230 */
/* 231 */
/* 232 */ if (keys instanceof UnsafeArrayData) {
/* 233 */
/* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
/* 235 */ // grow the global buffer before writing data.
/* 236 */ holder.grow(sizeInBytes1);
/* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
/* 238 */ holder.cursor += sizeInBytes1;
/* 239 */
/* 240 */ } else {
/* 241 */ final int numElements = keys.numElements();
/* 242 */ arrayWriter.initialize(holder, numElements, 8);
/* 243 */
/* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
/* 245 */ if (keys.isNullAt(index1)) {
/* 246 */ arrayWriter.setNull(index1);
/* 247 */ } else {
/* 248 */ final UTF8String element = keys.getUTF8String(index1);
/* 249 */ arrayWriter.write(index1, element);
/* 250 */ }
/* 251 */ }
/* 252 */ }
/* 253 */
/* 254 */ // Write the numBytes of key array into the first 8 bytes.
/* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
/* 256 */
/* 257 */
/* 258 */ if (values1 instanceof UnsafeArrayData) {
/* 259 */
/* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
/* 261 */ // grow the global buffer before writing data.
/* 262 */ holder.grow(sizeInBytes4);
/* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
/* 264 */ holder.cursor += sizeInBytes4;
/* 265 */
/* 266 */ } else {
/* 267 */ final int numElements1 = values1.numElements();
/* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
/* 269 */
/* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
/* 271 */ if (values1.isNullAt(index2)) {
/* 272 */ arrayWriter1.setNull(index2);
/* 273 */ } else {
/* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
/* 275 */
/* 276 */ final int tmpCursor3 = holder.cursor;
/* 277 */
/* 278 */ if (element1 instanceof UnsafeRow) {
/* 279 */
/* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
/* 281 */ // grow the global buffer before writing data.
/* 282 */ holder.grow(sizeInBytes2);
/* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
/* 284 */ holder.cursor += sizeInBytes2;
/* 285 */
/* 286 */ } else {
/* 287 */ rowWriter1.reset();
/* 288 */
/* 289 */ final ArrayData fieldName = element1.getArray(0);
/* 290 */ if (element1.isNullAt(0)) {
/* 291 */ rowWriter1.setNullAt(0);
/* 292 */ } else {
/* 293 */ // Remember the current cursor so that we can calculate how many bytes are
/* 294 */ // written later.
/* 295 */ final int tmpCursor4 = holder.cursor;
/* 296 */
/* 297 */ if (fieldName instanceof UnsafeArrayData) {
/* 298 */
/* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
/* 300 */ // grow the global buffer before writing data.
/* 301 */ holder.grow(sizeInBytes3);
/* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
/* 303 */ holder.cursor += sizeInBytes3;
/* 304 */
/* 305 */ } else {
/* 306 */ final int numElements2 = fieldName.numElements();
/* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
/* 308 */
/* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
/* 310 */ if (fieldName.isNullAt(index3)) {
/* 311 */ arrayWriter2.setNullDouble(index3);
/* 312 */ } else {
/* 313 */ final double element2 = fieldName.getDouble(index3);
/* 314 */ arrayWriter2.write(index3, element2);
/* 315 */ }
/* 316 */ }
/* 317 */ }
/* 318 */
/* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
/* 320 */ }
/* 321 */ }
/* 322 */
/* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
/* 324 */
/* 325 */ }
/* 326 */ }
/* 327 */ }
/* 328 */
/* 329 */ }
/* 330 */
/* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
/* 332 */ }
/* 333 */ result.setTotalSize(holder.totalSize());
/* 334 */ return result;
/* 335 */ }
/* 336 */ }
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:941)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:998)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:995)
at org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
... 30 more
Caused by: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
at org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
at org.codehaus.janino.UnitCompiler.toRvalueOrCompileException(UnitCompiler.java:6639)
at org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5001)
at org.codehaus.janino.UnitCompiler.access$10500(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$13.visitAmbiguousName(UnitCompiler.java:4984)
at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:3633)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:3563)
at org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4956)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4925)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3189)
at org.codehaus.janino.UnitCompiler.access$5100(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3143)
at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3139)
at org.codehaus.janino.Java$Assignment.accept(Java.java:3847)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3139)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2112)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1377)
at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2558)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894)
at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377)
at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369)
at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
at org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311)
at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:935)
... 34 more
17/07/12 21:49:31 INFO SparkContext: Invoking stop() from shutdown hook
17/07/12 21:49:31 INFO SparkUI: Stopped Spark web UI at http://192.168.0.101:4040
17/07/12 21:49:31 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
17/07/12 21:49:31 INFO MemoryStore: MemoryStore cleared
17/07/12 21:49:31 INFO BlockManager: BlockManager stopped
17/07/12 21:49:31 INFO BlockManagerMaster: BlockManagerMaster stopped
17/07/12 21:49:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
17/07/12 21:49:31 INFO SparkContext: Successfully stopped SparkContext
17/07/12 21:49:31 INFO ShutdownHookManager: Shutdown hook called
17/07/12 21:49:31 INFO ShutdownHookManager: Deleting directory /private/var/folders/fw/lg6m2xps12bfwd7x0hj3rpzm0000gn/T/spark-886b188a-a3c3-460e-96a0-b7a4d1aa09de
Process finished with exit code 1
> Cannot convert a Seq of Map whose value type is again a seq, into a dataset
> ----------------------------------------------------------------------------
>
> Key: SPARK-21391
> URL: https://issues.apache.org/jira/browse/SPARK-21391
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 2.1.0
> Environment: Seen on mac OSX, scala 2.11, java 8
> Reporter: indraneel rao
>
> There is an error while trying to create a dataset from a sequence of Maps, whose values have any kind of collections. Even when they are wrapped in a case class.
> Eg : The following piece of code throws an error:
>
> {code:java}
> case class Values(values: Option[Seq[Double]])
> case class ItemProperties(properties:Map[String,Values])
> val l1 = List(ItemProperties(
> Map(
> "A1" -> Values(Some(Seq(1.0,2.0))),
> "B1" -> Values(Some(Seq(44.0,55.0)))
> )
> ),
> ItemProperties(
> Map(
> "A2" -> Values(Some(Seq(123.0,25.0))),
> "B2" -> Values(Some(Seq(445.0,35.0)))
> )
> )
> )
> l1.toDS().show()
> {code}
> Heres the error:
> 17/07/12 21:49:31 ERROR CodeGenerator: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
> /* 001 */ public java.lang.Object generate(Object[] references) {
> /* 002 */ return new SpecificUnsafeProjection(references);
> /* 003 */ }
> /* 004 */
> /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
> /* 006 */
> /* 007 */ private Object[] references;
> /* 008 */ private boolean resultIsNull;
> /* 009 */ private java.lang.String argValue;
> /* 010 */ private Object[] values;
> /* 011 */ private boolean resultIsNull1;
> /* 012 */ private scala.collection.Seq argValue1;
> /* 013 */ private boolean isNull12;
> /* 014 */ private boolean value12;
> /* 015 */ private boolean isNull13;
> /* 016 */ private InternalRow value13;
> /* 017 */ private boolean isNull14;
> /* 018 */ private InternalRow value14;
> /* 019 */ private UnsafeRow result;
> /* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
> /* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
> /* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
> /* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
> /* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
> /* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
> /* 026 */
> /* 027 */ public SpecificUnsafeProjection(Object[] references) {
> /* 028 */ this.references = references;
> /* 029 */
> /* 030 */
> /* 031 */ this.values = null;
> /* 032 */
> /* 033 */
> /* 034 */ isNull12 = false;
> /* 035 */ value12 = false;
> /* 036 */ isNull13 = false;
> /* 037 */ value13 = null;
> /* 038 */ isNull14 = false;
> /* 039 */ value14 = null;
> /* 040 */ result = new UnsafeRow(1);
> /* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
> /* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
> /* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
> /* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 047 */
> /* 048 */ }
> /* 049 */
> /* 050 */ public void initialize(int partitionIndex) {
> /* 051 */
> /* 052 */ }
> /* 053 */
> /* 054 */
> /* 055 */ private void evalIfTrueExpr(InternalRow i) {
> /* 056 */ final InternalRow value7 = null;
> /* 057 */ isNull13 = true;
> /* 058 */ value13 = value7;
> /* 059 */ }
> /* 060 */
> /* 061 */
> /* 062 */ private void evalIfCondExpr(InternalRow i) {
> /* 063 */
> /* 064 */ isNull12 = false;
> /* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
> /* 066 */ }
> /* 067 */
> /* 068 */
> /* 069 */ private void evalIfFalseExpr(InternalRow i) {
> /* 070 */ values = new Object[1];
> /* 071 */ resultIsNull1 = false;
> /* 072 */ if (!resultIsNull1) {
> /* 073 */
> /* 074 */ boolean isNull11 = true;
> /* 075 */ scala.Option value11 = null;
> /* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
> /* 077 */
> /* 078 */ isNull11 = false;
> /* 079 */ if (!isNull11) {
> /* 080 */
> /* 081 */ Object funcResult1 = null;
> /* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
> /* 083 */ if (funcResult1 == null) {
> /* 084 */ isNull11 = true;
> /* 085 */ } else {
> /* 086 */ value11 = (scala.Option) funcResult1;
> /* 087 */ }
> /* 088 */
> /* 089 */ }
> /* 090 */ isNull11 = value11 == null;
> /* 091 */ }
> /* 092 */
> /* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
> /* 094 */ scala.collection.Seq value10 = isNull10 ?
> /* 095 */ null : (scala.collection.Seq) value11.get();
> /* 096 */ resultIsNull1 = isNull10;
> /* 097 */ argValue1 = value10;
> /* 098 */ }
> /* 099 */
> /* 100 */
> /* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
> /* 102 */ if (resultIsNull1) {
> /* 103 */ values[0] = null;
> /* 104 */ } else {
> /* 105 */ values[0] = value9;
> /* 106 */ }
> /* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
> /* 108 */ this.values = null;
> /* 109 */ isNull14 = false;
> /* 110 */ value14 = value8;
> /* 111 */ }
> /* 112 */
> /* 113 */
> /* 114 */ // Scala.Function1 need this
> /* 115 */ public java.lang.Object apply(java.lang.Object row) {
> /* 116 */ return apply((InternalRow) row);
> /* 117 */ }
> /* 118 */
> /* 119 */ public UnsafeRow apply(InternalRow i) {
> /* 120 */ holder.reset();
> /* 121 */
> /* 122 */ rowWriter.zeroOutNullBytes();
> /* 123 */
> /* 124 */
> /* 125 */ boolean isNull3 = i.isNullAt(0);
> /* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
> /* 127 */
> /* 128 */ if (isNull3) {
> /* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
> /* 130 */ }
> /* 131 */ boolean isNull1 = true;
> /* 132 */ scala.collection.immutable.Map value1 = null;
> /* 133 */ if (!false) {
> /* 134 */
> /* 135 */ isNull1 = false;
> /* 136 */ if (!isNull1) {
> /* 137 */
> /* 138 */ Object funcResult = null;
> /* 139 */ funcResult = value3.properties();
> /* 140 */ if (funcResult == null) {
> /* 141 */ isNull1 = true;
> /* 142 */ } else {
> /* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
> /* 144 */ }
> /* 145 */
> /* 146 */ }
> /* 147 */ isNull1 = value1 == null;
> /* 148 */ }
> /* 149 */ MapData value = null;
> /* 150 */ if (!isNull1) {
> /* 151 */ final int length = value1.size();
> /* 152 */ final Object[] convertedKeys = new Object[length];
> /* 153 */ final Object[] convertedValues = new Object[length];
> /* 154 */ int index = 0;
> /* 155 */ final scala.collection.Iterator entries = value1.iterator();
> /* 156 */ while(entries.hasNext()) {
> /* 157 */
> /* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
> /* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
> /* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
> /* 161 */
> /* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
> /* 163 */
> /* 164 */
> /* 165 */ resultIsNull = false;
> /* 166 */ if (!resultIsNull) {
> /* 167 */
> /* 168 */ resultIsNull = false;
> /* 169 */ argValue = ExternalMapToCatalyst_key0;
> /* 170 */ }
> /* 171 */
> /* 172 */ boolean isNull4 = resultIsNull;
> /* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
> /* 174 */ isNull4 = value4 == null;
> /* 175 */ if (isNull4) {
> /* 176 */ throw new RuntimeException("Cannot use null as map key!");
> /* 177 */ } else {
> /* 178 */ convertedKeys[index] = (UTF8String) value4;
> /* 179 */ }
> /* 180 */
> /* 181 */
> /* 182 */ evalIfCondExpr(i);
> /* 183 */ boolean isNull5 = false;
> /* 184 */ InternalRow value5 = null;
> /* 185 */ if (!isNull12 && value12) {
> /* 186 */ evalIfTrueExpr(i);
> /* 187 */ isNull5 = isNull13;
> /* 188 */ value5 = value13;
> /* 189 */ } else {
> /* 190 */ evalIfFalseExpr(i);
> /* 191 */ isNull5 = isNull14;
> /* 192 */ value5 = value14;
> /* 193 */ }
> /* 194 */ if (isNull5) {
> /* 195 */ convertedValues[index] = null;
> /* 196 */ } else {
> /* 197 */ convertedValues[index] = (InternalRow) value5;
> /* 198 */ }
> /* 199 */
> /* 200 */ index++;
> /* 201 */ }
> /* 202 */
> /* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
> /* 204 */ }
> /* 205 */ if (isNull1) {
> /* 206 */ rowWriter.setNullAt(0);
> /* 207 */ } else {
> /* 208 */ // Remember the current cursor so that we can calculate how many bytes are
> /* 209 */ // written later.
> /* 210 */ final int tmpCursor = holder.cursor;
> /* 211 */
> /* 212 */ if (value instanceof UnsafeMapData) {
> /* 213 */
> /* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
> /* 215 */ // grow the global buffer before writing data.
> /* 216 */ holder.grow(sizeInBytes);
> /* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
> /* 218 */ holder.cursor += sizeInBytes;
> /* 219 */
> /* 220 */ } else {
> /* 221 */ final ArrayData keys = value.keyArray();
> /* 222 */ final ArrayData values1 = value.valueArray();
> /* 223 */
> /* 224 */ // preserve 8 bytes to write the key array numBytes later.
> /* 225 */ holder.grow(8);
> /* 226 */ holder.cursor += 8;
> /* 227 */
> /* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
> /* 229 */ final int tmpCursor1 = holder.cursor;
> /* 230 */
> /* 231 */
> /* 232 */ if (keys instanceof UnsafeArrayData) {
> /* 233 */
> /* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
> /* 235 */ // grow the global buffer before writing data.
> /* 236 */ holder.grow(sizeInBytes1);
> /* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
> /* 238 */ holder.cursor += sizeInBytes1;
> /* 239 */
> /* 240 */ } else {
> /* 241 */ final int numElements = keys.numElements();
> /* 242 */ arrayWriter.initialize(holder, numElements, 8);
> /* 243 */
> /* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
> /* 245 */ if (keys.isNullAt(index1)) {
> /* 246 */ arrayWriter.setNull(index1);
> /* 247 */ } else {
> /* 248 */ final UTF8String element = keys.getUTF8String(index1);
> /* 249 */ arrayWriter.write(index1, element);
> /* 250 */ }
> /* 251 */ }
> /* 252 */ }
> /* 253 */
> /* 254 */ // Write the numBytes of key array into the first 8 bytes.
> /* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
> /* 256 */
> /* 257 */
> /* 258 */ if (values1 instanceof UnsafeArrayData) {
> /* 259 */
> /* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
> /* 261 */ // grow the global buffer before writing data.
> /* 262 */ holder.grow(sizeInBytes4);
> /* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
> /* 264 */ holder.cursor += sizeInBytes4;
> /* 265 */
> /* 266 */ } else {
> /* 267 */ final int numElements1 = values1.numElements();
> /* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
> /* 269 */
> /* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
> /* 271 */ if (values1.isNullAt(index2)) {
> /* 272 */ arrayWriter1.setNull(index2);
> /* 273 */ } else {
> /* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
> /* 275 */
> /* 276 */ final int tmpCursor3 = holder.cursor;
> /* 277 */
> /* 278 */ if (element1 instanceof UnsafeRow) {
> /* 279 */
> /* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
> /* 281 */ // grow the global buffer before writing data.
> /* 282 */ holder.grow(sizeInBytes2);
> /* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
> /* 284 */ holder.cursor += sizeInBytes2;
> /* 285 */
> /* 286 */ } else {
> /* 287 */ rowWriter1.reset();
> /* 288 */
> /* 289 */ final ArrayData fieldName = element1.getArray(0);
> /* 290 */ if (element1.isNullAt(0)) {
> /* 291 */ rowWriter1.setNullAt(0);
> /* 292 */ } else {
> /* 293 */ // Remember the current cursor so that we can calculate how many bytes are
> /* 294 */ // written later.
> /* 295 */ final int tmpCursor4 = holder.cursor;
> /* 296 */
> /* 297 */ if (fieldName instanceof UnsafeArrayData) {
> /* 298 */
> /* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
> /* 300 */ // grow the global buffer before writing data.
> /* 301 */ holder.grow(sizeInBytes3);
> /* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
> /* 303 */ holder.cursor += sizeInBytes3;
> /* 304 */
> /* 305 */ } else {
> /* 306 */ final int numElements2 = fieldName.numElements();
> /* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
> /* 308 */
> /* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
> /* 310 */ if (fieldName.isNullAt(index3)) {
> /* 311 */ arrayWriter2.setNullDouble(index3);
> /* 312 */ } else {
> /* 313 */ final double element2 = fieldName.getDouble(index3);
> /* 314 */ arrayWriter2.write(index3, element2);
> /* 315 */ }
> /* 316 */ }
> /* 317 */ }
> /* 318 */
> /* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
> /* 320 */ }
> /* 321 */ }
> /* 322 */
> /* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
> /* 324 */
> /* 325 */ }
> /* 326 */ }
> /* 327 */ }
> /* 328 */
> /* 329 */ }
> /* 330 */
> /* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
> /* 332 */ }
> /* 333 */ result.setTotalSize(holder.totalSize());
> /* 334 */ return result;
> /* 335 */ }
> /* 336 */ }
> org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
> at org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
> at org.codehaus.janino.UnitCompiler.toRvalueOrCompileException(UnitCompiler.java:6639)
> at org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5001)
> at org.codehaus.janino.UnitCompiler.access$10500(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$13.visitAmbiguousName(UnitCompiler.java:4984)
> at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:3633)
> at org.codehaus.janino.Java$Lvalue.accept(Java.java:3563)
> at org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4956)
> at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4925)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3189)
> at org.codehaus.janino.UnitCompiler.access$5100(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3143)
> at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3139)
> at org.codehaus.janino.Java$Assignment.accept(Java.java:3847)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3139)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2112)
> at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1377)
> at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1370)
> at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2558)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894)
> at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377)
> at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369)
> at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
> at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
> at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
> at org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
> at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
> at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
> at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311)
> at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:935)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:998)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:995)
> at org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
> at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
> at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
> at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
> at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
> at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
> at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:890)
> at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:405)
> at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:359)
> at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:874)
> at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection$lzycompute(ExpressionEncoder.scala:266)
> at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection(ExpressionEncoder.scala:266)
> at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:290)
> at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
> at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
> at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
> at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
> at scala.collection.immutable.List.foreach(List.scala:383)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:245)
> at scala.collection.immutable.List.map(List.scala:286)
> at org.apache.spark.sql.SparkSession.createDataset(SparkSession.scala:429)
> at org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:389)
> at org.apache.spark.sql.SQLImplicits.localSeqToDatasetHolder(SQLImplicits.scala:173)
> at local.TestApp$.main(TestApp.scala:63)
> at local.TestApp.main(TestApp.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
> Exception in thread "main" java.lang.RuntimeException: Error while encoding: java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
> /* 001 */ public java.lang.Object generate(Object[] references) {
> /* 002 */ return new SpecificUnsafeProjection(references);
> /* 003 */ }
> /* 004 */
> /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
> /* 006 */
> /* 007 */ private Object[] references;
> /* 008 */ private boolean resultIsNull;
> /* 009 */ private java.lang.String argValue;
> /* 010 */ private Object[] values;
> /* 011 */ private boolean resultIsNull1;
> /* 012 */ private scala.collection.Seq argValue1;
> /* 013 */ private boolean isNull12;
> /* 014 */ private boolean value12;
> /* 015 */ private boolean isNull13;
> /* 016 */ private InternalRow value13;
> /* 017 */ private boolean isNull14;
> /* 018 */ private InternalRow value14;
> /* 019 */ private UnsafeRow result;
> /* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
> /* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
> /* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
> /* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
> /* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
> /* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
> /* 026 */
> /* 027 */ public SpecificUnsafeProjection(Object[] references) {
> /* 028 */ this.references = references;
> /* 029 */
> /* 030 */
> /* 031 */ this.values = null;
> /* 032 */
> /* 033 */
> /* 034 */ isNull12 = false;
> /* 035 */ value12 = false;
> /* 036 */ isNull13 = false;
> /* 037 */ value13 = null;
> /* 038 */ isNull14 = false;
> /* 039 */ value14 = null;
> /* 040 */ result = new UnsafeRow(1);
> /* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
> /* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
> /* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
> /* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 047 */
> /* 048 */ }
> /* 049 */
> /* 050 */ public void initialize(int partitionIndex) {
> /* 051 */
> /* 052 */ }
> /* 053 */
> /* 054 */
> /* 055 */ private void evalIfTrueExpr(InternalRow i) {
> /* 056 */ final InternalRow value7 = null;
> /* 057 */ isNull13 = true;
> /* 058 */ value13 = value7;
> /* 059 */ }
> /* 060 */
> /* 061 */
> /* 062 */ private void evalIfCondExpr(InternalRow i) {
> /* 063 */
> /* 064 */ isNull12 = false;
> /* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
> /* 066 */ }
> /* 067 */
> /* 068 */
> /* 069 */ private void evalIfFalseExpr(InternalRow i) {
> /* 070 */ values = new Object[1];
> /* 071 */ resultIsNull1 = false;
> /* 072 */ if (!resultIsNull1) {
> /* 073 */
> /* 074 */ boolean isNull11 = true;
> /* 075 */ scala.Option value11 = null;
> /* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
> /* 077 */
> /* 078 */ isNull11 = false;
> /* 079 */ if (!isNull11) {
> /* 080 */
> /* 081 */ Object funcResult1 = null;
> /* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
> /* 083 */ if (funcResult1 == null) {
> /* 084 */ isNull11 = true;
> /* 085 */ } else {
> /* 086 */ value11 = (scala.Option) funcResult1;
> /* 087 */ }
> /* 088 */
> /* 089 */ }
> /* 090 */ isNull11 = value11 == null;
> /* 091 */ }
> /* 092 */
> /* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
> /* 094 */ scala.collection.Seq value10 = isNull10 ?
> /* 095 */ null : (scala.collection.Seq) value11.get();
> /* 096 */ resultIsNull1 = isNull10;
> /* 097 */ argValue1 = value10;
> /* 098 */ }
> /* 099 */
> /* 100 */
> /* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
> /* 102 */ if (resultIsNull1) {
> /* 103 */ values[0] = null;
> /* 104 */ } else {
> /* 105 */ values[0] = value9;
> /* 106 */ }
> /* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
> /* 108 */ this.values = null;
> /* 109 */ isNull14 = false;
> /* 110 */ value14 = value8;
> /* 111 */ }
> /* 112 */
> /* 113 */
> /* 114 */ // Scala.Function1 need this
> /* 115 */ public java.lang.Object apply(java.lang.Object row) {
> /* 116 */ return apply((InternalRow) row);
> /* 117 */ }
> /* 118 */
> /* 119 */ public UnsafeRow apply(InternalRow i) {
> /* 120 */ holder.reset();
> /* 121 */
> /* 122 */ rowWriter.zeroOutNullBytes();
> /* 123 */
> /* 124 */
> /* 125 */ boolean isNull3 = i.isNullAt(0);
> /* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
> /* 127 */
> /* 128 */ if (isNull3) {
> /* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
> /* 130 */ }
> /* 131 */ boolean isNull1 = true;
> /* 132 */ scala.collection.immutable.Map value1 = null;
> /* 133 */ if (!false) {
> /* 134 */
> /* 135 */ isNull1 = false;
> /* 136 */ if (!isNull1) {
> /* 137 */
> /* 138 */ Object funcResult = null;
> /* 139 */ funcResult = value3.properties();
> /* 140 */ if (funcResult == null) {
> /* 141 */ isNull1 = true;
> /* 142 */ } else {
> /* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
> /* 144 */ }
> /* 145 */
> /* 146 */ }
> /* 147 */ isNull1 = value1 == null;
> /* 148 */ }
> /* 149 */ MapData value = null;
> /* 150 */ if (!isNull1) {
> /* 151 */ final int length = value1.size();
> /* 152 */ final Object[] convertedKeys = new Object[length];
> /* 153 */ final Object[] convertedValues = new Object[length];
> /* 154 */ int index = 0;
> /* 155 */ final scala.collection.Iterator entries = value1.iterator();
> /* 156 */ while(entries.hasNext()) {
> /* 157 */
> /* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
> /* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
> /* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
> /* 161 */
> /* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
> /* 163 */
> /* 164 */
> /* 165 */ resultIsNull = false;
> /* 166 */ if (!resultIsNull) {
> /* 167 */
> /* 168 */ resultIsNull = false;
> /* 169 */ argValue = ExternalMapToCatalyst_key0;
> /* 170 */ }
> /* 171 */
> /* 172 */ boolean isNull4 = resultIsNull;
> /* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
> /* 174 */ isNull4 = value4 == null;
> /* 175 */ if (isNull4) {
> /* 176 */ throw new RuntimeException("Cannot use null as map key!");
> /* 177 */ } else {
> /* 178 */ convertedKeys[index] = (UTF8String) value4;
> /* 179 */ }
> /* 180 */
> /* 181 */
> /* 182 */ evalIfCondExpr(i);
> /* 183 */ boolean isNull5 = false;
> /* 184 */ InternalRow value5 = null;
> /* 185 */ if (!isNull12 && value12) {
> /* 186 */ evalIfTrueExpr(i);
> /* 187 */ isNull5 = isNull13;
> /* 188 */ value5 = value13;
> /* 189 */ } else {
> /* 190 */ evalIfFalseExpr(i);
> /* 191 */ isNull5 = isNull14;
> /* 192 */ value5 = value14;
> /* 193 */ }
> /* 194 */ if (isNull5) {
> /* 195 */ convertedValues[index] = null;
> /* 196 */ } else {
> /* 197 */ convertedValues[index] = (InternalRow) value5;
> /* 198 */ }
> /* 199 */
> /* 200 */ index++;
> /* 201 */ }
> /* 202 */
> /* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
> /* 204 */ }
> /* 205 */ if (isNull1) {
> /* 206 */ rowWriter.setNullAt(0);
> /* 207 */ } else {
> /* 208 */ // Remember the current cursor so that we can calculate how many bytes are
> /* 209 */ // written later.
> /* 210 */ final int tmpCursor = holder.cursor;
> /* 211 */
> /* 212 */ if (value instanceof UnsafeMapData) {
> /* 213 */
> /* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
> /* 215 */ // grow the global buffer before writing data.
> /* 216 */ holder.grow(sizeInBytes);
> /* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
> /* 218 */ holder.cursor += sizeInBytes;
> /* 219 */
> /* 220 */ } else {
> /* 221 */ final ArrayData keys = value.keyArray();
> /* 222 */ final ArrayData values1 = value.valueArray();
> /* 223 */
> /* 224 */ // preserve 8 bytes to write the key array numBytes later.
> /* 225 */ holder.grow(8);
> /* 226 */ holder.cursor += 8;
> /* 227 */
> /* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
> /* 229 */ final int tmpCursor1 = holder.cursor;
> /* 230 */
> /* 231 */
> /* 232 */ if (keys instanceof UnsafeArrayData) {
> /* 233 */
> /* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
> /* 235 */ // grow the global buffer before writing data.
> /* 236 */ holder.grow(sizeInBytes1);
> /* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
> /* 238 */ holder.cursor += sizeInBytes1;
> /* 239 */
> /* 240 */ } else {
> /* 241 */ final int numElements = keys.numElements();
> /* 242 */ arrayWriter.initialize(holder, numElements, 8);
> /* 243 */
> /* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
> /* 245 */ if (keys.isNullAt(index1)) {
> /* 246 */ arrayWriter.setNull(index1);
> /* 247 */ } else {
> /* 248 */ final UTF8String element = keys.getUTF8String(index1);
> /* 249 */ arrayWriter.write(index1, element);
> /* 250 */ }
> /* 251 */ }
> /* 252 */ }
> /* 253 */
> /* 254 */ // Write the numBytes of key array into the first 8 bytes.
> /* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
> /* 256 */
> /* 257 */
> /* 258 */ if (values1 instanceof UnsafeArrayData) {
> /* 259 */
> /* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
> /* 261 */ // grow the global buffer before writing data.
> /* 262 */ holder.grow(sizeInBytes4);
> /* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
> /* 264 */ holder.cursor += sizeInBytes4;
> /* 265 */
> /* 266 */ } else {
> /* 267 */ final int numElements1 = values1.numElements();
> /* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
> /* 269 */
> /* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
> /* 271 */ if (values1.isNullAt(index2)) {
> /* 272 */ arrayWriter1.setNull(index2);
> /* 273 */ } else {
> /* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
> /* 275 */
> /* 276 */ final int tmpCursor3 = holder.cursor;
> /* 277 */
> /* 278 */ if (element1 instanceof UnsafeRow) {
> /* 279 */
> /* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
> /* 281 */ // grow the global buffer before writing data.
> /* 282 */ holder.grow(sizeInBytes2);
> /* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
> /* 284 */ holder.cursor += sizeInBytes2;
> /* 285 */
> /* 286 */ } else {
> /* 287 */ rowWriter1.reset();
> /* 288 */
> /* 289 */ final ArrayData fieldName = element1.getArray(0);
> /* 290 */ if (element1.isNullAt(0)) {
> /* 291 */ rowWriter1.setNullAt(0);
> /* 292 */ } else {
> /* 293 */ // Remember the current cursor so that we can calculate how many bytes are
> /* 294 */ // written later.
> /* 295 */ final int tmpCursor4 = holder.cursor;
> /* 296 */
> /* 297 */ if (fieldName instanceof UnsafeArrayData) {
> /* 298 */
> /* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
> /* 300 */ // grow the global buffer before writing data.
> /* 301 */ holder.grow(sizeInBytes3);
> /* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
> /* 303 */ holder.cursor += sizeInBytes3;
> /* 304 */
> /* 305 */ } else {
> /* 306 */ final int numElements2 = fieldName.numElements();
> /* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
> /* 308 */
> /* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
> /* 310 */ if (fieldName.isNullAt(index3)) {
> /* 311 */ arrayWriter2.setNullDouble(index3);
> /* 312 */ } else {
> /* 313 */ final double element2 = fieldName.getDouble(index3);
> /* 314 */ arrayWriter2.write(index3, element2);
> /* 315 */ }
> /* 316 */ }
> /* 317 */ }
> /* 318 */
> /* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
> /* 320 */ }
> /* 321 */ }
> /* 322 */
> /* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
> /* 324 */
> /* 325 */ }
> /* 326 */ }
> /* 327 */ }
> /* 328 */
> /* 329 */ }
> /* 330 */
> /* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
> /* 332 */ }
> /* 333 */ result.setTotalSize(holder.totalSize());
> /* 334 */ return result;
> /* 335 */ }
> /* 336 */ }
> externalmaptocatalyst(ExternalMapToCatalyst_key0, ObjectType(class java.lang.String), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key0, false, ObjectType(class java.lang.String)), true), ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values), if (isnull(lambdavariable(ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values)))) null else named_struct(values, newInstance(class org.apache.spark.sql.catalyst.util.GenericArrayData)), assertnotnull(input[0, local.ItemProperties, true], top level Product input object).properties) AS properties#0
> +- externalmaptocatalyst(ExternalMapToCatalyst_key0, ObjectType(class java.lang.String), staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, lambdavariable(ExternalMapToCatalyst_key0, false, ObjectType(class java.lang.String)), true), ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values), if (isnull(lambdavariable(ExternalMapToCatalyst_value0, ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values)))) null else named_struct(values, newInstance(class org.apache.spark.sql.catalyst.util.GenericArrayData)), assertnotnull(input[0, local.ItemProperties, true], top level Product input object).properties)
> +- assertnotnull(input[0, local.ItemProperties, true], top level Product input object).properties
> +- assertnotnull(input[0, local.ItemProperties, true], top level Product input object)
> +- input[0, local.ItemProperties, true]
> at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:293)
> at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
> at org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429)
> at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
> at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245)
> at scala.collection.immutable.List.foreach(List.scala:383)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:245)
> at scala.collection.immutable.List.map(List.scala:286)
> at org.apache.spark.sql.SparkSession.createDataset(SparkSession.scala:429)
> at org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:389)
> at org.apache.spark.sql.SQLImplicits.localSeqToDatasetHolder(SQLImplicits.scala:173)
> at local.TestApp$.main(TestApp.scala:63)
> at local.TestApp.main(TestApp.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
> Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
> /* 001 */ public java.lang.Object generate(Object[] references) {
> /* 002 */ return new SpecificUnsafeProjection(references);
> /* 003 */ }
> /* 004 */
> /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
> /* 006 */
> /* 007 */ private Object[] references;
> /* 008 */ private boolean resultIsNull;
> /* 009 */ private java.lang.String argValue;
> /* 010 */ private Object[] values;
> /* 011 */ private boolean resultIsNull1;
> /* 012 */ private scala.collection.Seq argValue1;
> /* 013 */ private boolean isNull12;
> /* 014 */ private boolean value12;
> /* 015 */ private boolean isNull13;
> /* 016 */ private InternalRow value13;
> /* 017 */ private boolean isNull14;
> /* 018 */ private InternalRow value14;
> /* 019 */ private UnsafeRow result;
> /* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
> /* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
> /* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
> /* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
> /* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
> /* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
> /* 026 */
> /* 027 */ public SpecificUnsafeProjection(Object[] references) {
> /* 028 */ this.references = references;
> /* 029 */
> /* 030 */
> /* 031 */ this.values = null;
> /* 032 */
> /* 033 */
> /* 034 */ isNull12 = false;
> /* 035 */ value12 = false;
> /* 036 */ isNull13 = false;
> /* 037 */ value13 = null;
> /* 038 */ isNull14 = false;
> /* 039 */ value14 = null;
> /* 040 */ result = new UnsafeRow(1);
> /* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
> /* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
> /* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
> /* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 047 */
> /* 048 */ }
> /* 049 */
> /* 050 */ public void initialize(int partitionIndex) {
> /* 051 */
> /* 052 */ }
> /* 053 */
> /* 054 */
> /* 055 */ private void evalIfTrueExpr(InternalRow i) {
> /* 056 */ final InternalRow value7 = null;
> /* 057 */ isNull13 = true;
> /* 058 */ value13 = value7;
> /* 059 */ }
> /* 060 */
> /* 061 */
> /* 062 */ private void evalIfCondExpr(InternalRow i) {
> /* 063 */
> /* 064 */ isNull12 = false;
> /* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
> /* 066 */ }
> /* 067 */
> /* 068 */
> /* 069 */ private void evalIfFalseExpr(InternalRow i) {
> /* 070 */ values = new Object[1];
> /* 071 */ resultIsNull1 = false;
> /* 072 */ if (!resultIsNull1) {
> /* 073 */
> /* 074 */ boolean isNull11 = true;
> /* 075 */ scala.Option value11 = null;
> /* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
> /* 077 */
> /* 078 */ isNull11 = false;
> /* 079 */ if (!isNull11) {
> /* 080 */
> /* 081 */ Object funcResult1 = null;
> /* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
> /* 083 */ if (funcResult1 == null) {
> /* 084 */ isNull11 = true;
> /* 085 */ } else {
> /* 086 */ value11 = (scala.Option) funcResult1;
> /* 087 */ }
> /* 088 */
> /* 089 */ }
> /* 090 */ isNull11 = value11 == null;
> /* 091 */ }
> /* 092 */
> /* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
> /* 094 */ scala.collection.Seq value10 = isNull10 ?
> /* 095 */ null : (scala.collection.Seq) value11.get();
> /* 096 */ resultIsNull1 = isNull10;
> /* 097 */ argValue1 = value10;
> /* 098 */ }
> /* 099 */
> /* 100 */
> /* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
> /* 102 */ if (resultIsNull1) {
> /* 103 */ values[0] = null;
> /* 104 */ } else {
> /* 105 */ values[0] = value9;
> /* 106 */ }
> /* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
> /* 108 */ this.values = null;
> /* 109 */ isNull14 = false;
> /* 110 */ value14 = value8;
> /* 111 */ }
> /* 112 */
> /* 113 */
> /* 114 */ // Scala.Function1 need this
> /* 115 */ public java.lang.Object apply(java.lang.Object row) {
> /* 116 */ return apply((InternalRow) row);
> /* 117 */ }
> /* 118 */
> /* 119 */ public UnsafeRow apply(InternalRow i) {
> /* 120 */ holder.reset();
> /* 121 */
> /* 122 */ rowWriter.zeroOutNullBytes();
> /* 123 */
> /* 124 */
> /* 125 */ boolean isNull3 = i.isNullAt(0);
> /* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
> /* 127 */
> /* 128 */ if (isNull3) {
> /* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
> /* 130 */ }
> /* 131 */ boolean isNull1 = true;
> /* 132 */ scala.collection.immutable.Map value1 = null;
> /* 133 */ if (!false) {
> /* 134 */
> /* 135 */ isNull1 = false;
> /* 136 */ if (!isNull1) {
> /* 137 */
> /* 138 */ Object funcResult = null;
> /* 139 */ funcResult = value3.properties();
> /* 140 */ if (funcResult == null) {
> /* 141 */ isNull1 = true;
> /* 142 */ } else {
> /* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
> /* 144 */ }
> /* 145 */
> /* 146 */ }
> /* 147 */ isNull1 = value1 == null;
> /* 148 */ }
> /* 149 */ MapData value = null;
> /* 150 */ if (!isNull1) {
> /* 151 */ final int length = value1.size();
> /* 152 */ final Object[] convertedKeys = new Object[length];
> /* 153 */ final Object[] convertedValues = new Object[length];
> /* 154 */ int index = 0;
> /* 155 */ final scala.collection.Iterator entries = value1.iterator();
> /* 156 */ while(entries.hasNext()) {
> /* 157 */
> /* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
> /* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
> /* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
> /* 161 */
> /* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
> /* 163 */
> /* 164 */
> /* 165 */ resultIsNull = false;
> /* 166 */ if (!resultIsNull) {
> /* 167 */
> /* 168 */ resultIsNull = false;
> /* 169 */ argValue = ExternalMapToCatalyst_key0;
> /* 170 */ }
> /* 171 */
> /* 172 */ boolean isNull4 = resultIsNull;
> /* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
> /* 174 */ isNull4 = value4 == null;
> /* 175 */ if (isNull4) {
> /* 176 */ throw new RuntimeException("Cannot use null as map key!");
> /* 177 */ } else {
> /* 178 */ convertedKeys[index] = (UTF8String) value4;
> /* 179 */ }
> /* 180 */
> /* 181 */
> /* 182 */ evalIfCondExpr(i);
> /* 183 */ boolean isNull5 = false;
> /* 184 */ InternalRow value5 = null;
> /* 185 */ if (!isNull12 && value12) {
> /* 186 */ evalIfTrueExpr(i);
> /* 187 */ isNull5 = isNull13;
> /* 188 */ value5 = value13;
> /* 189 */ } else {
> /* 190 */ evalIfFalseExpr(i);
> /* 191 */ isNull5 = isNull14;
> /* 192 */ value5 = value14;
> /* 193 */ }
> /* 194 */ if (isNull5) {
> /* 195 */ convertedValues[index] = null;
> /* 196 */ } else {
> /* 197 */ convertedValues[index] = (InternalRow) value5;
> /* 198 */ }
> /* 199 */
> /* 200 */ index++;
> /* 201 */ }
> /* 202 */
> /* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
> /* 204 */ }
> /* 205 */ if (isNull1) {
> /* 206 */ rowWriter.setNullAt(0);
> /* 207 */ } else {
> /* 208 */ // Remember the current cursor so that we can calculate how many bytes are
> /* 209 */ // written later.
> /* 210 */ final int tmpCursor = holder.cursor;
> /* 211 */
> /* 212 */ if (value instanceof UnsafeMapData) {
> /* 213 */
> /* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
> /* 215 */ // grow the global buffer before writing data.
> /* 216 */ holder.grow(sizeInBytes);
> /* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
> /* 218 */ holder.cursor += sizeInBytes;
> /* 219 */
> /* 220 */ } else {
> /* 221 */ final ArrayData keys = value.keyArray();
> /* 222 */ final ArrayData values1 = value.valueArray();
> /* 223 */
> /* 224 */ // preserve 8 bytes to write the key array numBytes later.
> /* 225 */ holder.grow(8);
> /* 226 */ holder.cursor += 8;
> /* 227 */
> /* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
> /* 229 */ final int tmpCursor1 = holder.cursor;
> /* 230 */
> /* 231 */
> /* 232 */ if (keys instanceof UnsafeArrayData) {
> /* 233 */
> /* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
> /* 235 */ // grow the global buffer before writing data.
> /* 236 */ holder.grow(sizeInBytes1);
> /* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
> /* 238 */ holder.cursor += sizeInBytes1;
> /* 239 */
> /* 240 */ } else {
> /* 241 */ final int numElements = keys.numElements();
> /* 242 */ arrayWriter.initialize(holder, numElements, 8);
> /* 243 */
> /* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
> /* 245 */ if (keys.isNullAt(index1)) {
> /* 246 */ arrayWriter.setNull(index1);
> /* 247 */ } else {
> /* 248 */ final UTF8String element = keys.getUTF8String(index1);
> /* 249 */ arrayWriter.write(index1, element);
> /* 250 */ }
> /* 251 */ }
> /* 252 */ }
> /* 253 */
> /* 254 */ // Write the numBytes of key array into the first 8 bytes.
> /* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
> /* 256 */
> /* 257 */
> /* 258 */ if (values1 instanceof UnsafeArrayData) {
> /* 259 */
> /* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
> /* 261 */ // grow the global buffer before writing data.
> /* 262 */ holder.grow(sizeInBytes4);
> /* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
> /* 264 */ holder.cursor += sizeInBytes4;
> /* 265 */
> /* 266 */ } else {
> /* 267 */ final int numElements1 = values1.numElements();
> /* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
> /* 269 */
> /* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
> /* 271 */ if (values1.isNullAt(index2)) {
> /* 272 */ arrayWriter1.setNull(index2);
> /* 273 */ } else {
> /* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
> /* 275 */
> /* 276 */ final int tmpCursor3 = holder.cursor;
> /* 277 */
> /* 278 */ if (element1 instanceof UnsafeRow) {
> /* 279 */
> /* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
> /* 281 */ // grow the global buffer before writing data.
> /* 282 */ holder.grow(sizeInBytes2);
> /* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
> /* 284 */ holder.cursor += sizeInBytes2;
> /* 285 */
> /* 286 */ } else {
> /* 287 */ rowWriter1.reset();
> /* 288 */
> /* 289 */ final ArrayData fieldName = element1.getArray(0);
> /* 290 */ if (element1.isNullAt(0)) {
> /* 291 */ rowWriter1.setNullAt(0);
> /* 292 */ } else {
> /* 293 */ // Remember the current cursor so that we can calculate how many bytes are
> /* 294 */ // written later.
> /* 295 */ final int tmpCursor4 = holder.cursor;
> /* 296 */
> /* 297 */ if (fieldName instanceof UnsafeArrayData) {
> /* 298 */
> /* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
> /* 300 */ // grow the global buffer before writing data.
> /* 301 */ holder.grow(sizeInBytes3);
> /* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
> /* 303 */ holder.cursor += sizeInBytes3;
> /* 304 */
> /* 305 */ } else {
> /* 306 */ final int numElements2 = fieldName.numElements();
> /* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
> /* 308 */
> /* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
> /* 310 */ if (fieldName.isNullAt(index3)) {
> /* 311 */ arrayWriter2.setNullDouble(index3);
> /* 312 */ } else {
> /* 313 */ final double element2 = fieldName.getDouble(index3);
> /* 314 */ arrayWriter2.write(index3, element2);
> /* 315 */ }
> /* 316 */ }
> /* 317 */ }
> /* 318 */
> /* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
> /* 320 */ }
> /* 321 */ }
> /* 322 */
> /* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
> /* 324 */
> /* 325 */ }
> /* 326 */ }
> /* 327 */ }
> /* 328 */
> /* 329 */ }
> /* 330 */
> /* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
> /* 332 */ }
> /* 333 */ result.setTotalSize(holder.totalSize());
> /* 334 */ return result;
> /* 335 */ }
> /* 336 */ }
> at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306)
> at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293)
> at org.spark_project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
> at org.spark_project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135)
> at org.spark_project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410)
> at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380)
> at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
> at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
> at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
> at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
> at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:890)
> at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:405)
> at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:359)
> at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:874)
> at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection$lzycompute(ExpressionEncoder.scala:266)
> at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection(ExpressionEncoder.scala:266)
> at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:290)
> ... 17 more
> Caused by: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
> /* 001 */ public java.lang.Object generate(Object[] references) {
> /* 002 */ return new SpecificUnsafeProjection(references);
> /* 003 */ }
> /* 004 */
> /* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
> /* 006 */
> /* 007 */ private Object[] references;
> /* 008 */ private boolean resultIsNull;
> /* 009 */ private java.lang.String argValue;
> /* 010 */ private Object[] values;
> /* 011 */ private boolean resultIsNull1;
> /* 012 */ private scala.collection.Seq argValue1;
> /* 013 */ private boolean isNull12;
> /* 014 */ private boolean value12;
> /* 015 */ private boolean isNull13;
> /* 016 */ private InternalRow value13;
> /* 017 */ private boolean isNull14;
> /* 018 */ private InternalRow value14;
> /* 019 */ private UnsafeRow result;
> /* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
> /* 021 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
> /* 022 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter;
> /* 023 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter1;
> /* 024 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1;
> /* 025 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter arrayWriter2;
> /* 026 */
> /* 027 */ public SpecificUnsafeProjection(Object[] references) {
> /* 028 */ this.references = references;
> /* 029 */
> /* 030 */
> /* 031 */ this.values = null;
> /* 032 */
> /* 033 */
> /* 034 */ isNull12 = false;
> /* 035 */ value12 = false;
> /* 036 */ isNull13 = false;
> /* 037 */ value13 = null;
> /* 038 */ isNull14 = false;
> /* 039 */ value14 = null;
> /* 040 */ result = new UnsafeRow(1);
> /* 041 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32);
> /* 042 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
> /* 043 */ this.arrayWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 044 */ this.arrayWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 045 */ this.rowWriter1 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1);
> /* 046 */ this.arrayWriter2 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter();
> /* 047 */
> /* 048 */ }
> /* 049 */
> /* 050 */ public void initialize(int partitionIndex) {
> /* 051 */
> /* 052 */ }
> /* 053 */
> /* 054 */
> /* 055 */ private void evalIfTrueExpr(InternalRow i) {
> /* 056 */ final InternalRow value7 = null;
> /* 057 */ isNull13 = true;
> /* 058 */ value13 = value7;
> /* 059 */ }
> /* 060 */
> /* 061 */
> /* 062 */ private void evalIfCondExpr(InternalRow i) {
> /* 063 */
> /* 064 */ isNull12 = false;
> /* 065 */ value12 = ExternalMapToCatalyst_value_isNull0;
> /* 066 */ }
> /* 067 */
> /* 068 */
> /* 069 */ private void evalIfFalseExpr(InternalRow i) {
> /* 070 */ values = new Object[1];
> /* 071 */ resultIsNull1 = false;
> /* 072 */ if (!resultIsNull1) {
> /* 073 */
> /* 074 */ boolean isNull11 = true;
> /* 075 */ scala.Option value11 = null;
> /* 076 */ if (!ExternalMapToCatalyst_value_isNull0) {
> /* 077 */
> /* 078 */ isNull11 = false;
> /* 079 */ if (!isNull11) {
> /* 080 */
> /* 081 */ Object funcResult1 = null;
> /* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values();
> /* 083 */ if (funcResult1 == null) {
> /* 084 */ isNull11 = true;
> /* 085 */ } else {
> /* 086 */ value11 = (scala.Option) funcResult1;
> /* 087 */ }
> /* 088 */
> /* 089 */ }
> /* 090 */ isNull11 = value11 == null;
> /* 091 */ }
> /* 092 */
> /* 093 */ final boolean isNull10 = isNull11 || value11.isEmpty();
> /* 094 */ scala.collection.Seq value10 = isNull10 ?
> /* 095 */ null : (scala.collection.Seq) value11.get();
> /* 096 */ resultIsNull1 = isNull10;
> /* 097 */ argValue1 = value10;
> /* 098 */ }
> /* 099 */
> /* 100 */
> /* 101 */ final ArrayData value9 = resultIsNull1 ? null : new org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
> /* 102 */ if (resultIsNull1) {
> /* 103 */ values[0] = null;
> /* 104 */ } else {
> /* 105 */ values[0] = value9;
> /* 106 */ }
> /* 107 */ final InternalRow value8 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
> /* 108 */ this.values = null;
> /* 109 */ isNull14 = false;
> /* 110 */ value14 = value8;
> /* 111 */ }
> /* 112 */
> /* 113 */
> /* 114 */ // Scala.Function1 need this
> /* 115 */ public java.lang.Object apply(java.lang.Object row) {
> /* 116 */ return apply((InternalRow) row);
> /* 117 */ }
> /* 118 */
> /* 119 */ public UnsafeRow apply(InternalRow i) {
> /* 120 */ holder.reset();
> /* 121 */
> /* 122 */ rowWriter.zeroOutNullBytes();
> /* 123 */
> /* 124 */
> /* 125 */ boolean isNull3 = i.isNullAt(0);
> /* 126 */ local.ItemProperties value3 = isNull3 ? null : ((local.ItemProperties)i.get(0, null));
> /* 127 */
> /* 128 */ if (isNull3) {
> /* 129 */ throw new RuntimeException(((java.lang.String) references[0]));
> /* 130 */ }
> /* 131 */ boolean isNull1 = true;
> /* 132 */ scala.collection.immutable.Map value1 = null;
> /* 133 */ if (!false) {
> /* 134 */
> /* 135 */ isNull1 = false;
> /* 136 */ if (!isNull1) {
> /* 137 */
> /* 138 */ Object funcResult = null;
> /* 139 */ funcResult = value3.properties();
> /* 140 */ if (funcResult == null) {
> /* 141 */ isNull1 = true;
> /* 142 */ } else {
> /* 143 */ value1 = (scala.collection.immutable.Map) funcResult;
> /* 144 */ }
> /* 145 */
> /* 146 */ }
> /* 147 */ isNull1 = value1 == null;
> /* 148 */ }
> /* 149 */ MapData value = null;
> /* 150 */ if (!isNull1) {
> /* 151 */ final int length = value1.size();
> /* 152 */ final Object[] convertedKeys = new Object[length];
> /* 153 */ final Object[] convertedValues = new Object[length];
> /* 154 */ int index = 0;
> /* 155 */ final scala.collection.Iterator entries = value1.iterator();
> /* 156 */ while(entries.hasNext()) {
> /* 157 */
> /* 158 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next();
> /* 159 */ java.lang.String ExternalMapToCatalyst_key0 = (java.lang.String) entry._1();
> /* 160 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) entry._2();
> /* 161 */
> /* 162 */ boolean ExternalMapToCatalyst_value_isNull0 = ExternalMapToCatalyst_value0 == null;
> /* 163 */
> /* 164 */
> /* 165 */ resultIsNull = false;
> /* 166 */ if (!resultIsNull) {
> /* 167 */
> /* 168 */ resultIsNull = false;
> /* 169 */ argValue = ExternalMapToCatalyst_key0;
> /* 170 */ }
> /* 171 */
> /* 172 */ boolean isNull4 = resultIsNull;
> /* 173 */ final UTF8String value4 = resultIsNull ? null : org.apache.spark.unsafe.types.UTF8String.fromString(argValue);
> /* 174 */ isNull4 = value4 == null;
> /* 175 */ if (isNull4) {
> /* 176 */ throw new RuntimeException("Cannot use null as map key!");
> /* 177 */ } else {
> /* 178 */ convertedKeys[index] = (UTF8String) value4;
> /* 179 */ }
> /* 180 */
> /* 181 */
> /* 182 */ evalIfCondExpr(i);
> /* 183 */ boolean isNull5 = false;
> /* 184 */ InternalRow value5 = null;
> /* 185 */ if (!isNull12 && value12) {
> /* 186 */ evalIfTrueExpr(i);
> /* 187 */ isNull5 = isNull13;
> /* 188 */ value5 = value13;
> /* 189 */ } else {
> /* 190 */ evalIfFalseExpr(i);
> /* 191 */ isNull5 = isNull14;
> /* 192 */ value5 = value14;
> /* 193 */ }
> /* 194 */ if (isNull5) {
> /* 195 */ convertedValues[index] = null;
> /* 196 */ } else {
> /* 197 */ convertedValues[index] = (InternalRow) value5;
> /* 198 */ }
> /* 199 */
> /* 200 */ index++;
> /* 201 */ }
> /* 202 */
> /* 203 */ value = new org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues));
> /* 204 */ }
> /* 205 */ if (isNull1) {
> /* 206 */ rowWriter.setNullAt(0);
> /* 207 */ } else {
> /* 208 */ // Remember the current cursor so that we can calculate how many bytes are
> /* 209 */ // written later.
> /* 210 */ final int tmpCursor = holder.cursor;
> /* 211 */
> /* 212 */ if (value instanceof UnsafeMapData) {
> /* 213 */
> /* 214 */ final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
> /* 215 */ // grow the global buffer before writing data.
> /* 216 */ holder.grow(sizeInBytes);
> /* 217 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
> /* 218 */ holder.cursor += sizeInBytes;
> /* 219 */
> /* 220 */ } else {
> /* 221 */ final ArrayData keys = value.keyArray();
> /* 222 */ final ArrayData values1 = value.valueArray();
> /* 223 */
> /* 224 */ // preserve 8 bytes to write the key array numBytes later.
> /* 225 */ holder.grow(8);
> /* 226 */ holder.cursor += 8;
> /* 227 */
> /* 228 */ // Remember the current cursor so that we can write numBytes of key array later.
> /* 229 */ final int tmpCursor1 = holder.cursor;
> /* 230 */
> /* 231 */
> /* 232 */ if (keys instanceof UnsafeArrayData) {
> /* 233 */
> /* 234 */ final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
> /* 235 */ // grow the global buffer before writing data.
> /* 236 */ holder.grow(sizeInBytes1);
> /* 237 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
> /* 238 */ holder.cursor += sizeInBytes1;
> /* 239 */
> /* 240 */ } else {
> /* 241 */ final int numElements = keys.numElements();
> /* 242 */ arrayWriter.initialize(holder, numElements, 8);
> /* 243 */
> /* 244 */ for (int index1 = 0; index1 < numElements; index1++) {
> /* 245 */ if (keys.isNullAt(index1)) {
> /* 246 */ arrayWriter.setNull(index1);
> /* 247 */ } else {
> /* 248 */ final UTF8String element = keys.getUTF8String(index1);
> /* 249 */ arrayWriter.write(index1, element);
> /* 250 */ }
> /* 251 */ }
> /* 252 */ }
> /* 253 */
> /* 254 */ // Write the numBytes of key array into the first 8 bytes.
> /* 255 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);
> /* 256 */
> /* 257 */
> /* 258 */ if (values1 instanceof UnsafeArrayData) {
> /* 259 */
> /* 260 */ final int sizeInBytes4 = ((UnsafeArrayData) values1).getSizeInBytes();
> /* 261 */ // grow the global buffer before writing data.
> /* 262 */ holder.grow(sizeInBytes4);
> /* 263 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, holder.cursor);
> /* 264 */ holder.cursor += sizeInBytes4;
> /* 265 */
> /* 266 */ } else {
> /* 267 */ final int numElements1 = values1.numElements();
> /* 268 */ arrayWriter1.initialize(holder, numElements1, 8);
> /* 269 */
> /* 270 */ for (int index2 = 0; index2 < numElements1; index2++) {
> /* 271 */ if (values1.isNullAt(index2)) {
> /* 272 */ arrayWriter1.setNull(index2);
> /* 273 */ } else {
> /* 274 */ final InternalRow element1 = values1.getStruct(index2, 1);
> /* 275 */
> /* 276 */ final int tmpCursor3 = holder.cursor;
> /* 277 */
> /* 278 */ if (element1 instanceof UnsafeRow) {
> /* 279 */
> /* 280 */ final int sizeInBytes2 = ((UnsafeRow) element1).getSizeInBytes();
> /* 281 */ // grow the global buffer before writing data.
> /* 282 */ holder.grow(sizeInBytes2);
> /* 283 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, holder.cursor);
> /* 284 */ holder.cursor += sizeInBytes2;
> /* 285 */
> /* 286 */ } else {
> /* 287 */ rowWriter1.reset();
> /* 288 */
> /* 289 */ final ArrayData fieldName = element1.getArray(0);
> /* 290 */ if (element1.isNullAt(0)) {
> /* 291 */ rowWriter1.setNullAt(0);
> /* 292 */ } else {
> /* 293 */ // Remember the current cursor so that we can calculate how many bytes are
> /* 294 */ // written later.
> /* 295 */ final int tmpCursor4 = holder.cursor;
> /* 296 */
> /* 297 */ if (fieldName instanceof UnsafeArrayData) {
> /* 298 */
> /* 299 */ final int sizeInBytes3 = ((UnsafeArrayData) fieldName).getSizeInBytes();
> /* 300 */ // grow the global buffer before writing data.
> /* 301 */ holder.grow(sizeInBytes3);
> /* 302 */ ((UnsafeArrayData) fieldName).writeToMemory(holder.buffer, holder.cursor);
> /* 303 */ holder.cursor += sizeInBytes3;
> /* 304 */
> /* 305 */ } else {
> /* 306 */ final int numElements2 = fieldName.numElements();
> /* 307 */ arrayWriter2.initialize(holder, numElements2, 8);
> /* 308 */
> /* 309 */ for (int index3 = 0; index3 < numElements2; index3++) {
> /* 310 */ if (fieldName.isNullAt(index3)) {
> /* 311 */ arrayWriter2.setNullDouble(index3);
> /* 312 */ } else {
> /* 313 */ final double element2 = fieldName.getDouble(index3);
> /* 314 */ arrayWriter2.write(index3, element2);
> /* 315 */ }
> /* 316 */ }
> /* 317 */ }
> /* 318 */
> /* 319 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, holder.cursor - tmpCursor4);
> /* 320 */ }
> /* 321 */ }
> /* 322 */
> /* 323 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, holder.cursor - tmpCursor3);
> /* 324 */
> /* 325 */ }
> /* 326 */ }
> /* 327 */ }
> /* 328 */
> /* 329 */ }
> /* 330 */
> /* 331 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
> /* 332 */ }
> /* 333 */ result.setTotalSize(holder.totalSize());
> /* 334 */ return result;
> /* 335 */ }
> /* 336 */ }
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:941)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:998)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:995)
> at org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
> at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
> ... 30 more
> Caused by: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an rvalue
> at org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
> at org.codehaus.janino.UnitCompiler.toRvalueOrCompileException(UnitCompiler.java:6639)
> at org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5001)
> at org.codehaus.janino.UnitCompiler.access$10500(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$13.visitAmbiguousName(UnitCompiler.java:4984)
> at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:3633)
> at org.codehaus.janino.Java$Lvalue.accept(Java.java:3563)
> at org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4956)
> at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4925)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3189)
> at org.codehaus.janino.UnitCompiler.access$5100(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3143)
> at org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3139)
> at org.codehaus.janino.Java$Assignment.accept(Java.java:3847)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3139)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2112)
> at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1377)
> at org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1370)
> at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2558)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
> at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894)
> at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377)
> at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369)
> at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
> at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564)
> at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
> at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
> at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
> at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
> at org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
> at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
> at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
> at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
> at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311)
> at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229)
> at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
> at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:935)
> ... 34 more
> 17/07/12 21:49:31 INFO SparkContext: Invoking stop() from shutdown hook
> 17/07/12 21:49:31 INFO SparkUI: Stopped Spark web UI at http://192.168.0.101:4040
> 17/07/12 21:49:31 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
> 17/07/12 21:49:31 INFO MemoryStore: MemoryStore cleared
> 17/07/12 21:49:31 INFO BlockManager: BlockManager stopped
> 17/07/12 21:49:31 INFO BlockManagerMaster: BlockManagerMaster stopped
> 17/07/12 21:49:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
> 17/07/12 21:49:31 INFO SparkContext: Successfully stopped SparkContext
> 17/07/12 21:49:31 INFO ShutdownHookManager: Shutdown hook called
> 17/07/12 21:49:31 INFO ShutdownHookManager: Deleting directory /private/var/folders/fw/lg6m2xps12bfwd7x0hj3rpzm0000gn/T/spark-886b188a-a3c3-460e-96a0-b7a4d1aa09de
> Process finished with exit code 1
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org