You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@spark.apache.org by "Proust (Feng Guizhou) [Travel Search & Discovery]" <pf...@coupang.com> on 2020/02/23 08:08:41 UTC
[Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO
Field with Type Array[String]
Hi, Spark Users
I ecounter below NegativeArraySizeException when run Spark SQL. The catalyst generated code for "apply2_19" and "apply1_11" is attached and also the related DTO.
Difficult to understand how the problem could happen, please help if any idea.
I can see maybe https://issues.apache.org/jira/browse/SPARK-15062 is similar but my data type is Array[String] and Spark version is 2.1.2 which looks good both.
[SPARK-15062] Show on DataFrame causes OutOfMemoryError, NegativeArraySizeException or segfault - ASF JIRA<https://issues.apache.org/jira/browse/SPARK-15062>
By increasing memory to 8G one will instead get a NegativeArraySizeException or a segfault. See here for original discussion: http://apache-spark-developers-list ...
issues.apache.org
java.lang.NegativeArraySizeException
at org.apache.spark.unsafe.types.UTF8String.getBytes(UTF8String.java:229)
at org.apache.spark.unsafe.types.UTF8String.toString(UTF8String.java:1005)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply1_11$(generated.java:2467)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply2_19$(generated.java:1475)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply(generated.java:3881)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
at scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:1076)
at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:1091)
at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:1128)
at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:1132)
at scala.collection.Iterator$class.foreach(Iterator.scala:893)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:101)
at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:95)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:99)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
case class VendorItemDto(
rate_category_id: java.lang.Long,
product_id: java.lang.Long,
vendor_item_rate_category_id: String,
vendor_item_id: java.lang.Long,
name: String,
item_name: String,
vendor_item_order: java.lang.Integer,
option_codes: Array[String],
option_names: Array[String],
feature_values_ids: Array[String],
feature_values_names: Array[String],
benefit_policy_type: String,
benefit_discount_rate: java.lang.Double,
benefit_discount_amount: java.lang.Long,
available_stock: java.lang.Integer,
sale_price: java.lang.Double,
original_price: java.lang.Double,
supply_price: java.lang.Double,
period_id_set: Array[String],
use_start_set: Array[String],
use_end_set: Array[String]
)
/* 2430 */ private void apply1_11(InternalRow i) {
/* 2431 */
/* 2432 */
/* 2433 */ boolean isNull222 = MapObjects_loopIsNull379;
/* 2434 */ ArrayData value222 = null;
/* 2435 */
/* 2436 */ if (!MapObjects_loopIsNull379) {
/* 2437 */
/* 2438 */ if (MapObjects_loopValue378.isNullAt(19)) {
/* 2439 */ isNull222 = true;
/* 2440 */ } else {
/* 2441 */ value222 = MapObjects_loopValue378.getArray(19);
/* 2442 */ }
/* 2443 */
/* 2444 */ }
/* 2445 */ ArrayData value221 = null;
/* 2446 */
/* 2447 */ if (!isNull222) {
/* 2448 */
/* 2449 */ java.lang.String[] convertedArray17 = null;
/* 2450 */ int dataLength17 = value222.numElements();
/* 2451 */ convertedArray17 = new java.lang.String[dataLength17];
/* 2452 */
/* 2453 */ int loopIndex17 = 0;
/* 2454 */ while (loopIndex17 < dataLength17) {
/* 2455 */ MapObjects_loopValue390 = (UTF8String) (value222.getUTF8String(loopIndex17));
/* 2456 */ MapObjects_loopIsNull391 = value222.isNullAt(loopIndex17);
/* 2457 */
/* 2458 */
/* 2459 */ boolean isNull223 = true;
/* 2460 */ java.lang.String value223 = null;
/* 2461 */ if (!MapObjects_loopIsNull391) {
/* 2462 */
/* 2463 */ isNull223 = false;
/* 2464 */ if (!isNull223) {
/* 2465 */
/* 2466 */ Object funcResult52 = null;
/* 2467 */ funcResult52 = MapObjects_loopValue390.toString();
/* 2468 */ if (funcResult52 == null) {
/* 2469 */ isNull223 = true;
/* 2470 */ } else {
/* 2471 */ value223 = (java.lang.String) funcResult52;
/* 2472 */ }
/* 2473 */
/* 2474 */ }
/* 2475 */ isNull223 = value223 == null;
/* 2476 */ }
/* 2477 */ if (isNull223) {
/* 2478 */ convertedArray17[loopIndex17] = null;
/* 2479 */ } else {
/* 2480 */ convertedArray17[loopIndex17] = value223;
/* 2481 */ }
/* 2482 */
/* 2483 */ loopIndex17 += 1;
/* 2484 */ }
/* 2485 */
/* 2486 */ value221 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray17);
/* 2487 */ }
/* 2488 */ boolean isNull220 = true;
/* 2489 */ java.lang.String[] value220 = null;
/* 2490 */ if (!isNull222) {
/* 2491 */
/* 2492 */ isNull220 = false;
/* 2493 */ if (!isNull220) {
/* 2494 */
/* 2495 */ Object funcResult53 = null;
/* 2496 */ funcResult53 = value221.array();
/* 2497 */ if (funcResult53 == null) {
/* 2498 */ isNull220 = true;
/* 2499 */ } else {
/* 2500 */ value220 = (java.lang.String[]) funcResult53;
/* 2501 */ }
/* 2502 */
/* 2503 */ }
/* 2504 */ isNull220 = value220 == null;
/* 2505 */ }
/* 2506 */ argValue111 = value220;
/* 2507 */
/* 2508 */ }
/* 933 */ private void apply2_19(InternalRow i) {
/* 934 */
/* 935 */
/* 936 */ resultIsNull30 = false;
/* 937 */ if (!resultIsNull30) {
/* 938 */
/* 939 */ boolean isNull161 = i.isNullAt(36);
/* 940 */ long value161 = isNull161 ? -1L : (i.getLong(36));
/* 941 */ resultIsNull30 = isNull161;
/* 942 */ argValue90 = value161;
/* 943 */ }
/* 944 */
/* 945 */
/* 946 */ final java.lang.Long value160 = resultIsNull30 ? null : new java.lang.Long(argValue90);
/* 947 */ argValue35 = value160;
/* 948 */
/* 949 */
/* 950 */ resultIsNull31 = false;
/* 951 */ if (!resultIsNull31) {
/* 952 */
/* 953 */ boolean isNull163 = i.isNullAt(37);
/* 954 */ long value163 = isNull163 ? -1L : (i.getLong(37));
/* 955 */ resultIsNull31 = isNull163;
/* 956 */ argValue91 = value163;
/* 957 */ }
/* 958 */
/* 959 */
/* 960 */ final java.lang.Long value162 = resultIsNull31 ? null : new java.lang.Long(argValue91);
/* 961 */ argValue36 = value162;
/* 962 */
/* 963 */
/* 964 */ boolean isNull167 = i.isNullAt(35);
/* 965 */ ArrayData value167 = isNull167 ? null : (i.getArray(35));
/* 966 */ boolean isNull166 = isNull167;
/* 967 */ ArrayData value166 = null;
/* 968 */ if (!isNull167) {
/* 969 */
/* 970 */ final int n = value167.numElements();
/* 971 */ final Object[] values = new Object[n];
/* 972 */ for (int j = 0; j < n; j ++) {
/* 973 */ if (value167.isNullAt(j)) {
/* 974 */ values[j] = null;
/* 975 */ } else {
/* 976 */ boolean feNull = false;
/* 977 */ InternalRow fePrim =
/* 978 */ value167.getStruct(j, 21);
/* 979 */
/* 980 */ boolean teNull = feNull;
/* 981 */ InternalRow tePrim = null;
/* 982 */ if (!feNull) {
/* 983 */
/* 984 */ final org.apache.spark.sql.catalyst.expressions.GenericInternalRow result = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(21);
/* 985 */ final InternalRow tmpRow = fePrim;
/* 986 */
/* 987 */ boolean ffn = tmpRow.isNullAt(0);
/* 988 */ if (ffn) {
/* 989 */ result.setNullAt(0);
/* 990 */ } else {
/* 991 */ long ffp =
/* 992 */ tmpRow.getLong(0);
/* 993 */
/* 994 */ boolean tfn = ffn;
/* 995 */ long tfp = -1L;
/* 996 */ if (!ffn) {
/* 997 */ tfp = ffp;
/* 998 */ }
/* 999 */
/* 1000 */ if (tfn) {
/* 1001 */ result.setNullAt(0);
/* 1002 */ } else {
/* 1003 */ result.setLong(0, tfp);
/* 1004 */ }
/* 1005 */ }
/* 1006 */
/* 1007 */
/* 1008 */ boolean ffn1 = tmpRow.isNullAt(1);
/* 1009 */ if (ffn1) {
/* 1010 */ result.setNullAt(1);
/* 1011 */ } else {
/* 1012 */ long ffp1 =
/* 1013 */ tmpRow.getLong(1);
/* 1014 */
/* 1015 */ boolean tfn1 = ffn1;
/* 1016 */ long tfp1 = -1L;
/* 1017 */ if (!ffn1) {
/* 1018 */ tfp1 = ffp1;
/* 1019 */ }
/* 1020 */
/* 1021 */ if (tfn1) {
/* 1022 */ result.setNullAt(1);
/* 1023 */ } else {
/* 1024 */ result.setLong(1, tfp1);
/* 1025 */ }
/* 1026 */ }
/* 1027 */
/* 1028 */
/* 1029 */ boolean ffn2 = tmpRow.isNullAt(2);
/* 1030 */ if (ffn2) {
/* 1031 */ result.setNullAt(2);
/* 1032 */ } else {
/* 1033 */ UTF8String ffp2 =
/* 1034 */ tmpRow.getUTF8String(2);
/* 1035 */
/* 1036 */ boolean tfn2 = ffn2;
/* 1037 */ UTF8String tfp2 = null;
/* 1038 */ if (!ffn2) {
/* 1039 */ tfp2 = ffp2;
/* 1040 */ }
/* 1041 */
/* 1042 */ if (tfn2) {
/* 1043 */ result.setNullAt(2);
/* 1044 */ } else {
/* 1045 */ result.update(2, tfp2.clone());
/* 1046 */ }
/* 1047 */ }
/* 1048 */
/* 1049 */
/* 1050 */ boolean ffn3 = tmpRow.isNullAt(3);
/* 1051 */ if (ffn3) {
/* 1052 */ result.setNullAt(3);
/* 1053 */ } else {
/* 1054 */ long ffp3 =
/* 1055 */ tmpRow.getLong(3);
/* 1056 */
/* 1057 */ boolean tfn3 = ffn3;
/* 1058 */ long tfp3 = -1L;
/* 1059 */ if (!ffn3) {
/* 1060 */ tfp3 = ffp3;
/* 1061 */ }
/* 1062 */
/* 1063 */ if (tfn3) {
/* 1064 */ result.setNullAt(3);
/* 1065 */ } else {
/* 1066 */ result.setLong(3, tfp3);
/* 1067 */ }
/* 1068 */ }
/* 1069 */
/* 1070 */
/* 1071 */ boolean ffn4 = tmpRow.isNullAt(4);
/* 1072 */ if (ffn4) {
/* 1073 */ result.setNullAt(4);
/* 1074 */ } else {
/* 1075 */ UTF8String ffp4 =
/* 1076 */ tmpRow.getUTF8String(4);
/* 1077 */
/* 1078 */ boolean tfn4 = ffn4;
/* 1079 */ UTF8String tfp4 = null;
/* 1080 */ if (!ffn4) {
/* 1081 */ tfp4 = ffp4;
/* 1082 */ }
/* 1083 */
/* 1084 */ if (tfn4) {
/* 1085 */ result.setNullAt(4);
/* 1086 */ } else {
/* 1087 */ result.update(4, tfp4.clone());
/* 1088 */ }
/* 1089 */ }
/* 1090 */
/* 1091 */
/* 1092 */ boolean ffn5 = tmpRow.isNullAt(5);
/* 1093 */ if (ffn5) {
/* 1094 */ result.setNullAt(5);
/* 1095 */ } else {
/* 1096 */ UTF8String ffp5 =
/* 1097 */ tmpRow.getUTF8String(5);
/* 1098 */
/* 1099 */ boolean tfn5 = ffn5;
/* 1100 */ UTF8String tfp5 = null;
/* 1101 */ if (!ffn5) {
/* 1102 */ tfp5 = ffp5;
/* 1103 */ }
/* 1104 */
/* 1105 */ if (tfn5) {
/* 1106 */ result.setNullAt(5);
/* 1107 */ } else {
/* 1108 */ result.update(5, tfp5.clone());
/* 1109 */ }
/* 1110 */ }
/* 1111 */
/* 1112 */
/* 1113 */ boolean ffn6 = tmpRow.isNullAt(6);
/* 1114 */ if (ffn6) {
/* 1115 */ result.setNullAt(6);
/* 1116 */ } else {
/* 1117 */ int ffp6 =
/* 1118 */ tmpRow.getInt(6);
/* 1119 */
/* 1120 */ boolean tfn6 = ffn6;
/* 1121 */ int tfp6 = -1;
/* 1122 */ if (!ffn6) {
/* 1123 */ tfp6 = ffp6;
/* 1124 */ }
/* 1125 */
/* 1126 */ if (tfn6) {
/* 1127 */ result.setNullAt(6);
/* 1128 */ } else {
/* 1129 */ result.setInt(6, tfp6);
/* 1130 */ }
/* 1131 */ }
/* 1132 */
/* 1133 */
/* 1134 */ boolean ffn7 = tmpRow.isNullAt(7);
/* 1135 */ if (ffn7) {
/* 1136 */ result.setNullAt(7);
/* 1137 */ } else {
/* 1138 */ ArrayData ffp7 =
/* 1139 */ tmpRow.getArray(7);
/* 1140 */
/* 1141 */ boolean tfn7 = ffn7;
/* 1142 */ ArrayData tfp7 = null;
/* 1143 */ if (!ffn7) {
/* 1144 */ tfp7 = ffp7;
/* 1145 */ }
/* 1146 */
/* 1147 */ if (tfn7) {
/* 1148 */ result.setNullAt(7);
/* 1149 */ } else {
/* 1150 */ result.update(7, tfp7);
/* 1151 */ }
/* 1152 */ }
/* 1153 */
/* 1154 */
/* 1155 */ boolean ffn8 = tmpRow.isNullAt(8);
/* 1156 */ if (ffn8) {
/* 1157 */ result.setNullAt(8);
/* 1158 */ } else {
/* 1159 */ ArrayData ffp8 =
/* 1160 */ tmpRow.getArray(8);
/* 1161 */
/* 1162 */ boolean tfn8 = ffn8;
/* 1163 */ ArrayData tfp8 = null;
/* 1164 */ if (!ffn8) {
/* 1165 */ tfp8 = ffp8;
/* 1166 */ }
/* 1167 */
/* 1168 */ if (tfn8) {
/* 1169 */ result.setNullAt(8);
/* 1170 */ } else {
/* 1171 */ result.update(8, tfp8);
/* 1172 */ }
/* 1173 */ }
/* 1174 */
/* 1175 */
/* 1176 */ boolean ffn9 = tmpRow.isNullAt(9);
/* 1177 */ if (ffn9) {
/* 1178 */ result.setNullAt(9);
/* 1179 */ } else {
/* 1180 */ ArrayData ffp9 =
/* 1181 */ tmpRow.getArray(9);
/* 1182 */
/* 1183 */ boolean tfn9 = ffn9;
/* 1184 */ ArrayData tfp9 = null;
/* 1185 */ if (!ffn9) {
/* 1186 */ tfp9 = ffp9;
/* 1187 */ }
/* 1188 */
/* 1189 */ if (tfn9) {
/* 1190 */ result.setNullAt(9);
/* 1191 */ } else {
/* 1192 */ result.update(9, tfp9);
/* 1193 */ }
/* 1194 */ }
/* 1195 */
/* 1196 */
/* 1197 */ boolean ffn10 = tmpRow.isNullAt(10);
/* 1198 */ if (ffn10) {
/* 1199 */ result.setNullAt(10);
/* 1200 */ } else {
/* 1201 */ ArrayData ffp10 =
/* 1202 */ tmpRow.getArray(10);
/* 1203 */
/* 1204 */ boolean tfn10 = ffn10;
/* 1205 */ ArrayData tfp10 = null;
/* 1206 */ if (!ffn10) {
/* 1207 */ tfp10 = ffp10;
/* 1208 */ }
/* 1209 */
/* 1210 */ if (tfn10) {
/* 1211 */ result.setNullAt(10);
/* 1212 */ } else {
/* 1213 */ result.update(10, tfp10);
/* 1214 */ }
/* 1215 */ }
/* 1216 */
/* 1217 */
/* 1218 */ boolean ffn11 = tmpRow.isNullAt(11);
/* 1219 */ if (ffn11) {
/* 1220 */ result.setNullAt(11);
/* 1221 */ } else {
/* 1222 */ UTF8String ffp11 =
/* 1223 */ tmpRow.getUTF8String(11);
/* 1224 */
/* 1225 */ boolean tfn11 = ffn11;
/* 1226 */ UTF8String tfp11 = null;
/* 1227 */ if (!ffn11) {
/* 1228 */ tfp11 = ffp11;
/* 1229 */ }
/* 1230 */
/* 1231 */ if (tfn11) {
/* 1232 */ result.setNullAt(11);
/* 1233 */ } else {
/* 1234 */ result.update(11, tfp11.clone());
/* 1235 */ }
/* 1236 */ }
/* 1237 */
/* 1238 */
/* 1239 */ boolean ffn12 = tmpRow.isNullAt(12);
/* 1240 */ if (ffn12) {
/* 1241 */ result.setNullAt(12);
/* 1242 */ } else {
/* 1243 */ double ffp12 =
/* 1244 */ tmpRow.getDouble(12);
/* 1245 */
/* 1246 */ boolean tfn12 = ffn12;
/* 1247 */ double tfp12 = -1.0;
/* 1248 */ if (!ffn12) {
/* 1249 */ tfp12 = ffp12;
/* 1250 */ }
/* 1251 */
/* 1252 */ if (tfn12) {
/* 1253 */ result.setNullAt(12);
/* 1254 */ } else {
/* 1255 */ result.setDouble(12, tfp12);
/* 1256 */ }
/* 1257 */ }
/* 1258 */
/* 1259 */
/* 1260 */ boolean ffn13 = tmpRow.isNullAt(13);
/* 1261 */ if (ffn13) {
/* 1262 */ result.setNullAt(13);
/* 1263 */ } else {
/* 1264 */ long ffp13 =
/* 1265 */ tmpRow.getLong(13);
/* 1266 */
/* 1267 */ boolean tfn13 = ffn13;
/* 1268 */ long tfp13 = -1L;
/* 1269 */ if (!ffn13) {
/* 1270 */ tfp13 = ffp13;
/* 1271 */ }
/* 1272 */
/* 1273 */ if (tfn13) {
/* 1274 */ result.setNullAt(13);
/* 1275 */ } else {
/* 1276 */ result.setLong(13, tfp13);
/* 1277 */ }
/* 1278 */ }
/* 1279 */
/* 1280 */
/* 1281 */ boolean ffn14 = tmpRow.isNullAt(14);
/* 1282 */ if (ffn14) {
/* 1283 */ result.setNullAt(14);
/* 1284 */ } else {
/* 1285 */ int ffp14 =
/* 1286 */ tmpRow.getInt(14);
/* 1287 */
/* 1288 */ boolean tfn14 = ffn14;
/* 1289 */ int tfp14 = -1;
/* 1290 */ if (!ffn14) {
/* 1291 */ tfp14 = ffp14;
/* 1292 */ }
/* 1293 */
/* 1294 */ if (tfn14) {
/* 1295 */ result.setNullAt(14);
/* 1296 */ } else {
/* 1297 */ result.setInt(14, tfp14);
/* 1298 */ }
/* 1299 */ }
/* 1300 */
/* 1301 */
/* 1302 */ boolean ffn15 = tmpRow.isNullAt(15);
/* 1303 */ if (ffn15) {
/* 1304 */ result.setNullAt(15);
/* 1305 */ } else {
/* 1306 */ double ffp15 =
/* 1307 */ tmpRow.getDouble(15);
/* 1308 */
/* 1309 */ boolean tfn15 = ffn15;
/* 1310 */ double tfp15 = -1.0;
/* 1311 */ if (!ffn15) {
/* 1312 */ tfp15 = ffp15;
/* 1313 */ }
/* 1314 */
/* 1315 */ if (tfn15) {
/* 1316 */ result.setNullAt(15);
/* 1317 */ } else {
/* 1318 */ result.setDouble(15, tfp15);
/* 1319 */ }
/* 1320 */ }
/* 1321 */
/* 1322 */
/* 1323 */ boolean ffn16 = tmpRow.isNullAt(16);
/* 1324 */ if (ffn16) {
/* 1325 */ result.setNullAt(16);
/* 1326 */ } else {
/* 1327 */ double ffp16 =
/* 1328 */ tmpRow.getDouble(16);
/* 1329 */
/* 1330 */ boolean tfn16 = ffn16;
/* 1331 */ double tfp16 = -1.0;
/* 1332 */ if (!ffn16) {
/* 1333 */ tfp16 = ffp16;
/* 1334 */ }
/* 1335 */
/* 1336 */ if (tfn16) {
/* 1337 */ result.setNullAt(16);
/* 1338 */ } else {
/* 1339 */ result.setDouble(16, tfp16);
/* 1340 */ }
/* 1341 */ }
/* 1342 */
/* 1343 */
/* 1344 */ boolean ffn17 = tmpRow.isNullAt(17);
/* 1345 */ if (ffn17) {
/* 1346 */ result.setNullAt(17);
/* 1347 */ } else {
/* 1348 */ double ffp17 =
/* 1349 */ tmpRow.getDouble(17);
/* 1350 */
/* 1351 */ boolean tfn17 = ffn17;
/* 1352 */ double tfp17 = -1.0;
/* 1353 */ if (!ffn17) {
/* 1354 */ tfp17 = ffp17;
/* 1355 */ }
/* 1356 */
/* 1357 */ if (tfn17) {
/* 1358 */ result.setNullAt(17);
/* 1359 */ } else {
/* 1360 */ result.setDouble(17, tfp17);
/* 1361 */ }
/* 1362 */ }
/* 1363 */
/* 1364 */
/* 1365 */ boolean ffn18 = tmpRow.isNullAt(18);
/* 1366 */ if (ffn18) {
/* 1367 */ result.setNullAt(18);
/* 1368 */ } else {
/* 1369 */ ArrayData ffp18 =
/* 1370 */ tmpRow.getArray(18);
/* 1371 */
/* 1372 */ boolean tfn18 = ffn18;
/* 1373 */ ArrayData tfp18 = null;
/* 1374 */ if (!ffn18) {
/* 1375 */ tfp18 = ffp18;
/* 1376 */ }
/* 1377 */
/* 1378 */ if (tfn18) {
/* 1379 */ result.setNullAt(18);
/* 1380 */ } else {
/* 1381 */ result.update(18, tfp18);
/* 1382 */ }
/* 1383 */ }
/* 1384 */
/* 1385 */
/* 1386 */ boolean ffn19 = tmpRow.isNullAt(19);
/* 1387 */ if (ffn19) {
/* 1388 */ result.setNullAt(19);
/* 1389 */ } else {
/* 1390 */ ArrayData ffp19 =
/* 1391 */ tmpRow.getArray(19);
/* 1392 */
/* 1393 */ boolean tfn19 = ffn19;
/* 1394 */ ArrayData tfp19 = null;
/* 1395 */ if (!ffn19) {
/* 1396 */ tfp19 = ffp19;
/* 1397 */ }
/* 1398 */
/* 1399 */ if (tfn19) {
/* 1400 */ result.setNullAt(19);
/* 1401 */ } else {
/* 1402 */ result.update(19, tfp19);
/* 1403 */ }
/* 1404 */ }
/* 1405 */
/* 1406 */
/* 1407 */ boolean ffn20 = tmpRow.isNullAt(20);
/* 1408 */ if (ffn20) {
/* 1409 */ result.setNullAt(20);
/* 1410 */ } else {
/* 1411 */ ArrayData ffp20 =
/* 1412 */ tmpRow.getArray(20);
/* 1413 */
/* 1414 */ boolean tfn20 = ffn20;
/* 1415 */ ArrayData tfp20 = null;
/* 1416 */ if (!ffn20) {
/* 1417 */ tfp20 = ffp20;
/* 1418 */ }
/* 1419 */
/* 1420 */ if (tfn20) {
/* 1421 */ result.setNullAt(20);
/* 1422 */ } else {
/* 1423 */ result.update(20, tfp20);
/* 1424 */ }
/* 1425 */ }
/* 1426 */
/* 1427 */ tePrim = result.copy();
/* 1428 */
/* 1429 */ }
/* 1430 */
/* 1431 */ if (teNull) {
/* 1432 */ values[j] = null;
/* 1433 */ } else {
/* 1434 */ values[j] = tePrim;
/* 1435 */ }
/* 1436 */ }
/* 1437 */ }
/* 1438 */ value166 = new org.apache.spark.sql.catalyst.util.GenericArrayData(values);
/* 1439 */
/* 1440 */ }
/* 1441 */ ArrayData value165 = null;
/* 1442 */
/* 1443 */ if (!isNull166) {
/* 1444 */
/* 1445 */ indexer.dto.ticket.VendorItemDto[] convertedArray19 = null;
/* 1446 */ int dataLength19 = value166.numElements();
/* 1447 */ convertedArray19 = new indexer.dto.ticket.VendorItemDto[dataLength19];
/* 1448 */
/* 1449 */ int loopIndex19 = 0;
/* 1450 */ while (loopIndex19 < dataLength19) {
/* 1451 */ MapObjects_loopValue378 = (InternalRow) (value166.getStruct(loopIndex19, 21));
/* 1452 */ MapObjects_loopIsNull379 = value166.isNullAt(loopIndex19);
/* 1453 */
/* 1454 */
/* 1455 */ boolean isNull168 = false;
/* 1456 */ indexer.dto.ticket.VendorItemDto value168 = null;
/* 1457 */ if (!false && MapObjects_loopIsNull379) {
/* 1458 */
/* 1459 */ final indexer.dto.ticket.VendorItemDto value170 = null;
/* 1460 */ isNull168 = true;
/* 1461 */ value168 = value170;
/* 1462 */ } else {
/* 1463 */
/* 1464 */ apply1_0(i);
/* 1465 */ apply1_1(i);
/* 1466 */ apply1_2(i);
/* 1467 */ apply1_3(i);
/* 1468 */ apply1_4(i);
/* 1469 */ apply1_5(i);
/* 1470 */ apply1_6(i);
/* 1471 */ apply1_7(i);
/* 1472 */ apply1_8(i);
/* 1473 */ apply1_9(i);
/* 1474 */ apply1_10(i);
/* 1475 */ apply1_11(i);
/* 1476 */ apply1_12(i);
/* 1477 */
/* 1478 */ final indexer.dto.ticket.VendorItemDto value171 = false ? null : new indexer.dto.ticket.VendorItemDto(argValue92, argValue93, argValue94, argValue95, argValue96, argValue97, argValue98, argValue99, argValue100, argValue101, argValue102, argValue103, argValue104, argValue105, argValue106, argValue107, argValue108, argValue109, argValue110, argValue111, argValue112);
/* 1479 */ isNull168 = false;
/* 1480 */ value168 = value171;
/* 1481 */ }
/* 1482 */ if (isNull168) {
/* 1483 */ convertedArray19[loopIndex19] = null;
/* 1484 */ } else {
/* 1485 */ convertedArray19[loopIndex19] = value168;
/* 1486 */ }
/* 1487 */
/* 1488 */ loopIndex19 += 1;
/* 1489 */ }
/* 1490 */
/* 1491 */ value165 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray19);
/* 1492 */ }
/* 1493 */ boolean isNull164 = true;
/* 1494 */ indexer.dto.ticket.VendorItemDto[] value164 = null;
/* 1495 */ if (!isNull166) {
/* 1496 */
/* 1497 */ isNull164 = false;
/* 1498 */ if (!isNull164) {
/* 1499 */
/* 1500 */ Object funcResult56 = null;
/* 1501 */ funcResult56 = value165.array();
/* 1502 */ if (funcResult56 == null) {
/* 1503 */ isNull164 = true;
/* 1504 */ } else {
/* 1505 */ value164 = (indexer.dto.ticket.VendorItemDto[]) funcResult56;
/* 1506 */ }
/* 1507 */
/* 1508 */ }
/* 1509 */ isNull164 = value164 == null;
/* 1510 */ }
/* 1511 */ argValue37 = value164;
/* 1512 */
/* 1513 */ }
Thanks a lot
Proust
??: ??????????2889?????C?7?
Address: 7th floor, No#3, Jinke road 2889, Pudong district, Shanghai, China.
Mobile: +86 13621672634
Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to
DTO Field with Type Array[String]
Posted by "Proust (Feng Guizhou) [Travel Search & Discovery]" <pf...@coupang.com>.
Also I tried to disable the Kryo reference tracking, then the problem will simply change to Java stack overflow exception.
________________________________
From: Proust (Feng Guizhou) [Travel Search & Discovery] <pf...@coupang.com>
Sent: Tuesday, February 25, 2020 11:28 PM
To: Sandeep Patra <sp...@gmail.com>
Cc: user@spark.apache.org <us...@spark.apache.org>
Subject: Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]
Thanks for the information, tried both JavaSerializer and KryoSerializer, same problem encountered
And the stacktrace looks very different from the one mentioned in the stackoverflow link
________________________________
From: Sandeep Patra <sp...@gmail.com>
Sent: Sunday, February 23, 2020 8:04 PM
To: Proust (Feng Guizhou) [Travel Search & Discovery] <pf...@coupang.com>
Cc: user@spark.apache.org <us...@spark.apache.org>
Subject: Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]
[Warning]: This email originated from an external source. Do not open links or attachments unless you know the content is safe.
[경고]: 본 이메일은 회사 외부에서 유입되었습니다. 내용이 안전한지 확인하기 전까지는 링크나 첨부파일을 열지 마십시오.
This might be due to the serializer being used.
This stackoverflow answer might help: https://stackoverflow.com/questions/44414429/spark-negativearraysizeexception<https://kor01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fstackoverflow.com%2Fquestions%2F44414429%2Fspark-negativearraysizeexception&data=02%7C01%7Cpfeng%40coupang.com%7Ce27ecc57fb6b426c1c0d08d7b858a406%7Ce3098f96361b47c6a9f4ab7bafcaffe9%7C0%7C0%7C637180563158673718&sdata=%2FowfaBCtHA1DDj1pxLm6PsvP2cPjldqo%2BGmv8XTb5%2F0%3D&reserved=0>
On Sun, Feb 23, 2020 at 1:39 PM Proust (Feng Guizhou) [Travel Search & Discovery] <pf...@coupang.com>> wrote:
Hi, Spark Users
I ecounter below NegativeArraySizeException when run Spark SQL. The catalyst generated code for "apply2_19" and "apply1_11" is attached and also the related DTO.
Difficult to understand how the problem could happen, please help if any idea.
I can see maybe https://issues.apache.org/jira/browse/SPARK-15062<https://kor01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fissues.apache.org%2Fjira%2Fbrowse%2FSPARK-15062&data=02%7C01%7Cpfeng%40coupang.com%7Ce27ecc57fb6b426c1c0d08d7b858a406%7Ce3098f96361b47c6a9f4ab7bafcaffe9%7C0%7C0%7C637180563158683718&sdata=SWKK0Mh5PYLEEpXUqsuuWUT8cwWldGy2GlO5N5anuLQ%3D&reserved=0> is similar but my data type is Array[String] and Spark version is 2.1.2 which looks good both.
[SPARK-15062] Show on DataFrame causes OutOfMemoryError, NegativeArraySizeException or segfault - ASF JIRA<https://kor01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fissues.apache.org%2Fjira%2Fbrowse%2FSPARK-15062&data=02%7C01%7Cpfeng%40coupang.com%7Ce27ecc57fb6b426c1c0d08d7b858a406%7Ce3098f96361b47c6a9f4ab7bafcaffe9%7C0%7C0%7C637180563158683718&sdata=SWKK0Mh5PYLEEpXUqsuuWUT8cwWldGy2GlO5N5anuLQ%3D&reserved=0>
By increasing memory to 8G one will instead get a NegativeArraySizeException or a segfault. See here for original discussion: http://apache-spark-developers-list ...
issues.apache.org<https://kor01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fissues.apache.org%2F&data=02%7C01%7Cpfeng%40coupang.com%7Ce27ecc57fb6b426c1c0d08d7b858a406%7Ce3098f96361b47c6a9f4ab7bafcaffe9%7C0%7C0%7C637180563158693718&sdata=%2FsDqfepuI28hyv%2Fm4tyETjBI5fbMPdyNnEU6dU0AuT0%3D&reserved=0>
java.lang.NegativeArraySizeException
at org.apache.spark.unsafe.types.UTF8String.getBytes(UTF8String.java:229)
at org.apache.spark.unsafe.types.UTF8String.toString(UTF8String.java:1005)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply1_11$(generated.java:2467)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply2_19$(generated.java:1475)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply(generated.java:3881)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
at scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:1076)
at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:1091)
at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:1128)
at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:1132)
at scala.collection.Iterator$class.foreach(Iterator.scala:893)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:101)
at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:95)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:99)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
case class VendorItemDto(
rate_category_id: java.lang.Long,
product_id: java.lang.Long,
vendor_item_rate_category_id: String,
vendor_item_id: java.lang.Long,
name: String,
item_name: String,
vendor_item_order: java.lang.Integer,
option_codes: Array[String],
option_names: Array[String],
feature_values_ids: Array[String],
feature_values_names: Array[String],
benefit_policy_type: String,
benefit_discount_rate: java.lang.Double,
benefit_discount_amount: java.lang.Long,
available_stock: java.lang.Integer,
sale_price: java.lang.Double,
original_price: java.lang.Double,
supply_price: java.lang.Double,
period_id_set: Array[String],
use_start_set: Array[String],
use_end_set: Array[String]
)
/* 2430 */ private void apply1_11(InternalRow i) {
/* 2431 */
/* 2432 */
/* 2433 */ boolean isNull222 = MapObjects_loopIsNull379;
/* 2434 */ ArrayData value222 = null;
/* 2435 */
/* 2436 */ if (!MapObjects_loopIsNull379) {
/* 2437 */
/* 2438 */ if (MapObjects_loopValue378.isNullAt(19)) {
/* 2439 */ isNull222 = true;
/* 2440 */ } else {
/* 2441 */ value222 = MapObjects_loopValue378.getArray(19);
/* 2442 */ }
/* 2443 */
/* 2444 */ }
/* 2445 */ ArrayData value221 = null;
/* 2446 */
/* 2447 */ if (!isNull222) {
/* 2448 */
/* 2449 */ java.lang.String[] convertedArray17 = null;
/* 2450 */ int dataLength17 = value222.numElements();
/* 2451 */ convertedArray17 = new java.lang.String[dataLength17];
/* 2452 */
/* 2453 */ int loopIndex17 = 0;
/* 2454 */ while (loopIndex17 < dataLength17) {
/* 2455 */ MapObjects_loopValue390 = (UTF8String) (value222.getUTF8String(loopIndex17));
/* 2456 */ MapObjects_loopIsNull391 = value222.isNullAt(loopIndex17);
/* 2457 */
/* 2458 */
/* 2459 */ boolean isNull223 = true;
/* 2460 */ java.lang.String value223 = null;
/* 2461 */ if (!MapObjects_loopIsNull391) {
/* 2462 */
/* 2463 */ isNull223 = false;
/* 2464 */ if (!isNull223) {
/* 2465 */
/* 2466 */ Object funcResult52 = null;
/* 2467 */ funcResult52 = MapObjects_loopValue390.toString();
/* 2468 */ if (funcResult52 == null) {
/* 2469 */ isNull223 = true;
/* 2470 */ } else {
/* 2471 */ value223 = (java.lang.String) funcResult52;
/* 2472 */ }
/* 2473 */
/* 2474 */ }
/* 2475 */ isNull223 = value223 == null;
/* 2476 */ }
/* 2477 */ if (isNull223) {
/* 2478 */ convertedArray17[loopIndex17] = null;
/* 2479 */ } else {
/* 2480 */ convertedArray17[loopIndex17] = value223;
/* 2481 */ }
/* 2482 */
/* 2483 */ loopIndex17 += 1;
/* 2484 */ }
/* 2485 */
/* 2486 */ value221 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray17);
/* 2487 */ }
/* 2488 */ boolean isNull220 = true;
/* 2489 */ java.lang.String[] value220 = null;
/* 2490 */ if (!isNull222) {
/* 2491 */
/* 2492 */ isNull220 = false;
/* 2493 */ if (!isNull220) {
/* 2494 */
/* 2495 */ Object funcResult53 = null;
/* 2496 */ funcResult53 = value221.array();
/* 2497 */ if (funcResult53 == null) {
/* 2498 */ isNull220 = true;
/* 2499 */ } else {
/* 2500 */ value220 = (java.lang.String[]) funcResult53;
/* 2501 */ }
/* 2502 */
/* 2503 */ }
/* 2504 */ isNull220 = value220 == null;
/* 2505 */ }
/* 2506 */ argValue111 = value220;
/* 2507 */
/* 2508 */ }
/* 933 */ private void apply2_19(InternalRow i) {
/* 934 */
/* 935 */
/* 936 */ resultIsNull30 = false;
/* 937 */ if (!resultIsNull30) {
/* 938 */
/* 939 */ boolean isNull161 = i.isNullAt(36);
/* 940 */ long value161 = isNull161 ? -1L : (i.getLong(36));
/* 941 */ resultIsNull30 = isNull161;
/* 942 */ argValue90 = value161;
/* 943 */ }
/* 944 */
/* 945 */
/* 946 */ final java.lang.Long value160 = resultIsNull30 ? null : new java.lang.Long(argValue90);
/* 947 */ argValue35 = value160;
/* 948 */
/* 949 */
/* 950 */ resultIsNull31 = false;
/* 951 */ if (!resultIsNull31) {
/* 952 */
/* 953 */ boolean isNull163 = i.isNullAt(37);
/* 954 */ long value163 = isNull163 ? -1L : (i.getLong(37));
/* 955 */ resultIsNull31 = isNull163;
/* 956 */ argValue91 = value163;
/* 957 */ }
/* 958 */
/* 959 */
/* 960 */ final java.lang.Long value162 = resultIsNull31 ? null : new java.lang.Long(argValue91);
/* 961 */ argValue36 = value162;
/* 962 */
/* 963 */
/* 964 */ boolean isNull167 = i.isNullAt(35);
/* 965 */ ArrayData value167 = isNull167 ? null : (i.getArray(35));
/* 966 */ boolean isNull166 = isNull167;
/* 967 */ ArrayData value166 = null;
/* 968 */ if (!isNull167) {
/* 969 */
/* 970 */ final int n = value167.numElements();
/* 971 */ final Object[] values = new Object[n];
/* 972 */ for (int j = 0; j < n; j ++) {
/* 973 */ if (value167.isNullAt(j)) {
/* 974 */ values[j] = null;
/* 975 */ } else {
/* 976 */ boolean feNull = false;
/* 977 */ InternalRow fePrim =
/* 978 */ value167.getStruct(j, 21);
/* 979 */
/* 980 */ boolean teNull = feNull;
/* 981 */ InternalRow tePrim = null;
/* 982 */ if (!feNull) {
/* 983 */
/* 984 */ final org.apache.spark.sql.catalyst.expressions.GenericInternalRow result = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(21);
/* 985 */ final InternalRow tmpRow = fePrim;
/* 986 */
/* 987 */ boolean ffn = tmpRow.isNullAt(0);
/* 988 */ if (ffn) {
/* 989 */ result.setNullAt(0);
/* 990 */ } else {
/* 991 */ long ffp =
/* 992 */ tmpRow.getLong(0);
/* 993 */
/* 994 */ boolean tfn = ffn;
/* 995 */ long tfp = -1L;
/* 996 */ if (!ffn) {
/* 997 */ tfp = ffp;
/* 998 */ }
/* 999 */
/* 1000 */ if (tfn) {
/* 1001 */ result.setNullAt(0);
/* 1002 */ } else {
/* 1003 */ result.setLong(0, tfp);
/* 1004 */ }
/* 1005 */ }
/* 1006 */
/* 1007 */
/* 1008 */ boolean ffn1 = tmpRow.isNullAt(1);
/* 1009 */ if (ffn1) {
/* 1010 */ result.setNullAt(1);
/* 1011 */ } else {
/* 1012 */ long ffp1 =
/* 1013 */ tmpRow.getLong(1);
/* 1014 */
/* 1015 */ boolean tfn1 = ffn1;
/* 1016 */ long tfp1 = -1L;
/* 1017 */ if (!ffn1) {
/* 1018 */ tfp1 = ffp1;
/* 1019 */ }
/* 1020 */
/* 1021 */ if (tfn1) {
/* 1022 */ result.setNullAt(1);
/* 1023 */ } else {
/* 1024 */ result.setLong(1, tfp1);
/* 1025 */ }
/* 1026 */ }
/* 1027 */
/* 1028 */
/* 1029 */ boolean ffn2 = tmpRow.isNullAt(2);
/* 1030 */ if (ffn2) {
/* 1031 */ result.setNullAt(2);
/* 1032 */ } else {
/* 1033 */ UTF8String ffp2 =
/* 1034 */ tmpRow.getUTF8String(2);
/* 1035 */
/* 1036 */ boolean tfn2 = ffn2;
/* 1037 */ UTF8String tfp2 = null;
/* 1038 */ if (!ffn2) {
/* 1039 */ tfp2 = ffp2;
/* 1040 */ }
/* 1041 */
/* 1042 */ if (tfn2) {
/* 1043 */ result.setNullAt(2);
/* 1044 */ } else {
/* 1045 */ result.update(2, tfp2.clone());
/* 1046 */ }
/* 1047 */ }
/* 1048 */
/* 1049 */
/* 1050 */ boolean ffn3 = tmpRow.isNullAt(3);
/* 1051 */ if (ffn3) {
/* 1052 */ result.setNullAt(3);
/* 1053 */ } else {
/* 1054 */ long ffp3 =
/* 1055 */ tmpRow.getLong(3);
/* 1056 */
/* 1057 */ boolean tfn3 = ffn3;
/* 1058 */ long tfp3 = -1L;
/* 1059 */ if (!ffn3) {
/* 1060 */ tfp3 = ffp3;
/* 1061 */ }
/* 1062 */
/* 1063 */ if (tfn3) {
/* 1064 */ result.setNullAt(3);
/* 1065 */ } else {
/* 1066 */ result.setLong(3, tfp3);
/* 1067 */ }
/* 1068 */ }
/* 1069 */
/* 1070 */
/* 1071 */ boolean ffn4 = tmpRow.isNullAt(4);
/* 1072 */ if (ffn4) {
/* 1073 */ result.setNullAt(4);
/* 1074 */ } else {
/* 1075 */ UTF8String ffp4 =
/* 1076 */ tmpRow.getUTF8String(4);
/* 1077 */
/* 1078 */ boolean tfn4 = ffn4;
/* 1079 */ UTF8String tfp4 = null;
/* 1080 */ if (!ffn4) {
/* 1081 */ tfp4 = ffp4;
/* 1082 */ }
/* 1083 */
/* 1084 */ if (tfn4) {
/* 1085 */ result.setNullAt(4);
/* 1086 */ } else {
/* 1087 */ result.update(4, tfp4.clone());
/* 1088 */ }
/* 1089 */ }
/* 1090 */
/* 1091 */
/* 1092 */ boolean ffn5 = tmpRow.isNullAt(5);
/* 1093 */ if (ffn5) {
/* 1094 */ result.setNullAt(5);
/* 1095 */ } else {
/* 1096 */ UTF8String ffp5 =
/* 1097 */ tmpRow.getUTF8String(5);
/* 1098 */
/* 1099 */ boolean tfn5 = ffn5;
/* 1100 */ UTF8String tfp5 = null;
/* 1101 */ if (!ffn5) {
/* 1102 */ tfp5 = ffp5;
/* 1103 */ }
/* 1104 */
/* 1105 */ if (tfn5) {
/* 1106 */ result.setNullAt(5);
/* 1107 */ } else {
/* 1108 */ result.update(5, tfp5.clone());
/* 1109 */ }
/* 1110 */ }
/* 1111 */
/* 1112 */
/* 1113 */ boolean ffn6 = tmpRow.isNullAt(6);
/* 1114 */ if (ffn6) {
/* 1115 */ result.setNullAt(6);
/* 1116 */ } else {
/* 1117 */ int ffp6 =
/* 1118 */ tmpRow.getInt(6);
/* 1119 */
/* 1120 */ boolean tfn6 = ffn6;
/* 1121 */ int tfp6 = -1;
/* 1122 */ if (!ffn6) {
/* 1123 */ tfp6 = ffp6;
/* 1124 */ }
/* 1125 */
/* 1126 */ if (tfn6) {
/* 1127 */ result.setNullAt(6);
/* 1128 */ } else {
/* 1129 */ result.setInt(6, tfp6);
/* 1130 */ }
/* 1131 */ }
/* 1132 */
/* 1133 */
/* 1134 */ boolean ffn7 = tmpRow.isNullAt(7);
/* 1135 */ if (ffn7) {
/* 1136 */ result.setNullAt(7);
/* 1137 */ } else {
/* 1138 */ ArrayData ffp7 =
/* 1139 */ tmpRow.getArray(7);
/* 1140 */
/* 1141 */ boolean tfn7 = ffn7;
/* 1142 */ ArrayData tfp7 = null;
/* 1143 */ if (!ffn7) {
/* 1144 */ tfp7 = ffp7;
/* 1145 */ }
/* 1146 */
/* 1147 */ if (tfn7) {
/* 1148 */ result.setNullAt(7);
/* 1149 */ } else {
/* 1150 */ result.update(7, tfp7);
/* 1151 */ }
/* 1152 */ }
/* 1153 */
/* 1154 */
/* 1155 */ boolean ffn8 = tmpRow.isNullAt(8);
/* 1156 */ if (ffn8) {
/* 1157 */ result.setNullAt(8);
/* 1158 */ } else {
/* 1159 */ ArrayData ffp8 =
/* 1160 */ tmpRow.getArray(8);
/* 1161 */
/* 1162 */ boolean tfn8 = ffn8;
/* 1163 */ ArrayData tfp8 = null;
/* 1164 */ if (!ffn8) {
/* 1165 */ tfp8 = ffp8;
/* 1166 */ }
/* 1167 */
/* 1168 */ if (tfn8) {
/* 1169 */ result.setNullAt(8);
/* 1170 */ } else {
/* 1171 */ result.update(8, tfp8);
/* 1172 */ }
/* 1173 */ }
/* 1174 */
/* 1175 */
/* 1176 */ boolean ffn9 = tmpRow.isNullAt(9);
/* 1177 */ if (ffn9) {
/* 1178 */ result.setNullAt(9);
/* 1179 */ } else {
/* 1180 */ ArrayData ffp9 =
/* 1181 */ tmpRow.getArray(9);
/* 1182 */
/* 1183 */ boolean tfn9 = ffn9;
/* 1184 */ ArrayData tfp9 = null;
/* 1185 */ if (!ffn9) {
/* 1186 */ tfp9 = ffp9;
/* 1187 */ }
/* 1188 */
/* 1189 */ if (tfn9) {
/* 1190 */ result.setNullAt(9);
/* 1191 */ } else {
/* 1192 */ result.update(9, tfp9);
/* 1193 */ }
/* 1194 */ }
/* 1195 */
/* 1196 */
/* 1197 */ boolean ffn10 = tmpRow.isNullAt(10);
/* 1198 */ if (ffn10) {
/* 1199 */ result.setNullAt(10);
/* 1200 */ } else {
/* 1201 */ ArrayData ffp10 =
/* 1202 */ tmpRow.getArray(10);
/* 1203 */
/* 1204 */ boolean tfn10 = ffn10;
/* 1205 */ ArrayData tfp10 = null;
/* 1206 */ if (!ffn10) {
/* 1207 */ tfp10 = ffp10;
/* 1208 */ }
/* 1209 */
/* 1210 */ if (tfn10) {
/* 1211 */ result.setNullAt(10);
/* 1212 */ } else {
/* 1213 */ result.update(10, tfp10);
/* 1214 */ }
/* 1215 */ }
/* 1216 */
/* 1217 */
/* 1218 */ boolean ffn11 = tmpRow.isNullAt(11);
/* 1219 */ if (ffn11) {
/* 1220 */ result.setNullAt(11);
/* 1221 */ } else {
/* 1222 */ UTF8String ffp11 =
/* 1223 */ tmpRow.getUTF8String(11);
/* 1224 */
/* 1225 */ boolean tfn11 = ffn11;
/* 1226 */ UTF8String tfp11 = null;
/* 1227 */ if (!ffn11) {
/* 1228 */ tfp11 = ffp11;
/* 1229 */ }
/* 1230 */
/* 1231 */ if (tfn11) {
/* 1232 */ result.setNullAt(11);
/* 1233 */ } else {
/* 1234 */ result.update(11, tfp11.clone());
/* 1235 */ }
/* 1236 */ }
/* 1237 */
/* 1238 */
/* 1239 */ boolean ffn12 = tmpRow.isNullAt(12);
/* 1240 */ if (ffn12) {
/* 1241 */ result.setNullAt(12);
/* 1242 */ } else {
/* 1243 */ double ffp12 =
/* 1244 */ tmpRow.getDouble(12);
/* 1245 */
/* 1246 */ boolean tfn12 = ffn12;
/* 1247 */ double tfp12 = -1.0;
/* 1248 */ if (!ffn12) {
/* 1249 */ tfp12 = ffp12;
/* 1250 */ }
/* 1251 */
/* 1252 */ if (tfn12) {
/* 1253 */ result.setNullAt(12);
/* 1254 */ } else {
/* 1255 */ result.setDouble(12, tfp12);
/* 1256 */ }
/* 1257 */ }
/* 1258 */
/* 1259 */
/* 1260 */ boolean ffn13 = tmpRow.isNullAt(13);
/* 1261 */ if (ffn13) {
/* 1262 */ result.setNullAt(13);
/* 1263 */ } else {
/* 1264 */ long ffp13 =
/* 1265 */ tmpRow.getLong(13);
/* 1266 */
/* 1267 */ boolean tfn13 = ffn13;
/* 1268 */ long tfp13 = -1L;
/* 1269 */ if (!ffn13) {
/* 1270 */ tfp13 = ffp13;
/* 1271 */ }
/* 1272 */
/* 1273 */ if (tfn13) {
/* 1274 */ result.setNullAt(13);
/* 1275 */ } else {
/* 1276 */ result.setLong(13, tfp13);
/* 1277 */ }
/* 1278 */ }
/* 1279 */
/* 1280 */
/* 1281 */ boolean ffn14 = tmpRow.isNullAt(14);
/* 1282 */ if (ffn14) {
/* 1283 */ result.setNullAt(14);
/* 1284 */ } else {
/* 1285 */ int ffp14 =
/* 1286 */ tmpRow.getInt(14);
/* 1287 */
/* 1288 */ boolean tfn14 = ffn14;
/* 1289 */ int tfp14 = -1;
/* 1290 */ if (!ffn14) {
/* 1291 */ tfp14 = ffp14;
/* 1292 */ }
/* 1293 */
/* 1294 */ if (tfn14) {
/* 1295 */ result.setNullAt(14);
/* 1296 */ } else {
/* 1297 */ result.setInt(14, tfp14);
/* 1298 */ }
/* 1299 */ }
/* 1300 */
/* 1301 */
/* 1302 */ boolean ffn15 = tmpRow.isNullAt(15);
/* 1303 */ if (ffn15) {
/* 1304 */ result.setNullAt(15);
/* 1305 */ } else {
/* 1306 */ double ffp15 =
/* 1307 */ tmpRow.getDouble(15);
/* 1308 */
/* 1309 */ boolean tfn15 = ffn15;
/* 1310 */ double tfp15 = -1.0;
/* 1311 */ if (!ffn15) {
/* 1312 */ tfp15 = ffp15;
/* 1313 */ }
/* 1314 */
/* 1315 */ if (tfn15) {
/* 1316 */ result.setNullAt(15);
/* 1317 */ } else {
/* 1318 */ result.setDouble(15, tfp15);
/* 1319 */ }
/* 1320 */ }
/* 1321 */
/* 1322 */
/* 1323 */ boolean ffn16 = tmpRow.isNullAt(16);
/* 1324 */ if (ffn16) {
/* 1325 */ result.setNullAt(16);
/* 1326 */ } else {
/* 1327 */ double ffp16 =
/* 1328 */ tmpRow.getDouble(16);
/* 1329 */
/* 1330 */ boolean tfn16 = ffn16;
/* 1331 */ double tfp16 = -1.0;
/* 1332 */ if (!ffn16) {
/* 1333 */ tfp16 = ffp16;
/* 1334 */ }
/* 1335 */
/* 1336 */ if (tfn16) {
/* 1337 */ result.setNullAt(16);
/* 1338 */ } else {
/* 1339 */ result.setDouble(16, tfp16);
/* 1340 */ }
/* 1341 */ }
/* 1342 */
/* 1343 */
/* 1344 */ boolean ffn17 = tmpRow.isNullAt(17);
/* 1345 */ if (ffn17) {
/* 1346 */ result.setNullAt(17);
/* 1347 */ } else {
/* 1348 */ double ffp17 =
/* 1349 */ tmpRow.getDouble(17);
/* 1350 */
/* 1351 */ boolean tfn17 = ffn17;
/* 1352 */ double tfp17 = -1.0;
/* 1353 */ if (!ffn17) {
/* 1354 */ tfp17 = ffp17;
/* 1355 */ }
/* 1356 */
/* 1357 */ if (tfn17) {
/* 1358 */ result.setNullAt(17);
/* 1359 */ } else {
/* 1360 */ result.setDouble(17, tfp17);
/* 1361 */ }
/* 1362 */ }
/* 1363 */
/* 1364 */
/* 1365 */ boolean ffn18 = tmpRow.isNullAt(18);
/* 1366 */ if (ffn18) {
/* 1367 */ result.setNullAt(18);
/* 1368 */ } else {
/* 1369 */ ArrayData ffp18 =
/* 1370 */ tmpRow.getArray(18);
/* 1371 */
/* 1372 */ boolean tfn18 = ffn18;
/* 1373 */ ArrayData tfp18 = null;
/* 1374 */ if (!ffn18) {
/* 1375 */ tfp18 = ffp18;
/* 1376 */ }
/* 1377 */
/* 1378 */ if (tfn18) {
/* 1379 */ result.setNullAt(18);
/* 1380 */ } else {
/* 1381 */ result.update(18, tfp18);
/* 1382 */ }
/* 1383 */ }
/* 1384 */
/* 1385 */
/* 1386 */ boolean ffn19 = tmpRow.isNullAt(19);
/* 1387 */ if (ffn19) {
/* 1388 */ result.setNullAt(19);
/* 1389 */ } else {
/* 1390 */ ArrayData ffp19 =
/* 1391 */ tmpRow.getArray(19);
/* 1392 */
/* 1393 */ boolean tfn19 = ffn19;
/* 1394 */ ArrayData tfp19 = null;
/* 1395 */ if (!ffn19) {
/* 1396 */ tfp19 = ffp19;
/* 1397 */ }
/* 1398 */
/* 1399 */ if (tfn19) {
/* 1400 */ result.setNullAt(19);
/* 1401 */ } else {
/* 1402 */ result.update(19, tfp19);
/* 1403 */ }
/* 1404 */ }
/* 1405 */
/* 1406 */
/* 1407 */ boolean ffn20 = tmpRow.isNullAt(20);
/* 1408 */ if (ffn20) {
/* 1409 */ result.setNullAt(20);
/* 1410 */ } else {
/* 1411 */ ArrayData ffp20 =
/* 1412 */ tmpRow.getArray(20);
/* 1413 */
/* 1414 */ boolean tfn20 = ffn20;
/* 1415 */ ArrayData tfp20 = null;
/* 1416 */ if (!ffn20) {
/* 1417 */ tfp20 = ffp20;
/* 1418 */ }
/* 1419 */
/* 1420 */ if (tfn20) {
/* 1421 */ result.setNullAt(20);
/* 1422 */ } else {
/* 1423 */ result.update(20, tfp20);
/* 1424 */ }
/* 1425 */ }
/* 1426 */
/* 1427 */ tePrim = result.copy();
/* 1428 */
/* 1429 */ }
/* 1430 */
/* 1431 */ if (teNull) {
/* 1432 */ values[j] = null;
/* 1433 */ } else {
/* 1434 */ values[j] = tePrim;
/* 1435 */ }
/* 1436 */ }
/* 1437 */ }
/* 1438 */ value166 = new org.apache.spark.sql.catalyst.util.GenericArrayData(values);
/* 1439 */
/* 1440 */ }
/* 1441 */ ArrayData value165 = null;
/* 1442 */
/* 1443 */ if (!isNull166) {
/* 1444 */
/* 1445 */ indexer.dto.ticket.VendorItemDto[] convertedArray19 = null;
/* 1446 */ int dataLength19 = value166.numElements();
/* 1447 */ convertedArray19 = new indexer.dto.ticket.VendorItemDto[dataLength19];
/* 1448 */
/* 1449 */ int loopIndex19 = 0;
/* 1450 */ while (loopIndex19 < dataLength19) {
/* 1451 */ MapObjects_loopValue378 = (InternalRow) (value166.getStruct(loopIndex19, 21));
/* 1452 */ MapObjects_loopIsNull379 = value166.isNullAt(loopIndex19);
/* 1453 */
/* 1454 */
/* 1455 */ boolean isNull168 = false;
/* 1456 */ indexer.dto.ticket.VendorItemDto value168 = null;
/* 1457 */ if (!false && MapObjects_loopIsNull379) {
/* 1458 */
/* 1459 */ final indexer.dto.ticket.VendorItemDto value170 = null;
/* 1460 */ isNull168 = true;
/* 1461 */ value168 = value170;
/* 1462 */ } else {
/* 1463 */
/* 1464 */ apply1_0(i);
/* 1465 */ apply1_1(i);
/* 1466 */ apply1_2(i);
/* 1467 */ apply1_3(i);
/* 1468 */ apply1_4(i);
/* 1469 */ apply1_5(i);
/* 1470 */ apply1_6(i);
/* 1471 */ apply1_7(i);
/* 1472 */ apply1_8(i);
/* 1473 */ apply1_9(i);
/* 1474 */ apply1_10(i);
/* 1475 */ apply1_11(i);
/* 1476 */ apply1_12(i);
/* 1477 */
/* 1478 */ final indexer.dto.ticket.VendorItemDto value171 = false ? null : new indexer.dto.ticket.VendorItemDto(argValue92, argValue93, argValue94, argValue95, argValue96, argValue97, argValue98, argValue99, argValue100, argValue101, argValue102, argValue103, argValue104, argValue105, argValue106, argValue107, argValue108, argValue109, argValue110, argValue111, argValue112);
/* 1479 */ isNull168 = false;
/* 1480 */ value168 = value171;
/* 1481 */ }
/* 1482 */ if (isNull168) {
/* 1483 */ convertedArray19[loopIndex19] = null;
/* 1484 */ } else {
/* 1485 */ convertedArray19[loopIndex19] = value168;
/* 1486 */ }
/* 1487 */
/* 1488 */ loopIndex19 += 1;
/* 1489 */ }
/* 1490 */
/* 1491 */ value165 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray19);
/* 1492 */ }
/* 1493 */ boolean isNull164 = true;
/* 1494 */ indexer.dto.ticket.VendorItemDto[] value164 = null;
/* 1495 */ if (!isNull166) {
/* 1496 */
/* 1497 */ isNull164 = false;
/* 1498 */ if (!isNull164) {
/* 1499 */
/* 1500 */ Object funcResult56 = null;
/* 1501 */ funcResult56 = value165.array();
/* 1502 */ if (funcResult56 == null) {
/* 1503 */ isNull164 = true;
/* 1504 */ } else {
/* 1505 */ value164 = (indexer.dto.ticket.VendorItemDto[]) funcResult56;
/* 1506 */ }
/* 1507 */
/* 1508 */ }
/* 1509 */ isNull164 = value164 == null;
/* 1510 */ }
/* 1511 */ argValue37 = value164;
/* 1512 */
/* 1513 */ }
Thanks a lot
Proust
地址: 上海市浦东新区金科路2889弄长泰广场C座7楼
Address: 7th floor, No#3, Jinke road 2889, Pudong district, Shanghai, China.
Mobile: +86 13621672634
Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to
DTO Field with Type Array[String]
Posted by "Proust (Feng Guizhou) [Travel Search & Discovery]" <pf...@coupang.com>.
Thanks for the information, tried both JavaSerializer and KryoSerializer, same problem encountered
And the stacktrace looks very different from the one mentioned in the stackoverflow link
________________________________
From: Sandeep Patra <sp...@gmail.com>
Sent: Sunday, February 23, 2020 8:04 PM
To: Proust (Feng Guizhou) [Travel Search & Discovery] <pf...@coupang.com>
Cc: user@spark.apache.org <us...@spark.apache.org>
Subject: Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to DTO Field with Type Array[String]
[Warning]: This email originated from an external source. Do not open links or attachments unless you know the content is safe.
[경고]: 본 이메일은 회사 외부에서 유입되었습니다. 내용이 안전한지 확인하기 전까지는 링크나 첨부파일을 열지 마십시오.
This might be due to the serializer being used.
This stackoverflow answer might help: https://stackoverflow.com/questions/44414429/spark-negativearraysizeexception<https://kor01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fstackoverflow.com%2Fquestions%2F44414429%2Fspark-negativearraysizeexception&data=02%7C01%7Cpfeng%40coupang.com%7Ce27ecc57fb6b426c1c0d08d7b858a406%7Ce3098f96361b47c6a9f4ab7bafcaffe9%7C0%7C0%7C637180563158673718&sdata=%2FowfaBCtHA1DDj1pxLm6PsvP2cPjldqo%2BGmv8XTb5%2F0%3D&reserved=0>
On Sun, Feb 23, 2020 at 1:39 PM Proust (Feng Guizhou) [Travel Search & Discovery] <pf...@coupang.com>> wrote:
Hi, Spark Users
I ecounter below NegativeArraySizeException when run Spark SQL. The catalyst generated code for "apply2_19" and "apply1_11" is attached and also the related DTO.
Difficult to understand how the problem could happen, please help if any idea.
I can see maybe https://issues.apache.org/jira/browse/SPARK-15062<https://kor01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fissues.apache.org%2Fjira%2Fbrowse%2FSPARK-15062&data=02%7C01%7Cpfeng%40coupang.com%7Ce27ecc57fb6b426c1c0d08d7b858a406%7Ce3098f96361b47c6a9f4ab7bafcaffe9%7C0%7C0%7C637180563158683718&sdata=SWKK0Mh5PYLEEpXUqsuuWUT8cwWldGy2GlO5N5anuLQ%3D&reserved=0> is similar but my data type is Array[String] and Spark version is 2.1.2 which looks good both.
[SPARK-15062] Show on DataFrame causes OutOfMemoryError, NegativeArraySizeException or segfault - ASF JIRA<https://kor01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fissues.apache.org%2Fjira%2Fbrowse%2FSPARK-15062&data=02%7C01%7Cpfeng%40coupang.com%7Ce27ecc57fb6b426c1c0d08d7b858a406%7Ce3098f96361b47c6a9f4ab7bafcaffe9%7C0%7C0%7C637180563158683718&sdata=SWKK0Mh5PYLEEpXUqsuuWUT8cwWldGy2GlO5N5anuLQ%3D&reserved=0>
By increasing memory to 8G one will instead get a NegativeArraySizeException or a segfault. See here for original discussion: http://apache-spark-developers-list ...
issues.apache.org<https://kor01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fissues.apache.org%2F&data=02%7C01%7Cpfeng%40coupang.com%7Ce27ecc57fb6b426c1c0d08d7b858a406%7Ce3098f96361b47c6a9f4ab7bafcaffe9%7C0%7C0%7C637180563158693718&sdata=%2FsDqfepuI28hyv%2Fm4tyETjBI5fbMPdyNnEU6dU0AuT0%3D&reserved=0>
java.lang.NegativeArraySizeException
at org.apache.spark.unsafe.types.UTF8String.getBytes(UTF8String.java:229)
at org.apache.spark.unsafe.types.UTF8String.toString(UTF8String.java:1005)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply1_11$(generated.java:2467)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply2_19$(generated.java:1475)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply(generated.java:3881)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
at scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:1076)
at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:1091)
at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:1128)
at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:1132)
at scala.collection.Iterator$class.foreach(Iterator.scala:893)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:101)
at indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:95)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:99)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
case class VendorItemDto(
rate_category_id: java.lang.Long,
product_id: java.lang.Long,
vendor_item_rate_category_id: String,
vendor_item_id: java.lang.Long,
name: String,
item_name: String,
vendor_item_order: java.lang.Integer,
option_codes: Array[String],
option_names: Array[String],
feature_values_ids: Array[String],
feature_values_names: Array[String],
benefit_policy_type: String,
benefit_discount_rate: java.lang.Double,
benefit_discount_amount: java.lang.Long,
available_stock: java.lang.Integer,
sale_price: java.lang.Double,
original_price: java.lang.Double,
supply_price: java.lang.Double,
period_id_set: Array[String],
use_start_set: Array[String],
use_end_set: Array[String]
)
/* 2430 */ private void apply1_11(InternalRow i) {
/* 2431 */
/* 2432 */
/* 2433 */ boolean isNull222 = MapObjects_loopIsNull379;
/* 2434 */ ArrayData value222 = null;
/* 2435 */
/* 2436 */ if (!MapObjects_loopIsNull379) {
/* 2437 */
/* 2438 */ if (MapObjects_loopValue378.isNullAt(19)) {
/* 2439 */ isNull222 = true;
/* 2440 */ } else {
/* 2441 */ value222 = MapObjects_loopValue378.getArray(19);
/* 2442 */ }
/* 2443 */
/* 2444 */ }
/* 2445 */ ArrayData value221 = null;
/* 2446 */
/* 2447 */ if (!isNull222) {
/* 2448 */
/* 2449 */ java.lang.String[] convertedArray17 = null;
/* 2450 */ int dataLength17 = value222.numElements();
/* 2451 */ convertedArray17 = new java.lang.String[dataLength17];
/* 2452 */
/* 2453 */ int loopIndex17 = 0;
/* 2454 */ while (loopIndex17 < dataLength17) {
/* 2455 */ MapObjects_loopValue390 = (UTF8String) (value222.getUTF8String(loopIndex17));
/* 2456 */ MapObjects_loopIsNull391 = value222.isNullAt(loopIndex17);
/* 2457 */
/* 2458 */
/* 2459 */ boolean isNull223 = true;
/* 2460 */ java.lang.String value223 = null;
/* 2461 */ if (!MapObjects_loopIsNull391) {
/* 2462 */
/* 2463 */ isNull223 = false;
/* 2464 */ if (!isNull223) {
/* 2465 */
/* 2466 */ Object funcResult52 = null;
/* 2467 */ funcResult52 = MapObjects_loopValue390.toString();
/* 2468 */ if (funcResult52 == null) {
/* 2469 */ isNull223 = true;
/* 2470 */ } else {
/* 2471 */ value223 = (java.lang.String) funcResult52;
/* 2472 */ }
/* 2473 */
/* 2474 */ }
/* 2475 */ isNull223 = value223 == null;
/* 2476 */ }
/* 2477 */ if (isNull223) {
/* 2478 */ convertedArray17[loopIndex17] = null;
/* 2479 */ } else {
/* 2480 */ convertedArray17[loopIndex17] = value223;
/* 2481 */ }
/* 2482 */
/* 2483 */ loopIndex17 += 1;
/* 2484 */ }
/* 2485 */
/* 2486 */ value221 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray17);
/* 2487 */ }
/* 2488 */ boolean isNull220 = true;
/* 2489 */ java.lang.String[] value220 = null;
/* 2490 */ if (!isNull222) {
/* 2491 */
/* 2492 */ isNull220 = false;
/* 2493 */ if (!isNull220) {
/* 2494 */
/* 2495 */ Object funcResult53 = null;
/* 2496 */ funcResult53 = value221.array();
/* 2497 */ if (funcResult53 == null) {
/* 2498 */ isNull220 = true;
/* 2499 */ } else {
/* 2500 */ value220 = (java.lang.String[]) funcResult53;
/* 2501 */ }
/* 2502 */
/* 2503 */ }
/* 2504 */ isNull220 = value220 == null;
/* 2505 */ }
/* 2506 */ argValue111 = value220;
/* 2507 */
/* 2508 */ }
/* 933 */ private void apply2_19(InternalRow i) {
/* 934 */
/* 935 */
/* 936 */ resultIsNull30 = false;
/* 937 */ if (!resultIsNull30) {
/* 938 */
/* 939 */ boolean isNull161 = i.isNullAt(36);
/* 940 */ long value161 = isNull161 ? -1L : (i.getLong(36));
/* 941 */ resultIsNull30 = isNull161;
/* 942 */ argValue90 = value161;
/* 943 */ }
/* 944 */
/* 945 */
/* 946 */ final java.lang.Long value160 = resultIsNull30 ? null : new java.lang.Long(argValue90);
/* 947 */ argValue35 = value160;
/* 948 */
/* 949 */
/* 950 */ resultIsNull31 = false;
/* 951 */ if (!resultIsNull31) {
/* 952 */
/* 953 */ boolean isNull163 = i.isNullAt(37);
/* 954 */ long value163 = isNull163 ? -1L : (i.getLong(37));
/* 955 */ resultIsNull31 = isNull163;
/* 956 */ argValue91 = value163;
/* 957 */ }
/* 958 */
/* 959 */
/* 960 */ final java.lang.Long value162 = resultIsNull31 ? null : new java.lang.Long(argValue91);
/* 961 */ argValue36 = value162;
/* 962 */
/* 963 */
/* 964 */ boolean isNull167 = i.isNullAt(35);
/* 965 */ ArrayData value167 = isNull167 ? null : (i.getArray(35));
/* 966 */ boolean isNull166 = isNull167;
/* 967 */ ArrayData value166 = null;
/* 968 */ if (!isNull167) {
/* 969 */
/* 970 */ final int n = value167.numElements();
/* 971 */ final Object[] values = new Object[n];
/* 972 */ for (int j = 0; j < n; j ++) {
/* 973 */ if (value167.isNullAt(j)) {
/* 974 */ values[j] = null;
/* 975 */ } else {
/* 976 */ boolean feNull = false;
/* 977 */ InternalRow fePrim =
/* 978 */ value167.getStruct(j, 21);
/* 979 */
/* 980 */ boolean teNull = feNull;
/* 981 */ InternalRow tePrim = null;
/* 982 */ if (!feNull) {
/* 983 */
/* 984 */ final org.apache.spark.sql.catalyst.expressions.GenericInternalRow result = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(21);
/* 985 */ final InternalRow tmpRow = fePrim;
/* 986 */
/* 987 */ boolean ffn = tmpRow.isNullAt(0);
/* 988 */ if (ffn) {
/* 989 */ result.setNullAt(0);
/* 990 */ } else {
/* 991 */ long ffp =
/* 992 */ tmpRow.getLong(0);
/* 993 */
/* 994 */ boolean tfn = ffn;
/* 995 */ long tfp = -1L;
/* 996 */ if (!ffn) {
/* 997 */ tfp = ffp;
/* 998 */ }
/* 999 */
/* 1000 */ if (tfn) {
/* 1001 */ result.setNullAt(0);
/* 1002 */ } else {
/* 1003 */ result.setLong(0, tfp);
/* 1004 */ }
/* 1005 */ }
/* 1006 */
/* 1007 */
/* 1008 */ boolean ffn1 = tmpRow.isNullAt(1);
/* 1009 */ if (ffn1) {
/* 1010 */ result.setNullAt(1);
/* 1011 */ } else {
/* 1012 */ long ffp1 =
/* 1013 */ tmpRow.getLong(1);
/* 1014 */
/* 1015 */ boolean tfn1 = ffn1;
/* 1016 */ long tfp1 = -1L;
/* 1017 */ if (!ffn1) {
/* 1018 */ tfp1 = ffp1;
/* 1019 */ }
/* 1020 */
/* 1021 */ if (tfn1) {
/* 1022 */ result.setNullAt(1);
/* 1023 */ } else {
/* 1024 */ result.setLong(1, tfp1);
/* 1025 */ }
/* 1026 */ }
/* 1027 */
/* 1028 */
/* 1029 */ boolean ffn2 = tmpRow.isNullAt(2);
/* 1030 */ if (ffn2) {
/* 1031 */ result.setNullAt(2);
/* 1032 */ } else {
/* 1033 */ UTF8String ffp2 =
/* 1034 */ tmpRow.getUTF8String(2);
/* 1035 */
/* 1036 */ boolean tfn2 = ffn2;
/* 1037 */ UTF8String tfp2 = null;
/* 1038 */ if (!ffn2) {
/* 1039 */ tfp2 = ffp2;
/* 1040 */ }
/* 1041 */
/* 1042 */ if (tfn2) {
/* 1043 */ result.setNullAt(2);
/* 1044 */ } else {
/* 1045 */ result.update(2, tfp2.clone());
/* 1046 */ }
/* 1047 */ }
/* 1048 */
/* 1049 */
/* 1050 */ boolean ffn3 = tmpRow.isNullAt(3);
/* 1051 */ if (ffn3) {
/* 1052 */ result.setNullAt(3);
/* 1053 */ } else {
/* 1054 */ long ffp3 =
/* 1055 */ tmpRow.getLong(3);
/* 1056 */
/* 1057 */ boolean tfn3 = ffn3;
/* 1058 */ long tfp3 = -1L;
/* 1059 */ if (!ffn3) {
/* 1060 */ tfp3 = ffp3;
/* 1061 */ }
/* 1062 */
/* 1063 */ if (tfn3) {
/* 1064 */ result.setNullAt(3);
/* 1065 */ } else {
/* 1066 */ result.setLong(3, tfp3);
/* 1067 */ }
/* 1068 */ }
/* 1069 */
/* 1070 */
/* 1071 */ boolean ffn4 = tmpRow.isNullAt(4);
/* 1072 */ if (ffn4) {
/* 1073 */ result.setNullAt(4);
/* 1074 */ } else {
/* 1075 */ UTF8String ffp4 =
/* 1076 */ tmpRow.getUTF8String(4);
/* 1077 */
/* 1078 */ boolean tfn4 = ffn4;
/* 1079 */ UTF8String tfp4 = null;
/* 1080 */ if (!ffn4) {
/* 1081 */ tfp4 = ffp4;
/* 1082 */ }
/* 1083 */
/* 1084 */ if (tfn4) {
/* 1085 */ result.setNullAt(4);
/* 1086 */ } else {
/* 1087 */ result.update(4, tfp4.clone());
/* 1088 */ }
/* 1089 */ }
/* 1090 */
/* 1091 */
/* 1092 */ boolean ffn5 = tmpRow.isNullAt(5);
/* 1093 */ if (ffn5) {
/* 1094 */ result.setNullAt(5);
/* 1095 */ } else {
/* 1096 */ UTF8String ffp5 =
/* 1097 */ tmpRow.getUTF8String(5);
/* 1098 */
/* 1099 */ boolean tfn5 = ffn5;
/* 1100 */ UTF8String tfp5 = null;
/* 1101 */ if (!ffn5) {
/* 1102 */ tfp5 = ffp5;
/* 1103 */ }
/* 1104 */
/* 1105 */ if (tfn5) {
/* 1106 */ result.setNullAt(5);
/* 1107 */ } else {
/* 1108 */ result.update(5, tfp5.clone());
/* 1109 */ }
/* 1110 */ }
/* 1111 */
/* 1112 */
/* 1113 */ boolean ffn6 = tmpRow.isNullAt(6);
/* 1114 */ if (ffn6) {
/* 1115 */ result.setNullAt(6);
/* 1116 */ } else {
/* 1117 */ int ffp6 =
/* 1118 */ tmpRow.getInt(6);
/* 1119 */
/* 1120 */ boolean tfn6 = ffn6;
/* 1121 */ int tfp6 = -1;
/* 1122 */ if (!ffn6) {
/* 1123 */ tfp6 = ffp6;
/* 1124 */ }
/* 1125 */
/* 1126 */ if (tfn6) {
/* 1127 */ result.setNullAt(6);
/* 1128 */ } else {
/* 1129 */ result.setInt(6, tfp6);
/* 1130 */ }
/* 1131 */ }
/* 1132 */
/* 1133 */
/* 1134 */ boolean ffn7 = tmpRow.isNullAt(7);
/* 1135 */ if (ffn7) {
/* 1136 */ result.setNullAt(7);
/* 1137 */ } else {
/* 1138 */ ArrayData ffp7 =
/* 1139 */ tmpRow.getArray(7);
/* 1140 */
/* 1141 */ boolean tfn7 = ffn7;
/* 1142 */ ArrayData tfp7 = null;
/* 1143 */ if (!ffn7) {
/* 1144 */ tfp7 = ffp7;
/* 1145 */ }
/* 1146 */
/* 1147 */ if (tfn7) {
/* 1148 */ result.setNullAt(7);
/* 1149 */ } else {
/* 1150 */ result.update(7, tfp7);
/* 1151 */ }
/* 1152 */ }
/* 1153 */
/* 1154 */
/* 1155 */ boolean ffn8 = tmpRow.isNullAt(8);
/* 1156 */ if (ffn8) {
/* 1157 */ result.setNullAt(8);
/* 1158 */ } else {
/* 1159 */ ArrayData ffp8 =
/* 1160 */ tmpRow.getArray(8);
/* 1161 */
/* 1162 */ boolean tfn8 = ffn8;
/* 1163 */ ArrayData tfp8 = null;
/* 1164 */ if (!ffn8) {
/* 1165 */ tfp8 = ffp8;
/* 1166 */ }
/* 1167 */
/* 1168 */ if (tfn8) {
/* 1169 */ result.setNullAt(8);
/* 1170 */ } else {
/* 1171 */ result.update(8, tfp8);
/* 1172 */ }
/* 1173 */ }
/* 1174 */
/* 1175 */
/* 1176 */ boolean ffn9 = tmpRow.isNullAt(9);
/* 1177 */ if (ffn9) {
/* 1178 */ result.setNullAt(9);
/* 1179 */ } else {
/* 1180 */ ArrayData ffp9 =
/* 1181 */ tmpRow.getArray(9);
/* 1182 */
/* 1183 */ boolean tfn9 = ffn9;
/* 1184 */ ArrayData tfp9 = null;
/* 1185 */ if (!ffn9) {
/* 1186 */ tfp9 = ffp9;
/* 1187 */ }
/* 1188 */
/* 1189 */ if (tfn9) {
/* 1190 */ result.setNullAt(9);
/* 1191 */ } else {
/* 1192 */ result.update(9, tfp9);
/* 1193 */ }
/* 1194 */ }
/* 1195 */
/* 1196 */
/* 1197 */ boolean ffn10 = tmpRow.isNullAt(10);
/* 1198 */ if (ffn10) {
/* 1199 */ result.setNullAt(10);
/* 1200 */ } else {
/* 1201 */ ArrayData ffp10 =
/* 1202 */ tmpRow.getArray(10);
/* 1203 */
/* 1204 */ boolean tfn10 = ffn10;
/* 1205 */ ArrayData tfp10 = null;
/* 1206 */ if (!ffn10) {
/* 1207 */ tfp10 = ffp10;
/* 1208 */ }
/* 1209 */
/* 1210 */ if (tfn10) {
/* 1211 */ result.setNullAt(10);
/* 1212 */ } else {
/* 1213 */ result.update(10, tfp10);
/* 1214 */ }
/* 1215 */ }
/* 1216 */
/* 1217 */
/* 1218 */ boolean ffn11 = tmpRow.isNullAt(11);
/* 1219 */ if (ffn11) {
/* 1220 */ result.setNullAt(11);
/* 1221 */ } else {
/* 1222 */ UTF8String ffp11 =
/* 1223 */ tmpRow.getUTF8String(11);
/* 1224 */
/* 1225 */ boolean tfn11 = ffn11;
/* 1226 */ UTF8String tfp11 = null;
/* 1227 */ if (!ffn11) {
/* 1228 */ tfp11 = ffp11;
/* 1229 */ }
/* 1230 */
/* 1231 */ if (tfn11) {
/* 1232 */ result.setNullAt(11);
/* 1233 */ } else {
/* 1234 */ result.update(11, tfp11.clone());
/* 1235 */ }
/* 1236 */ }
/* 1237 */
/* 1238 */
/* 1239 */ boolean ffn12 = tmpRow.isNullAt(12);
/* 1240 */ if (ffn12) {
/* 1241 */ result.setNullAt(12);
/* 1242 */ } else {
/* 1243 */ double ffp12 =
/* 1244 */ tmpRow.getDouble(12);
/* 1245 */
/* 1246 */ boolean tfn12 = ffn12;
/* 1247 */ double tfp12 = -1.0;
/* 1248 */ if (!ffn12) {
/* 1249 */ tfp12 = ffp12;
/* 1250 */ }
/* 1251 */
/* 1252 */ if (tfn12) {
/* 1253 */ result.setNullAt(12);
/* 1254 */ } else {
/* 1255 */ result.setDouble(12, tfp12);
/* 1256 */ }
/* 1257 */ }
/* 1258 */
/* 1259 */
/* 1260 */ boolean ffn13 = tmpRow.isNullAt(13);
/* 1261 */ if (ffn13) {
/* 1262 */ result.setNullAt(13);
/* 1263 */ } else {
/* 1264 */ long ffp13 =
/* 1265 */ tmpRow.getLong(13);
/* 1266 */
/* 1267 */ boolean tfn13 = ffn13;
/* 1268 */ long tfp13 = -1L;
/* 1269 */ if (!ffn13) {
/* 1270 */ tfp13 = ffp13;
/* 1271 */ }
/* 1272 */
/* 1273 */ if (tfn13) {
/* 1274 */ result.setNullAt(13);
/* 1275 */ } else {
/* 1276 */ result.setLong(13, tfp13);
/* 1277 */ }
/* 1278 */ }
/* 1279 */
/* 1280 */
/* 1281 */ boolean ffn14 = tmpRow.isNullAt(14);
/* 1282 */ if (ffn14) {
/* 1283 */ result.setNullAt(14);
/* 1284 */ } else {
/* 1285 */ int ffp14 =
/* 1286 */ tmpRow.getInt(14);
/* 1287 */
/* 1288 */ boolean tfn14 = ffn14;
/* 1289 */ int tfp14 = -1;
/* 1290 */ if (!ffn14) {
/* 1291 */ tfp14 = ffp14;
/* 1292 */ }
/* 1293 */
/* 1294 */ if (tfn14) {
/* 1295 */ result.setNullAt(14);
/* 1296 */ } else {
/* 1297 */ result.setInt(14, tfp14);
/* 1298 */ }
/* 1299 */ }
/* 1300 */
/* 1301 */
/* 1302 */ boolean ffn15 = tmpRow.isNullAt(15);
/* 1303 */ if (ffn15) {
/* 1304 */ result.setNullAt(15);
/* 1305 */ } else {
/* 1306 */ double ffp15 =
/* 1307 */ tmpRow.getDouble(15);
/* 1308 */
/* 1309 */ boolean tfn15 = ffn15;
/* 1310 */ double tfp15 = -1.0;
/* 1311 */ if (!ffn15) {
/* 1312 */ tfp15 = ffp15;
/* 1313 */ }
/* 1314 */
/* 1315 */ if (tfn15) {
/* 1316 */ result.setNullAt(15);
/* 1317 */ } else {
/* 1318 */ result.setDouble(15, tfp15);
/* 1319 */ }
/* 1320 */ }
/* 1321 */
/* 1322 */
/* 1323 */ boolean ffn16 = tmpRow.isNullAt(16);
/* 1324 */ if (ffn16) {
/* 1325 */ result.setNullAt(16);
/* 1326 */ } else {
/* 1327 */ double ffp16 =
/* 1328 */ tmpRow.getDouble(16);
/* 1329 */
/* 1330 */ boolean tfn16 = ffn16;
/* 1331 */ double tfp16 = -1.0;
/* 1332 */ if (!ffn16) {
/* 1333 */ tfp16 = ffp16;
/* 1334 */ }
/* 1335 */
/* 1336 */ if (tfn16) {
/* 1337 */ result.setNullAt(16);
/* 1338 */ } else {
/* 1339 */ result.setDouble(16, tfp16);
/* 1340 */ }
/* 1341 */ }
/* 1342 */
/* 1343 */
/* 1344 */ boolean ffn17 = tmpRow.isNullAt(17);
/* 1345 */ if (ffn17) {
/* 1346 */ result.setNullAt(17);
/* 1347 */ } else {
/* 1348 */ double ffp17 =
/* 1349 */ tmpRow.getDouble(17);
/* 1350 */
/* 1351 */ boolean tfn17 = ffn17;
/* 1352 */ double tfp17 = -1.0;
/* 1353 */ if (!ffn17) {
/* 1354 */ tfp17 = ffp17;
/* 1355 */ }
/* 1356 */
/* 1357 */ if (tfn17) {
/* 1358 */ result.setNullAt(17);
/* 1359 */ } else {
/* 1360 */ result.setDouble(17, tfp17);
/* 1361 */ }
/* 1362 */ }
/* 1363 */
/* 1364 */
/* 1365 */ boolean ffn18 = tmpRow.isNullAt(18);
/* 1366 */ if (ffn18) {
/* 1367 */ result.setNullAt(18);
/* 1368 */ } else {
/* 1369 */ ArrayData ffp18 =
/* 1370 */ tmpRow.getArray(18);
/* 1371 */
/* 1372 */ boolean tfn18 = ffn18;
/* 1373 */ ArrayData tfp18 = null;
/* 1374 */ if (!ffn18) {
/* 1375 */ tfp18 = ffp18;
/* 1376 */ }
/* 1377 */
/* 1378 */ if (tfn18) {
/* 1379 */ result.setNullAt(18);
/* 1380 */ } else {
/* 1381 */ result.update(18, tfp18);
/* 1382 */ }
/* 1383 */ }
/* 1384 */
/* 1385 */
/* 1386 */ boolean ffn19 = tmpRow.isNullAt(19);
/* 1387 */ if (ffn19) {
/* 1388 */ result.setNullAt(19);
/* 1389 */ } else {
/* 1390 */ ArrayData ffp19 =
/* 1391 */ tmpRow.getArray(19);
/* 1392 */
/* 1393 */ boolean tfn19 = ffn19;
/* 1394 */ ArrayData tfp19 = null;
/* 1395 */ if (!ffn19) {
/* 1396 */ tfp19 = ffp19;
/* 1397 */ }
/* 1398 */
/* 1399 */ if (tfn19) {
/* 1400 */ result.setNullAt(19);
/* 1401 */ } else {
/* 1402 */ result.update(19, tfp19);
/* 1403 */ }
/* 1404 */ }
/* 1405 */
/* 1406 */
/* 1407 */ boolean ffn20 = tmpRow.isNullAt(20);
/* 1408 */ if (ffn20) {
/* 1409 */ result.setNullAt(20);
/* 1410 */ } else {
/* 1411 */ ArrayData ffp20 =
/* 1412 */ tmpRow.getArray(20);
/* 1413 */
/* 1414 */ boolean tfn20 = ffn20;
/* 1415 */ ArrayData tfp20 = null;
/* 1416 */ if (!ffn20) {
/* 1417 */ tfp20 = ffp20;
/* 1418 */ }
/* 1419 */
/* 1420 */ if (tfn20) {
/* 1421 */ result.setNullAt(20);
/* 1422 */ } else {
/* 1423 */ result.update(20, tfp20);
/* 1424 */ }
/* 1425 */ }
/* 1426 */
/* 1427 */ tePrim = result.copy();
/* 1428 */
/* 1429 */ }
/* 1430 */
/* 1431 */ if (teNull) {
/* 1432 */ values[j] = null;
/* 1433 */ } else {
/* 1434 */ values[j] = tePrim;
/* 1435 */ }
/* 1436 */ }
/* 1437 */ }
/* 1438 */ value166 = new org.apache.spark.sql.catalyst.util.GenericArrayData(values);
/* 1439 */
/* 1440 */ }
/* 1441 */ ArrayData value165 = null;
/* 1442 */
/* 1443 */ if (!isNull166) {
/* 1444 */
/* 1445 */ indexer.dto.ticket.VendorItemDto[] convertedArray19 = null;
/* 1446 */ int dataLength19 = value166.numElements();
/* 1447 */ convertedArray19 = new indexer.dto.ticket.VendorItemDto[dataLength19];
/* 1448 */
/* 1449 */ int loopIndex19 = 0;
/* 1450 */ while (loopIndex19 < dataLength19) {
/* 1451 */ MapObjects_loopValue378 = (InternalRow) (value166.getStruct(loopIndex19, 21));
/* 1452 */ MapObjects_loopIsNull379 = value166.isNullAt(loopIndex19);
/* 1453 */
/* 1454 */
/* 1455 */ boolean isNull168 = false;
/* 1456 */ indexer.dto.ticket.VendorItemDto value168 = null;
/* 1457 */ if (!false && MapObjects_loopIsNull379) {
/* 1458 */
/* 1459 */ final indexer.dto.ticket.VendorItemDto value170 = null;
/* 1460 */ isNull168 = true;
/* 1461 */ value168 = value170;
/* 1462 */ } else {
/* 1463 */
/* 1464 */ apply1_0(i);
/* 1465 */ apply1_1(i);
/* 1466 */ apply1_2(i);
/* 1467 */ apply1_3(i);
/* 1468 */ apply1_4(i);
/* 1469 */ apply1_5(i);
/* 1470 */ apply1_6(i);
/* 1471 */ apply1_7(i);
/* 1472 */ apply1_8(i);
/* 1473 */ apply1_9(i);
/* 1474 */ apply1_10(i);
/* 1475 */ apply1_11(i);
/* 1476 */ apply1_12(i);
/* 1477 */
/* 1478 */ final indexer.dto.ticket.VendorItemDto value171 = false ? null : new indexer.dto.ticket.VendorItemDto(argValue92, argValue93, argValue94, argValue95, argValue96, argValue97, argValue98, argValue99, argValue100, argValue101, argValue102, argValue103, argValue104, argValue105, argValue106, argValue107, argValue108, argValue109, argValue110, argValue111, argValue112);
/* 1479 */ isNull168 = false;
/* 1480 */ value168 = value171;
/* 1481 */ }
/* 1482 */ if (isNull168) {
/* 1483 */ convertedArray19[loopIndex19] = null;
/* 1484 */ } else {
/* 1485 */ convertedArray19[loopIndex19] = value168;
/* 1486 */ }
/* 1487 */
/* 1488 */ loopIndex19 += 1;
/* 1489 */ }
/* 1490 */
/* 1491 */ value165 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray19);
/* 1492 */ }
/* 1493 */ boolean isNull164 = true;
/* 1494 */ indexer.dto.ticket.VendorItemDto[] value164 = null;
/* 1495 */ if (!isNull166) {
/* 1496 */
/* 1497 */ isNull164 = false;
/* 1498 */ if (!isNull164) {
/* 1499 */
/* 1500 */ Object funcResult56 = null;
/* 1501 */ funcResult56 = value165.array();
/* 1502 */ if (funcResult56 == null) {
/* 1503 */ isNull164 = true;
/* 1504 */ } else {
/* 1505 */ value164 = (indexer.dto.ticket.VendorItemDto[]) funcResult56;
/* 1506 */ }
/* 1507 */
/* 1508 */ }
/* 1509 */ isNull164 = value164 == null;
/* 1510 */ }
/* 1511 */ argValue37 = value164;
/* 1512 */
/* 1513 */ }
Thanks a lot
Proust
地址: 上海市浦东新区金科路2889弄长泰广场C座7楼
Address: 7th floor, No#3, Jinke road 2889, Pudong district, Shanghai, China.
Mobile: +86 13621672634
Re: [Spark SQL] NegativeArraySizeException When Parse InternalRow to
DTO Field with Type Array[String]
Posted by Sandeep Patra <sp...@gmail.com>.
This might be due to the serializer being used.
This stackoverflow answer might help:
https://stackoverflow.com/questions/44414429/spark-negativearraysizeexception
On Sun, Feb 23, 2020 at 1:39 PM Proust (Feng Guizhou) [Travel Search &
Discovery] <pf...@coupang.com> wrote:
> Hi, Spark Users
>
> I ecounter below NegativeArraySizeException when run Spark SQL. The
> catalyst generated code for "apply2_19" and "apply1_11" is attached and
> also the related DTO.
> Difficult to understand how the problem could happen, please help if any
> idea.
>
> I can see maybe https://issues.apache.org/jira/browse/SPARK-15062 is
> similar but my data type is Array[String] and Spark version is 2.1.2 which
> looks good both.
> [SPARK-15062] Show on DataFrame causes OutOfMemoryError,
> NegativeArraySizeException or segfault - ASF JIRA
> <https://issues.apache.org/jira/browse/SPARK-15062>
> By increasing memory to 8G one will instead get a
> NegativeArraySizeException or a segfault. See here for original discussion:
> http://apache-spark-developers-list ...
> issues.apache.org
>
>
> java.lang.NegativeArraySizeException
> at org.apache.spark.unsafe.types.UTF8String.getBytes(UTF8String.java:229)
> at
> org.apache.spark.unsafe.types.UTF8String.toString(UTF8String.java:1005)
> at
> org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply1_11$(generated.java:2467)
> at
> org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply2_19$(generated.java:1475)
> at
> org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificSafeProjection.apply(generated.java:3881)
> at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
> at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
> at
> scala.collection.Iterator$GroupedIterator.takeDestructively(Iterator.scala:1076)
> at scala.collection.Iterator$GroupedIterator.go(Iterator.scala:1091)
> at scala.collection.Iterator$GroupedIterator.fill(Iterator.scala:1128)
> at scala.collection.Iterator$GroupedIterator.hasNext(Iterator.scala:1132)
> at scala.collection.Iterator$class.foreach(Iterator.scala:893)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
> at
> indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:101)
> at
> indexer.executor.TicketIndexerExecutorV2$$anonfun$indexData$2.apply(TicketIndexerExecutorV2.scala:95)
> at
> org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
> at
> org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:926)
> at
> org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
> at
> org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954)
> at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
> at org.apache.spark.scheduler.Task.run(Task.scala:99)
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
>
>
> case class VendorItemDto(
> rate_category_id: java.lang.Long,
> product_id: java.lang.Long,
> vendor_item_rate_category_id: String,
> vendor_item_id: java.lang.Long,
> name: String,
> item_name: String,
> vendor_item_order: java.lang.Integer,
> option_codes: Array[String],
> option_names: Array[String],
> feature_values_ids: Array[String],
> feature_values_names: Array[String],
> benefit_policy_type: String,
> benefit_discount_rate: java.lang.Double,
> benefit_discount_amount: java.lang.Long,
> available_stock: java.lang.Integer,
> sale_price: java.lang.Double,
> original_price: java.lang.Double,
> supply_price: java.lang.Double,
> period_id_set: Array[String],
> use_start_set: Array[String],
> use_end_set: Array[String]
> )
>
>
>
> /* 2430 */ private void apply1_11(InternalRow i) {
> /* 2431 */
> /* 2432 */
> /* 2433 */ boolean isNull222 = MapObjects_loopIsNull379;
> /* 2434 */ ArrayData value222 = null;
> /* 2435 */
> /* 2436 */ if (!MapObjects_loopIsNull379) {
> /* 2437 */
> /* 2438 */ if (MapObjects_loopValue378.isNullAt(19)) {
> /* 2439 */ isNull222 = true;
> /* 2440 */ } else {
> /* 2441 */ value222 = MapObjects_loopValue378.getArray(19);
> /* 2442 */ }
> /* 2443 */
> /* 2444 */ }
> /* 2445 */ ArrayData value221 = null;
> /* 2446 */
> /* 2447 */ if (!isNull222) {
> /* 2448 */
> /* 2449 */ java.lang.String[] convertedArray17 = null;
> /* 2450 */ int dataLength17 = value222.numElements();
> /* 2451 */ convertedArray17 = new java.lang.String[dataLength17];
> /* 2452 */
> /* 2453 */ int loopIndex17 = 0;
> /* 2454 */ while (loopIndex17 < dataLength17) {
> /* 2455 */ MapObjects_loopValue390 = (UTF8String)
> (value222.getUTF8String(loopIndex17));
> /* 2456 */ MapObjects_loopIsNull391 =
> value222.isNullAt(loopIndex17);
> /* 2457 */
> /* 2458 */
> /* 2459 */ boolean isNull223 = true;
> /* 2460 */ java.lang.String value223 = null;
> /* 2461 */ if (!MapObjects_loopIsNull391) {
> /* 2462 */
> /* 2463 */ isNull223 = false;
> /* 2464 */ if (!isNull223) {
> /* 2465 */
> /* 2466 */ Object funcResult52 = null;
> /* 2467 */ funcResult52 = MapObjects_loopValue390.toString();
> /* 2468 */ if (funcResult52 == null) {
> /* 2469 */ isNull223 = true;
> /* 2470 */ } else {
> /* 2471 */ value223 = (java.lang.String) funcResult52;
> /* 2472 */ }
> /* 2473 */
> /* 2474 */ }
> /* 2475 */ isNull223 = value223 == null;
> /* 2476 */ }
> /* 2477 */ if (isNull223) {
> /* 2478 */ convertedArray17[loopIndex17] = null;
> /* 2479 */ } else {
> /* 2480 */ convertedArray17[loopIndex17] = value223;
> /* 2481 */ }
> /* 2482 */
> /* 2483 */ loopIndex17 += 1;
> /* 2484 */ }
> /* 2485 */
> /* 2486 */ value221 = new
> org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray17);
> /* 2487 */ }
> /* 2488 */ boolean isNull220 = true;
> /* 2489 */ java.lang.String[] value220 = null;
> /* 2490 */ if (!isNull222) {
> /* 2491 */
> /* 2492 */ isNull220 = false;
> /* 2493 */ if (!isNull220) {
> /* 2494 */
> /* 2495 */ Object funcResult53 = null;
> /* 2496 */ funcResult53 = value221.array();
> /* 2497 */ if (funcResult53 == null) {
> /* 2498 */ isNull220 = true;
> /* 2499 */ } else {
> /* 2500 */ value220 = (java.lang.String[]) funcResult53;
> /* 2501 */ }
> /* 2502 */
> /* 2503 */ }
> /* 2504 */ isNull220 = value220 == null;
> /* 2505 */ }
> /* 2506 */ argValue111 = value220;
> /* 2507 */
> /* 2508 */ }
>
>
> /* 933 */ private void apply2_19(InternalRow i) {
> /* 934 */
> /* 935 */
> /* 936 */ resultIsNull30 = false;
> /* 937 */ if (!resultIsNull30) {
> /* 938 */
> /* 939 */ boolean isNull161 = i.isNullAt(36);
> /* 940 */ long value161 = isNull161 ? -1L : (i.getLong(36));
> /* 941 */ resultIsNull30 = isNull161;
> /* 942 */ argValue90 = value161;
> /* 943 */ }
> /* 944 */
> /* 945 */
> /* 946 */ final java.lang.Long value160 = resultIsNull30 ? null : new
> java.lang.Long(argValue90);
> /* 947 */ argValue35 = value160;
> /* 948 */
> /* 949 */
> /* 950 */ resultIsNull31 = false;
> /* 951 */ if (!resultIsNull31) {
> /* 952 */
> /* 953 */ boolean isNull163 = i.isNullAt(37);
> /* 954 */ long value163 = isNull163 ? -1L : (i.getLong(37));
> /* 955 */ resultIsNull31 = isNull163;
> /* 956 */ argValue91 = value163;
> /* 957 */ }
> /* 958 */
> /* 959 */
> /* 960 */ final java.lang.Long value162 = resultIsNull31 ? null : new
> java.lang.Long(argValue91);
> /* 961 */ argValue36 = value162;
> /* 962 */
> /* 963 */
> /* 964 */ boolean isNull167 = i.isNullAt(35);
> /* 965 */ ArrayData value167 = isNull167 ? null : (i.getArray(35));
> /* 966 */ boolean isNull166 = isNull167;
> /* 967 */ ArrayData value166 = null;
> /* 968 */ if (!isNull167) {
> /* 969 */
> /* 970 */ final int n = value167.numElements();
> /* 971 */ final Object[] values = new Object[n];
> /* 972 */ for (int j = 0; j < n; j ++) {
> /* 973 */ if (value167.isNullAt(j)) {
> /* 974 */ values[j] = null;
> /* 975 */ } else {
> /* 976 */ boolean feNull = false;
> /* 977 */ InternalRow fePrim =
> /* 978 */ value167.getStruct(j, 21);
> /* 979 */
> /* 980 */ boolean teNull = feNull;
> /* 981 */ InternalRow tePrim = null;
> /* 982 */ if (!feNull) {
> /* 983 */
> /* 984 */ final
> org.apache.spark.sql.catalyst.expressions.GenericInternalRow result = new
> org.apache.spark.sql.catalyst.expressions.GenericInternalRow(21);
> /* 985 */ final InternalRow tmpRow = fePrim;
> /* 986 */
> /* 987 */ boolean ffn = tmpRow.isNullAt(0);
> /* 988 */ if (ffn) {
> /* 989 */ result.setNullAt(0);
> /* 990 */ } else {
> /* 991 */ long ffp =
> /* 992 */ tmpRow.getLong(0);
> /* 993 */
> /* 994 */ boolean tfn = ffn;
> /* 995 */ long tfp = -1L;
> /* 996 */ if (!ffn) {
> /* 997 */ tfp = ffp;
> /* 998 */ }
> /* 999 */
> /* 1000 */ if (tfn) {
> /* 1001 */ result.setNullAt(0);
> /* 1002 */ } else {
> /* 1003 */ result.setLong(0, tfp);
> /* 1004 */ }
> /* 1005 */ }
> /* 1006 */
> /* 1007 */
> /* 1008 */ boolean ffn1 = tmpRow.isNullAt(1);
> /* 1009 */ if (ffn1) {
> /* 1010 */ result.setNullAt(1);
> /* 1011 */ } else {
> /* 1012 */ long ffp1 =
> /* 1013 */ tmpRow.getLong(1);
> /* 1014 */
> /* 1015 */ boolean tfn1 = ffn1;
> /* 1016 */ long tfp1 = -1L;
> /* 1017 */ if (!ffn1) {
> /* 1018 */ tfp1 = ffp1;
> /* 1019 */ }
> /* 1020 */
> /* 1021 */ if (tfn1) {
> /* 1022 */ result.setNullAt(1);
> /* 1023 */ } else {
> /* 1024 */ result.setLong(1, tfp1);
> /* 1025 */ }
> /* 1026 */ }
> /* 1027 */
> /* 1028 */
> /* 1029 */ boolean ffn2 = tmpRow.isNullAt(2);
> /* 1030 */ if (ffn2) {
> /* 1031 */ result.setNullAt(2);
> /* 1032 */ } else {
> /* 1033 */ UTF8String ffp2 =
> /* 1034 */ tmpRow.getUTF8String(2);
> /* 1035 */
> /* 1036 */ boolean tfn2 = ffn2;
> /* 1037 */ UTF8String tfp2 = null;
> /* 1038 */ if (!ffn2) {
> /* 1039 */ tfp2 = ffp2;
> /* 1040 */ }
> /* 1041 */
> /* 1042 */ if (tfn2) {
> /* 1043 */ result.setNullAt(2);
> /* 1044 */ } else {
> /* 1045 */ result.update(2, tfp2.clone());
> /* 1046 */ }
> /* 1047 */ }
> /* 1048 */
> /* 1049 */
> /* 1050 */ boolean ffn3 = tmpRow.isNullAt(3);
> /* 1051 */ if (ffn3) {
> /* 1052 */ result.setNullAt(3);
> /* 1053 */ } else {
> /* 1054 */ long ffp3 =
> /* 1055 */ tmpRow.getLong(3);
> /* 1056 */
> /* 1057 */ boolean tfn3 = ffn3;
> /* 1058 */ long tfp3 = -1L;
> /* 1059 */ if (!ffn3) {
> /* 1060 */ tfp3 = ffp3;
> /* 1061 */ }
> /* 1062 */
> /* 1063 */ if (tfn3) {
> /* 1064 */ result.setNullAt(3);
> /* 1065 */ } else {
> /* 1066 */ result.setLong(3, tfp3);
> /* 1067 */ }
> /* 1068 */ }
> /* 1069 */
> /* 1070 */
> /* 1071 */ boolean ffn4 = tmpRow.isNullAt(4);
> /* 1072 */ if (ffn4) {
> /* 1073 */ result.setNullAt(4);
> /* 1074 */ } else {
> /* 1075 */ UTF8String ffp4 =
> /* 1076 */ tmpRow.getUTF8String(4);
> /* 1077 */
> /* 1078 */ boolean tfn4 = ffn4;
> /* 1079 */ UTF8String tfp4 = null;
> /* 1080 */ if (!ffn4) {
> /* 1081 */ tfp4 = ffp4;
> /* 1082 */ }
> /* 1083 */
> /* 1084 */ if (tfn4) {
> /* 1085 */ result.setNullAt(4);
> /* 1086 */ } else {
> /* 1087 */ result.update(4, tfp4.clone());
> /* 1088 */ }
> /* 1089 */ }
> /* 1090 */
> /* 1091 */
> /* 1092 */ boolean ffn5 = tmpRow.isNullAt(5);
> /* 1093 */ if (ffn5) {
> /* 1094 */ result.setNullAt(5);
> /* 1095 */ } else {
> /* 1096 */ UTF8String ffp5 =
> /* 1097 */ tmpRow.getUTF8String(5);
> /* 1098 */
> /* 1099 */ boolean tfn5 = ffn5;
> /* 1100 */ UTF8String tfp5 = null;
> /* 1101 */ if (!ffn5) {
> /* 1102 */ tfp5 = ffp5;
> /* 1103 */ }
> /* 1104 */
> /* 1105 */ if (tfn5) {
> /* 1106 */ result.setNullAt(5);
> /* 1107 */ } else {
> /* 1108 */ result.update(5, tfp5.clone());
> /* 1109 */ }
> /* 1110 */ }
> /* 1111 */
> /* 1112 */
> /* 1113 */ boolean ffn6 = tmpRow.isNullAt(6);
> /* 1114 */ if (ffn6) {
> /* 1115 */ result.setNullAt(6);
> /* 1116 */ } else {
> /* 1117 */ int ffp6 =
> /* 1118 */ tmpRow.getInt(6);
> /* 1119 */
> /* 1120 */ boolean tfn6 = ffn6;
> /* 1121 */ int tfp6 = -1;
> /* 1122 */ if (!ffn6) {
> /* 1123 */ tfp6 = ffp6;
> /* 1124 */ }
> /* 1125 */
> /* 1126 */ if (tfn6) {
> /* 1127 */ result.setNullAt(6);
> /* 1128 */ } else {
> /* 1129 */ result.setInt(6, tfp6);
> /* 1130 */ }
> /* 1131 */ }
> /* 1132 */
> /* 1133 */
> /* 1134 */ boolean ffn7 = tmpRow.isNullAt(7);
> /* 1135 */ if (ffn7) {
> /* 1136 */ result.setNullAt(7);
> /* 1137 */ } else {
> /* 1138 */ ArrayData ffp7 =
> /* 1139 */ tmpRow.getArray(7);
> /* 1140 */
> /* 1141 */ boolean tfn7 = ffn7;
> /* 1142 */ ArrayData tfp7 = null;
> /* 1143 */ if (!ffn7) {
> /* 1144 */ tfp7 = ffp7;
> /* 1145 */ }
> /* 1146 */
> /* 1147 */ if (tfn7) {
> /* 1148 */ result.setNullAt(7);
> /* 1149 */ } else {
> /* 1150 */ result.update(7, tfp7);
> /* 1151 */ }
> /* 1152 */ }
> /* 1153 */
> /* 1154 */
> /* 1155 */ boolean ffn8 = tmpRow.isNullAt(8);
> /* 1156 */ if (ffn8) {
> /* 1157 */ result.setNullAt(8);
> /* 1158 */ } else {
> /* 1159 */ ArrayData ffp8 =
> /* 1160 */ tmpRow.getArray(8);
> /* 1161 */
> /* 1162 */ boolean tfn8 = ffn8;
> /* 1163 */ ArrayData tfp8 = null;
> /* 1164 */ if (!ffn8) {
> /* 1165 */ tfp8 = ffp8;
> /* 1166 */ }
> /* 1167 */
> /* 1168 */ if (tfn8) {
> /* 1169 */ result.setNullAt(8);
> /* 1170 */ } else {
> /* 1171 */ result.update(8, tfp8);
> /* 1172 */ }
> /* 1173 */ }
> /* 1174 */
> /* 1175 */
> /* 1176 */ boolean ffn9 = tmpRow.isNullAt(9);
> /* 1177 */ if (ffn9) {
> /* 1178 */ result.setNullAt(9);
> /* 1179 */ } else {
> /* 1180 */ ArrayData ffp9 =
> /* 1181 */ tmpRow.getArray(9);
> /* 1182 */
> /* 1183 */ boolean tfn9 = ffn9;
> /* 1184 */ ArrayData tfp9 = null;
> /* 1185 */ if (!ffn9) {
> /* 1186 */ tfp9 = ffp9;
> /* 1187 */ }
> /* 1188 */
> /* 1189 */ if (tfn9) {
> /* 1190 */ result.setNullAt(9);
> /* 1191 */ } else {
> /* 1192 */ result.update(9, tfp9);
> /* 1193 */ }
> /* 1194 */ }
> /* 1195 */
> /* 1196 */
> /* 1197 */ boolean ffn10 = tmpRow.isNullAt(10);
> /* 1198 */ if (ffn10) {
> /* 1199 */ result.setNullAt(10);
> /* 1200 */ } else {
> /* 1201 */ ArrayData ffp10 =
> /* 1202 */ tmpRow.getArray(10);
> /* 1203 */
> /* 1204 */ boolean tfn10 = ffn10;
> /* 1205 */ ArrayData tfp10 = null;
> /* 1206 */ if (!ffn10) {
> /* 1207 */ tfp10 = ffp10;
> /* 1208 */ }
> /* 1209 */
> /* 1210 */ if (tfn10) {
> /* 1211 */ result.setNullAt(10);
> /* 1212 */ } else {
> /* 1213 */ result.update(10, tfp10);
> /* 1214 */ }
> /* 1215 */ }
> /* 1216 */
> /* 1217 */
> /* 1218 */ boolean ffn11 = tmpRow.isNullAt(11);
> /* 1219 */ if (ffn11) {
> /* 1220 */ result.setNullAt(11);
> /* 1221 */ } else {
> /* 1222 */ UTF8String ffp11 =
> /* 1223 */ tmpRow.getUTF8String(11);
> /* 1224 */
> /* 1225 */ boolean tfn11 = ffn11;
> /* 1226 */ UTF8String tfp11 = null;
> /* 1227 */ if (!ffn11) {
> /* 1228 */ tfp11 = ffp11;
> /* 1229 */ }
> /* 1230 */
> /* 1231 */ if (tfn11) {
> /* 1232 */ result.setNullAt(11);
> /* 1233 */ } else {
> /* 1234 */ result.update(11, tfp11.clone());
> /* 1235 */ }
> /* 1236 */ }
> /* 1237 */
> /* 1238 */
> /* 1239 */ boolean ffn12 = tmpRow.isNullAt(12);
> /* 1240 */ if (ffn12) {
> /* 1241 */ result.setNullAt(12);
> /* 1242 */ } else {
> /* 1243 */ double ffp12 =
> /* 1244 */ tmpRow.getDouble(12);
> /* 1245 */
> /* 1246 */ boolean tfn12 = ffn12;
> /* 1247 */ double tfp12 = -1.0;
> /* 1248 */ if (!ffn12) {
> /* 1249 */ tfp12 = ffp12;
> /* 1250 */ }
> /* 1251 */
> /* 1252 */ if (tfn12) {
> /* 1253 */ result.setNullAt(12);
> /* 1254 */ } else {
> /* 1255 */ result.setDouble(12, tfp12);
> /* 1256 */ }
> /* 1257 */ }
> /* 1258 */
> /* 1259 */
> /* 1260 */ boolean ffn13 = tmpRow.isNullAt(13);
> /* 1261 */ if (ffn13) {
> /* 1262 */ result.setNullAt(13);
> /* 1263 */ } else {
> /* 1264 */ long ffp13 =
> /* 1265 */ tmpRow.getLong(13);
> /* 1266 */
> /* 1267 */ boolean tfn13 = ffn13;
> /* 1268 */ long tfp13 = -1L;
> /* 1269 */ if (!ffn13) {
> /* 1270 */ tfp13 = ffp13;
> /* 1271 */ }
> /* 1272 */
> /* 1273 */ if (tfn13) {
> /* 1274 */ result.setNullAt(13);
> /* 1275 */ } else {
> /* 1276 */ result.setLong(13, tfp13);
> /* 1277 */ }
> /* 1278 */ }
> /* 1279 */
> /* 1280 */
> /* 1281 */ boolean ffn14 = tmpRow.isNullAt(14);
> /* 1282 */ if (ffn14) {
> /* 1283 */ result.setNullAt(14);
> /* 1284 */ } else {
> /* 1285 */ int ffp14 =
> /* 1286 */ tmpRow.getInt(14);
> /* 1287 */
> /* 1288 */ boolean tfn14 = ffn14;
> /* 1289 */ int tfp14 = -1;
> /* 1290 */ if (!ffn14) {
> /* 1291 */ tfp14 = ffp14;
> /* 1292 */ }
> /* 1293 */
> /* 1294 */ if (tfn14) {
> /* 1295 */ result.setNullAt(14);
> /* 1296 */ } else {
> /* 1297 */ result.setInt(14, tfp14);
> /* 1298 */ }
> /* 1299 */ }
> /* 1300 */
> /* 1301 */
> /* 1302 */ boolean ffn15 = tmpRow.isNullAt(15);
> /* 1303 */ if (ffn15) {
> /* 1304 */ result.setNullAt(15);
> /* 1305 */ } else {
> /* 1306 */ double ffp15 =
> /* 1307 */ tmpRow.getDouble(15);
> /* 1308 */
> /* 1309 */ boolean tfn15 = ffn15;
> /* 1310 */ double tfp15 = -1.0;
> /* 1311 */ if (!ffn15) {
> /* 1312 */ tfp15 = ffp15;
> /* 1313 */ }
> /* 1314 */
> /* 1315 */ if (tfn15) {
> /* 1316 */ result.setNullAt(15);
> /* 1317 */ } else {
> /* 1318 */ result.setDouble(15, tfp15);
> /* 1319 */ }
> /* 1320 */ }
> /* 1321 */
> /* 1322 */
> /* 1323 */ boolean ffn16 = tmpRow.isNullAt(16);
> /* 1324 */ if (ffn16) {
> /* 1325 */ result.setNullAt(16);
> /* 1326 */ } else {
> /* 1327 */ double ffp16 =
> /* 1328 */ tmpRow.getDouble(16);
> /* 1329 */
> /* 1330 */ boolean tfn16 = ffn16;
> /* 1331 */ double tfp16 = -1.0;
> /* 1332 */ if (!ffn16) {
> /* 1333 */ tfp16 = ffp16;
> /* 1334 */ }
> /* 1335 */
> /* 1336 */ if (tfn16) {
> /* 1337 */ result.setNullAt(16);
> /* 1338 */ } else {
> /* 1339 */ result.setDouble(16, tfp16);
> /* 1340 */ }
> /* 1341 */ }
> /* 1342 */
> /* 1343 */
> /* 1344 */ boolean ffn17 = tmpRow.isNullAt(17);
> /* 1345 */ if (ffn17) {
> /* 1346 */ result.setNullAt(17);
> /* 1347 */ } else {
> /* 1348 */ double ffp17 =
> /* 1349 */ tmpRow.getDouble(17);
> /* 1350 */
> /* 1351 */ boolean tfn17 = ffn17;
> /* 1352 */ double tfp17 = -1.0;
> /* 1353 */ if (!ffn17) {
> /* 1354 */ tfp17 = ffp17;
> /* 1355 */ }
> /* 1356 */
> /* 1357 */ if (tfn17) {
> /* 1358 */ result.setNullAt(17);
> /* 1359 */ } else {
> /* 1360 */ result.setDouble(17, tfp17);
> /* 1361 */ }
> /* 1362 */ }
> /* 1363 */
> /* 1364 */
> /* 1365 */ boolean ffn18 = tmpRow.isNullAt(18);
> /* 1366 */ if (ffn18) {
> /* 1367 */ result.setNullAt(18);
> /* 1368 */ } else {
> /* 1369 */ ArrayData ffp18 =
> /* 1370 */ tmpRow.getArray(18);
> /* 1371 */
> /* 1372 */ boolean tfn18 = ffn18;
> /* 1373 */ ArrayData tfp18 = null;
> /* 1374 */ if (!ffn18) {
> /* 1375 */ tfp18 = ffp18;
> /* 1376 */ }
> /* 1377 */
> /* 1378 */ if (tfn18) {
> /* 1379 */ result.setNullAt(18);
> /* 1380 */ } else {
> /* 1381 */ result.update(18, tfp18);
> /* 1382 */ }
> /* 1383 */ }
> /* 1384 */
> /* 1385 */
> /* 1386 */ boolean ffn19 = tmpRow.isNullAt(19);
> /* 1387 */ if (ffn19) {
> /* 1388 */ result.setNullAt(19);
> /* 1389 */ } else {
> /* 1390 */ ArrayData ffp19 =
> /* 1391 */ tmpRow.getArray(19);
> /* 1392 */
> /* 1393 */ boolean tfn19 = ffn19;
> /* 1394 */ ArrayData tfp19 = null;
> /* 1395 */ if (!ffn19) {
> /* 1396 */ tfp19 = ffp19;
> /* 1397 */ }
> /* 1398 */
> /* 1399 */ if (tfn19) {
> /* 1400 */ result.setNullAt(19);
> /* 1401 */ } else {
> /* 1402 */ result.update(19, tfp19);
> /* 1403 */ }
> /* 1404 */ }
> /* 1405 */
> /* 1406 */
> /* 1407 */ boolean ffn20 = tmpRow.isNullAt(20);
> /* 1408 */ if (ffn20) {
> /* 1409 */ result.setNullAt(20);
> /* 1410 */ } else {
> /* 1411 */ ArrayData ffp20 =
> /* 1412 */ tmpRow.getArray(20);
> /* 1413 */
> /* 1414 */ boolean tfn20 = ffn20;
> /* 1415 */ ArrayData tfp20 = null;
> /* 1416 */ if (!ffn20) {
> /* 1417 */ tfp20 = ffp20;
> /* 1418 */ }
> /* 1419 */
> /* 1420 */ if (tfn20) {
> /* 1421 */ result.setNullAt(20);
> /* 1422 */ } else {
> /* 1423 */ result.update(20, tfp20);
> /* 1424 */ }
> /* 1425 */ }
> /* 1426 */
> /* 1427 */ tePrim = result.copy();
> /* 1428 */
> /* 1429 */ }
> /* 1430 */
> /* 1431 */ if (teNull) {
> /* 1432 */ values[j] = null;
> /* 1433 */ } else {
> /* 1434 */ values[j] = tePrim;
> /* 1435 */ }
> /* 1436 */ }
> /* 1437 */ }
> /* 1438 */ value166 = new
> org.apache.spark.sql.catalyst.util.GenericArrayData(values);
> /* 1439 */
> /* 1440 */ }
> /* 1441 */ ArrayData value165 = null;
> /* 1442 */
> /* 1443 */ if (!isNull166) {
> /* 1444 */
> /* 1445 */ indexer.dto.ticket.VendorItemDto[] convertedArray19 =
> null;
> /* 1446 */ int dataLength19 = value166.numElements();
> /* 1447 */ convertedArray19 = new
> indexer.dto.ticket.VendorItemDto[dataLength19];
> /* 1448 */
> /* 1449 */ int loopIndex19 = 0;
> /* 1450 */ while (loopIndex19 < dataLength19) {
> /* 1451 */ MapObjects_loopValue378 = (InternalRow)
> (value166.getStruct(loopIndex19, 21));
> /* 1452 */ MapObjects_loopIsNull379 =
> value166.isNullAt(loopIndex19);
> /* 1453 */
> /* 1454 */
> /* 1455 */ boolean isNull168 = false;
> /* 1456 */ indexer.dto.ticket.VendorItemDto value168 = null;
> /* 1457 */ if (!false && MapObjects_loopIsNull379) {
> /* 1458 */
> /* 1459 */ final indexer.dto.ticket.VendorItemDto value170 =
> null;
> /* 1460 */ isNull168 = true;
> /* 1461 */ value168 = value170;
> /* 1462 */ } else {
> /* 1463 */
> /* 1464 */ apply1_0(i);
> /* 1465 */ apply1_1(i);
> /* 1466 */ apply1_2(i);
> /* 1467 */ apply1_3(i);
> /* 1468 */ apply1_4(i);
> /* 1469 */ apply1_5(i);
> /* 1470 */ apply1_6(i);
> /* 1471 */ apply1_7(i);
> /* 1472 */ apply1_8(i);
> /* 1473 */ apply1_9(i);
> /* 1474 */ apply1_10(i);
> /* 1475 */ apply1_11(i);
> /* 1476 */ apply1_12(i);
> /* 1477 */
> /* 1478 */ final indexer.dto.ticket.VendorItemDto value171 =
> false ? null : new indexer.dto.ticket.VendorItemDto(argValue92, argValue93,
> argValue94, argValue95, argValue96, argValue97, argValue98, argValue99,
> argValue100, argValue101, argValue102, argValue103, argValue104,
> argValue105, argValue106, argValue107, argValue108, argValue109,
> argValue110, argValue111, argValue112);
> /* 1479 */ isNull168 = false;
> /* 1480 */ value168 = value171;
> /* 1481 */ }
> /* 1482 */ if (isNull168) {
> /* 1483 */ convertedArray19[loopIndex19] = null;
> /* 1484 */ } else {
> /* 1485 */ convertedArray19[loopIndex19] = value168;
> /* 1486 */ }
> /* 1487 */
> /* 1488 */ loopIndex19 += 1;
> /* 1489 */ }
> /* 1490 */
> /* 1491 */ value165 = new
> org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray19);
> /* 1492 */ }
> /* 1493 */ boolean isNull164 = true;
> /* 1494 */ indexer.dto.ticket.VendorItemDto[] value164 = null;
> /* 1495 */ if (!isNull166) {
> /* 1496 */
> /* 1497 */ isNull164 = false;
> /* 1498 */ if (!isNull164) {
> /* 1499 */
> /* 1500 */ Object funcResult56 = null;
> /* 1501 */ funcResult56 = value165.array();
> /* 1502 */ if (funcResult56 == null) {
> /* 1503 */ isNull164 = true;
> /* 1504 */ } else {
> /* 1505 */ value164 = (indexer.dto.ticket.VendorItemDto[])
> funcResult56;
> /* 1506 */ }
> /* 1507 */
> /* 1508 */ }
> /* 1509 */ isNull164 = value164 == null;
> /* 1510 */ }
> /* 1511 */ argValue37 = value164;
> /* 1512 */
> /* 1513 */ }
>
>
> Thanks a lot
> Proust
>
>
>
> 地址: 上海市浦东新区金科路2889弄长泰广场C座7楼
> Address: 7th floor, No#3, Jinke road 2889, Pudong district, Shanghai,
> China.
> Mobile: +86 13621672634
>