You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2019/03/13 20:52:38 UTC
[spark] branch master updated: [MINOR][SQL] Deduplicate huge if
statements in get between specialized getters
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 733f2c0 [MINOR][SQL] Deduplicate huge if statements in get between specialized getters
733f2c0 is described below
commit 733f2c0b98208815f8408e36ab669d7c07e3767f
Author: Jungtaek Lim (HeartSaVioR) <ka...@gmail.com>
AuthorDate: Wed Mar 13 15:52:21 2019 -0500
[MINOR][SQL] Deduplicate huge if statements in get between specialized getters
## What changes were proposed in this pull request?
This patch deduplicates the huge if statements regarding getting value between specialized getters.
## How was this patch tested?
Existing UT.
Closes #24016 from HeartSaVioR/MINOR-deduplicate-get-from-specialized-getters.
Authored-by: Jungtaek Lim (HeartSaVioR) <ka...@gmail.com>
Signed-off-by: Sean Owen <se...@databricks.com>
---
.../expressions/SpecializedGettersReader.java | 89 ++++++++++++++++++++++
.../sql/catalyst/expressions/UnsafeArrayData.java | 42 +---------
.../spark/sql/catalyst/expressions/UnsafeRow.java | 41 +---------
.../apache/spark/sql/vectorized/ColumnarArray.java | 38 +--------
4 files changed, 93 insertions(+), 117 deletions(-)
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java
new file mode 100644
index 0000000..ea0648a
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions;
+
+import org.apache.spark.sql.types.*;
+
+public final class SpecializedGettersReader {
+ private SpecializedGettersReader() {}
+
+ public static Object read(
+ SpecializedGetters obj,
+ int ordinal,
+ DataType dataType,
+ boolean handleNull,
+ boolean handleUserDefinedType) {
+ if (handleNull && (obj.isNullAt(ordinal) || dataType instanceof NullType)) {
+ return null;
+ }
+ if (dataType instanceof BooleanType) {
+ return obj.getBoolean(ordinal);
+ }
+ if (dataType instanceof ByteType) {
+ return obj.getByte(ordinal);
+ }
+ if (dataType instanceof ShortType) {
+ return obj.getShort(ordinal);
+ }
+ if (dataType instanceof IntegerType) {
+ return obj.getInt(ordinal);
+ }
+ if (dataType instanceof LongType) {
+ return obj.getLong(ordinal);
+ }
+ if (dataType instanceof FloatType) {
+ return obj.getFloat(ordinal);
+ }
+ if (dataType instanceof DoubleType) {
+ return obj.getDouble(ordinal);
+ }
+ if (dataType instanceof StringType) {
+ return obj.getUTF8String(ordinal);
+ }
+ if (dataType instanceof DecimalType) {
+ DecimalType dt = (DecimalType) dataType;
+ return obj.getDecimal(ordinal, dt.precision(), dt.scale());
+ }
+ if (dataType instanceof DateType) {
+ return obj.getInt(ordinal);
+ }
+ if (dataType instanceof TimestampType) {
+ return obj.getLong(ordinal);
+ }
+ if (dataType instanceof CalendarIntervalType) {
+ return obj.getInterval(ordinal);
+ }
+ if (dataType instanceof BinaryType) {
+ return obj.getBinary(ordinal);
+ }
+ if (dataType instanceof StructType) {
+ return obj.getStruct(ordinal, ((StructType) dataType).size());
+ }
+ if (dataType instanceof ArrayType) {
+ return obj.getArray(ordinal);
+ }
+ if (dataType instanceof MapType) {
+ return obj.getMap(ordinal);
+ }
+ if (handleUserDefinedType && dataType instanceof UserDefinedType) {
+ return obj.get(ordinal, ((UserDefinedType)dataType).sqlType());
+ }
+
+ throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString());
+ }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
index a07d6de..04fc986 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
@@ -53,7 +53,6 @@ import org.apache.spark.unsafe.types.UTF8String;
*/
public final class UnsafeArrayData extends ArrayData {
-
public static int calculateHeaderPortionInBytes(int numFields) {
return (int)calculateHeaderPortionInBytes((long)numFields);
}
@@ -137,46 +136,7 @@ public final class UnsafeArrayData extends ArrayData {
@Override
public Object get(int ordinal, DataType dataType) {
- if (isNullAt(ordinal) || dataType instanceof NullType) {
- return null;
- } else if (dataType instanceof BooleanType) {
- return getBoolean(ordinal);
- } else if (dataType instanceof ByteType) {
- return getByte(ordinal);
- } else if (dataType instanceof ShortType) {
- return getShort(ordinal);
- } else if (dataType instanceof IntegerType) {
- return getInt(ordinal);
- } else if (dataType instanceof LongType) {
- return getLong(ordinal);
- } else if (dataType instanceof FloatType) {
- return getFloat(ordinal);
- } else if (dataType instanceof DoubleType) {
- return getDouble(ordinal);
- } else if (dataType instanceof DecimalType) {
- DecimalType dt = (DecimalType) dataType;
- return getDecimal(ordinal, dt.precision(), dt.scale());
- } else if (dataType instanceof DateType) {
- return getInt(ordinal);
- } else if (dataType instanceof TimestampType) {
- return getLong(ordinal);
- } else if (dataType instanceof BinaryType) {
- return getBinary(ordinal);
- } else if (dataType instanceof StringType) {
- return getUTF8String(ordinal);
- } else if (dataType instanceof CalendarIntervalType) {
- return getInterval(ordinal);
- } else if (dataType instanceof StructType) {
- return getStruct(ordinal, ((StructType) dataType).size());
- } else if (dataType instanceof ArrayType) {
- return getArray(ordinal);
- } else if (dataType instanceof MapType) {
- return getMap(ordinal);
- } else if (dataType instanceof UserDefinedType) {
- return get(ordinal, ((UserDefinedType)dataType).sqlType());
- } else {
- throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString());
- }
+ return SpecializedGettersReader.read(this, ordinal, dataType, true, true);
}
@Override
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
index 9bf9452..11561fa 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
@@ -299,46 +299,7 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo
@Override
public Object get(int ordinal, DataType dataType) {
- if (isNullAt(ordinal) || dataType instanceof NullType) {
- return null;
- } else if (dataType instanceof BooleanType) {
- return getBoolean(ordinal);
- } else if (dataType instanceof ByteType) {
- return getByte(ordinal);
- } else if (dataType instanceof ShortType) {
- return getShort(ordinal);
- } else if (dataType instanceof IntegerType) {
- return getInt(ordinal);
- } else if (dataType instanceof LongType) {
- return getLong(ordinal);
- } else if (dataType instanceof FloatType) {
- return getFloat(ordinal);
- } else if (dataType instanceof DoubleType) {
- return getDouble(ordinal);
- } else if (dataType instanceof DecimalType) {
- DecimalType dt = (DecimalType) dataType;
- return getDecimal(ordinal, dt.precision(), dt.scale());
- } else if (dataType instanceof DateType) {
- return getInt(ordinal);
- } else if (dataType instanceof TimestampType) {
- return getLong(ordinal);
- } else if (dataType instanceof BinaryType) {
- return getBinary(ordinal);
- } else if (dataType instanceof StringType) {
- return getUTF8String(ordinal);
- } else if (dataType instanceof CalendarIntervalType) {
- return getInterval(ordinal);
- } else if (dataType instanceof StructType) {
- return getStruct(ordinal, ((StructType) dataType).size());
- } else if (dataType instanceof ArrayType) {
- return getArray(ordinal);
- } else if (dataType instanceof MapType) {
- return getMap(ordinal);
- } else if (dataType instanceof UserDefinedType) {
- return get(ordinal, ((UserDefinedType)dataType).sqlType());
- } else {
- throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString());
- }
+ return SpecializedGettersReader.read(this, ordinal, dataType, true, true);
}
@Override
diff --git a/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java b/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java
index 8dc7b11..147dd24 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java
@@ -17,6 +17,7 @@
package org.apache.spark.sql.vectorized;
import org.apache.spark.annotation.Evolving;
+import org.apache.spark.sql.catalyst.expressions.SpecializedGettersReader;
import org.apache.spark.sql.catalyst.expressions.UnsafeArrayData;
import org.apache.spark.sql.catalyst.util.ArrayData;
import org.apache.spark.sql.catalyst.util.GenericArrayData;
@@ -174,42 +175,7 @@ public final class ColumnarArray extends ArrayData {
@Override
public Object get(int ordinal, DataType dataType) {
- if (dataType instanceof BooleanType) {
- return getBoolean(ordinal);
- } else if (dataType instanceof ByteType) {
- return getByte(ordinal);
- } else if (dataType instanceof ShortType) {
- return getShort(ordinal);
- } else if (dataType instanceof IntegerType) {
- return getInt(ordinal);
- } else if (dataType instanceof LongType) {
- return getLong(ordinal);
- } else if (dataType instanceof FloatType) {
- return getFloat(ordinal);
- } else if (dataType instanceof DoubleType) {
- return getDouble(ordinal);
- } else if (dataType instanceof StringType) {
- return getUTF8String(ordinal);
- } else if (dataType instanceof BinaryType) {
- return getBinary(ordinal);
- } else if (dataType instanceof DecimalType) {
- DecimalType t = (DecimalType) dataType;
- return getDecimal(ordinal, t.precision(), t.scale());
- } else if (dataType instanceof DateType) {
- return getInt(ordinal);
- } else if (dataType instanceof TimestampType) {
- return getLong(ordinal);
- } else if (dataType instanceof ArrayType) {
- return getArray(ordinal);
- } else if (dataType instanceof StructType) {
- return getStruct(ordinal, ((StructType)dataType).fields().length);
- } else if (dataType instanceof MapType) {
- return getMap(ordinal);
- } else if (dataType instanceof CalendarIntervalType) {
- return getInterval(ordinal);
- } else {
- throw new UnsupportedOperationException("Datatype not supported " + dataType);
- }
+ return SpecializedGettersReader.read(this, ordinal, dataType, false, false);
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org