You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2019/02/13 01:23:49 UTC
[spark] branch master updated: [SPARK-26857][SQL] Return
UnsafeArrayData for date/timestamp type in ColumnarArray.copy()
This is an automated email from the ASF dual-hosted git repository.
yamamuro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 72a349a [SPARK-26857][SQL] Return UnsafeArrayData for date/timestamp type in ColumnarArray.copy()
72a349a is described below
commit 72a349a95d7661cd442851b62e7e830f213ed05c
Author: Gengliang Wang <ge...@databricks.com>
AuthorDate: Wed Feb 13 10:23:31 2019 +0900
[SPARK-26857][SQL] Return UnsafeArrayData for date/timestamp type in ColumnarArray.copy()
## What changes were proposed in this pull request?
In https://github.com/apache/spark/issues/23569, the copy method of `ColumnarArray` is implemented.
To further improve it, we can return `UnsafeArrayData` for `date`/`timestamp` type in `ColumnarArray.copy()`.
## How was this patch tested?
Unit test
Closes #23761 from gengliangwang/copyDateAndTS.
Authored-by: Gengliang Wang <ge...@databricks.com>
Signed-off-by: Takeshi Yamamuro <ya...@apache.org>
---
.../apache/spark/sql/vectorized/ColumnarArray.java | 4 ++--
.../execution/vectorized/ColumnVectorSuite.scala | 28 ++++++++++++++++++++++
2 files changed, 30 insertions(+), 2 deletions(-)
diff --git a/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java b/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java
index 1471627..8dc7b11 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java
@@ -56,9 +56,9 @@ public final class ColumnarArray extends ArrayData {
return UnsafeArrayData.fromPrimitiveArray(toByteArray());
} else if (dt instanceof ShortType) {
return UnsafeArrayData.fromPrimitiveArray(toShortArray());
- } else if (dt instanceof IntegerType) {
+ } else if (dt instanceof IntegerType || dt instanceof DateType) {
return UnsafeArrayData.fromPrimitiveArray(toIntArray());
- } else if (dt instanceof LongType) {
+ } else if (dt instanceof LongType || dt instanceof TimestampType) {
return UnsafeArrayData.fromPrimitiveArray(toLongArray());
} else if (dt instanceof FloatType) {
return UnsafeArrayData.fromPrimitiveArray(toFloatArray());
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala
index 866fcb1..c2e783d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala
@@ -108,6 +108,20 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach {
}
}
+ testVectors("date", 10, DateType) { testVector =>
+ (0 until 10).foreach { i =>
+ testVector.appendInt(i)
+ }
+
+ val array = new ColumnarArray(testVector, 0, 10)
+ val arrayCopy = array.copy()
+
+ (0 until 10).foreach { i =>
+ assert(array.get(i, DateType) === i)
+ assert(arrayCopy.get(i, DateType) === i)
+ }
+ }
+
testVectors("long", 10, LongType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendLong(i)
@@ -122,6 +136,20 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach {
}
}
+ testVectors("timestamp", 10, TimestampType) { testVector =>
+ (0 until 10).foreach { i =>
+ testVector.appendLong(i)
+ }
+
+ val array = new ColumnarArray(testVector, 0, 10)
+ val arrayCopy = array.copy()
+
+ (0 until 10).foreach { i =>
+ assert(array.get(i, TimestampType) === i)
+ assert(arrayCopy.get(i, TimestampType) === i)
+ }
+ }
+
testVectors("float", 10, FloatType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendFloat(i.toFloat)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org