You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2023/05/11 00:44:54 UTC

[spark] branch branch-3.4 updated: [SPARK-43425][SQL][3.4] Add `TimestampNTZType` to `ColumnarBatchRow`

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.4 by this push:
     new 95af6f6e3a2 [SPARK-43425][SQL][3.4] Add `TimestampNTZType` to `ColumnarBatchRow`
95af6f6e3a2 is described below

commit 95af6f6e3a22a038c4c0d37817fd511e7ee3395b
Author: Fokko Driesprong <fo...@tabular.io>
AuthorDate: Thu May 11 09:44:39 2023 +0900

    [SPARK-43425][SQL][3.4] Add `TimestampNTZType` to `ColumnarBatchRow`
    
    ### What changes were proposed in this pull request?
    
    Noticed this one was missing when implementing `TimestampNTZType` in Iceberg.
    
    ### Why are the changes needed?
    
    Able to read `TimestampNTZType` using the `ColumnarBatchRow`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No
    
    ### How was this patch tested?
    
    Closes #41103 from Fokko/fd-add-timestampntz.
    
    Authored-by: Fokko Driesprong <fokkotabular.io>
    
    Closes #41112 from Fokko/fd-backport.
    
    Authored-by: Fokko Driesprong <fo...@tabular.io>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 .../org/apache/spark/sql/vectorized/ColumnarBatchRow.java |  2 ++
 .../sql/execution/vectorized/ColumnarBatchSuite.scala     | 15 +++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java
index 05922a0cd5d..1494564dcfb 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarBatchRow.java
@@ -176,6 +176,8 @@ public final class ColumnarBatchRow extends InternalRow {
       return getInt(ordinal);
     } else if (dataType instanceof TimestampType) {
       return getLong(ordinal);
+    } else if (dataType instanceof TimestampNTZType) {
+      return getLong(ordinal);
     } else if (dataType instanceof ArrayType) {
       return getArray(ordinal);
     } else if (dataType instanceof StructType) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
index 9b2c2c34c7e..b34af3c8c63 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
@@ -1957,6 +1957,21 @@ class ColumnarBatchSuite extends SparkFunSuite {
       column.close()
   }
 
+  testVector("Timestamp without timezone", 10, TimestampNTZType) {
+    column =>
+      val dt = TimestampNTZType
+      (0 until 10).foreach { i =>
+        column.putLong(i, i)
+      }
+      val bachRow = new ColumnarBatchRow(Array(column))
+      (0 until 10).foreach { i =>
+        bachRow.rowId = i
+        assert(bachRow.get(0, dt) === i)
+        val batchRowCopy = bachRow.copy()
+        assert(batchRowCopy.get(0, dt) === i)
+      }
+  }
+
   testVector("WritableColumnVector.reserve(): requested capacity is negative", 1024, ByteType) {
     column =>
       val ex = intercept[RuntimeException] { column.reserve(-1) }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org