You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ur...@apache.org on 2026/07/31 09:11:34 UTC

(spark) branch master updated: [SPARK-57815][SQL] Reject nanosecond-precision timestamps in Hive-serde tables

This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new d1828e710251 [SPARK-57815][SQL] Reject nanosecond-precision timestamps in Hive-serde tables
d1828e710251 is described below

commit d1828e710251d27d7245b5fedf48ccd313ecc585
Author: Stevo Mitric <st...@gmail.com>
AuthorDate: Fri Jul 31 09:11:15 2026 +0000

    [SPARK-57815][SQL] Reject nanosecond-precision timestamps in Hive-serde tables
    
    ### What changes were proposed in this pull request?
    
    Handle nanosecond-precision timestamp columns (`TIMESTAMP_LTZ(p)` / `TIMESTAMP_NTZ(p)`, `p` in [7, 9]) in the Hive-serde (`STORED AS`) write path by rejecting them explicitly, mirroring how the TIME type is handled (SPARK-57556):
    
    - `HiveFileFormat.supportDataType` now returns `false` for `AnyTimestampNanoType` (recursing into nested types), so writing such a column fails with a clear `UNSUPPORTED_DATA_TYPE_FOR_DATASOURCE` error at `verifySchema` time instead of silently downgrading to microsecond precision.
    - `HiveInspectors.toInspector(dataType)`, `toInspector(expr)` (for `Literal`), and `toTypeInfo` now throw `UNSUPPORTED_DATATYPE` for `AnyTimestampNanoType`, since Hive has no nanosecond-precision timestamp type to map to.
    
    Rejection (rather than full support) is the intentional behavior chosen by the umbrella: Hive has no nanosecond timestamp type, and the metastore-compatibility side already treats these types as Hive-incompatible (SPARK-57831).
    
    ### Why are the changes needed?
    
    Sub-task of SPARK-56822. Without this, the Hive ORC serde silently downgrades nanosecond timestamps to microsecond `TimestampType` on write, losing precision with no warning. Silent downgrades must be eliminated.
    
    ### Does this PR introduce any user-facing change?
    
    Yes. Writing a nanosecond-precision timestamp column through the Hive serde now fails with a clear error instead of silently losing sub-microsecond precision.
    
    ### How was this patch tested?
    
    New unit test in `HiveInspectorSuite` covering `toInspector`/`toTypeInfo` for all four nanos type/precision combinations, and a new `InsertSuite` test that writes to a Hive-serde directory `STORED AS ORC` and `STORED AS PARQUET`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Code (Claude Opus 4.8)
    
    Closes #57635 from stevomitric/stevomitric/spark-57815-nanos.
    
    Authored-by: Stevo Mitric <st...@gmail.com>
    Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
 .../org/apache/spark/sql/hive/HiveInspectors.scala |  9 ++++++
 .../spark/sql/hive/execution/HiveFileFormat.scala  |  6 +++-
 .../apache/spark/sql/hive/HiveInspectorSuite.scala | 20 +++++++++++++
 .../org/apache/spark/sql/hive/InsertSuite.scala    | 34 ++++++++++++++++++++++
 4 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 7f76132e2572..662e55dac543 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -967,6 +967,9 @@ private[hive] trait HiveInspectors {
       toInspector(sqlType)
     // Hive has no TIME type, so it cannot be represented by any Hive object inspector.
     case _: TimeType => throw unsupportedHiveType(dataType)
+    // Hive has no nanosecond-precision timestamp type, so it cannot be represented by any Hive
+    // object inspector. Reject it instead of silently downgrading to microsecond precision.
+    case _: AnyTimestampNanoType => throw unsupportedHiveType(dataType)
   }
 
   private def unsupportedHiveType(dataType: DataType): AnalysisException = {
@@ -1044,6 +1047,10 @@ private[hive] trait HiveInspectors {
     // Hive has no TIME type, so a TIME constant cannot be mapped to a Hive object inspector.
     case Literal(_, dt: TimeType) =>
       throw unsupportedHiveType(dt)
+    // Hive has no nanosecond-precision timestamp type, so such a constant cannot be mapped to a
+    // Hive object inspector.
+    case Literal(_, dt: AnyTimestampNanoType) =>
+      throw unsupportedHiveType(dt)
     // We will enumerate all of the possible constant expressions, throw exception if we missed
     case Literal(_, dt) =>
       throw SparkException.internalError(s"Hive doesn't support the constant type [$dt].")
@@ -1297,6 +1304,8 @@ private[hive] trait HiveInspectors {
       case _: YearMonthIntervalType => intervalYearMonthTypeInfo
       // Hive has no TIME type, so there is no Hive TypeInfo to map it to.
       case _: TimeType => throw unsupportedHiveType(dt)
+      // Hive has no nanosecond-precision timestamp type, so there is no Hive TypeInfo to map it to.
+      case _: AnyTimestampNanoType => throw unsupportedHiveType(dt)
       case dt => throw unsupportedHiveType(dt)
     }
   }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
index acbc72fbf7e0..4545acc7f203 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
@@ -41,7 +41,7 @@ import org.apache.spark.sql.execution.datasources.{FileFormat, OutputWriter, Out
 import org.apache.spark.sql.hive.{HiveInspectors, HiveTableUtil}
 import org.apache.spark.sql.internal.SessionStateHelper
 import org.apache.spark.sql.sources.DataSourceRegister
-import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType, TimeType, UserDefinedType}
+import org.apache.spark.sql.types.{AnyTimestampNanoType, ArrayType, DataType, MapType, StructType, TimeType, UserDefinedType}
 import org.apache.spark.util.SerializableJobConf
 
 /**
@@ -120,6 +120,10 @@ case class HiveFileFormat(fileSinkConf: FileSinkDesc)
     // (recursing into nested types) while preserving the default behavior for all other types.
     case _: TimeType => false
 
+    // Hive has no nanosecond-precision timestamp type. Reject it explicitly rather than silently
+    // downgrading to microsecond precision (which the Hive serde would otherwise do).
+    case _: AnyTimestampNanoType => false
+
     case st: StructType => st.forall { f => supportDataType(f.dataType) }
 
     case ArrayType(elementType, _) => supportDataType(elementType)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index b7fb506f07b5..a88dcd0ef56a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -308,4 +308,24 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
       condition = "UNSUPPORTED_DATATYPE",
       parameters = expectedParams)
   }
+
+  test("SPARK-57815: nanosecond timestamp types are unsupported in Hive object inspectors") {
+    Seq(
+      TimestampNTZNanosType(9), TimestampNTZNanosType(7),
+      TimestampLTZNanosType(9), TimestampLTZNanosType(8)).foreach { nanosType =>
+      val expectedParams = Map("typeName" -> s"\"${nanosType.sql}\"")
+      checkError(
+        exception = intercept[AnalysisException](toInspector(nanosType)),
+        condition = "UNSUPPORTED_DATATYPE",
+        parameters = expectedParams)
+      checkError(
+        exception = intercept[AnalysisException](toInspector(Literal.create(null, nanosType))),
+        condition = "UNSUPPORTED_DATATYPE",
+        parameters = expectedParams)
+      checkError(
+        exception = intercept[AnalysisException](nanosType.toTypeInfo),
+        condition = "UNSUPPORTED_DATATYPE",
+        parameters = expectedParams)
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
index 7385410e1a71..db1743862e90 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
@@ -708,6 +708,40 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter {
     }
   }
 
+  test("SPARK-57815: nanosecond timestamp is unsupported when writing to a Hive serde directory") {
+    // Disable native data source conversion so that the write goes through the Hive serde path
+    // (HiveFileFormat) instead of a native data source that supports nanosecond timestamps.
+    withSQLConf(
+        HiveUtils.CONVERT_METASTORE_INSERT_DIR.key -> "false",
+        SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") {
+      Seq("ORC", "PARQUET").foreach { fileFormat =>
+        Seq(
+          "TIMESTAMP_NTZ(9)" -> TimestampNTZNanosType(9),
+          "TIMESTAMP_LTZ(9)" -> TimestampLTZNanosType(9)).foreach { case (typeStr, dt) =>
+          withTempDir { dir =>
+            // InsertIntoHiveDirCommand wraps the failure in a SparkException, so assert on the
+            // cause. Rejecting here avoids a silent downgrade to microsecond precision.
+            val e = intercept[SparkException] {
+              sql(
+                s"""
+                   |INSERT OVERWRITE LOCAL DIRECTORY '${dir.toURI.getPath}'
+                   |STORED AS $fileFormat
+                   |SELECT CAST('2025-01-06 12:30:45.123456789' AS $typeStr) AS c
+                 """.stripMargin)
+            }
+            checkError(
+              exception = e.getCause.asInstanceOf[AnalysisException],
+              condition = "UNSUPPORTED_DATA_TYPE_FOR_DATASOURCE",
+              parameters = Map(
+                "columnName" -> "`c`",
+                "columnType" -> s"\"${dt.sql}\"",
+                "format" -> "Hive"))
+          }
+        }
+      }
+    }
+  }
+
   test("insert overwrite to dir from temp table") {
     withTempView("test_insert_table") {
       spark.range(10).selectExpr("id", "id AS str").createOrReplaceTempView("test_insert_table")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org