You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2021/01/26 04:27:53 UTC
[spark] branch branch-3.0 updated: [SPARK-34223][SQL] FIX NPE for
static partition with null in InsertIntoHadoopFsRelationCommand
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 3b18d03 [SPARK-34223][SQL] FIX NPE for static partition with null in InsertIntoHadoopFsRelationCommand
3b18d03 is described below
commit 3b18d0383ab03b411e3c493ceda38781df9ec8dc
Author: Kent Yao <ya...@apache.org>
AuthorDate: Tue Jan 26 12:05:58 2021 +0800
[SPARK-34223][SQL] FIX NPE for static partition with null in InsertIntoHadoopFsRelationCommand
with a simple case, the null will be passed to InsertIntoHadoopFsRelationCommand blindly, we should avoid the npe
```scala
test("NPE") {
withTable("t") {
sql(s"CREATE TABLE t(i STRING, c string) USING $format PARTITIONED BY (c)")
sql("INSERT OVERWRITE t PARTITION (c=null) VALUES ('1')")
checkAnswer(spark.table("t"), Row("1", null))
}
}
```
```logtalk
java.lang.NullPointerException
at scala.collection.immutable.StringOps$.length(StringOps.scala:51)
at scala.collection.immutable.StringOps.length(StringOps.scala:51)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:35)
at scala.collection.IndexedSeqOptimized.foreach
at scala.collection.immutable.StringOps.foreach(StringOps.scala:33)
at org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils$.escapePathName(ExternalCatalogUtils.scala:69)
at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.orig-s0.0000030000-r30676-expand-or-complete(InsertIntoHadoopFsRelationCommand.scala:231)
```
a bug fix
no
new tests
Closes #31320 from yaooqinn/SPARK-34223.
Authored-by: Kent Yao <ya...@apache.org>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit b3915ddd919fac11254084a2b138ad730fa8e5b0)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../datasources/InsertIntoHadoopFsRelationCommand.scala | 9 ++-------
.../test/scala/org/apache/spark/sql/sources/InsertSuite.scala | 8 ++++++++
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
index fe733f4..89899c5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
@@ -25,6 +25,7 @@ import org.apache.spark.internal.io.FileCommitProtocol
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogTablePartition}
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils._
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
@@ -59,7 +60,6 @@ case class InsertIntoHadoopFsRelationCommand(
fileIndex: Option[FileIndex],
outputColumnNames: Seq[String])
extends DataWritingCommand {
- import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName
private lazy val parameters = CaseInsensitiveMap(options)
@@ -216,12 +216,7 @@ case class InsertIntoHadoopFsRelationCommand(
committer: FileCommitProtocol): Unit = {
val staticPartitionPrefix = if (staticPartitions.nonEmpty) {
"/" + partitionColumns.flatMap { p =>
- staticPartitions.get(p.name) match {
- case Some(value) =>
- Some(escapePathName(p.name) + "=" + escapePathName(value))
- case None =>
- None
- }
+ staticPartitions.get(p.name).map(getPartitionPathString(p.name, _))
}.mkString("/")
} else {
""
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index 995d4b0..8fda0c8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -851,6 +851,14 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
}
}
}
+
+ test("SPARK-34223: static partition with null raise NPE") {
+ withTable("t") {
+ sql(s"CREATE TABLE t(i STRING, c string) USING PARQUET PARTITIONED BY (c)")
+ sql("INSERT OVERWRITE t PARTITION (c=null) VALUES ('1')")
+ checkAnswer(spark.table("t"), Row("1", null))
+ }
+ }
}
class FileExistingTestFileSystem extends RawLocalFileSystem {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org