You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/08/01 00:19:58 UTC
spark git commit: [SPARK-18950][SQL] Report conflicting fields when
merging two StructTypes
Repository: spark
Updated Branches:
refs/heads/master 106eaa9b9 -> 6b186c9d6
[SPARK-18950][SQL] Report conflicting fields when merging two StructTypes
## What changes were proposed in this pull request?
Currently, StructType.merge() only reports data types of conflicting fields when merging two incompatible schemas. It would be nice to also report the field names for easier debugging.
## How was this patch tested?
Unit test in DataTypeSuite.
Print exception message when conflict is triggered.
Author: bravo-zhang <mz...@gmail.com>
Closes #16365 from bravo-zhang/spark-18950.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6b186c9d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6b186c9d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6b186c9d
Branch: refs/heads/master
Commit: 6b186c9d606d598435a5c1981b0a54afea188513
Parents: 106eaa9
Author: bravo-zhang <mz...@gmail.com>
Authored: Mon Jul 31 17:19:55 2017 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Mon Jul 31 17:19:55 2017 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/types/StructType.scala | 15 +++++++++++----
.../org/apache/spark/sql/types/DataTypeSuite.scala | 6 ++++--
2 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6b186c9d/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index 54006e2..b314ef4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.types
import scala.collection.mutable.ArrayBuffer
+import scala.util.control.NonFatal
import scala.util.Try
import org.json4s.JsonDSL._
@@ -467,10 +468,16 @@ object StructType extends AbstractDataType {
leftFields.foreach {
case leftField @ StructField(leftName, leftType, leftNullable, _) =>
rightMapped.get(leftName)
- .map { case rightField @ StructField(_, rightType, rightNullable, _) =>
- leftField.copy(
- dataType = merge(leftType, rightType),
- nullable = leftNullable || rightNullable)
+ .map { case rightField @ StructField(rightName, rightType, rightNullable, _) =>
+ try {
+ leftField.copy(
+ dataType = merge(leftType, rightType),
+ nullable = leftNullable || rightNullable)
+ } catch {
+ case NonFatal(e) =>
+ throw new SparkException(s"Failed to merge fields '$leftName' and " +
+ s"'$rightName'. " + e.getMessage)
+ }
}
.orElse {
Some(leftField)
http://git-wip-us.apache.org/repos/asf/spark/blob/6b186c9d/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
index c4635c8..193826d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
@@ -142,9 +142,11 @@ class DataTypeSuite extends SparkFunSuite {
val right = StructType(
StructField("b", LongType) :: Nil)
- intercept[SparkException] {
+ val message = intercept[SparkException] {
left.merge(right)
- }
+ }.getMessage
+ assert(message.equals("Failed to merge fields 'b' and 'b'. " +
+ "Failed to merge incompatible data types FloatType and LongType"))
}
test("existsRecursively") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org