You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/07/04 20:55:08 UTC
spark git commit: [SQL] More unit tests for implicit type cast & add
simpleString to AbstractDataType.
Repository: spark
Updated Branches:
refs/heads/master 48f7aed68 -> 347cab85c
[SQL] More unit tests for implicit type cast & add simpleString to AbstractDataType.
Author: Reynold Xin <rx...@databricks.com>
Closes #7221 from rxin/implicit-cast-tests and squashes the following commits:
64b13bd [Reynold Xin] Fixed a bug ..
489b732 [Reynold Xin] [SQL] More unit tests for implicit type cast & add simpleString to AbstractDataType.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/347cab85
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/347cab85
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/347cab85
Branch: refs/heads/master
Commit: 347cab85cd924ffd326f3d1367b3b156ee08052d
Parents: 48f7aed
Author: Reynold Xin <rx...@databricks.com>
Authored: Sat Jul 4 11:55:04 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Sat Jul 4 11:55:04 2015 -0700
----------------------------------------------------------------------
.../sql/catalyst/analysis/CheckAnalysis.scala | 6 ++---
.../spark/sql/types/AbstractDataType.scala | 7 ++++++
.../org/apache/spark/sql/types/ArrayType.scala | 2 ++
.../apache/spark/sql/types/DecimalType.scala | 2 ++
.../org/apache/spark/sql/types/MapType.scala | 2 ++
.../org/apache/spark/sql/types/StructType.scala | 2 ++
.../analysis/HiveTypeCoercionSuite.scala | 25 +++++++++++++++++++-
7 files changed, 42 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/347cab85/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 583338d..476ac2b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -40,7 +40,7 @@ trait CheckAnalysis {
def containsMultipleGenerators(exprs: Seq[Expression]): Boolean = {
exprs.flatMap(_.collect {
case e: Generator => true
- }).length >= 1
+ }).nonEmpty
}
def checkAnalysis(plan: LogicalPlan): Unit = {
@@ -85,12 +85,12 @@ trait CheckAnalysis {
case Aggregate(groupingExprs, aggregateExprs, child) =>
def checkValidAggregateExpression(expr: Expression): Unit = expr match {
case _: AggregateExpression => // OK
- case e: Attribute if groupingExprs.find(_ semanticEquals e).isEmpty =>
+ case e: Attribute if !groupingExprs.exists(_.semanticEquals(e)) =>
failAnalysis(
s"expression '${e.prettyString}' is neither present in the group by, " +
s"nor is it an aggregate function. " +
"Add to group by or wrap in first() if you don't care which value you get.")
- case e if groupingExprs.find(_ semanticEquals e).isDefined => // OK
+ case e if groupingExprs.exists(_.semanticEquals(e)) => // OK
case e if e.references.isEmpty => // OK
case e => e.children.foreach(checkValidAggregateExpression)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/347cab85/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
index e5dc99f..ffefb0e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
@@ -37,6 +37,9 @@ private[sql] abstract class AbstractDataType {
* Returns true if this data type is a parent of the `childCandidate`.
*/
private[sql] def isParentOf(childCandidate: DataType): Boolean
+
+ /** Readable string representation for the type. */
+ private[sql] def simpleString: String
}
@@ -56,6 +59,10 @@ private[sql] class TypeCollection(private val types: Seq[DataType]) extends Abst
private[sql] override def defaultConcreteType: DataType = types.head
private[sql] override def isParentOf(childCandidate: DataType): Boolean = false
+
+ private[sql] override def simpleString: String = {
+ types.map(_.simpleString).mkString("(", " or ", ")")
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/347cab85/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
index 8ea6cb1..43413ec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
@@ -31,6 +31,8 @@ object ArrayType extends AbstractDataType {
private[sql] override def isParentOf(childCandidate: DataType): Boolean = {
childCandidate.isInstanceOf[ArrayType]
}
+
+ private[sql] override def simpleString: String = "array"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/347cab85/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
index 434fc03..127b16f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
@@ -90,6 +90,8 @@ object DecimalType extends AbstractDataType {
childCandidate.isInstanceOf[DecimalType]
}
+ private[sql] override def simpleString: String = "decimal"
+
val Unlimited: DecimalType = DecimalType(None)
private[sql] object Fixed {
http://git-wip-us.apache.org/repos/asf/spark/blob/347cab85/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala
index 2b25617..868dea1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/MapType.scala
@@ -75,6 +75,8 @@ object MapType extends AbstractDataType {
childCandidate.isInstanceOf[MapType]
}
+ private[sql] override def simpleString: String = "map"
+
/**
* Construct a [[MapType]] object with the given key type and value type.
* The `valueContainsNull` is true.
http://git-wip-us.apache.org/repos/asf/spark/blob/347cab85/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index 7e77b77..3b17566 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -309,6 +309,8 @@ object StructType extends AbstractDataType {
childCandidate.isInstanceOf[StructType]
}
+ private[sql] override def simpleString: String = "struct"
+
def apply(fields: Seq[StructField]): StructType = StructType(fields.toArray)
def apply(fields: java.util.List[StructField]): StructType = {
http://git-wip-us.apache.org/repos/asf/spark/blob/347cab85/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
index 60e727c..67d05ab 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.types._
class HiveTypeCoercionSuite extends PlanTest {
- test("implicit type cast") {
+ test("eligible implicit type cast") {
def shouldCast(from: DataType, to: AbstractDataType, expected: DataType): Unit = {
val got = HiveTypeCoercion.ImplicitTypeCasts.implicitCast(Literal.create(null, from), to)
assert(got.map(_.dataType) == Option(expected),
@@ -68,6 +68,29 @@ class HiveTypeCoercionSuite extends PlanTest {
shouldCast(IntegerType, TypeCollection(BinaryType, IntegerType), IntegerType)
shouldCast(BinaryType, TypeCollection(BinaryType, IntegerType), BinaryType)
shouldCast(BinaryType, TypeCollection(IntegerType, BinaryType), BinaryType)
+
+ shouldCast(IntegerType, TypeCollection(StringType, BinaryType), StringType)
+ shouldCast(IntegerType, TypeCollection(BinaryType, StringType), StringType)
+ }
+
+ test("ineligible implicit type cast") {
+ def shouldNotCast(from: DataType, to: AbstractDataType): Unit = {
+ val got = HiveTypeCoercion.ImplicitTypeCasts.implicitCast(Literal.create(null, from), to)
+ assert(got.isEmpty, s"Should not be able to cast $from to $to, but got $got")
+ }
+
+ shouldNotCast(IntegerType, DateType)
+ shouldNotCast(IntegerType, TimestampType)
+ shouldNotCast(LongType, DateType)
+ shouldNotCast(LongType, TimestampType)
+ shouldNotCast(DecimalType.Unlimited, DateType)
+ shouldNotCast(DecimalType.Unlimited, TimestampType)
+
+ shouldNotCast(IntegerType, TypeCollection(DateType, TimestampType))
+
+ shouldNotCast(IntegerType, ArrayType)
+ shouldNotCast(IntegerType, MapType)
+ shouldNotCast(IntegerType, StructType)
}
test("tightest common bound for types") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org