You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/04/24 06:21:07 UTC
spark git commit: [SQL] Fixed expression data type matching.
Repository: spark
Updated Branches:
refs/heads/master 67bccbda1 -> d3a302def
[SQL] Fixed expression data type matching.
Also took the chance to improve documentation for various types.
Author: Reynold Xin <rx...@databricks.com>
Closes #5675 from rxin/data-type-matching-expr and squashes the following commits:
0f31856 [Reynold Xin] One more function documentation.
27c1973 [Reynold Xin] Added more documentation.
336a36d [Reynold Xin] [SQL] Fixed expression data type matching.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d3a302de
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d3a302de
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d3a302de
Branch: refs/heads/master
Commit: d3a302defc45768492dec9da4c40d78d28997a65
Parents: 67bccbd
Author: Reynold Xin <rx...@databricks.com>
Authored: Thu Apr 23 21:21:03 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu Apr 23 21:21:03 2015 -0700
----------------------------------------------------------------------
.../expressions/codegen/CodeGenerator.scala | 2 +-
.../org/apache/spark/sql/types/DataType.scala | 50 ++++++++++++++++----
2 files changed, 42 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d3a302de/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index cbe5203..dbc92fb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -279,7 +279,7 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
org.apache.spark.sql.types.UTF8String(${eval.primitiveTerm}.toString)
""".children
- case EqualTo(e1: BinaryType, e2: BinaryType) =>
+ case EqualTo(e1 @ BinaryType(), e2 @ BinaryType()) =>
(e1, e2).evaluateAs (BooleanType) {
case (eval1, eval2) =>
q"""
http://git-wip-us.apache.org/repos/asf/spark/blob/d3a302de/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
index e6bfcd9..06bff7d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -40,32 +40,46 @@ import org.apache.spark.util.Utils
*/
@DeveloperApi
abstract class DataType {
- /** Matches any expression that evaluates to this DataType */
- def unapply(a: Expression): Boolean = a match {
+ /**
+ * Enables matching against NumericType for expressions:
+ * {{{
+ * case Cast(child @ BinaryType(), StringType) =>
+ * ...
+ * }}}
+ */
+ private[sql] def unapply(a: Expression): Boolean = a match {
case e: Expression if e.dataType == this => true
case _ => false
}
- /** The default size of a value of this data type. */
+ /**
+ * The default size of a value of this data type, used internally for size estimation.
+ */
def defaultSize: Int
+ /** Name of the type used in JSON serialization. */
def typeName: String = this.getClass.getSimpleName.stripSuffix("$").dropRight(4).toLowerCase
private[sql] def jsonValue: JValue = typeName
+ /** The compact JSON representation of this data type. */
def json: String = compact(render(jsonValue))
+ /** The pretty (i.e. indented) JSON representation of this data type. */
def prettyJson: String = pretty(render(jsonValue))
+ /** Readable string representation for the type. */
def simpleString: String = typeName
- /** Check if `this` and `other` are the same data type when ignoring nullability
- * (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
+ /**
+ * Check if `this` and `other` are the same data type when ignoring nullability
+ * (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
*/
private[spark] def sameType(other: DataType): Boolean =
DataType.equalsIgnoreNullability(this, other)
- /** Returns the same data type but set all nullability fields are true
+ /**
+ * Returns the same data type but set all nullability fields are true
* (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
*/
private[spark] def asNullable: DataType
@@ -104,12 +118,25 @@ abstract class NumericType extends AtomicType {
private[sql] object NumericType {
+ /**
+ * Enables matching against NumericType for expressions:
+ * {{{
+ * case Cast(child @ NumericType(), StringType) =>
+ * ...
+ * }}}
+ */
def unapply(e: Expression): Boolean = e.dataType.isInstanceOf[NumericType]
}
-/** Matcher for any expressions that evaluate to [[IntegralType]]s */
private[sql] object IntegralType {
+ /**
+ * Enables matching against IntegralType for expressions:
+ * {{{
+ * case Cast(child @ IntegralType(), StringType) =>
+ * ...
+ * }}}
+ */
def unapply(a: Expression): Boolean = a match {
case e: Expression if e.dataType.isInstanceOf[IntegralType] => true
case _ => false
@@ -122,9 +149,14 @@ private[sql] abstract class IntegralType extends NumericType {
}
-
-/** Matcher for any expressions that evaluate to [[FractionalType]]s */
private[sql] object FractionalType {
+ /**
+ * Enables matching against FractionalType for expressions:
+ * {{{
+ * case Cast(child @ FractionalType(), StringType) =>
+ * ...
+ * }}}
+ */
def unapply(a: Expression): Boolean = a match {
case e: Expression if e.dataType.isInstanceOf[FractionalType] => true
case _ => false
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org