You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/05/27 10:13:02 UTC
spark git commit: [SPARK-7887][SQL] Remove EvaluatedType from SQL
Expression.
Repository: spark
Updated Branches:
refs/heads/master 4f98d7a7f -> 9f48bf6b3
[SPARK-7887][SQL] Remove EvaluatedType from SQL Expression.
This type is not really used. Might as well remove it.
Author: Reynold Xin <rx...@databricks.com>
Closes #6427 from rxin/evalutedType and squashes the following commits:
51a319a [Reynold Xin] [SPARK-7887][SQL] Remove EvaluatedType from SQL Expression.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9f48bf6b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9f48bf6b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9f48bf6b
Branch: refs/heads/master
Commit: 9f48bf6b3761d66c7dc50f076ed92aff21b7eea0
Parents: 4f98d7a
Author: Reynold Xin <rx...@databricks.com>
Authored: Wed May 27 01:12:59 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Wed May 27 01:12:59 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/unresolved.scala | 10 +++++-----
.../sql/catalyst/expressions/BoundAttribute.scala | 2 --
.../spark/sql/catalyst/expressions/Cast.scala | 2 --
.../spark/sql/catalyst/expressions/Expression.scala | 8 ++------
.../sql/catalyst/expressions/ExtractValue.scala | 2 --
.../spark/sql/catalyst/expressions/ScalaUdf.scala | 2 --
.../spark/sql/catalyst/expressions/SortOrder.scala | 2 +-
.../spark/sql/catalyst/expressions/aggregates.scala | 4 +---
.../spark/sql/catalyst/expressions/arithmetic.scala | 16 ++++------------
.../sql/catalyst/expressions/complexTypes.scala | 6 ++----
.../sql/catalyst/expressions/decimalFunctions.scala | 2 --
.../spark/sql/catalyst/expressions/generators.scala | 2 --
.../spark/sql/catalyst/expressions/literals.scala | 2 --
.../sql/catalyst/expressions/mathfuncs/binary.scala | 3 ++-
.../sql/catalyst/expressions/mathfuncs/unary.scala | 1 -
.../sql/catalyst/expressions/namedExpressions.scala | 6 ++----
.../sql/catalyst/expressions/nullFunctions.scala | 1 -
.../spark/sql/catalyst/expressions/predicates.scala | 6 ------
.../spark/sql/catalyst/expressions/random.scala | 2 --
.../spark/sql/catalyst/expressions/sets.scala | 4 ----
.../sql/catalyst/expressions/stringOperations.scala | 8 --------
.../catalyst/expressions/windowExpressions.scala | 12 +++++-------
.../sql/catalyst/plans/physical/partitioning.scala | 4 ++--
.../spark/sql/catalyst/trees/TreeNodeSuite.scala | 9 ++++-----
.../expressions/MonotonicallyIncreasingID.scala | 2 --
.../execution/expressions/SparkPartitionID.scala | 2 --
.../org/apache/spark/sql/execution/pythonUdfs.scala | 2 +-
.../scala/org/apache/spark/sql/hive/hiveUdfs.scala | 5 +----
28 files changed, 32 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index 2999c2e..bbb150c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -67,7 +67,7 @@ case class UnresolvedAttribute(nameParts: Seq[String])
override def withName(newName: String): UnresolvedAttribute = UnresolvedAttribute.quoted(newName)
// Unresolved attributes are transient at compile time and don't get evaluated during execution.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"'$name"
@@ -85,7 +85,7 @@ case class UnresolvedFunction(name: String, children: Seq[Expression]) extends E
override lazy val resolved = false
// Unresolved functions are transient at compile time and don't get evaluated during execution.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"'$name(${children.mkString(",")})"
@@ -107,7 +107,7 @@ trait Star extends NamedExpression with trees.LeafNode[Expression] {
override lazy val resolved = false
// Star gets expanded at runtime so we never evaluate a Star.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
def expand(input: Seq[Attribute], resolver: Resolver): Seq[NamedExpression]
@@ -166,7 +166,7 @@ case class MultiAlias(child: Expression, names: Seq[String])
override lazy val resolved = false
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"$child AS $names"
@@ -200,7 +200,7 @@ case class UnresolvedExtractValue(child: Expression, extraction: Expression)
override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
override lazy val resolved = false
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"$child[$extraction]"
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala
index c6217f0..1ffc95c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala
@@ -30,8 +30,6 @@ import org.apache.spark.sql.catalyst.trees
case class BoundReference(ordinal: Int, dataType: DataType, nullable: Boolean)
extends NamedExpression with trees.LeafNode[Expression] {
- type EvaluatedType = Any
-
override def toString: String = s"input[$ordinal]"
override def eval(input: Row): Any = input(ordinal)
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index d8cf2b2..df3cdf2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -105,8 +105,6 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w
override def toString: String = s"CAST($child, $dataType)"
- type EvaluatedType = Any
-
// [[func]] assumes the input is no longer null because eval already does the null check.
@inline private[this] def buildCast[T](a: Any, func: T => Any): Any = func(a.asInstanceOf[T])
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index c7ae9da..d199287 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -25,9 +25,6 @@ import org.apache.spark.sql.types._
abstract class Expression extends TreeNode[Expression] {
self: Product =>
- /** The narrowest possible type that is produced when this expression is evaluated. */
- type EvaluatedType <: Any
-
/**
* Returns true when an expression is a candidate for static evaluation before the query is
* executed.
@@ -44,7 +41,7 @@ abstract class Expression extends TreeNode[Expression] {
def references: AttributeSet = AttributeSet(children.flatMap(_.references.iterator))
/** Returns the result of evaluating this expression on a given input Row */
- def eval(input: Row = null): EvaluatedType
+ def eval(input: Row = null): Any
/**
* Returns `true` if this expression and all its children have been resolved to a specific schema
@@ -117,8 +114,7 @@ abstract class UnaryExpression extends Expression with trees.UnaryNode[Expressio
// not like a real expressions.
case class GroupExpression(children: Seq[Expression]) extends Expression {
self: Product =>
- type EvaluatedType = Seq[Any]
- override def eval(input: Row): EvaluatedType = throw new UnsupportedOperationException
+ override def eval(input: Row): Any = throw new UnsupportedOperationException
override def nullable: Boolean = false
override def foldable: Boolean = false
override def dataType: DataType = throw new UnsupportedOperationException
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtractValue.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtractValue.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtractValue.scala
index e05926c..b5f4e16 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtractValue.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtractValue.scala
@@ -92,8 +92,6 @@ object ExtractValue {
trait ExtractValue extends UnaryExpression {
self: Product =>
-
- type EvaluatedType = Any
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
index fe2873e..5b45347 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
@@ -27,8 +27,6 @@ import org.apache.spark.sql.types.DataType
case class ScalaUdf(function: AnyRef, dataType: DataType, children: Seq[Expression])
extends Expression {
- type EvaluatedType = Any
-
override def nullable: Boolean = true
override def toString: String = s"scalaUDF(${children.mkString(",")})"
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
index 83074eb..195eec8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
@@ -36,7 +36,7 @@ case class SortOrder(child: Expression, direction: SortDirection) extends Expres
override def nullable: Boolean = child.nullable
// SortOrder itself is never evaluated.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"$child ${if (direction == Ascending) "ASC" else "DESC"}"
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala
index f3830c6..72eff5f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala
@@ -37,7 +37,7 @@ abstract class AggregateExpression extends Expression {
* [[AggregateExpression.eval]] should never be invoked because [[AggregateExpression]]'s are
* replaced with a physical aggregate operator at runtime.
*/
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
}
@@ -74,8 +74,6 @@ abstract class AggregateFunction
extends AggregateExpression with Serializable with trees.LeafNode[Expression] {
self: Product =>
- override type EvaluatedType = Any
-
/** Base should return the generic aggregate expression that this function is computing */
val base: AggregateExpression
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index c7a37ad..34c833b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -22,7 +22,6 @@ import org.apache.spark.sql.catalyst.errors.TreeNodeException
import org.apache.spark.sql.types._
case class UnaryMinus(child: Expression) extends UnaryExpression {
- type EvaluatedType = Any
override def dataType: DataType = child.dataType
override def foldable: Boolean = child.foldable
@@ -45,7 +44,6 @@ case class UnaryMinus(child: Expression) extends UnaryExpression {
}
case class Sqrt(child: Expression) extends UnaryExpression {
- type EvaluatedType = Any
override def dataType: DataType = DoubleType
override def foldable: Boolean = child.foldable
@@ -72,8 +70,6 @@ case class Sqrt(child: Expression) extends UnaryExpression {
abstract class BinaryArithmetic extends BinaryExpression {
self: Product =>
- type EvaluatedType = Any
-
override lazy val resolved =
left.resolved && right.resolved &&
left.dataType == right.dataType &&
@@ -101,7 +97,7 @@ abstract class BinaryArithmetic extends BinaryExpression {
}
}
- def evalInternal(evalE1: EvaluatedType, evalE2: EvaluatedType): Any =
+ def evalInternal(evalE1: Any, evalE2: Any): Any =
sys.error(s"BinaryExpressions must either override eval or evalInternal")
}
@@ -244,7 +240,7 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme
case other => sys.error(s"Unsupported bitwise & operation on $other")
}
- override def evalInternal(evalE1: EvaluatedType, evalE2: EvaluatedType): Any = and(evalE1, evalE2)
+ override def evalInternal(evalE1: Any, evalE2: Any): Any = and(evalE1, evalE2)
}
/**
@@ -265,7 +261,7 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet
case other => sys.error(s"Unsupported bitwise | operation on $other")
}
- override def evalInternal(evalE1: EvaluatedType, evalE2: EvaluatedType): Any = or(evalE1, evalE2)
+ override def evalInternal(evalE1: Any, evalE2: Any): Any = or(evalE1, evalE2)
}
/**
@@ -286,14 +282,13 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme
case other => sys.error(s"Unsupported bitwise ^ operation on $other")
}
- override def evalInternal(evalE1: EvaluatedType, evalE2: EvaluatedType): Any = xor(evalE1, evalE2)
+ override def evalInternal(evalE1: Any, evalE2: Any): Any = xor(evalE1, evalE2)
}
/**
* A function that calculates bitwise not(~) of a number.
*/
case class BitwiseNot(child: Expression) extends UnaryExpression {
- type EvaluatedType = Any
override def dataType: DataType = child.dataType
override def foldable: Boolean = child.foldable
@@ -323,7 +318,6 @@ case class BitwiseNot(child: Expression) extends UnaryExpression {
}
case class MaxOf(left: Expression, right: Expression) extends Expression {
- type EvaluatedType = Any
override def foldable: Boolean = left.foldable && right.foldable
@@ -368,7 +362,6 @@ case class MaxOf(left: Expression, right: Expression) extends Expression {
}
case class MinOf(left: Expression, right: Expression) extends Expression {
- type EvaluatedType = Any
override def foldable: Boolean = left.foldable && right.foldable
@@ -416,7 +409,6 @@ case class MinOf(left: Expression, right: Expression) extends Expression {
* A function that get the absolute value of the numeric value.
*/
case class Abs(child: Expression) extends UnaryExpression {
- type EvaluatedType = Any
override def dataType: DataType = child.dataType
override def foldable: Boolean = child.foldable
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
index 956a242..e7cd713 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
@@ -24,8 +24,7 @@ import org.apache.spark.sql.types._
* Returns an Array containing the evaluation of all children expressions.
*/
case class CreateArray(children: Seq[Expression]) extends Expression {
- override type EvaluatedType = Any
-
+
override def foldable: Boolean = children.forall(_.foldable)
lazy val childTypes = children.map(_.dataType).distinct
@@ -54,7 +53,6 @@ case class CreateArray(children: Seq[Expression]) extends Expression {
* TODO: [[CreateStruct]] does not support codegen.
*/
case class CreateStruct(children: Seq[NamedExpression]) extends Expression {
- override type EvaluatedType = Row
override def foldable: Boolean = children.forall(_.foldable)
@@ -71,7 +69,7 @@ case class CreateStruct(children: Seq[NamedExpression]) extends Expression {
override def nullable: Boolean = false
- override def eval(input: Row): EvaluatedType = {
+ override def eval(input: Row): Any = {
Row(children.map(_.eval(input)): _*)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalFunctions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalFunctions.scala
index adb94df..65ba189 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalFunctions.scala
@@ -21,7 +21,6 @@ import org.apache.spark.sql.types._
/** Return the unscaled Long value of a Decimal, assuming it fits in a Long */
case class UnscaledValue(child: Expression) extends UnaryExpression {
- override type EvaluatedType = Any
override def dataType: DataType = LongType
override def foldable: Boolean = child.foldable
@@ -40,7 +39,6 @@ case class UnscaledValue(child: Expression) extends UnaryExpression {
/** Create a Decimal from an unscaled Long value */
case class MakeDecimal(child: Expression, precision: Int, scale: Int) extends UnaryExpression {
- override type EvaluatedType = Decimal
override def dataType: DataType = DecimalType(precision, scale)
override def foldable: Boolean = child.foldable
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index 747a47b..cab40fe 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -40,8 +40,6 @@ import org.apache.spark.sql.types._
abstract class Generator extends Expression {
self: Product =>
- override type EvaluatedType = TraversableOnce[Row]
-
// TODO ideally we should return the type of ArrayType(StructType),
// however, we don't keep the output field names in the Generator.
override def dataType: DataType = throw new UnsupportedOperationException
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 5f8c735..d3ca3d9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -78,14 +78,12 @@ case class Literal protected (value: Any, dataType: DataType) extends LeafExpres
override def toString: String = if (value != null) value.toString else "null"
- type EvaluatedType = Any
override def eval(input: Row): Any = value
}
// TODO: Specialize
case class MutableLiteral(var value: Any, dataType: DataType, nullable: Boolean = true)
extends LeafExpression {
- type EvaluatedType = Any
def update(expression: Expression, input: Row): Unit = {
value = expression.eval(input)
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
index fcc06d3..d5be446 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.types._
*/
abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
extends BinaryExpression with Serializable with ExpectsInputTypes { self: Product =>
- type EvaluatedType = Any
+
override def symbol: String = null
override def expectedChildTypes: Seq[DataType] = Seq(DoubleType, DoubleType)
@@ -68,6 +68,7 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
case class Atan2(
left: Expression,
right: Expression) extends BinaryMathExpression(math.atan2, "ATAN2") {
+
override def eval(input: Row): Any = {
val evalE1 = left.eval(input)
if (evalE1 == null) {
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
index dc68469..cdcb8e2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
@@ -28,7 +28,6 @@ import org.apache.spark.sql.types._
abstract class MathematicalExpression(f: Double => Double, name: String)
extends UnaryExpression with Serializable with ExpectsInputTypes {
self: Product =>
- type EvaluatedType = Any
override def expectedChildTypes: Seq[DataType] = Seq(DoubleType)
override def dataType: DataType = DoubleType
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 50be26d..00565ec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -111,7 +111,6 @@ case class Alias(child: Expression, name: String)(
val explicitMetadata: Option[Metadata] = None)
extends NamedExpression with trees.UnaryNode[Expression] {
- override type EvaluatedType = Any
// Alias(Generator, xx) need to be transformed into Generate(generator, ...)
override lazy val resolved = childrenResolved && !child.isInstanceOf[Generator]
@@ -229,7 +228,7 @@ case class AttributeReference(
}
// Unresolved attributes are transient at compile time and don't get evaluated during execution.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"$name#${exprId.id}$typeSuffix"
@@ -240,7 +239,6 @@ case class AttributeReference(
* expression id or the unresolved indicator.
*/
case class PrettyAttribute(name: String) extends Attribute with trees.LeafNode[Expression] {
- type EvaluatedType = Any
override def toString: String = name
@@ -252,7 +250,7 @@ case class PrettyAttribute(name: String) extends Attribute with trees.LeafNode[E
override def withName(newName: String): Attribute = throw new UnsupportedOperationException
override def qualifiers: Seq[String] = throw new UnsupportedOperationException
override def exprId: ExprId = throw new UnsupportedOperationException
- override def eval(input: Row): EvaluatedType = throw new UnsupportedOperationException
+ override def eval(input: Row): Any = throw new UnsupportedOperationException
override def nullable: Boolean = throw new UnsupportedOperationException
override def dataType: DataType = NullType
}
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullFunctions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullFunctions.scala
index f9161cf..5070570 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullFunctions.scala
@@ -22,7 +22,6 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedException
import org.apache.spark.sql.types.DataType
case class Coalesce(children: Seq[Expression]) extends Expression {
- type EvaluatedType = Any
/** Coalesce is nullable if all of its children are nullable, or if it has no children. */
override def nullable: Boolean = !children.exists(!_.nullable)
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index 1d72a9e..e2d1c81 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -35,8 +35,6 @@ trait Predicate extends Expression {
self: Product =>
override def dataType: DataType = BooleanType
-
- type EvaluatedType = Any
}
trait PredicateHelper {
@@ -341,8 +339,6 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi
trueValue.dataType
}
- type EvaluatedType = Any
-
override def eval(input: Row): Any = {
if (true == predicate.eval(input)) {
trueValue.eval(input)
@@ -357,8 +353,6 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi
trait CaseWhenLike extends Expression {
self: Product =>
- type EvaluatedType = Any
-
// Note that `branches` are considered in consecutive pairs (cond, val), and the optional last
// element is the value for the default catch-all case (if provided).
// Hence, `branches` consists of at least two elements, and can have an odd or even length.
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala
index 66d7c8b..de82c15 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala
@@ -38,8 +38,6 @@ abstract class RDG(seed: Long) extends LeafExpression with Serializable {
*/
@transient protected lazy val rng = new XORShiftRandom(seed + TaskContext.get().partitionId())
- override type EvaluatedType = Double
-
override def nullable: Boolean = false
override def dataType: DataType = DoubleType
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala
index 4c44182..b65bf16 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala
@@ -51,7 +51,6 @@ private[sql] class OpenHashSetUDT(
* Creates a new set of the specified type
*/
case class NewSet(elementType: DataType) extends LeafExpression {
- type EvaluatedType = Any
override def nullable: Boolean = false
@@ -69,7 +68,6 @@ case class NewSet(elementType: DataType) extends LeafExpression {
* For performance, this expression mutates its input during evaluation.
*/
case class AddItemToSet(item: Expression, set: Expression) extends Expression {
- type EvaluatedType = Any
override def children: Seq[Expression] = item :: set :: Nil
@@ -101,7 +99,6 @@ case class AddItemToSet(item: Expression, set: Expression) extends Expression {
* For performance, this expression mutates its left input set during evaluation.
*/
case class CombineSets(left: Expression, right: Expression) extends BinaryExpression {
- type EvaluatedType = Any
override def nullable: Boolean = left.nullable || right.nullable
@@ -133,7 +130,6 @@ case class CombineSets(left: Expression, right: Expression) extends BinaryExpres
* Returns the number of elements in the input set.
*/
case class CountSet(child: Expression) extends UnaryExpression {
- type EvaluatedType = Any
override def nullable: Boolean = child.nullable
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
index 7683e09..5da93fe 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
@@ -25,8 +25,6 @@ import org.apache.spark.sql.types._
trait StringRegexExpression extends ExpectsInputTypes {
self: BinaryExpression =>
- type EvaluatedType = Any
-
def escape(v: String): String
def matches(regex: Pattern, str: String): Boolean
@@ -114,8 +112,6 @@ case class RLike(left: Expression, right: Expression)
trait CaseConversionExpression extends ExpectsInputTypes {
self: UnaryExpression =>
- type EvaluatedType = Any
-
def convert(v: UTF8String): UTF8String
override def foldable: Boolean = child.foldable
@@ -159,8 +155,6 @@ trait StringComparison extends ExpectsInputTypes {
def compare(l: UTF8String, r: UTF8String): Boolean
- override type EvaluatedType = Any
-
override def nullable: Boolean = left.nullable || right.nullable
override def expectedChildTypes: Seq[DataType] = Seq(StringType, StringType)
@@ -211,8 +205,6 @@ case class EndsWith(left: Expression, right: Expression)
*/
case class Substring(str: Expression, pos: Expression, len: Expression)
extends Expression with ExpectsInputTypes {
-
- type EvaluatedType = Any
override def foldable: Boolean = str.foldable && pos.foldable && len.foldable
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
index 099d67c..2729b34 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
@@ -66,8 +66,6 @@ case class WindowSpecDefinition(
}
}
- type EvaluatedType = Any
-
override def children: Seq[Expression] = partitionSpec ++ orderSpec
override lazy val resolved: Boolean =
@@ -76,7 +74,7 @@ case class WindowSpecDefinition(
override def toString: String = simpleString
- override def eval(input: Row): EvaluatedType = throw new UnsupportedOperationException
+ override def eval(input: Row): Any = throw new UnsupportedOperationException
override def nullable: Boolean = true
override def foldable: Boolean = false
override def dataType: DataType = throw new UnsupportedOperationException
@@ -299,7 +297,7 @@ case class UnresolvedWindowFunction(
override def get(index: Int): Any =
throw new UnresolvedException(this, "get")
// Unresolved functions are transient at compile time and don't get evaluated during execution.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"'$name(${children.mkString(",")})"
@@ -311,25 +309,25 @@ case class UnresolvedWindowFunction(
case class UnresolvedWindowExpression(
child: UnresolvedWindowFunction,
windowSpec: WindowSpecReference) extends UnaryExpression {
+
override def dataType: DataType = throw new UnresolvedException(this, "dataType")
override def foldable: Boolean = throw new UnresolvedException(this, "foldable")
override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
override lazy val resolved = false
// Unresolved functions are transient at compile time and don't get evaluated during execution.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
}
case class WindowExpression(
windowFunction: WindowFunction,
windowSpec: WindowSpecDefinition) extends Expression {
- override type EvaluatedType = Any
override def children: Seq[Expression] =
windowFunction :: windowSpec :: Nil
- override def eval(input: Row): EvaluatedType =
+ override def eval(input: Row): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def dataType: DataType = windowFunction.dataType
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
index fb4217a..80ba57a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
@@ -169,7 +169,7 @@ case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int)
override def keyExpressions: Seq[Expression] = expressions
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
}
@@ -213,6 +213,6 @@ case class RangePartitioning(ordering: Seq[SortOrder], numPartitions: Int)
override def keyExpressions: Seq[Expression] = ordering.map(_.child)
- override def eval(input: Row): EvaluatedType =
+ override def eval(input: Row): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index 3d10dab..e5f77dc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -25,12 +25,11 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types.{IntegerType, StringType, NullType}
case class Dummy(optKey: Option[Expression]) extends Expression {
- def children: Seq[Expression] = optKey.toSeq
- def nullable: Boolean = true
- def dataType: NullType = NullType
+ override def children: Seq[Expression] = optKey.toSeq
+ override def nullable: Boolean = true
+ override def dataType: NullType = NullType
override lazy val resolved = true
- type EvaluatedType = Any
- def eval(input: Row): Any = null.asInstanceOf[Any]
+ override def eval(input: Row): Any = null.asInstanceOf[Any]
}
class TreeNodeSuite extends FunSuite {
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/MonotonicallyIncreasingID.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/MonotonicallyIncreasingID.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/MonotonicallyIncreasingID.scala
index 9ac732b..e228a60 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/MonotonicallyIncreasingID.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/MonotonicallyIncreasingID.scala
@@ -39,8 +39,6 @@ private[sql] case class MonotonicallyIncreasingID() extends LeafExpression {
*/
@transient private[this] var count: Long = 0L
- override type EvaluatedType = Long
-
override def nullable: Boolean = false
override def dataType: DataType = LongType
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/SparkPartitionID.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/SparkPartitionID.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/SparkPartitionID.scala
index c2c6cbd..1272793 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/SparkPartitionID.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/SparkPartitionID.scala
@@ -27,8 +27,6 @@ import org.apache.spark.sql.types.{IntegerType, DataType}
*/
private[sql] case object SparkPartitionID extends LeafExpression {
- override type EvaluatedType = Int
-
override def nullable: Boolean = false
override def dataType: DataType = IntegerType
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
index 11b2897..55f3ff4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
@@ -56,7 +56,7 @@ private[spark] case class PythonUDF(
def nullable: Boolean = true
- override def eval(input: Row): PythonUDF.this.EvaluatedType = {
+ override def eval(input: Row): Any = {
sys.error("PythonUDFs can not be directly evaluated.")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/9f48bf6b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala
index bc6b3a2..7ec4f73 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala
@@ -75,7 +75,7 @@ private[hive] abstract class HiveFunctionRegistry
private[hive] case class HiveSimpleUdf(funcWrapper: HiveFunctionWrapper, children: Seq[Expression])
extends Expression with HiveInspectors with Logging {
- type EvaluatedType = Any
+
type UDFType = UDF
override def nullable: Boolean = true
@@ -139,7 +139,6 @@ private[hive] class DeferredObjectAdapter(oi: ObjectInspector)
private[hive] case class HiveGenericUdf(funcWrapper: HiveFunctionWrapper, children: Seq[Expression])
extends Expression with HiveInspectors with Logging {
type UDFType = GenericUDF
- type EvaluatedType = Any
override def nullable: Boolean = true
@@ -336,8 +335,6 @@ private[hive] case class HiveWindowFunction(
def nullable: Boolean = true
- override type EvaluatedType = Any
-
override def eval(input: Row): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org