You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/11/02 22:53:05 UTC
spark git commit: [SPARK-18214][SQL] Simplify RuntimeReplaceable type
coercion
Repository: spark
Updated Branches:
refs/heads/master 37d95227a -> fd90541c3
[SPARK-18214][SQL] Simplify RuntimeReplaceable type coercion
## What changes were proposed in this pull request?
RuntimeReplaceable is used to create aliases for expressions, but the way it deals with type coercion is pretty weird (each expression is responsible for how to handle type coercion, which does not obey the normal implicit type cast rules).
This patch simplifies its handling by allowing the analyzer to traverse into the actual expression of a RuntimeReplaceable.
## How was this patch tested?
- Correctness should be guaranteed by existing unit tests already
- Removed SQLCompatibilityFunctionSuite and moved it sql-compatibility-functions.sql
- Added a new test case in sql-compatibility-functions.sql for verifying explain behavior.
Author: Reynold Xin <rx...@databricks.com>
Closes #15723 from rxin/SPARK-18214.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fd90541c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fd90541c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fd90541c
Branch: refs/heads/master
Commit: fd90541c35af2bccf0155467bec8cea7c8865046
Parents: 37d9522
Author: Reynold Xin <rx...@databricks.com>
Authored: Wed Nov 2 15:53:02 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Wed Nov 2 15:53:02 2016 -0700
----------------------------------------------------------------------
.../sql/catalyst/analysis/TypeCoercion.scala | 2 -
.../sql/catalyst/expressions/Expression.scala | 30 ++---
.../expressions/datetimeExpressions.scala | 2 -
.../catalyst/expressions/nullExpressions.scala | 75 ++++-------
.../sql/catalyst/optimizer/finishAnalysis.scala | 2 +-
.../expressions/NullFunctionsSuite.scala | 19 ++-
.../inputs/sql-compatibility-functions.sql | 25 ++++
.../resources/sql-tests/results/array.sql.out | 5 +-
.../results/sql-compatibility-functions.sql.out | 124 +++++++++++++++++++
.../sql/SQLCompatibilityFunctionSuite.scala | 98 ---------------
.../apache/spark/sql/SQLQueryTestSuite.scala | 4 +-
11 files changed, 204 insertions(+), 182 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index 01b04c0..6662a9e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -528,8 +528,6 @@ object TypeCoercion {
NaNvl(l, Cast(r, DoubleType))
case NaNvl(l, r) if l.dataType == FloatType && r.dataType == DoubleType =>
NaNvl(Cast(l, DoubleType), r)
-
- case e: RuntimeReplaceable => e.replaceForTypeCoercion()
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index 726a231..221f830 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -186,7 +186,7 @@ abstract class Expression extends TreeNode[Expression] {
*/
def prettyName: String = nodeName.toLowerCase
- protected def flatArguments = productIterator.flatMap {
+ protected def flatArguments: Iterator[Any] = productIterator.flatMap {
case t: Traversable[_] => t
case single => single :: Nil
}
@@ -229,26 +229,16 @@ trait Unevaluable extends Expression {
* An expression that gets replaced at runtime (currently by the optimizer) into a different
* expression for evaluation. This is mainly used to provide compatibility with other databases.
* For example, we use this to support "nvl" by replacing it with "coalesce".
+ *
+ * A RuntimeReplaceable should have the original parameters along with a "child" expression in the
+ * case class constructor, and define a normal constructor that accepts only the original
+ * parameters. For an example, see [[Nvl]]. To make sure the explain plan and expression SQL
+ * works correctly, the implementation should also override flatArguments method and sql method.
*/
-trait RuntimeReplaceable extends Unevaluable {
- /**
- * Method for concrete implementations to override that specifies how to construct the expression
- * that should replace the current one.
- */
- def replaceForEvaluation(): Expression
-
- /**
- * Method for concrete implementations to override that specifies how to coerce the input types.
- */
- def replaceForTypeCoercion(): Expression
-
- /** The expression that should be used during evaluation. */
- lazy val replaced: Expression = replaceForEvaluation()
-
- override def nullable: Boolean = replaced.nullable
- override def foldable: Boolean = replaced.foldable
- override def dataType: DataType = replaced.dataType
- override def checkInputDataTypes(): TypeCheckResult = replaced.checkInputDataTypes()
+trait RuntimeReplaceable extends UnaryExpression with Unevaluable {
+ override def nullable: Boolean = child.nullable
+ override def foldable: Boolean = child.foldable
+ override def dataType: DataType = child.dataType
}
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 67c078a..05bfa7d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -488,8 +488,6 @@ abstract class UnixTime extends BinaryExpression with ExpectsInputTypes {
}""")
}
}
-
- override def prettyName: String = "unix_time"
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
index 1c18265..70862a8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
@@ -89,78 +89,53 @@ case class Coalesce(children: Seq[Expression]) extends Expression {
@ExpressionDescription(usage = "_FUNC_(a,b) - Returns b if a is null, or a otherwise.")
-case class IfNull(left: Expression, right: Expression) extends RuntimeReplaceable {
- override def children: Seq[Expression] = Seq(left, right)
-
- override def replaceForEvaluation(): Expression = Coalesce(Seq(left, right))
+case class IfNull(left: Expression, right: Expression, child: Expression)
+ extends RuntimeReplaceable {
- override def replaceForTypeCoercion(): Expression = {
- if (left.dataType != right.dataType) {
- TypeCoercion.findTightestCommonTypeOfTwo(left.dataType, right.dataType).map { dtype =>
- copy(left = Cast(left, dtype), right = Cast(right, dtype))
- }.getOrElse(this)
- } else {
- this
- }
+ def this(left: Expression, right: Expression) = {
+ this(left, right, Coalesce(Seq(left, right)))
}
+
+ override def flatArguments: Iterator[Any] = Iterator(left, right)
+ override def sql: String = s"$prettyName(${left.sql}, ${right.sql})"
}
@ExpressionDescription(usage = "_FUNC_(a,b) - Returns null if a equals to b, or a otherwise.")
-case class NullIf(left: Expression, right: Expression) extends RuntimeReplaceable {
- override def children: Seq[Expression] = Seq(left, right)
+case class NullIf(left: Expression, right: Expression, child: Expression)
+ extends RuntimeReplaceable {
- override def replaceForEvaluation(): Expression = {
- If(EqualTo(left, right), Literal.create(null, left.dataType), left)
+ def this(left: Expression, right: Expression) = {
+ this(left, right, If(EqualTo(left, right), Literal.create(null, left.dataType), left))
}
- override def replaceForTypeCoercion(): Expression = {
- if (left.dataType != right.dataType) {
- TypeCoercion.findTightestCommonTypeOfTwo(left.dataType, right.dataType).map { dtype =>
- copy(left = Cast(left, dtype), right = Cast(right, dtype))
- }.getOrElse(this)
- } else {
- this
- }
- }
+ override def flatArguments: Iterator[Any] = Iterator(left, right)
+ override def sql: String = s"$prettyName(${left.sql}, ${right.sql})"
}
@ExpressionDescription(usage = "_FUNC_(a,b) - Returns b if a is null, or a otherwise.")
-case class Nvl(left: Expression, right: Expression) extends RuntimeReplaceable {
- override def children: Seq[Expression] = Seq(left, right)
+case class Nvl(left: Expression, right: Expression, child: Expression) extends RuntimeReplaceable {
- override def replaceForEvaluation(): Expression = Coalesce(Seq(left, right))
-
- override def replaceForTypeCoercion(): Expression = {
- if (left.dataType != right.dataType) {
- TypeCoercion.findTightestCommonTypeToString(left.dataType, right.dataType).map { dtype =>
- copy(left = Cast(left, dtype), right = Cast(right, dtype))
- }.getOrElse(this)
- } else {
- this
- }
+ def this(left: Expression, right: Expression) = {
+ this(left, right, Coalesce(Seq(left, right)))
}
+
+ override def flatArguments: Iterator[Any] = Iterator(left, right)
+ override def sql: String = s"$prettyName(${left.sql}, ${right.sql})"
}
@ExpressionDescription(usage = "_FUNC_(a,b,c) - Returns b if a is not null, or c otherwise.")
-case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression)
+case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, child: Expression)
extends RuntimeReplaceable {
- override def replaceForEvaluation(): Expression = If(IsNotNull(expr1), expr2, expr3)
-
- override def children: Seq[Expression] = Seq(expr1, expr2, expr3)
-
- override def replaceForTypeCoercion(): Expression = {
- if (expr2.dataType != expr3.dataType) {
- TypeCoercion.findTightestCommonTypeOfTwo(expr2.dataType, expr3.dataType).map { dtype =>
- copy(expr2 = Cast(expr2, dtype), expr3 = Cast(expr3, dtype))
- }.getOrElse(this)
- } else {
- this
- }
+ def this(expr1: Expression, expr2: Expression, expr3: Expression) = {
+ this(expr1, expr2, expr3, If(IsNotNull(expr1), expr2, expr3))
}
+
+ override def flatArguments: Iterator[Any] = Iterator(expr1, expr2, expr3)
+ override def sql: String = s"$prettyName(${expr1.sql}, ${expr2.sql}, ${expr3.sql})"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
index 7c66731..f20eb95 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
@@ -31,7 +31,7 @@ import org.apache.spark.sql.types._
*/
object ReplaceExpressions extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
- case e: RuntimeReplaceable => e.replaced
+ case e: RuntimeReplaceable => e.child
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
index e736379..62c9ab3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
@@ -18,7 +18,9 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.analysis.SimpleAnalyzer
import org.apache.spark.sql.catalyst.expressions.objects.AssertNotNull
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project}
import org.apache.spark.sql.types._
class NullFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
@@ -86,18 +88,23 @@ class NullFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
}
test("SPARK-16602 Nvl should support numeric-string cases") {
+ def analyze(expr: Expression): Expression = {
+ val relation = LocalRelation()
+ SimpleAnalyzer.execute(Project(Seq(Alias(expr, "c")()), relation)).expressions.head
+ }
+
val intLit = Literal.create(1, IntegerType)
val doubleLit = Literal.create(2.2, DoubleType)
val stringLit = Literal.create("c", StringType)
val nullLit = Literal.create(null, NullType)
- assert(Nvl(intLit, doubleLit).replaceForTypeCoercion().dataType == DoubleType)
- assert(Nvl(intLit, stringLit).replaceForTypeCoercion().dataType == StringType)
- assert(Nvl(stringLit, doubleLit).replaceForTypeCoercion().dataType == StringType)
+ assert(analyze(new Nvl(intLit, doubleLit)).dataType == DoubleType)
+ assert(analyze(new Nvl(intLit, stringLit)).dataType == StringType)
+ assert(analyze(new Nvl(stringLit, doubleLit)).dataType == StringType)
- assert(Nvl(nullLit, intLit).replaceForTypeCoercion().dataType == IntegerType)
- assert(Nvl(doubleLit, nullLit).replaceForTypeCoercion().dataType == DoubleType)
- assert(Nvl(nullLit, stringLit).replaceForTypeCoercion().dataType == StringType)
+ assert(analyze(new Nvl(nullLit, intLit)).dataType == IntegerType)
+ assert(analyze(new Nvl(doubleLit, nullLit)).dataType == DoubleType)
+ assert(analyze(new Nvl(nullLit, stringLit)).dataType == StringType)
}
test("AtLeastNNonNulls") {
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/core/src/test/resources/sql-tests/inputs/sql-compatibility-functions.sql
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/inputs/sql-compatibility-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/sql-compatibility-functions.sql
new file mode 100644
index 0000000..2b5b692
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/sql-compatibility-functions.sql
@@ -0,0 +1,25 @@
+-- A test suite for functions added for compatibility with other databases such as Oracle, MSSQL.
+-- These functions are typically implemented using the trait RuntimeReplaceable.
+
+SELECT ifnull(null, 'x'), ifnull('y', 'x'), ifnull(null, null);
+SELECT nullif('x', 'x'), nullif('x', 'y');
+SELECT nvl(null, 'x'), nvl('y', 'x'), nvl(null, null);
+SELECT nvl2(null, 'x', 'y'), nvl2('n', 'x', 'y'), nvl2(null, null, null);
+
+-- type coercion
+SELECT ifnull(1, 2.1d), ifnull(null, 2.1d);
+SELECT nullif(1, 2.1d), nullif(1, 1.0d);
+SELECT nvl(1, 2.1d), nvl(null, 2.1d);
+SELECT nvl2(null, 1, 2.1d), nvl2('n', 1, 2.1d);
+
+-- explain for these functions; use range to avoid constant folding
+explain extended
+select ifnull(id, 'x'), nullif(id, 'x'), nvl(id, 'x'), nvl2(id, 'x', 'y')
+from range(2);
+
+-- SPARK-16730 cast alias functions for Hive compatibility
+SELECT boolean(1), tinyint(1), smallint(1), int(1), bigint(1);
+SELECT float(1), double(1), decimal(1);
+SELECT date("2014-04-04"), timestamp(date("2014-04-04"));
+-- error handling: only one argument
+SELECT string(1, 2);
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/core/src/test/resources/sql-tests/results/array.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/array.sql.out b/sql/core/src/test/resources/sql-tests/results/array.sql.out
index 499a3d5..981b250 100644
--- a/sql/core/src/test/resources/sql-tests/results/array.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/array.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 10
+-- Number of queries: 12
-- !query 0
@@ -124,6 +124,7 @@ struct<sort_array(boolean_array, true):array<boolean>,sort_array(tinyint_array,
-- !query 8 output
[true] [1,2] [1,2] [1,2] [1,2] [9223372036854775808,9223372036854775809] [1.0,2.0] [1.0,2.0] [2016-03-13,2016-03-14] [2016-11-12 20:54:00.0,2016-11-15 20:54:00.0]
+
-- !query 9
select sort_array(array('b', 'd'), '1')
-- !query 9 schema
@@ -132,6 +133,7 @@ struct<>
org.apache.spark.sql.AnalysisException
cannot resolve 'sort_array(array('b', 'd'), '1')' due to data type mismatch: Sort order in second argument requires a boolean literal.; line 1 pos 7
+
-- !query 10
select sort_array(array('b', 'd'), cast(NULL as boolean))
-- !query 10 schema
@@ -140,6 +142,7 @@ struct<>
org.apache.spark.sql.AnalysisException
cannot resolve 'sort_array(array('b', 'd'), CAST(NULL AS BOOLEAN))' due to data type mismatch: Sort order in second argument requires a boolean literal.; line 1 pos 7
+
-- !query 11
select
size(boolean_array),
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out
new file mode 100644
index 0000000..9f0b959
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out
@@ -0,0 +1,124 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 13
+
+
+-- !query 0
+SELECT ifnull(null, 'x'), ifnull('y', 'x'), ifnull(null, null)
+-- !query 0 schema
+struct<ifnull(NULL, 'x'):string,ifnull('y', 'x'):string,ifnull(NULL, NULL):null>
+-- !query 0 output
+x y NULL
+
+
+-- !query 1
+SELECT nullif('x', 'x'), nullif('x', 'y')
+-- !query 1 schema
+struct<nullif('x', 'x'):string,nullif('x', 'y'):string>
+-- !query 1 output
+NULL x
+
+
+-- !query 2
+SELECT nvl(null, 'x'), nvl('y', 'x'), nvl(null, null)
+-- !query 2 schema
+struct<nvl(NULL, 'x'):string,nvl('y', 'x'):string,nvl(NULL, NULL):null>
+-- !query 2 output
+x y NULL
+
+
+-- !query 3
+SELECT nvl2(null, 'x', 'y'), nvl2('n', 'x', 'y'), nvl2(null, null, null)
+-- !query 3 schema
+struct<nvl2(NULL, 'x', 'y'):string,nvl2('n', 'x', 'y'):string,nvl2(NULL, NULL, NULL):null>
+-- !query 3 output
+y x NULL
+
+
+-- !query 4
+SELECT ifnull(1, 2.1d), ifnull(null, 2.1d)
+-- !query 4 schema
+struct<ifnull(1, 2.1D):double,ifnull(NULL, 2.1D):double>
+-- !query 4 output
+1.0 2.1
+
+
+-- !query 5
+SELECT nullif(1, 2.1d), nullif(1, 1.0d)
+-- !query 5 schema
+struct<nullif(1, 2.1D):int,nullif(1, 1.0D):int>
+-- !query 5 output
+1 NULL
+
+
+-- !query 6
+SELECT nvl(1, 2.1d), nvl(null, 2.1d)
+-- !query 6 schema
+struct<nvl(1, 2.1D):double,nvl(NULL, 2.1D):double>
+-- !query 6 output
+1.0 2.1
+
+
+-- !query 7
+SELECT nvl2(null, 1, 2.1d), nvl2('n', 1, 2.1d)
+-- !query 7 schema
+struct<nvl2(NULL, 1, 2.1D):double,nvl2('n', 1, 2.1D):double>
+-- !query 7 output
+2.1 1.0
+
+
+-- !query 8
+explain extended
+select ifnull(id, 'x'), nullif(id, 'x'), nvl(id, 'x'), nvl2(id, 'x', 'y')
+from range(2)
+-- !query 8 schema
+struct<plan:string>
+-- !query 8 output
+== Parsed Logical Plan ==
+'Project [unresolvedalias('ifnull('id, x), None), unresolvedalias('nullif('id, x), None), unresolvedalias('nvl('id, x), None), unresolvedalias('nvl2('id, x, y), None)]
++- 'UnresolvedTableValuedFunction range, [2]
+
+== Analyzed Logical Plan ==
+ifnull(`id`, 'x'): string, nullif(`id`, 'x'): bigint, nvl(`id`, 'x'): string, nvl2(`id`, 'x', 'y'): string
+Project [ifnull(id#xL, x) AS ifnull(`id`, 'x')#x, nullif(id#xL, x) AS nullif(`id`, 'x')#xL, nvl(id#xL, x) AS nvl(`id`, 'x')#x, nvl2(id#xL, x, y) AS nvl2(`id`, 'x', 'y')#x]
++- Range (0, 2, step=1, splits=None)
+
+== Optimized Logical Plan ==
+Project [coalesce(cast(id#xL as string), x) AS ifnull(`id`, 'x')#x, id#xL AS nullif(`id`, 'x')#xL, coalesce(cast(id#xL as string), x) AS nvl(`id`, 'x')#x, x AS nvl2(`id`, 'x', 'y')#x]
++- Range (0, 2, step=1, splits=None)
+
+== Physical Plan ==
+*Project [coalesce(cast(id#xL as string), x) AS ifnull(`id`, 'x')#x, id#xL AS nullif(`id`, 'x')#xL, coalesce(cast(id#xL as string), x) AS nvl(`id`, 'x')#x, x AS nvl2(`id`, 'x', 'y')#x]
++- *Range (0, 2, step=1, splits=None)
+
+
+-- !query 9
+SELECT boolean(1), tinyint(1), smallint(1), int(1), bigint(1)
+-- !query 9 schema
+struct<CAST(1 AS BOOLEAN):boolean,CAST(1 AS TINYINT):tinyint,CAST(1 AS SMALLINT):smallint,CAST(1 AS INT):int,CAST(1 AS BIGINT):bigint>
+-- !query 9 output
+true 1 1 1 1
+
+
+-- !query 10
+SELECT float(1), double(1), decimal(1)
+-- !query 10 schema
+struct<CAST(1 AS FLOAT):float,CAST(1 AS DOUBLE):double,CAST(1 AS DECIMAL(10,0)):decimal(10,0)>
+-- !query 10 output
+1.0 1.0 1
+
+
+-- !query 11
+SELECT date("2014-04-04"), timestamp(date("2014-04-04"))
+-- !query 11 schema
+struct<CAST(2014-04-04 AS DATE):date,CAST(CAST(2014-04-04 AS DATE) AS TIMESTAMP):timestamp>
+-- !query 11 output
+2014-04-04 2014-04-04 00:00:00
+
+
+-- !query 12
+SELECT string(1, 2)
+-- !query 12 schema
+struct<>
+-- !query 12 output
+org.apache.spark.sql.AnalysisException
+Function string accepts only one argument; line 1 pos 7
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala
deleted file mode 100644
index 27b60e0..0000000
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLCompatibilityFunctionSuite.scala
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import java.math.BigDecimal
-import java.sql.Timestamp
-
-import org.apache.spark.sql.test.SharedSQLContext
-
-/**
- * A test suite for functions added for compatibility with other databases such as Oracle, MSSQL.
- *
- * These functions are typically implemented using the trait
- * [[org.apache.spark.sql.catalyst.expressions.RuntimeReplaceable]].
- */
-class SQLCompatibilityFunctionSuite extends QueryTest with SharedSQLContext {
-
- test("ifnull") {
- checkAnswer(
- sql("SELECT ifnull(null, 'x'), ifnull('y', 'x'), ifnull(null, null)"),
- Row("x", "y", null))
-
- // Type coercion
- checkAnswer(
- sql("SELECT ifnull(1, 2.1d), ifnull(null, 2.1d)"),
- Row(1.0, 2.1))
- }
-
- test("nullif") {
- checkAnswer(
- sql("SELECT nullif('x', 'x'), nullif('x', 'y')"),
- Row(null, "x"))
-
- // Type coercion
- checkAnswer(
- sql("SELECT nullif(1, 2.1d), nullif(1, 1.0d)"),
- Row(1.0, null))
- }
-
- test("nvl") {
- checkAnswer(
- sql("SELECT nvl(null, 'x'), nvl('y', 'x'), nvl(null, null)"),
- Row("x", "y", null))
-
- // Type coercion
- checkAnswer(
- sql("SELECT nvl(1, 2.1d), nvl(null, 2.1d)"),
- Row(1.0, 2.1))
- }
-
- test("nvl2") {
- checkAnswer(
- sql("SELECT nvl2(null, 'x', 'y'), nvl2('n', 'x', 'y'), nvl2(null, null, null)"),
- Row("y", "x", null))
-
- // Type coercion
- checkAnswer(
- sql("SELECT nvl2(null, 1, 2.1d), nvl2('n', 1, 2.1d)"),
- Row(2.1, 1.0))
- }
-
- test("SPARK-16730 cast alias functions for Hive compatibility") {
- checkAnswer(
- sql("SELECT boolean(1), tinyint(1), smallint(1), int(1), bigint(1)"),
- Row(true, 1.toByte, 1.toShort, 1, 1L))
-
- checkAnswer(
- sql("SELECT float(1), double(1), decimal(1)"),
- Row(1.toFloat, 1.0, new BigDecimal(1)))
-
- checkAnswer(
- sql("SELECT date(\"2014-04-04\"), timestamp(date(\"2014-04-04\"))"),
- Row(new java.util.Date(114, 3, 4), new Timestamp(114, 3, 4, 0, 0, 0, 0)))
-
- checkAnswer(
- sql("SELECT string(1)"),
- Row("1"))
-
- // Error handling: only one argument
- val errorMsg = intercept[AnalysisException](sql("SELECT string(1, 2)")).getMessage
- assert(errorMsg.contains("Function string accepts only one argument"))
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/fd90541c/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 2d73d9f..1a4049f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.planning.PhysicalOperation
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile}
-import org.apache.spark.sql.execution.command.ShowColumnsCommand
import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.sql.types.StructType
@@ -215,7 +214,8 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext {
try {
val df = session.sql(sql)
val schema = df.schema
- val answer = df.queryExecution.hiveResultString()
+ // Get answer, but also get rid of the #1234 expression ids that show up in explain plans
+ val answer = df.queryExecution.hiveResultString().map(_.replaceAll("#\\d+", "#x"))
// If the output is not pre-sorted, sort it.
if (isSorted(df.queryExecution.analyzed)) (schema, answer) else (schema, answer.sorted)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org