You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2016/02/21 15:53:46 UTC
[2/2] spark git commit: [SPARK-12799] Simplify various string output
for expressions
[SPARK-12799] Simplify various string output for expressions
This PR introduces several major changes:
1. Replacing `Expression.prettyString` with `Expression.sql`
The `prettyString` method is mostly an internal, developer faced facility for debugging purposes, and shouldn't be exposed to users.
1. Using SQL-like representation as column names for selected fields that are not named expression (back-ticks and double quotes should be removed)
Before, we were using `prettyString` as column names when possible, and sometimes the result column names can be weird. Here are several examples:
Expression | `prettyString` | `sql` | Note
------------------ | -------------- | ---------- | ---------------
`a && b` | `a && b` | `a AND b` |
`a.getField("f")` | `a[f]` | `a.f` | `a` is a struct
1. Adding trait `NonSQLExpression` extending from `Expression` for expressions that don't have a SQL representation (e.g. Scala UDF/UDAF and Java/Scala object expressions used for encoders)
`NonSQLExpression.sql` may return an arbitrary user facing string representation of the expression.
Author: Cheng Lian <li...@databricks.com>
Closes #10757 from liancheng/spark-12799.simplify-expression-string-methods.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d9efe63e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d9efe63e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d9efe63e
Branch: refs/heads/master
Commit: d9efe63ecdc60a9955f1924de0e8a00bcb6a559d
Parents: d806ed3
Author: Cheng Lian <li...@databricks.com>
Authored: Sun Feb 21 22:53:15 2016 +0800
Committer: Cheng Lian <li...@databricks.com>
Committed: Sun Feb 21 22:53:15 2016 +0800
----------------------------------------------------------------------
R/pkg/inst/tests/testthat/test_sparkSQL.R | 4 +-
python/pyspark/sql/column.py | 32 ++++-----
python/pyspark/sql/context.py | 10 +--
python/pyspark/sql/functions.py | 30 ++++-----
.../spark/sql/catalyst/analysis/Analyzer.scala | 8 ++-
.../sql/catalyst/analysis/CheckAnalysis.scala | 26 +++----
.../analysis/DistinctAggregationRewriter.scala | 6 +-
.../sql/catalyst/analysis/unresolved.scala | 17 ++---
.../expressions/ExpectsInputTypes.scala | 2 +-
.../sql/catalyst/expressions/Expression.scala | 71 ++++++++------------
.../sql/catalyst/expressions/ScalaUDF.scala | 4 +-
.../expressions/aggregate/interfaces.scala | 5 +-
.../sql/catalyst/expressions/arithmetic.scala | 8 ++-
.../expressions/bitwiseExpressions.scala | 2 +
.../expressions/codegen/CodegenFallback.scala | 5 +-
.../expressions/complexTypeExtractors.scala | 22 +++---
.../expressions/conditionalExpressions.scala | 2 +-
.../sql/catalyst/expressions/literals.scala | 39 +++--------
.../catalyst/expressions/mathExpressions.scala | 6 +-
.../catalyst/expressions/namedExpressions.scala | 20 ++++--
.../catalyst/expressions/nullExpressions.scala | 2 -
.../sql/catalyst/expressions/objects.scala | 25 +++----
.../sql/catalyst/expressions/predicates.scala | 8 +--
.../expressions/stringExpressions.scala | 5 --
.../expressions/windowExpressions.scala | 6 +-
.../catalyst/plans/logical/basicOperators.scala | 2 +-
.../spark/sql/catalyst/util/package.scala | 37 +++++++---
.../apache/spark/sql/types/DecimalType.scala | 2 +
.../org/apache/spark/sql/types/StructType.scala | 4 +-
.../catalyst/analysis/AnalysisErrorSuite.scala | 26 +++----
.../sql/catalyst/analysis/AnalysisTest.scala | 15 ++++-
.../analysis/ExpressionTypeCheckingSuite.scala | 4 +-
.../encoders/EncoderResolutionSuite.scala | 2 +-
.../scala/org/apache/spark/sql/Column.scala | 17 ++---
.../scala/org/apache/spark/sql/DataFrame.scala | 4 +-
.../org/apache/spark/sql/GroupedData.scala | 3 +-
.../spark/sql/execution/WholeStageCodegen.scala | 3 +-
.../spark/sql/execution/aggregate/udaf.scala | 4 +-
.../parquet/CatalystSchemaConverter.scala | 2 +-
.../spark/sql/execution/python/PythonUDF.scala | 9 ++-
.../org/apache/spark/sql/DatasetSuite.scala | 2 +-
.../datasources/parquet/ParquetIOSuite.scala | 4 +-
.../org/apache/spark/sql/hive/SQLBuilder.scala | 22 ++++--
.../org/apache/spark/sql/hive/hiveUDFs.scala | 43 +++++++-----
.../sql/hive/ExpressionSQLBuilderSuite.scala | 15 +++--
.../spark/sql/hive/LogicalPlanToSQLSuite.scala | 64 ++++++++++++++++++
.../spark/sql/hive/execution/HivePlanTest.scala | 4 +-
.../spark/sql/hive/orc/OrcSourceSuite.scala | 22 +++---
.../apache/spark/sql/hive/parquetSuites.scala | 5 +-
49 files changed, 402 insertions(+), 278 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/R/pkg/inst/tests/testthat/test_sparkSQL.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R
index 7b57137..cc11810 100644
--- a/R/pkg/inst/tests/testthat/test_sparkSQL.R
+++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R
@@ -1047,13 +1047,13 @@ test_that("column functions", {
schema = c("a", "b", "c"))
result <- collect(select(df, struct("a", "c")))
expected <- data.frame(row.names = 1:2)
- expected$"struct(a,c)" <- list(listToStruct(list(a = 1L, c = 3L)),
+ expected$"struct(a, c)" <- list(listToStruct(list(a = 1L, c = 3L)),
listToStruct(list(a = 4L, c = 6L)))
expect_equal(result, expected)
result <- collect(select(df, struct(df$a, df$b)))
expected <- data.frame(row.names = 1:2)
- expected$"struct(a,b)" <- list(listToStruct(list(a = 1L, b = 2L)),
+ expected$"struct(a, b)" <- list(listToStruct(list(a = 1L, b = 2L)),
listToStruct(list(a = 4L, b = 5L)))
expect_equal(result, expected)
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/python/pyspark/sql/column.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index 320451c..3866a49 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -219,17 +219,17 @@ class Column(object):
>>> from pyspark.sql import Row
>>> df = sc.parallelize([Row(r=Row(a=1, b="b"))]).toDF()
>>> df.select(df.r.getField("b")).show()
- +----+
- |r[b]|
- +----+
- | b|
- +----+
+ +---+
+ |r.b|
+ +---+
+ | b|
+ +---+
>>> df.select(df.r.a).show()
- +----+
- |r[a]|
- +----+
- | 1|
- +----+
+ +---+
+ |r.a|
+ +---+
+ | 1|
+ +---+
"""
return self[name]
@@ -346,12 +346,12 @@ class Column(object):
expression is between the given columns.
>>> df.select(df.name, df.age.between(2, 4)).show()
- +-----+--------------------------+
- | name|((age >= 2) && (age <= 4))|
- +-----+--------------------------+
- |Alice| true|
- | Bob| false|
- +-----+--------------------------+
+ +-----+---------------------------+
+ | name|((age >= 2) AND (age <= 4))|
+ +-----+---------------------------+
+ |Alice| true|
+ | Bob| false|
+ +-----+---------------------------+
"""
return (self >= lowerBound) & (self <= upperBound)
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/python/pyspark/sql/context.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 7f6fb41..89bf144 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -92,8 +92,8 @@ class SQLContext(object):
>>> df.registerTempTable("allTypes")
>>> sqlContext.sql('select i+1, d+1, not b, list[1], dict["s"], time, row.a '
... 'from allTypes where b and i > 0').collect()
- [Row(_c0=2, _c1=2.0, _c2=False, _c3=2, _c4=0, \
- time=datetime.datetime(2014, 8, 1, 14, 1, 5), a=1)]
+ [Row((i + CAST(1 AS BIGINT))=2, (d + CAST(1 AS DOUBLE))=2.0, (NOT b)=False, list[1]=2, \
+ dict[s]=0, time=datetime.datetime(2014, 8, 1, 14, 1, 5), a=1)]
>>> df.map(lambda x: (x.i, x.s, x.d, x.l, x.b, x.time, x.row.a, x.list)).collect()
[(1, u'string', 1.0, 1, True, datetime.datetime(2014, 8, 1, 14, 1, 5), 1, [1, 2, 3])]
"""
@@ -210,17 +210,17 @@ class SQLContext(object):
>>> sqlContext.registerFunction("stringLengthString", lambda x: len(x))
>>> sqlContext.sql("SELECT stringLengthString('test')").collect()
- [Row(_c0=u'4')]
+ [Row(stringLengthString(test)=u'4')]
>>> from pyspark.sql.types import IntegerType
>>> sqlContext.registerFunction("stringLengthInt", lambda x: len(x), IntegerType())
>>> sqlContext.sql("SELECT stringLengthInt('test')").collect()
- [Row(_c0=4)]
+ [Row(stringLengthInt(test)=4)]
>>> from pyspark.sql.types import IntegerType
>>> sqlContext.udf.register("stringLengthInt", lambda x: len(x), IntegerType())
>>> sqlContext.sql("SELECT stringLengthInt('test')").collect()
- [Row(_c0=4)]
+ [Row(stringLengthInt(test)=4)]
"""
udf = UserDefinedFunction(f, returnType, name)
self._ssql_ctx.udf().registerPython(name, udf._judf)
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/python/pyspark/sql/functions.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 5fc1cc2..fdae05d 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -223,22 +223,22 @@ def coalesce(*cols):
+----+----+
>>> cDf.select(coalesce(cDf["a"], cDf["b"])).show()
- +-------------+
- |coalesce(a,b)|
- +-------------+
- | null|
- | 1|
- | 2|
- +-------------+
+ +--------------+
+ |coalesce(a, b)|
+ +--------------+
+ | null|
+ | 1|
+ | 2|
+ +--------------+
>>> cDf.select('*', coalesce(cDf["a"], lit(0.0))).show()
- +----+----+---------------+
- | a| b|coalesce(a,0.0)|
- +----+----+---------------+
- |null|null| 0.0|
- | 1|null| 1.0|
- |null| 2| 0.0|
- +----+----+---------------+
+ +----+----+----------------+
+ | a| b|coalesce(a, 0.0)|
+ +----+----+----------------+
+ |null|null| 0.0|
+ | 1|null| 1.0|
+ |null| 2| 0.0|
+ +----+----+----------------+
"""
sc = SparkContext._active_spark_context
jc = sc._jvm.functions.coalesce(_to_seq(sc, cols, _to_java_column))
@@ -1528,7 +1528,7 @@ def array_contains(col, value):
>>> df = sqlContext.createDataFrame([(["a", "b", "c"],), ([],)], ['data'])
>>> df.select(array_contains(df.data, "a")).collect()
- [Row(array_contains(data,a)=True), Row(array_contains(data,a)=False)]
+ [Row(array_contains(data, a)=True), Row(array_contains(data, a)=False)]
"""
sc = SparkContext._active_spark_context
return Column(sc._jvm.functions.array_contains(_to_java_column(col), value))
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index ba306f8..e153f4d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
import org.apache.spark.sql.catalyst.trees.TreeNodeRef
+import org.apache.spark.sql.catalyst.util.usePrettyExpression
import org.apache.spark.sql.types._
/**
@@ -165,7 +166,8 @@ class Analyzer(
case e if !e.resolved => u
case g: Generator => MultiAlias(g, Nil)
case c @ Cast(ne: NamedExpression, _) => Alias(c, ne.name)()
- case other => Alias(other, optionalAliasName.getOrElse(s"_c$i"))()
+ case e: ExtractValue => Alias(e, usePrettyExpression(e).sql)()
+ case e => Alias(e, optionalAliasName.getOrElse(usePrettyExpression(e).sql))()
}
}
}.asInstanceOf[Seq[NamedExpression]]
@@ -328,7 +330,7 @@ class Analyzer(
throw new AnalysisException(
s"Aggregate expression required for pivot, found '$aggregate'")
}
- val name = if (singleAgg) value.toString else value + "_" + aggregate.prettyString
+ val name = if (singleAgg) value.toString else value + "_" + aggregate.sql
Alias(filteredAggregate, name)()
}
}
@@ -1456,7 +1458,7 @@ object CleanupAliases extends Rule[LogicalPlan] {
*/
object ResolveUpCast extends Rule[LogicalPlan] {
private def fail(from: Expression, to: DataType, walkedTypePath: Seq[String]) = {
- throw new AnalysisException(s"Cannot up cast `${from.prettyString}` from " +
+ throw new AnalysisException(s"Cannot up cast ${from.sql} from " +
s"${from.dataType.simpleString} to ${to.simpleString} as it may truncate\n" +
"The type path of the target object is:\n" + walkedTypePath.mkString("", "\n", "\n") +
"You can either add an explicit cast to the input data or choose a higher precision " +
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index fe053b9..1e430c1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -57,13 +57,13 @@ trait CheckAnalysis {
operator transformExpressionsUp {
case a: Attribute if !a.resolved =>
val from = operator.inputSet.map(_.name).mkString(", ")
- a.failAnalysis(s"cannot resolve '${a.prettyString}' given input columns: [$from]")
+ a.failAnalysis(s"cannot resolve '${a.sql}' given input columns: [$from]")
case e: Expression if e.checkInputDataTypes().isFailure =>
e.checkInputDataTypes() match {
case TypeCheckResult.TypeCheckFailure(message) =>
e.failAnalysis(
- s"cannot resolve '${e.prettyString}' due to data type mismatch: $message")
+ s"cannot resolve '${e.sql}' due to data type mismatch: $message")
}
case c: Cast if !c.resolved =>
@@ -106,12 +106,12 @@ trait CheckAnalysis {
operator match {
case f: Filter if f.condition.dataType != BooleanType =>
failAnalysis(
- s"filter expression '${f.condition.prettyString}' " +
+ s"filter expression '${f.condition.sql}' " +
s"of type ${f.condition.dataType.simpleString} is not a boolean.")
case j @ Join(_, _, _, Some(condition)) if condition.dataType != BooleanType =>
failAnalysis(
- s"join condition '${condition.prettyString}' " +
+ s"join condition '${condition.sql}' " +
s"of type ${condition.dataType.simpleString} is not a boolean.")
case j @ Join(_, _, _, Some(condition)) =>
@@ -119,10 +119,10 @@ trait CheckAnalysis {
case p: Predicate =>
p.asInstanceOf[Expression].children.foreach(checkValidJoinConditionExprs)
case e if e.dataType.isInstanceOf[BinaryType] =>
- failAnalysis(s"binary type expression ${e.prettyString} cannot be used " +
+ failAnalysis(s"binary type expression ${e.sql} cannot be used " +
"in join conditions")
case e if e.dataType.isInstanceOf[MapType] =>
- failAnalysis(s"map type expression ${e.prettyString} cannot be used " +
+ failAnalysis(s"map type expression ${e.sql} cannot be used " +
"in join conditions")
case _ => // OK
}
@@ -144,13 +144,13 @@ trait CheckAnalysis {
if (!child.deterministic) {
failAnalysis(
- s"nondeterministic expression ${expr.prettyString} should not " +
+ s"nondeterministic expression ${expr.sql} should not " +
s"appear in the arguments of an aggregate function.")
}
}
case e: Attribute if !groupingExprs.exists(_.semanticEquals(e)) =>
failAnalysis(
- s"expression '${e.prettyString}' is neither present in the group by, " +
+ s"expression '${e.sql}' is neither present in the group by, " +
s"nor is it an aggregate function. " +
"Add to group by or wrap in first() (or first_value) if you don't care " +
"which value you get.")
@@ -163,7 +163,7 @@ trait CheckAnalysis {
// Check if the data type of expr is orderable.
if (!RowOrdering.isOrderable(expr.dataType)) {
failAnalysis(
- s"expression ${expr.prettyString} cannot be used as a grouping expression " +
+ s"expression ${expr.sql} cannot be used as a grouping expression " +
s"because its data type ${expr.dataType.simpleString} is not a orderable " +
s"data type.")
}
@@ -172,7 +172,7 @@ trait CheckAnalysis {
// This is just a sanity check, our analysis rule PullOutNondeterministic should
// already pull out those nondeterministic expressions and evaluate them in
// a Project node.
- failAnalysis(s"nondeterministic expression ${expr.prettyString} should not " +
+ failAnalysis(s"nondeterministic expression ${expr.sql} should not " +
s"appear in grouping expression.")
}
}
@@ -217,7 +217,7 @@ trait CheckAnalysis {
case p @ Project(exprs, _) if containsMultipleGenerators(exprs) =>
failAnalysis(
s"""Only a single table generating function is allowed in a SELECT clause, found:
- | ${exprs.map(_.prettyString).mkString(",")}""".stripMargin)
+ | ${exprs.map(_.sql).mkString(",")}""".stripMargin)
case j: Join if !j.duplicateResolved =>
val conflictingAttributes = j.left.outputSet.intersect(j.right.outputSet)
@@ -248,9 +248,9 @@ trait CheckAnalysis {
failAnalysis(
s"""nondeterministic expressions are only allowed in
|Project, Filter, Aggregate or Window, found:
- | ${o.expressions.map(_.prettyString).mkString(",")}
+ | ${o.expressions.map(_.sql).mkString(",")}
|in operator ${operator.simpleString}
- """.stripMargin)
+ """.stripMargin)
case _ => // Analysis successful!
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala
index 5dfce89..b49885d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala
@@ -130,7 +130,7 @@ case class DistinctAggregationRewriter(conf: CatalystConf) extends Rule[LogicalP
new AttributeReference("gid", IntegerType, false)(isGenerated = true)
val groupByMap = a.groupingExpressions.collect {
case ne: NamedExpression => ne -> ne.toAttribute
- case e => e -> new AttributeReference(e.prettyString, e.dataType, e.nullable)()
+ case e => e -> new AttributeReference(e.sql, e.dataType, e.nullable)()
}
val groupByAttrs = groupByMap.map(_._2)
@@ -184,7 +184,7 @@ case class DistinctAggregationRewriter(conf: CatalystConf) extends Rule[LogicalP
val regularAggOperatorMap = regularAggExprs.map { e =>
// Perform the actual aggregation in the initial aggregate.
val af = patchAggregateFunctionChildren(e.aggregateFunction)(regularAggChildAttrLookup)
- val operator = Alias(e.copy(aggregateFunction = af), e.prettyString)()
+ val operator = Alias(e.copy(aggregateFunction = af), e.sql)()
// Select the result of the first aggregate in the last aggregate.
val result = AggregateExpression(
@@ -269,5 +269,5 @@ case class DistinctAggregationRewriter(conf: CatalystConf) extends Rule[LogicalP
// NamedExpression. This is done to prevent collisions between distinct and regular aggregate
// children, in this case attribute reuse causes the input of the regular aggregate to bound to
// the (nulled out) input of the distinct aggregate.
- e -> new AttributeReference(e.prettyString, e.dataType, true)()
+ e -> new AttributeReference(e.sql, e.dataType, true)()
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index 79eebbf..01afa01 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan}
import org.apache.spark.sql.catalyst.trees.TreeNode
+import org.apache.spark.sql.catalyst.util.quoteIdentifier
import org.apache.spark.sql.types.{DataType, StructType}
/**
@@ -67,6 +68,8 @@ case class UnresolvedAttribute(nameParts: Seq[String]) extends Attribute with Un
override def withName(newName: String): UnresolvedAttribute = UnresolvedAttribute.quoted(newName)
override def toString: String = s"'$name"
+
+ override def sql: String = quoteIdentifier(name)
}
object UnresolvedAttribute {
@@ -141,11 +144,8 @@ case class UnresolvedFunction(
override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
override lazy val resolved = false
- override def prettyString: String = {
- s"${name}(${children.map(_.prettyString).mkString(",")})"
- }
-
- override def toString: String = s"'$name(${children.mkString(",")})"
+ override def prettyName: String = name
+ override def toString: String = s"'$name(${children.mkString(", ")})"
}
/**
@@ -208,10 +208,9 @@ case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevalu
Alias(extract, f.name)()
}
- case _ => {
+ case _ =>
throw new AnalysisException("Can only star expand struct data types. Attribute: `" +
target.get + "`")
- }
}
} else {
val from = input.inputSet.map(_.name).mkString(", ")
@@ -228,6 +227,7 @@ case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevalu
* For example the SQL expression "stack(2, key, value, key, value) as (a, b)" could be represented
* as follows:
* MultiAlias(stack_function, Seq(a, b))
+ *
* @param child the computation being performed
* @param names the names to be associated with each output of computing [[child]].
@@ -284,13 +284,14 @@ case class UnresolvedExtractValue(child: Expression, extraction: Expression)
override lazy val resolved = false
override def toString: String = s"$child[$extraction]"
+ override def sql: String = s"${child.sql}[${extraction.sql}]"
}
/**
* Holds the expression that has yet to be aliased.
*
* @param child The computation that is needs to be resolved during analysis.
- * @param aliasName The name if specified to be asoosicated with the result of computing [[child]]
+ * @param aliasName The name if specified to be associated with the result of computing [[child]]
*
*/
case class UnresolvedAlias(child: Expression, aliasName: Option[String] = None)
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpectsInputTypes.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpectsInputTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpectsInputTypes.scala
index 04650d8..c7be8e8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpectsInputTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpectsInputTypes.scala
@@ -45,7 +45,7 @@ trait ExpectsInputTypes extends Expression {
val mismatches = children.zip(inputTypes).zipWithIndex.collect {
case ((child, expected), idx) if !expected.acceptsType(child.dataType) =>
s"argument ${idx + 1} requires ${expected.simpleString} type, " +
- s"however, '${child.prettyString}' is of ${child.dataType.simpleString} type."
+ s"however, '${child.sql}' is of ${child.dataType.simpleString} type."
}
if (mismatches.isEmpty) {
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index c73b2f8..119496c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -18,10 +18,10 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, TypeCheckResult, UnresolvedAttribute}
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.trees.TreeNode
-import org.apache.spark.sql.catalyst.util.sequenceOption
+import org.apache.spark.sql.catalyst.util.toCommentSafeString
import org.apache.spark.sql.types._
////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -96,7 +96,7 @@ abstract class Expression extends TreeNode[Expression] {
ctx.subExprEliminationExprs.get(this).map { subExprState =>
// This expression is repeated meaning the code to evaluated has already been added
// as a function and called in advance. Just use it.
- val code = s"/* ${this.toCommentSafeString} */"
+ val code = s"/* ${toCommentSafeString(this.toString)} */"
ExprCode(code, subExprState.isNull, subExprState.value)
}.getOrElse {
val isNull = ctx.freshName("isNull")
@@ -105,7 +105,7 @@ abstract class Expression extends TreeNode[Expression] {
ve.code = genCode(ctx, ve)
if (ve.code != "") {
// Add `this` in the comment.
- ve.copy(s"/* ${this.toCommentSafeString} */\n" + ve.code.trim)
+ ve.copy(s"/* ${toCommentSafeString(this.toString)} */\n" + ve.code.trim)
} else {
ve
}
@@ -201,17 +201,6 @@ abstract class Expression extends TreeNode[Expression] {
*/
def prettyName: String = getClass.getSimpleName.toLowerCase
- /**
- * Returns a user-facing string representation of this expression, i.e. does not have developer
- * centric debugging information like the expression id.
- */
- def prettyString: String = {
- transform {
- case a: AttributeReference => PrettyAttribute(a.name, a.dataType)
- case u: UnresolvedAttribute => PrettyAttribute(u.name)
- }.toString
- }
-
private def flatArguments = productIterator.flatMap {
case t: Traversable[_] => t
case single => single :: Nil
@@ -219,24 +208,16 @@ abstract class Expression extends TreeNode[Expression] {
override def simpleString: String = toString
- override def toString: String = prettyName + flatArguments.mkString("(", ",", ")")
+ override def toString: String = prettyName + flatArguments.mkString("(", ", ", ")")
/**
- * Returns the string representation of this expression that is safe to be put in
- * code comments of generated code.
+ * Returns SQL representation of this expression. For expressions extending [[NonSQLExpression]],
+ * this method may return an arbitrary user facing string.
*/
- protected def toCommentSafeString: String = this.toString
- .replace("*/", "\\*\\/")
- .replace("\\u", "\\\\u")
-
- /**
- * Returns SQL representation of this expression. For expressions that don't have a SQL
- * representation (e.g. `ScalaUDF`), this method should throw an `UnsupportedOperationException`.
- */
- @throws[UnsupportedOperationException](cause = "Expression doesn't have a SQL representation")
- def sql: String = throw new UnsupportedOperationException(
- s"Cannot map expression $this to its SQL representation"
- )
+ def sql: String = {
+ val childrenSQL = children.map(_.sql).mkString(", ")
+ s"$prettyName($childrenSQL)"
+ }
}
@@ -255,6 +236,19 @@ trait Unevaluable extends Expression {
/**
+ * Expressions that don't have SQL representation should extend this trait. Examples are
+ * `ScalaUDF`, `ScalaUDAF`, and object expressions like `MapObjects` and `Invoke`.
+ */
+trait NonSQLExpression extends Expression {
+ override def sql: String = {
+ transform {
+ case a: Attribute => new PrettyAttribute(a)
+ }.toString
+ }
+}
+
+
+/**
* An expression that is nondeterministic.
*/
trait Nondeterministic extends Expression {
@@ -373,8 +367,6 @@ abstract class UnaryExpression extends Expression {
"""
}
}
-
- override def sql: String = s"($prettyName(${child.sql}))"
}
@@ -477,8 +469,6 @@ abstract class BinaryExpression extends Expression {
"""
}
}
-
- override def sql: String = s"$prettyName(${left.sql}, ${right.sql})"
}
@@ -499,6 +489,8 @@ abstract class BinaryOperator extends BinaryExpression with ExpectsInputTypes {
def symbol: String
+ def sqlOperator: String = symbol
+
override def toString: String = s"($left $symbol $right)"
override def inputTypes: Seq[AbstractDataType] = Seq(inputType, inputType)
@@ -506,17 +498,17 @@ abstract class BinaryOperator extends BinaryExpression with ExpectsInputTypes {
override def checkInputDataTypes(): TypeCheckResult = {
// First check whether left and right have the same type, then check if the type is acceptable.
if (left.dataType != right.dataType) {
- TypeCheckResult.TypeCheckFailure(s"differing types in '$prettyString' " +
+ TypeCheckResult.TypeCheckFailure(s"differing types in '$sql' " +
s"(${left.dataType.simpleString} and ${right.dataType.simpleString}).")
} else if (!inputType.acceptsType(left.dataType)) {
- TypeCheckResult.TypeCheckFailure(s"'$prettyString' requires ${inputType.simpleString} type," +
+ TypeCheckResult.TypeCheckFailure(s"'$sql' requires ${inputType.simpleString} type," +
s" not ${left.dataType.simpleString}")
} else {
TypeCheckResult.TypeCheckSuccess
}
}
- override def sql: String = s"(${left.sql} $symbol ${right.sql})"
+ override def sql: String = s"(${left.sql} $sqlOperator ${right.sql})"
}
@@ -623,9 +615,4 @@ abstract class TernaryExpression extends Expression {
"""
}
}
-
- override def sql: String = {
- val childrenSQL = children.map(_.sql).mkString(", ")
- s"$prettyName($childrenSQL)"
- }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index 6816947..22184f1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -39,11 +39,11 @@ case class ScalaUDF(
dataType: DataType,
children: Seq[Expression],
inputTypes: Seq[DataType] = Nil)
- extends Expression with ImplicitCastInputTypes {
+ extends Expression with ImplicitCastInputTypes with NonSQLExpression {
override def nullable: Boolean = true
- override def toString: String = s"UDF(${children.mkString(",")})"
+ override def toString: String = s"UDF(${children.mkString(", ")})"
// scalastyle:off line.size.limit
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala
index f88a57a..ff3064a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.catalyst.expressions.aggregate
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
-import org.apache.spark.sql.catalyst.util.sequenceOption
import org.apache.spark.sql.types._
/** The mode of an [[AggregateFunction]]. */
@@ -92,8 +91,6 @@ private[sql] case class AggregateExpression(
AttributeSet(childReferences)
}
- override def prettyString: String = aggregateFunction.prettyString
-
override def toString: String = s"($aggregateFunction,mode=$mode,isDistinct=$isDistinct)"
override def sql: String = aggregateFunction.sql(isDistinct)
@@ -168,7 +165,7 @@ sealed abstract class AggregateFunction extends Expression with ImplicitCastInpu
}
def sql(isDistinct: Boolean): String = {
- val distinct = if (isDistinct) "DISTINCT " else " "
+ val distinct = if (isDistinct) "DISTINCT " else ""
s"$prettyName($distinct${children.map(_.sql).mkString(", ")})"
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 1cacd3f..5af2346 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -319,7 +319,9 @@ case class Remainder(left: Expression, right: Expression) extends BinaryArithmet
}
}
-case class MaxOf(left: Expression, right: Expression) extends BinaryArithmetic {
+case class MaxOf(left: Expression, right: Expression)
+ extends BinaryArithmetic with NonSQLExpression {
+
// TODO: Remove MaxOf and MinOf, and replace its usage with Greatest and Least.
override def inputType: AbstractDataType = TypeCollection.Ordered
@@ -373,7 +375,9 @@ case class MaxOf(left: Expression, right: Expression) extends BinaryArithmetic {
override def symbol: String = "max"
}
-case class MinOf(left: Expression, right: Expression) extends BinaryArithmetic {
+case class MinOf(left: Expression, right: Expression)
+ extends BinaryArithmetic with NonSQLExpression {
+
// TODO: Remove MaxOf and MinOf, and replace its usage with Greatest and Least.
override def inputType: AbstractDataType = TypeCollection.Ordered
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
index a97bd9e..4c90b3f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
@@ -123,4 +123,6 @@ case class BitwiseNot(child: Expression) extends UnaryExpression with ExpectsInp
}
protected override def nullSafeEval(input: Any): Any = not(input)
+
+ override def sql: String = s"~${child.sql}"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenFallback.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenFallback.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenFallback.scala
index f58a2da..1365ee4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenFallback.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenFallback.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.catalyst.expressions.codegen
import org.apache.spark.sql.catalyst.expressions.{Expression, LeafExpression, Nondeterministic}
+import org.apache.spark.sql.catalyst.util.toCommentSafeString
/**
* A trait that can be used to provide a fallback mode for expression code generation.
@@ -37,7 +38,7 @@ trait CodegenFallback extends Expression {
val objectTerm = ctx.freshName("obj")
if (nullable) {
s"""
- /* expression: ${this.toCommentSafeString} */
+ /* expression: ${toCommentSafeString(this.toString)} */
Object $objectTerm = ((Expression) references[$idx]).eval($input);
boolean ${ev.isNull} = $objectTerm == null;
${ctx.javaType(this.dataType)} ${ev.value} = ${ctx.defaultValue(this.dataType)};
@@ -48,7 +49,7 @@ trait CodegenFallback extends Expression {
} else {
ev.isNull = "false"
s"""
- /* expression: ${this.toCommentSafeString} */
+ /* expression: ${toCommentSafeString(this.toString)} */
Object $objectTerm = ((Expression) references[$idx]).eval($input);
${ctx.javaType(this.dataType)} ${ev.value} = (${ctx.boxedType(this.dataType)}) $objectTerm;
"""
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index 6b24fae..44cdc8d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -21,7 +21,7 @@ import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
-import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData, MapData}
+import org.apache.spark.sql.catalyst.util.{quoteIdentifier, ArrayData, GenericArrayData, MapData}
import org.apache.spark.sql.types._
////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -93,6 +93,8 @@ object ExtractValue {
}
}
+trait ExtractValue extends Expression
+
/**
* Returns the value of fields in the Struct `child`.
*
@@ -102,13 +104,15 @@ object ExtractValue {
* For example, when get field `yEAr` from `<year: int, month: int>`, we should pass in `yEAr`.
*/
case class GetStructField(child: Expression, ordinal: Int, name: Option[String] = None)
- extends UnaryExpression {
+ extends UnaryExpression with ExtractValue {
private[sql] lazy val childSchema = child.dataType.asInstanceOf[StructType]
override def dataType: DataType = childSchema(ordinal).dataType
override def nullable: Boolean = child.nullable || childSchema(ordinal).nullable
override def toString: String = s"$child.${name.getOrElse(childSchema(ordinal).name)}"
+ override def sql: String =
+ child.sql + s".${quoteIdentifier(name.getOrElse(childSchema(ordinal).name))}"
protected override def nullSafeEval(input: Any): Any =
input.asInstanceOf[InternalRow].get(ordinal, childSchema(ordinal).dataType)
@@ -130,12 +134,11 @@ case class GetStructField(child: Expression, ordinal: Int, name: Option[String]
}
})
}
-
- override def sql: String = child.sql + s".`${childSchema(ordinal).name}`"
}
/**
- * Returns the array of value of fields in the Array of Struct `child`.
+ * For a child whose data type is an array of structs, extracts the `ordinal`-th fields of all array
+ * elements, and returns them as a new array.
*
* No need to do type checking since it is handled by [[ExtractValue]].
*/
@@ -144,10 +147,11 @@ case class GetArrayStructFields(
field: StructField,
ordinal: Int,
numFields: Int,
- containsNull: Boolean) extends UnaryExpression {
+ containsNull: Boolean) extends UnaryExpression with ExtractValue {
override def dataType: DataType = ArrayType(field.dataType, containsNull)
override def toString: String = s"$child.${field.name}"
+ override def sql: String = s"${child.sql}.${quoteIdentifier(field.name)}"
protected override def nullSafeEval(input: Any): Any = {
val array = input.asInstanceOf[ArrayData]
@@ -204,12 +208,13 @@ case class GetArrayStructFields(
* We need to do type checking here as `ordinal` expression maybe unresolved.
*/
case class GetArrayItem(child: Expression, ordinal: Expression)
- extends BinaryExpression with ExpectsInputTypes {
+ extends BinaryExpression with ExpectsInputTypes with ExtractValue {
// We have done type checking for child in `ExtractValue`, so only need to check the `ordinal`.
override def inputTypes: Seq[AbstractDataType] = Seq(AnyDataType, IntegralType)
override def toString: String = s"$child[$ordinal]"
+ override def sql: String = s"${child.sql}[${ordinal.sql}]"
override def left: Expression = child
override def right: Expression = ordinal
@@ -250,7 +255,7 @@ case class GetArrayItem(child: Expression, ordinal: Expression)
* We need to do type checking here as `key` expression maybe unresolved.
*/
case class GetMapValue(child: Expression, key: Expression)
- extends BinaryExpression with ExpectsInputTypes {
+ extends BinaryExpression with ExpectsInputTypes with ExtractValue {
private def keyType = child.dataType.asInstanceOf[MapType].keyType
@@ -258,6 +263,7 @@ case class GetMapValue(child: Expression, key: Expression)
override def inputTypes: Seq[AbstractDataType] = Seq(AnyDataType, keyType)
override def toString: String = s"$child[$key]"
+ override def sql: String = s"${child.sql}[${key.sql}]"
override def left: Expression = child
override def right: Expression = key
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
index 1eff2c4..200c6a0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
@@ -35,7 +35,7 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi
TypeCheckResult.TypeCheckFailure(
s"type of predicate expression in If should be boolean, not ${predicate.dataType}")
} else if (trueValue.dataType != falseValue.dataType) {
- TypeCheckResult.TypeCheckFailure(s"differing types in '$prettyString' " +
+ TypeCheckResult.TypeCheckFailure(s"differing types in '$sql' " +
s"(${trueValue.dataType.simpleString} and ${falseValue.dataType.simpleString}).")
} else {
TypeCheckResult.TypeCheckSuccess
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index ca0892e..d7d768b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -238,37 +238,20 @@ case class Literal protected (value: Any, dataType: DataType)
}
override def sql: String = (value, dataType) match {
- case (_, NullType | _: ArrayType | _: MapType | _: StructType) if value == null =>
- "NULL"
-
- case _ if value == null =>
- s"CAST(NULL AS ${dataType.sql})"
-
+ case (_, NullType | _: ArrayType | _: MapType | _: StructType) if value == null => "NULL"
+ case _ if value == null => s"CAST(NULL AS ${dataType.sql})"
case (v: UTF8String, StringType) =>
// Escapes all backslashes and double quotes.
"\"" + v.toString.replace("\\", "\\\\").replace("\"", "\\\"") + "\""
-
- case (v: Byte, ByteType) =>
- s"CAST($v AS ${ByteType.simpleString.toUpperCase})"
-
- case (v: Short, ShortType) =>
- s"CAST($v AS ${ShortType.simpleString.toUpperCase})"
-
- case (v: Long, LongType) =>
- s"CAST($v AS ${LongType.simpleString.toUpperCase})"
-
- case (v: Float, FloatType) =>
- s"CAST($v AS ${FloatType.simpleString.toUpperCase})"
-
- case (v: Decimal, DecimalType.Fixed(precision, scale)) =>
- s"CAST($v AS ${DecimalType.simpleString.toUpperCase}($precision, $scale))"
-
- case (v: Int, DateType) =>
- s"DATE '${DateTimeUtils.toJavaDate(v)}'"
-
- case (v: Long, TimestampType) =>
- s"TIMESTAMP('${DateTimeUtils.toJavaTimestamp(v)}')"
-
+ case (v: Byte, ByteType) => v + "Y"
+ case (v: Short, ShortType) => v + "S"
+ case (v: Long, LongType) => v + "L"
+ // Float type doesn't have a suffix
+ case (v: Float, FloatType) => s"CAST($v AS ${FloatType.sql})"
+ case (v: Double, DoubleType) => v + "D"
+ case (v: Decimal, t: DecimalType) => s"CAST($v AS ${t.sql})"
+ case (v: Int, DateType) => s"DATE '${DateTimeUtils.toJavaDate(v)}'"
+ case (v: Long, TimestampType) => s"TIMESTAMP('${DateTimeUtils.toJavaTimestamp(v)}')"
case _ => value.toString
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
index 8b9a60f..bc2df0f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
@@ -42,6 +42,7 @@ abstract class LeafMathExpression(c: Double, name: String)
override def foldable: Boolean = true
override def nullable: Boolean = false
override def toString: String = s"$name()"
+ override def prettyName: String = name
override def eval(input: InternalRow): Any = c
}
@@ -59,6 +60,7 @@ abstract class UnaryMathExpression(val f: Double => Double, name: String)
override def dataType: DataType = DoubleType
override def nullable: Boolean = true
override def toString: String = s"$name($child)"
+ override def prettyName: String = name
protected override def nullSafeEval(input: Any): Any = {
f(input.asInstanceOf[Double])
@@ -70,8 +72,6 @@ abstract class UnaryMathExpression(val f: Double => Double, name: String)
override def genCode(ctx: CodegenContext, ev: ExprCode): String = {
defineCodeGen(ctx, ev, c => s"java.lang.Math.${funcName}($c)")
}
-
- override def sql: String = s"$name(${child.sql})"
}
abstract class UnaryLogExpression(f: Double => Double, name: String)
@@ -113,6 +113,8 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
override def toString: String = s"$name($left, $right)"
+ override def prettyName: String = name
+
override def dataType: DataType = DoubleType
protected override def nullSafeEval(input1: Any, input2: Any): Any = {
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 207b8a0..1af5437 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -22,6 +22,7 @@ import java.util.UUID
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.catalyst.util.quoteIdentifier
import org.apache.spark.sql.types._
object NamedExpression {
@@ -183,9 +184,9 @@ case class Alias(child: Expression, name: String)(
override def sql: String = {
val qualifiersString =
- if (qualifiers.isEmpty) "" else qualifiers.map("`" + _ + "`").mkString("", ".", ".")
+ if (qualifiers.isEmpty) "" else qualifiers.map(quoteIdentifier).mkString("", ".", ".")
val aliasName = if (isGenerated) s"$name#${exprId.id}" else s"$name"
- s"${child.sql} AS $qualifiersString`$aliasName`"
+ s"${child.sql} AS $qualifiersString${quoteIdentifier(aliasName)}"
}
}
@@ -300,9 +301,9 @@ case class AttributeReference(
override def sql: String = {
val qualifiersString =
- if (qualifiers.isEmpty) "" else qualifiers.map("`" + _ + "`").mkString("", ".", ".")
+ if (qualifiers.isEmpty) "" else qualifiers.map(quoteIdentifier).mkString("", ".", ".")
val attrRefName = if (isGenerated) s"$name#${exprId.id}" else s"$name"
- s"$qualifiersString`$attrRefName`"
+ s"$qualifiersString${quoteIdentifier(attrRefName)}"
}
}
@@ -310,10 +311,19 @@ case class AttributeReference(
* A place holder used when printing expressions without debugging information such as the
* expression id or the unresolved indicator.
*/
-case class PrettyAttribute(name: String, dataType: DataType = NullType)
+case class PrettyAttribute(
+ name: String,
+ dataType: DataType = NullType)
extends Attribute with Unevaluable {
+ def this(attribute: Attribute) = this(attribute.name, attribute match {
+ case a: AttributeReference => a.dataType
+ case a: PrettyAttribute => a.dataType
+ case _ => NullType
+ })
+
override def toString: String = name
+ override def sql: String = toString
override def withNullability(newNullability: Boolean): Attribute =
throw new UnsupportedOperationException
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
index 667d351..e22026d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
@@ -83,8 +83,6 @@ case class Coalesce(children: Seq[Expression]) extends Expression {
"""
}.mkString("\n")
}
-
- override def sql: String = s"$prettyName(${children.map(_.sql).mkString(", ")})"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
index fef6825..737346d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
@@ -46,7 +46,7 @@ case class StaticInvoke(
dataType: DataType,
functionName: String,
arguments: Seq[Expression] = Nil,
- propagateNull: Boolean = true) extends Expression {
+ propagateNull: Boolean = true) extends Expression with NonSQLExpression {
val objectName = staticObject.getName.stripSuffix("$")
@@ -108,7 +108,7 @@ case class Invoke(
targetObject: Expression,
functionName: String,
dataType: DataType,
- arguments: Seq[Expression] = Nil) extends Expression {
+ arguments: Seq[Expression] = Nil) extends Expression with NonSQLExpression {
override def nullable: Boolean = true
override def children: Seq[Expression] = arguments.+:(targetObject)
@@ -204,7 +204,7 @@ case class NewInstance(
arguments: Seq[Expression],
propagateNull: Boolean,
dataType: DataType,
- outerPointer: Option[Literal]) extends Expression {
+ outerPointer: Option[Literal]) extends Expression with NonSQLExpression {
private val className = cls.getName
override def nullable: Boolean = propagateNull
@@ -268,7 +268,7 @@ case class NewInstance(
*/
case class UnwrapOption(
dataType: DataType,
- child: Expression) extends UnaryExpression with ExpectsInputTypes {
+ child: Expression) extends UnaryExpression with NonSQLExpression with ExpectsInputTypes {
override def nullable: Boolean = true
@@ -298,7 +298,7 @@ case class UnwrapOption(
* @param optType The type of this option.
*/
case class WrapOption(child: Expression, optType: DataType)
- extends UnaryExpression with ExpectsInputTypes {
+ extends UnaryExpression with NonSQLExpression with ExpectsInputTypes {
override def dataType: DataType = ObjectType(classOf[Option[_]])
@@ -328,7 +328,7 @@ case class WrapOption(child: Expression, optType: DataType)
* manually, but will instead be passed into the provided lambda function.
*/
case class LambdaVariable(value: String, isNull: String, dataType: DataType) extends LeafExpression
- with Unevaluable {
+ with Unevaluable with NonSQLExpression {
override def nullable: Boolean = true
@@ -368,7 +368,7 @@ object MapObjects {
case class MapObjects private(
loopVar: LambdaVariable,
lambdaFunction: Expression,
- inputData: Expression) extends Expression {
+ inputData: Expression) extends Expression with NonSQLExpression {
private def itemAccessorMethod(dataType: DataType): String => String = dataType match {
case NullType =>
@@ -483,7 +483,7 @@ case class MapObjects private(
*
* @param children A list of expression to use as content of the external row.
*/
-case class CreateExternalRow(children: Seq[Expression]) extends Expression {
+case class CreateExternalRow(children: Seq[Expression]) extends Expression with NonSQLExpression {
override def dataType: DataType = ObjectType(classOf[Row])
override def nullable: Boolean = false
@@ -516,7 +516,8 @@ case class CreateExternalRow(children: Seq[Expression]) extends Expression {
* Serializes an input object using a generic serializer (Kryo or Java).
* @param kryo if true, use Kryo. Otherwise, use Java.
*/
-case class EncodeUsingSerializer(child: Expression, kryo: Boolean) extends UnaryExpression {
+case class EncodeUsingSerializer(child: Expression, kryo: Boolean)
+ extends UnaryExpression with NonSQLExpression {
override def eval(input: InternalRow): Any =
throw new UnsupportedOperationException("Only code-generated evaluation is supported")
@@ -558,7 +559,7 @@ case class EncodeUsingSerializer(child: Expression, kryo: Boolean) extends Unary
* @param kryo if true, use Kryo. Otherwise, use Java.
*/
case class DecodeUsingSerializer[T](child: Expression, tag: ClassTag[T], kryo: Boolean)
- extends UnaryExpression {
+ extends UnaryExpression with NonSQLExpression {
override protected def genCode(ctx: CodegenContext, ev: ExprCode): String = {
// Code to initialize the serializer.
@@ -596,7 +597,7 @@ case class DecodeUsingSerializer[T](child: Expression, tag: ClassTag[T], kryo: B
* Initialize a Java Bean instance by setting its field values via setters.
*/
case class InitializeJavaBean(beanInstance: Expression, setters: Map[String, Expression])
- extends Expression {
+ extends Expression with NonSQLExpression {
override def nullable: Boolean = beanInstance.nullable
override def children: Seq[Expression] = beanInstance +: setters.values.toSeq
@@ -638,7 +639,7 @@ case class InitializeJavaBean(beanInstance: Expression, setters: Map[String, Exp
* non-null `s`, `s.i` can't be null.
*/
case class AssertNotNull(child: Expression, walkedTypePath: Seq[String])
- extends UnaryExpression {
+ extends UnaryExpression with NonSQLExpression {
override def dataType: DataType = child.dataType
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index c290aa8..20818bf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -249,6 +249,8 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with
override def symbol: String = "&&"
+ override def sqlOperator: String = "AND"
+
override def eval(input: InternalRow): Any = {
val input1 = left.eval(input)
if (input1 == false) {
@@ -289,8 +291,6 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with
}
"""
}
-
- override def sql: String = s"(${left.sql} AND ${right.sql})"
}
@@ -300,6 +300,8 @@ case class Or(left: Expression, right: Expression) extends BinaryOperator with P
override def symbol: String = "||"
+ override def sqlOperator: String = "OR"
+
override def eval(input: InternalRow): Any = {
val input1 = left.eval(input)
if (input1 == true) {
@@ -340,8 +342,6 @@ case class Or(left: Expression, right: Expression) extends BinaryOperator with P
}
"""
}
-
- override def sql: String = s"(${left.sql} OR ${right.sql})"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index b965212..4be065b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -23,7 +23,6 @@ import java.util.{HashMap, Locale, Map => JMap}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.util.ArrayData
-import org.apache.spark.sql.catalyst.util.sequenceOption
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.{ByteArray, UTF8String}
@@ -62,8 +61,6 @@ case class Concat(children: Seq[Expression]) extends Expression with ImplicitCas
}
"""
}
-
- override def sql: String = s"$prettyName(${children.map(_.sql).mkString(", ")})"
}
@@ -156,8 +153,6 @@ case class ConcatWs(children: Seq[Expression])
"""
}
}
-
- override def sql: String = s"$prettyName(${children.map(_.sql).mkString(", ")})"
}
trait String2StringExpression extends ImplicitCastInputTypes {
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
index afe122f..9e6bd0e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
@@ -556,8 +556,8 @@ abstract class RankLike extends AggregateWindowFunction {
override def inputTypes: Seq[AbstractDataType] = children.map(_ => AnyDataType)
/** Store the values of the window 'order' expressions. */
- protected val orderAttrs = children.map{ expr =>
- AttributeReference(expr.prettyString, expr.dataType)()
+ protected val orderAttrs = children.map { expr =>
+ AttributeReference(expr.sql, expr.dataType)()
}
/** Predicate that detects if the order attributes have changed. */
@@ -636,7 +636,7 @@ case class DenseRank(children: Seq[Expression]) extends RankLike {
/**
* The PercentRank function computes the percentage ranking of a value in a group of values. The
- * result the rank of the minus one divided by the total number of rows in the partitiion minus one:
+ * result the rank of the minus one divided by the total number of rows in the partition minus one:
* (r - 1) / (n - 1). If a partition only contains one row, the function will return 0.
*
* The PercentRank function is similar to the CumeDist function, but it uses rank values instead of
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
index 7d155ac..a19ba38 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
@@ -575,7 +575,7 @@ case class Pivot(
override def output: Seq[Attribute] = groupByExprs.map(_.toAttribute) ++ aggregates match {
case agg :: Nil => pivotValues.map(value => AttributeReference(value.toString, agg.dataType)())
case _ => pivotValues.flatMap{ value =>
- aggregates.map(agg => AttributeReference(value + "_" + agg.prettyString, agg.dataType)())
+ aggregates.map(agg => AttributeReference(value + "_" + agg.sql, agg.dataType)())
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
index 7a0d0de..43f707f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
@@ -19,6 +19,9 @@ package org.apache.spark.sql.catalyst
import java.io._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.types.{NumericType, StringType}
+import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.Utils
package object util {
@@ -130,19 +133,31 @@ package object util {
ret
}
+ // Replaces attributes, string literals, complex type extractors with their pretty form so that
+ // generated column names don't contain back-ticks or double-quotes.
+ def usePrettyExpression(e: Expression): Expression = e transform {
+ case a: Attribute => new PrettyAttribute(a)
+ case Literal(s: UTF8String, StringType) => PrettyAttribute(s.toString, StringType)
+ case Literal(v, t: NumericType) if v != null => PrettyAttribute(v.toString, t)
+ case e: GetStructField =>
+ val name = e.name.getOrElse(e.childSchema(e.ordinal).name)
+ PrettyAttribute(usePrettyExpression(e.child).sql + "." + name, e.dataType)
+ case e: GetArrayStructFields =>
+ PrettyAttribute(usePrettyExpression(e.child) + "." + e.field.name, e.dataType)
+ }
+
+ def quoteIdentifier(name: String): String = {
+ // Escapes back-ticks within the identifier name with double-back-ticks, and then quote the
+ // identifier with back-ticks.
+ "`" + name.replace("`", "``") + "`"
+ }
+
/**
- * Converts a `Seq` of `Option[T]` to an `Option` of `Seq[T]`.
+ * Returns the string representation of this expression that is safe to be put in
+ * code comments of generated code.
*/
- def sequenceOption[T](seq: Seq[Option[T]]): Option[Seq[T]] = seq match {
- case xs if xs.isEmpty =>
- Option(Seq.empty[T])
-
- case xs =>
- for {
- head <- xs.head
- tail <- sequenceOption(xs.tail)
- } yield head +: tail
- }
+ def toCommentSafeString(str: String): String =
+ str.replace("*/", "\\*\\/").replace("\\u", "\\\\u")
/* FIX ME
implicit class debugLogging(a: Any) {
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
index 5dd661e..71ea5b8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
@@ -64,6 +64,8 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType {
override def toString: String = s"DecimalType($precision,$scale)"
+ override def sql: String = typeName.toUpperCase
+
/**
* Returns whether this DecimalType is wider than `other`. If yes, it means `other`
* can be casted into `this` safely without losing any precision or range.
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index e797d83..5ff5435 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -25,7 +25,7 @@ import org.json4s.JsonDSL._
import org.apache.spark.SparkException
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, InterpretedOrdering}
-import org.apache.spark.sql.catalyst.util.{DataTypeParser, LegacyTypeStringParser}
+import org.apache.spark.sql.catalyst.util.{quoteIdentifier, DataTypeParser, LegacyTypeStringParser}
/**
* :: DeveloperApi ::
@@ -280,7 +280,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
}
override def sql: String = {
- val fieldTypes = fields.map(f => s"`${f.name}`: ${f.dataType.sql}")
+ val fieldTypes = fields.map(f => s"${quoteIdentifier(f.name)}: ${f.dataType.sql}")
s"STRUCT<${fieldTypes.mkString(", ")}>"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 69f78a0..de9a56d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -127,22 +127,24 @@ class AnalysisErrorSuite extends AnalysisTest {
errorTest(
"single invalid type, single arg",
testRelation.select(TestFunction(dateLit :: Nil, IntegerType :: Nil).as('a)),
- "cannot resolve" :: "testfunction" :: "argument 1" :: "requires int type" ::
- "'null' is of date type" :: Nil)
+ "cannot resolve" :: "testfunction(CAST(NULL AS DATE))" :: "argument 1" :: "requires int type" ::
+ "'CAST(NULL AS DATE)' is of date type" :: Nil)
errorTest(
"single invalid type, second arg",
testRelation.select(
TestFunction(dateLit :: dateLit :: Nil, DateType :: IntegerType :: Nil).as('a)),
- "cannot resolve" :: "testfunction" :: "argument 2" :: "requires int type" ::
- "'null' is of date type" :: Nil)
+ "cannot resolve" :: "testfunction(CAST(NULL AS DATE), CAST(NULL AS DATE))" ::
+ "argument 2" :: "requires int type" ::
+ "'CAST(NULL AS DATE)' is of date type" :: Nil)
errorTest(
"multiple invalid type",
testRelation.select(
TestFunction(dateLit :: dateLit :: Nil, IntegerType :: IntegerType :: Nil).as('a)),
- "cannot resolve" :: "testfunction" :: "argument 1" :: "argument 2" ::
- "requires int type" :: "'null' is of date type" :: Nil)
+ "cannot resolve" :: "testfunction(CAST(NULL AS DATE), CAST(NULL AS DATE))" ::
+ "argument 1" :: "argument 2" :: "requires int type" ::
+ "'CAST(NULL AS DATE)' is of date type" :: Nil)
errorTest(
"invalid window function",
@@ -207,7 +209,7 @@ class AnalysisErrorSuite extends AnalysisTest {
errorTest(
"sorting by attributes are not from grouping expressions",
testRelation2.groupBy('a, 'c)('a, 'c, count('a).as("a3")).orderBy('b.asc),
- "cannot resolve" :: "'b'" :: "given input columns" :: "[a, c, a3]" :: Nil)
+ "cannot resolve" :: "'`b`'" :: "given input columns" :: "[a, c, a3]" :: Nil)
errorTest(
"non-boolean filters",
@@ -222,7 +224,7 @@ class AnalysisErrorSuite extends AnalysisTest {
errorTest(
"missing group by",
testRelation2.groupBy('a)('b),
- "'b'" :: "group by" :: Nil
+ "'`b`'" :: "group by" :: Nil
)
errorTest(
@@ -270,7 +272,7 @@ class AnalysisErrorSuite extends AnalysisTest {
"SPARK-9955: correct error message for aggregate",
// When parse SQL string, we will wrap aggregate expressions with UnresolvedAlias.
testRelation2.where('bad_column > 1).groupBy('a)(UnresolvedAlias(max('b))),
- "cannot resolve 'bad_column'" :: Nil)
+ "cannot resolve '`bad_column`'" :: Nil)
test("SPARK-6452 regression test") {
// CheckAnalysis should throw AnalysisException when Aggregate contains missing attribute(s)
@@ -311,7 +313,7 @@ class AnalysisErrorSuite extends AnalysisTest {
case true =>
assertAnalysisSuccess(plan, true)
case false =>
- assertAnalysisError(plan, "expression a cannot be used as a grouping expression" :: Nil)
+ assertAnalysisError(plan, "expression `a` cannot be used as a grouping expression" :: Nil)
}
}
@@ -372,7 +374,7 @@ class AnalysisErrorSuite extends AnalysisTest {
Some(EqualTo(AttributeReference("a", BinaryType)(exprId = ExprId(2)),
AttributeReference("c", BinaryType)(exprId = ExprId(4)))))
- assertAnalysisError(plan, "binary type expression a cannot be used in join conditions" :: Nil)
+ assertAnalysisError(plan, "binary type expression `a` cannot be used in join conditions" :: Nil)
val plan2 =
Join(
@@ -386,6 +388,6 @@ class AnalysisErrorSuite extends AnalysisTest {
Some(EqualTo(AttributeReference("a", MapType(IntegerType, StringType))(exprId = ExprId(2)),
AttributeReference("c", MapType(IntegerType, StringType))(exprId = ExprId(4)))))
- assertAnalysisError(plan2, "map type expression a cannot be used in join conditions" :: Nil)
+ assertAnalysisError(plan2, "map type expression `a` cannot be used in join conditions" :: Nil)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
index af214b7..2756c46 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
@@ -71,8 +71,17 @@ trait AnalysisTest extends PlanTest {
val e = intercept[AnalysisException] {
analyzer.checkAnalysis(analyzer.execute(inputPlan))
}
- assert(expectedErrors.map(_.toLowerCase).forall(e.getMessage.toLowerCase.contains),
- s"Expected to throw Exception contains: ${expectedErrors.mkString(", ")}, " +
- s"actually we get ${e.getMessage}")
+
+ if (!expectedErrors.map(_.toLowerCase).forall(e.getMessage.toLowerCase.contains)) {
+ fail(
+ s"""Exception message should contain the following substrings:
+ |
+ | ${expectedErrors.mkString("\n ")}
+ |
+ |Actual exception message:
+ |
+ | ${e.getMessage}
+ """.stripMargin)
+ }
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
index 59549e3..92c8496 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
@@ -41,7 +41,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
assertSuccess(expr)
}
assert(e.getMessage.contains(
- s"cannot resolve '${expr.prettyString}' due to data type mismatch:"))
+ s"cannot resolve '${expr.sql}' due to data type mismatch:"))
assert(e.getMessage.contains(errorMessage))
}
@@ -52,7 +52,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
def assertErrorForDifferingTypes(expr: Expression): Unit = {
assertError(expr,
- s"differing types in '${expr.prettyString}'")
+ s"differing types in '${expr.sql}'")
}
test("check types for unary arithmetic") {
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
index 8b02b63..3ad0dae 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
@@ -142,7 +142,7 @@ class EncoderResolutionSuite extends PlanTest {
}.message
assert(msg2 ==
s"""
- |Cannot up cast `b.b` from decimal(38,18) to bigint as it may truncate
+ |Cannot up cast `b`.`b` from decimal(38,18) to bigint as it may truncate
|The type path of the target object is:
|- field (class: "scala.Long", name: "b")
|- field (class: "org.apache.spark.sql.catalyst.encoders.StringLongClass", name: "b")
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 2ab091e..6c7929c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -24,7 +24,7 @@ import org.apache.spark.annotation.Experimental
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder}
import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.util.DataTypeParser
+import org.apache.spark.sql.catalyst.util.{usePrettyExpression, DataTypeParser}
import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
import org.apache.spark.sql.functions.lit
import org.apache.spark.sql.types._
@@ -104,10 +104,9 @@ class Column(protected[sql] val expr: Expression) extends Logging {
def this(name: String) = this(name match {
case "*" => UnresolvedStar(None)
- case _ if name.endsWith(".*") => {
+ case _ if name.endsWith(".*") =>
val parts = UnresolvedAttribute.parseAttributeName(name.substring(0, name.length - 2))
UnresolvedStar(Some(parts))
- }
case _ => UnresolvedAttribute.quotedString(name)
})
@@ -123,6 +122,8 @@ class Column(protected[sql] val expr: Expression) extends Logging {
// make it a NamedExpression.
case u: UnresolvedAttribute => UnresolvedAlias(u)
+ case u: UnresolvedExtractValue => UnresolvedAlias(u)
+
case expr: NamedExpression => expr
// Leave an unaliased generator with an empty list of names since the analyzer will generate
@@ -131,7 +132,7 @@ class Column(protected[sql] val expr: Expression) extends Logging {
case jt: JsonTuple => MultiAlias(jt, Nil)
- case func: UnresolvedFunction => UnresolvedAlias(func, Some(func.prettyString))
+ case func: UnresolvedFunction => UnresolvedAlias(func, Some(usePrettyExpression(func).sql))
// If we have a top level Cast, there is a chance to give it a better alias, if there is a
// NamedExpression under this Cast.
@@ -139,13 +140,13 @@ class Column(protected[sql] val expr: Expression) extends Logging {
case Cast(ne: NamedExpression, to) => UnresolvedAlias(Cast(ne, to))
} match {
case ne: NamedExpression => ne
- case other => Alias(expr, expr.prettyString)()
+ case other => Alias(expr, usePrettyExpression(expr).sql)()
}
- case expr: Expression => Alias(expr, expr.prettyString)()
+ case expr: Expression => Alias(expr, usePrettyExpression(expr).sql)()
}
- override def toString: String = expr.prettyString
+ override def toString: String = usePrettyExpression(expr).sql
override def equals(that: Any): Boolean = that match {
case that: Column => that.expr.equals(this.expr)
@@ -987,7 +988,7 @@ class Column(protected[sql] val expr: Expression) extends Logging {
if (extended) {
println(expr)
} else {
- println(expr.prettyString)
+ println(expr.sql)
}
// scalastyle:on println
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 76c09a2..9674450 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.optimizer.CombineUnions
import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.util.usePrettyExpression
import org.apache.spark.sql.execution.{ExplainCommand, FileRelation, LogicalRDD, Queryable, QueryExecution, SQLExecution}
import org.apache.spark.sql.execution.datasources.{CreateTableUsingAsSelect, LogicalRelation}
import org.apache.spark.sql.execution.datasources.json.JacksonGenerator
@@ -1359,7 +1360,8 @@ class DataFrame private[sql](
"min" -> ((child: Expression) => Min(child).toAggregateExpression()),
"max" -> ((child: Expression) => Max(child).toAggregateExpression()))
- val outputCols = (if (cols.isEmpty) numericColumns.map(_.prettyString) else cols).toList
+ val outputCols =
+ (if (cols.isEmpty) numericColumns.map(usePrettyExpression(_).sql) else cols).toList
val ret: Seq[Row] = if (outputCols.nonEmpty) {
val aggExprs = statistics.flatMap { case (_, colToAgg) =>
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/core/src/main/scala/org/apache/spark/sql/GroupedData.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/GroupedData.scala b/sql/core/src/main/scala/org/apache/spark/sql/GroupedData.scala
index c74ef2c..66ec0e7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/GroupedData.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/GroupedData.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.analysis.{Star, UnresolvedAlias, Unresolved
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Pivot}
+import org.apache.spark.sql.catalyst.util.usePrettyExpression
import org.apache.spark.sql.types.NumericType
/**
@@ -74,7 +75,7 @@ class GroupedData protected[sql](
private[this] def alias(expr: Expression): NamedExpression = expr match {
case u: UnresolvedAttribute => UnresolvedAlias(u)
case expr: NamedExpression => expr
- case expr: Expression => Alias(expr, expr.prettyString)()
+ case expr: Expression => Alias(expr, usePrettyExpression(expr).sql)()
}
private[this] def aggregateNumericColumns(colNames: String*)(f: Expression => AggregateFunction)
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala
index ad8564e..990eeb2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.plans.physical.Partitioning
import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.util.toCommentSafeString
import org.apache.spark.sql.execution.aggregate.TungstenAggregate
import org.apache.spark.sql.execution.joins.{BroadcastHashJoin, BuildLeft, BuildRight}
import org.apache.spark.sql.execution.metric.LongSQLMetricValue
@@ -252,7 +253,7 @@ case class WholeStageCodegen(plan: CodegenSupport, children: Seq[SparkPlan])
}
/** Codegened pipeline for:
- * ${plan.treeString.trim}
+ * ${toCommentSafeString(plan.treeString.trim)}
*/
class GeneratedIterator extends org.apache.spark.sql.execution.BufferedRowIterator {
http://git-wip-us.apache.org/repos/asf/spark/blob/d9efe63e/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
index 812e696..ab17844 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.aggregate
import org.apache.spark.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, MutableRow}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, MutableRow, _}
import org.apache.spark.sql.catalyst.expressions.aggregate.ImperativeAggregate
import org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
@@ -324,7 +324,7 @@ private[sql] case class ScalaUDAF(
udaf: UserDefinedAggregateFunction,
mutableAggBufferOffset: Int = 0,
inputAggBufferOffset: Int = 0)
- extends ImperativeAggregate with Logging {
+ extends ImperativeAggregate with NonSQLExpression with Logging {
override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
copy(mutableAggBufferOffset = newMutableAggBufferOffset)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org