You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2018/09/12 12:31:01 UTC
spark git commit: [SPARK-25371][SQL][BACKPORT-2.3] struct() should
allow being called with 0 args
Repository: spark
Updated Branches:
refs/heads/branch-2.3 18688d370 -> d8ec5ff75
[SPARK-25371][SQL][BACKPORT-2.3] struct() should allow being called with 0 args
## What changes were proposed in this pull request?
SPARK-21281 introduced a check for the inputs of `CreateStructLike` to be non-empty. This means that `struct()`, which was previously considered valid, now throws an Exception. This behavior change was introduced in 2.3.0. The change may break users' application on upgrade and it causes `VectorAssembler` to fail when an empty `inputCols` is defined.
The PR removes the added check making `struct()` valid again.
## How was this patch tested?
added UT
Closes #22391 from mgaido91/SPARK-25371_2.3.
Authored-by: Marco Gaido <ma...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d8ec5ff7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d8ec5ff7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d8ec5ff7
Branch: refs/heads/branch-2.3
Commit: d8ec5ff7502451f61f2be3871d9bd1461b26accd
Parents: 18688d3
Author: Marco Gaido <ma...@gmail.com>
Authored: Wed Sep 12 20:30:18 2018 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Wed Sep 12 20:30:18 2018 +0800
----------------------------------------------------------------------
.../org/apache/spark/ml/feature/VectorAssemblerSuite.scala | 6 ++++++
.../spark/sql/catalyst/expressions/complexTypeCreator.scala | 5 +----
.../scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala | 2 --
3 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d8ec5ff7/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
index eca065f..96e3a60 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
@@ -147,4 +147,10 @@ class VectorAssemblerSuite
.filter(vectorUDF($"features") > 1)
.count() == 1)
}
+
+ test("SPARK-25371: VectorAssembler with empty inputCols") {
+ val vectorAssembler = new VectorAssembler().setInputCols(Array()).setOutputCol("a")
+ val output = vectorAssembler.transform(Seq(1).toDF("x"))
+ assert(output.select("a").limit(1).collect().head == Row(Vectors.sparse(0, Seq.empty)))
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d8ec5ff7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 047b80a..1f87ee7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -305,10 +305,7 @@ trait CreateNamedStructLike extends Expression {
}
override def checkInputDataTypes(): TypeCheckResult = {
- if (children.length < 1) {
- TypeCheckResult.TypeCheckFailure(
- s"input to function $prettyName requires at least one argument")
- } else if (children.size % 2 != 0) {
+ if (children.size % 2 != 0) {
TypeCheckResult.TypeCheckFailure(s"$prettyName expects an even number of arguments.")
} else {
val invalidNames = nameExprs.filterNot(e => e.foldable && e.dataType == StringType)
http://git-wip-us.apache.org/repos/asf/spark/blob/d8ec5ff7/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index 50e4759..67d8ccb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -465,8 +465,6 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
val funcsMustHaveAtLeastOneArg =
("coalesce", (df: DataFrame) => df.select(coalesce())) ::
("coalesce", (df: DataFrame) => df.selectExpr("coalesce()")) ::
- ("named_struct", (df: DataFrame) => df.select(struct())) ::
- ("named_struct", (df: DataFrame) => df.selectExpr("named_struct()")) ::
("hash", (df: DataFrame) => df.select(hash())) ::
("hash", (df: DataFrame) => df.selectExpr("hash()")) :: Nil
funcsMustHaveAtLeastOneArg.foreach { case (name, func) =>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org