You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/09/17 12:40:46 UTC
spark git commit: [SPARK-25431][SQL][EXAMPLES] Fix function examples
and the example results.
Repository: spark
Updated Branches:
refs/heads/master 0dd61ec47 -> 8cf6fd1c2
[SPARK-25431][SQL][EXAMPLES] Fix function examples and the example results.
## What changes were proposed in this pull request?
There are some mistakes in examples of newly added functions. Also the format of the example results are not unified. We should fix them.
## How was this patch tested?
Manually executed the examples.
Closes #22437 from ueshin/issues/SPARK-25431/fix_examples_2.
Authored-by: Takuya UESHIN <ue...@databricks.com>
Signed-off-by: hyukjinkwon <gu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8cf6fd1c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8cf6fd1c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8cf6fd1c
Branch: refs/heads/master
Commit: 8cf6fd1c2342949916fedb5a7f712177b22585fa
Parents: 0dd61ec
Author: Takuya UESHIN <ue...@databricks.com>
Authored: Mon Sep 17 20:40:42 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Mon Sep 17 20:40:42 2018 +0800
----------------------------------------------------------------------
.../expressions/collectionOperations.scala | 37 ++++++++++----------
.../expressions/complexTypeCreator.scala | 2 +-
.../expressions/higherOrderFunctions.scala | 32 ++++++++---------
3 files changed, 36 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8cf6fd1c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index ea6fccc..cc9edcf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -159,9 +159,9 @@ case class MapKeys(child: Expression)
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(2, 3, 4));
- [[1, 2], [2, 3], [3, 4]]
+ [{"0":1,"1":2},{"0":2,"1":3},{"0":3,"1":4}]
> SELECT _FUNC_(array(1, 2), array(2, 3), array(3, 4));
- [[1, 2, 3], [2, 3, 4]]
+ [{"0":1,"1":2,"2":3},{"0":2,"1":3,"2":4}]
""",
since = "2.4.0")
case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsInputTypes {
@@ -348,7 +348,7 @@ case class MapValues(child: Expression)
examples = """
Examples:
> SELECT _FUNC_(map(1, 'a', 2, 'b'));
- [(1,"a"),(2,"b")]
+ [{"key":1,"value":"a"},{"key":2,"value":"b"}]
""",
since = "2.4.0")
case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInputTypes {
@@ -516,7 +516,7 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp
examples = """
Examples:
> SELECT _FUNC_(map(1, 'a', 2, 'b'), map(2, 'c', 3, 'd'));
- [[1 -> "a"], [2 -> "b"], [2 -> "c"], [3 -> "d"]]
+ {1:"a",2:"c",3:"d"}
""", since = "2.4.0")
case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
@@ -1171,9 +1171,9 @@ case class ArraySort(child: Expression) extends UnaryExpression with ArraySortLi
examples = """
Examples:
> SELECT _FUNC_(array(1, 20, 3, 5));
- [3, 1, 5, 20]
+ [3,1,5,20]
> SELECT _FUNC_(array(1, 20, null, 3));
- [20, null, 3, 1]
+ [20,null,3,1]
""",
note = "The function is non-deterministic.",
since = "2.4.0")
@@ -1256,7 +1256,7 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None)
> SELECT _FUNC_('Spark SQL');
LQS krapS
> SELECT _FUNC_(array(2, 1, 4, 3));
- [3, 4, 1, 2]
+ [3,4,1,2]
""",
since = "1.5.0",
note = "Reverse logic for arrays is available since 2.4.0."
@@ -2123,7 +2123,7 @@ case class ArrayPosition(left: Expression, right: Expression)
> SELECT _FUNC_(array(1, 2, 3), 2);
2
> SELECT _FUNC_(map(1, 'a', 2, 'b'), 2);
- "b"
+ b
""",
since = "2.4.0")
case class ElementAt(left: Expression, right: Expression) extends GetMapValueUtil {
@@ -2238,8 +2238,9 @@ case class ElementAt(left: Expression, right: Expression) extends GetMapValueUti
> SELECT _FUNC_('Spark', 'SQL');
SparkSQL
> SELECT _FUNC_(array(1, 2, 3), array(4, 5), array(6));
- | [1,2,3,4,5,6]
- """)
+ [1,2,3,4,5,6]
+ """,
+ note = "Concat logic for arrays is available since 2.4.0.")
case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
private def allowedTypes: Seq[AbstractDataType] = Seq(StringType, BinaryType, ArrayType)
@@ -2427,7 +2428,7 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio
usage = "_FUNC_(arrayOfArrays) - Transforms an array of arrays into a single array.",
examples = """
Examples:
- > SELECT _FUNC_(array(array(1, 2), array(3, 4));
+ > SELECT _FUNC_(array(array(1, 2), array(3, 4)));
[1,2,3,4]
""",
since = "2.4.0")
@@ -2556,11 +2557,11 @@ case class Flatten(child: Expression) extends UnaryExpression {
examples = """
Examples:
> SELECT _FUNC_(1, 5);
- [1, 2, 3, 4, 5]
+ [1,2,3,4,5]
> SELECT _FUNC_(5, 1);
- [5, 4, 3, 2, 1]
+ [5,4,3,2,1]
> SELECT _FUNC_(to_date('2018-01-01'), to_date('2018-03-01'), interval 1 month);
- [2018-01-01, 2018-02-01, 2018-03-01]
+ [2018-01-01,2018-02-01,2018-03-01]
""",
since = "2.4.0"
)
@@ -2934,7 +2935,7 @@ object Sequence {
examples = """
Examples:
> SELECT _FUNC_('123', 2);
- ['123', '123']
+ ["123","123"]
""",
since = "2.4.0")
case class ArrayRepeat(left: Expression, right: Expression)
@@ -3421,7 +3422,7 @@ object ArrayBinaryLike {
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
- array(1, 2, 3, 5)
+ [1,2,3,5]
""",
since = "2.4.0")
case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLike
@@ -3632,7 +3633,7 @@ object ArrayUnion {
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
- array(1, 3)
+ [1,3]
""",
since = "2.4.0")
case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBinaryLike
@@ -3873,7 +3874,7 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
- array(2)
+ [2]
""",
since = "2.4.0")
case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryLike
http://git-wip-us.apache.org/repos/asf/spark/blob/8cf6fd1c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index aba9c6c..fd8b5e9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -248,7 +248,7 @@ case class CreateMap(children: Seq[Expression]) extends Expression {
in keys should not be null""",
examples = """
Examples:
- > SELECT _FUNC_([1.0, 3.0], ['2', '4']);
+ > SELECT _FUNC_(array(1.0, 3.0), array('2', '4'));
{1.0:"2",3.0:"4"}
""", since = "2.4.0")
case class MapFromArrays(left: Expression, right: Expression)
http://git-wip-us.apache.org/repos/asf/spark/blob/8cf6fd1c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index 2bb6b20..b07d946 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -209,9 +209,9 @@ trait MapBasedSimpleHigherOrderFunction extends SimpleHigherOrderFunction {
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), x -> x + 1);
- array(2, 3, 4)
+ [2,3,4]
> SELECT _FUNC_(array(1, 2, 3), (x, i) -> x + i);
- array(1, 3, 5)
+ [1,3,5]
""",
since = "2.4.0")
case class ArrayTransform(
@@ -268,7 +268,7 @@ usage = "_FUNC_(expr, func) - Filters entries in a map using the function.",
examples = """
Examples:
> SELECT _FUNC_(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v);
- [1 -> 0, 3 -> -1]
+ {1:0,3:-1}
""",
since = "2.4.0")
case class MapFilter(
@@ -318,7 +318,7 @@ case class MapFilter(
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 1);
- array(1, 3)
+ [1,3]
""",
since = "2.4.0")
case class ArrayFilter(
@@ -499,10 +499,10 @@ case class ArrayAggregate(
usage = "_FUNC_(expr, func) - Transforms elements in a map using the function.",
examples = """
Examples:
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1);
- map(array(2, 3, 4), array(1, 2, 3))
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
- map(array(2, 4, 6), array(1, 2, 3))
+ > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1);
+ {2:1,3:2,4:3}
+ > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
+ {2:1,4:2,6:3}
""",
since = "2.4.0")
case class TransformKeys(
@@ -549,10 +549,10 @@ case class TransformKeys(
usage = "_FUNC_(expr, func) - Transforms values in the map using the function.",
examples = """
Examples:
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1);
- map(array(1, 2, 3), array(2, 3, 4))
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
- map(array(1, 2, 3), array(2, 4, 6))
+ > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1);
+ {1:2,2:3,3:4}
+ > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
+ {1:2,2:4,3:6}
""",
since = "2.4.0")
case class TransformValues(
@@ -777,11 +777,11 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression)
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x));
- array(('a', 1), ('b', 2), ('c', 3))
- > SELECT _FUNC_(array(1, 2), array(3, 4), (x, y) -> x + y));
- array(4, 6)
+ [{"y":"a","x":1},{"y":"b","x":2},{"y":"c","x":3}]
+ > SELECT _FUNC_(array(1, 2), array(3, 4), (x, y) -> x + y);
+ [4,6]
> SELECT _FUNC_(array('a', 'b', 'c'), array('d', 'e', 'f'), (x, y) -> concat(x, y));
- array('ad', 'be', 'cf')
+ ["ad","be","cf"]
""",
since = "2.4.0")
// scalastyle:on line.size.limit
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org