You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/09/14 16:25:31 UTC
spark git commit: [SPARK-25431][SQL][EXAMPLES] Fix function examples
and unify the format of the example results.
Repository: spark
Updated Branches:
refs/heads/master a81ef9e1f -> 9c25d7f73
[SPARK-25431][SQL][EXAMPLES] Fix function examples and unify the format of the example results.
## What changes were proposed in this pull request?
There are some mistakes in examples of newly added functions. Also the format of the example results are not unified. We should fix and unify them.
## How was this patch tested?
Manually executed the examples.
Closes #22421 from ueshin/issues/SPARK-25431/fix_examples.
Authored-by: Takuya UESHIN <ue...@databricks.com>
Signed-off-by: gatorsmile <ga...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9c25d7f7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9c25d7f7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9c25d7f7
Branch: refs/heads/master
Commit: 9c25d7f735ed8c49c795babea3fda3cab226e7cb
Parents: a81ef9e
Author: Takuya UESHIN <ue...@databricks.com>
Authored: Fri Sep 14 09:25:27 2018 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Fri Sep 14 09:25:27 2018 -0700
----------------------------------------------------------------------
.../expressions/collectionOperations.scala | 49 ++++++++++----------
.../expressions/complexTypeCreator.scala | 4 +-
.../expressions/higherOrderFunctions.scala | 32 ++++++-------
3 files changed, 43 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9c25d7f7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index ea6fccc..3ad21ec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -131,7 +131,7 @@ case class Size(child: Expression) extends UnaryExpression with ExpectsInputType
examples = """
Examples:
> SELECT _FUNC_(map(1, 'a', 2, 'b'));
- [1,2]
+ [1, 2]
""")
case class MapKeys(child: Expression)
extends UnaryExpression with ExpectsInputTypes {
@@ -320,7 +320,7 @@ case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsI
examples = """
Examples:
> SELECT _FUNC_(map(1, 'a', 2, 'b'));
- ["a","b"]
+ ["a", "b"]
""")
case class MapValues(child: Expression)
extends UnaryExpression with ExpectsInputTypes {
@@ -348,7 +348,7 @@ case class MapValues(child: Expression)
examples = """
Examples:
> SELECT _FUNC_(map(1, 'a', 2, 'b'));
- [(1,"a"),(2,"b")]
+ [[1, "a"], [2, "b"]]
""",
since = "2.4.0")
case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInputTypes {
@@ -516,7 +516,7 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp
examples = """
Examples:
> SELECT _FUNC_(map(1, 'a', 2, 'b'), map(2, 'c', 3, 'd'));
- [[1 -> "a"], [2 -> "b"], [2 -> "c"], [3 -> "d"]]
+ [1 -> "a", 2 -> "b", 2 -> "c", 3 -> "d"]
""", since = "2.4.0")
case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
@@ -718,7 +718,7 @@ case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpres
examples = """
Examples:
> SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b')));
- {1:"a",2:"b"}
+ [1 -> "a", 2 -> "b"]
""",
since = "2.4.0")
case class MapFromEntries(child: Expression) extends UnaryExpression {
@@ -1071,7 +1071,7 @@ object ArraySortLike {
examples = """
Examples:
> SELECT _FUNC_(array('b', 'd', null, 'c', 'a'), true);
- [null,"a","b","c","d"]
+ [null, "a", "b", "c", "d"]
""")
// scalastyle:on line.size.limit
case class SortArray(base: Expression, ascendingOrder: Expression)
@@ -1129,7 +1129,7 @@ case class SortArray(base: Expression, ascendingOrder: Expression)
examples = """
Examples:
> SELECT _FUNC_(array('b', 'd', null, 'c', 'a'));
- ["a","b","c","d",null]
+ ["a", "b", "c", "d", null]
""",
since = "2.4.0")
// scalastyle:on line.size.limit
@@ -1254,7 +1254,7 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None)
examples = """
Examples:
> SELECT _FUNC_('Spark SQL');
- LQS krapS
+ "LQS krapS"
> SELECT _FUNC_(array(2, 1, 4, 3));
[3, 4, 1, 2]
""",
@@ -1634,9 +1634,9 @@ case class ArraysOverlap(left: Expression, right: Expression)
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3, 4), 2, 2);
- [2,3]
+ [2, 3]
> SELECT _FUNC_(array(1, 2, 3, 4), -2, 2);
- [3,4]
+ [3, 4]
""", since = "2.4.0")
// scalastyle:on line.size.limit
case class Slice(x: Expression, start: Expression, length: Expression)
@@ -1745,11 +1745,11 @@ case class Slice(x: Expression, start: Expression, length: Expression)
examples = """
Examples:
> SELECT _FUNC_(array('hello', 'world'), ' ');
- hello world
+ "hello world"
> SELECT _FUNC_(array('hello', null ,'world'), ' ');
- hello world
+ "hello world"
> SELECT _FUNC_(array('hello', null ,'world'), ' ', ',');
- hello , world
+ "hello , world"
""", since = "2.4.0")
case class ArrayJoin(
array: Expression,
@@ -2236,10 +2236,11 @@ case class ElementAt(left: Expression, right: Expression) extends GetMapValueUti
examples = """
Examples:
> SELECT _FUNC_('Spark', 'SQL');
- SparkSQL
+ "SparkSQL"
> SELECT _FUNC_(array(1, 2, 3), array(4, 5), array(6));
- | [1,2,3,4,5,6]
- """)
+ [1, 2, 3, 4, 5, 6]
+ """,
+ note = "Concat logic for arrays is available since 2.4.0.")
case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
private def allowedTypes: Seq[AbstractDataType] = Seq(StringType, BinaryType, ArrayType)
@@ -2427,8 +2428,8 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio
usage = "_FUNC_(arrayOfArrays) - Transforms an array of arrays into a single array.",
examples = """
Examples:
- > SELECT _FUNC_(array(array(1, 2), array(3, 4));
- [1,2,3,4]
+ > SELECT _FUNC_(array(array(1, 2), array(3, 4)));
+ [1, 2, 3, 4]
""",
since = "2.4.0")
case class Flatten(child: Expression) extends UnaryExpression {
@@ -2934,7 +2935,7 @@ object Sequence {
examples = """
Examples:
> SELECT _FUNC_('123', 2);
- ['123', '123']
+ ["123", "123"]
""",
since = "2.4.0")
case class ArrayRepeat(left: Expression, right: Expression)
@@ -3055,7 +3056,7 @@ case class ArrayRepeat(left: Expression, right: Expression)
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3, null, 3), 3);
- [1,2,null]
+ [1, 2, null]
""", since = "2.4.0")
case class ArrayRemove(left: Expression, right: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -3245,7 +3246,7 @@ trait ArraySetLike {
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3, null, 3));
- [1,2,3,null]
+ [1, 2, 3, null]
""", since = "2.4.0")
case class ArrayDistinct(child: Expression)
extends UnaryExpression with ArraySetLike with ExpectsInputTypes {
@@ -3421,7 +3422,7 @@ object ArrayBinaryLike {
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
- array(1, 2, 3, 5)
+ [1, 2, 3, 5]
""",
since = "2.4.0")
case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLike
@@ -3632,7 +3633,7 @@ object ArrayUnion {
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
- array(1, 3)
+ [1, 3]
""",
since = "2.4.0")
case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBinaryLike
@@ -3873,7 +3874,7 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
- array(2)
+ [2]
""",
since = "2.4.0")
case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryLike
http://git-wip-us.apache.org/repos/asf/spark/blob/9c25d7f7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index aba9c6c..117fa3e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -248,8 +248,8 @@ case class CreateMap(children: Seq[Expression]) extends Expression {
in keys should not be null""",
examples = """
Examples:
- > SELECT _FUNC_([1.0, 3.0], ['2', '4']);
- {1.0:"2",3.0:"4"}
+ > SELECT _FUNC_(array(1.0, 3.0), array('2', '4'));
+ [1.0 -> "2", 3.0 -> "4"]
""", since = "2.4.0")
case class MapFromArrays(left: Expression, right: Expression)
extends BinaryExpression with ExpectsInputTypes {
http://git-wip-us.apache.org/repos/asf/spark/blob/9c25d7f7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index 2bb6b20..3ef2ec0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -209,9 +209,9 @@ trait MapBasedSimpleHigherOrderFunction extends SimpleHigherOrderFunction {
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), x -> x + 1);
- array(2, 3, 4)
+ [2, 3, 4]
> SELECT _FUNC_(array(1, 2, 3), (x, i) -> x + i);
- array(1, 3, 5)
+ [1, 3, 5]
""",
since = "2.4.0")
case class ArrayTransform(
@@ -318,7 +318,7 @@ case class MapFilter(
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 1);
- array(1, 3)
+ [1, 3]
""",
since = "2.4.0")
case class ArrayFilter(
@@ -499,10 +499,10 @@ case class ArrayAggregate(
usage = "_FUNC_(expr, func) - Transforms elements in a map using the function.",
examples = """
Examples:
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1);
- map(array(2, 3, 4), array(1, 2, 3))
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
- map(array(2, 4, 6), array(1, 2, 3))
+ > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1);
+ [2 -> 1, 3 -> 2, 4 -> 3]
+ > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
+ [2 -> 1, 4 -> 2, 6 -> 3]
""",
since = "2.4.0")
case class TransformKeys(
@@ -549,10 +549,10 @@ case class TransformKeys(
usage = "_FUNC_(expr, func) - Transforms values in the map using the function.",
examples = """
Examples:
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1);
- map(array(1, 2, 3), array(2, 3, 4))
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
- map(array(1, 2, 3), array(2, 4, 6))
+ > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1);
+ [1 -> 2, 2 -> 3, 3 -> 4]
+ > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
+ [1 -> 2, 2 -> 4, 3 -> 6]
""",
since = "2.4.0")
case class TransformValues(
@@ -603,7 +603,7 @@ case class TransformValues(
examples = """
Examples:
> SELECT _FUNC_(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2));
- {1:"ax",2:"by"}
+ [1 -> "ax", 2 -> "by"]
""",
since = "2.4.0")
case class MapZipWith(left: Expression, right: Expression, function: Expression)
@@ -777,11 +777,11 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression)
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x));
- array(('a', 1), ('b', 2), ('c', 3))
- > SELECT _FUNC_(array(1, 2), array(3, 4), (x, y) -> x + y));
- array(4, 6)
+ [["a", 1], ["b", 2], ["c", 3]]
+ > SELECT _FUNC_(array(1, 2), array(3, 4), (x, y) -> x + y);
+ [4, 6]
> SELECT _FUNC_(array('a', 'b', 'c'), array('d', 'e', 'f'), (x, y) -> concat(x, y));
- array('ad', 'be', 'cf')
+ ["ad", "be", "cf"]
""",
since = "2.4.0")
// scalastyle:on line.size.limit
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org