You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2016/11/03 03:56:41 UTC
[2/3] spark git commit: [SPARK-17963][SQL][DOCUMENTATION] Add
examples (extend) in each expression and improve documentation
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb2ca8e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
index 5152265..a60494a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
@@ -139,8 +139,12 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
* evaluated by the optimizer during constant folding.
*/
@ExpressionDescription(
- usage = "_FUNC_() - Returns Euler's number, E.",
- extended = "> SELECT _FUNC_();\n 2.718281828459045")
+ usage = "_FUNC_() - Returns Euler's number, e.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_();
+ 2.718281828459045
+ """)
case class EulerNumber() extends LeafMathExpression(math.E, "E")
/**
@@ -148,8 +152,12 @@ case class EulerNumber() extends LeafMathExpression(math.E, "E")
* evaluated by the optimizer during constant folding.
*/
@ExpressionDescription(
- usage = "_FUNC_() - Returns PI.",
- extended = "> SELECT _FUNC_();\n 3.141592653589793")
+ usage = "_FUNC_() - Returns pi.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_();
+ 3.141592653589793
+ """)
case class Pi() extends LeafMathExpression(math.Pi, "PI")
////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -158,29 +166,61 @@ case class Pi() extends LeafMathExpression(math.Pi, "PI")
////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the arc cosine of x if -1<=x<=1 or NaN otherwise.",
- extended = "> SELECT _FUNC_(1);\n 0.0\n> SELECT _FUNC_(2);\n NaN")
+ usage = "_FUNC_(expr) - Returns the inverse cosine (a.k.a. arccosine) of `expr` if -1<=`expr`<=1 or NaN otherwise.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(1);
+ 0.0
+ > SELECT _FUNC_(2);
+ NaN
+ """)
+// scalastyle:on line.size.limit
case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS")
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the arc sin of x if -1<=x<=1 or NaN otherwise.",
- extended = "> SELECT _FUNC_(0);\n 0.0\n> SELECT _FUNC_(2);\n NaN")
+ usage = "_FUNC_(expr) - Returns the inverse sine (a.k.a. arcsine) the arc sin of `expr` if -1<=`expr`<=1 or NaN otherwise.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 0.0
+ > SELECT _FUNC_(2);
+ NaN
+ """)
+// scalastyle:on line.size.limit
case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN")
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the arc tangent.",
- extended = "> SELECT _FUNC_(0);\n 0.0")
+ usage = "_FUNC_(expr) - Returns the inverse tangent (a.k.a. arctangent).",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 0.0
+ """)
+// scalastyle:on line.size.limit
case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the cube root of a double value.",
- extended = "> SELECT _FUNC_(27.0);\n 3.0")
+ usage = "_FUNC_(expr) - Returns the cube root of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(27.0);
+ 3.0
+ """)
case class Cbrt(child: Expression) extends UnaryMathExpression(math.cbrt, "CBRT")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the smallest integer not smaller than x.",
- extended = "> SELECT _FUNC_(-0.1);\n 0\n> SELECT _FUNC_(5);\n 5")
+ usage = "_FUNC_(expr) - Returns the smallest integer not smaller than `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(-0.1);
+ 0
+ > SELECT _FUNC_(5);
+ 5
+ """)
case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL") {
override def dataType: DataType = child.dataType match {
case dt @ DecimalType.Fixed(_, 0) => dt
@@ -208,13 +248,21 @@ case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL"
}
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the cosine of x.",
- extended = "> SELECT _FUNC_(0);\n 1.0")
+ usage = "_FUNC_(expr) - Returns the cosine of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 1.0
+ """)
case class Cos(child: Expression) extends UnaryMathExpression(math.cos, "COS")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the hyperbolic cosine of x.",
- extended = "> SELECT _FUNC_(0);\n 1.0")
+ usage = "_FUNC_(expr) - Returns the hyperbolic cosine of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 1.0
+ """)
case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH")
/**
@@ -225,8 +273,14 @@ case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH"
* @param toBaseExpr to which base
*/
@ExpressionDescription(
- usage = "_FUNC_(num, from_base, to_base) - Convert num from from_base to to_base.",
- extended = "> SELECT _FUNC_('100', 2, 10);\n '4'\n> SELECT _FUNC_(-10, 16, -10);\n '16'")
+ usage = "_FUNC_(num, from_base, to_base) - Convert `num` from `from_base` to `to_base`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('100', 2, 10);
+ 4
+ > SELECT _FUNC_(-10, 16, -10);
+ 16
+ """)
case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -256,18 +310,32 @@ case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expre
}
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns e to the power of x.",
- extended = "> SELECT _FUNC_(0);\n 1.0")
+ usage = "_FUNC_(expr) - Returns e to the power of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 1.0
+ """)
case class Exp(child: Expression) extends UnaryMathExpression(math.exp, "EXP")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns exp(x) - 1.",
- extended = "> SELECT _FUNC_(0);\n 0.0")
+ usage = "_FUNC_(expr) - Returns exp(`expr`) - 1.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 0.0
+ """)
case class Expm1(child: Expression) extends UnaryMathExpression(math.expm1, "EXPM1")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the largest integer not greater than x.",
- extended = "> SELECT _FUNC_(-0.1);\n -1\n> SELECT _FUNC_(5);\n 5")
+ usage = "_FUNC_(expr) - Returns the largest integer not greater than `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(-0.1);
+ -1
+ > SELECT _FUNC_(5);
+ 5
+ """)
case class Floor(child: Expression) extends UnaryMathExpression(math.floor, "FLOOR") {
override def dataType: DataType = child.dataType match {
case dt @ DecimalType.Fixed(_, 0) => dt
@@ -326,8 +394,12 @@ object Factorial {
}
@ExpressionDescription(
- usage = "_FUNC_(n) - Returns n factorial for n is [0..20]. Otherwise, NULL.",
- extended = "> SELECT _FUNC_(5);\n 120")
+ usage = "_FUNC_(expr) - Returns the factorial of `expr`. `expr` is [0..20]. Otherwise, null.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(5);
+ 120
+ """)
case class Factorial(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def inputTypes: Seq[DataType] = Seq(IntegerType)
@@ -361,13 +433,21 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas
}
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the natural logarithm of x with base e.",
- extended = "> SELECT _FUNC_(1);\n 0.0")
+ usage = "_FUNC_(expr) - Returns the natural logarithm (base e) of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(1);
+ 0.0
+ """)
case class Log(child: Expression) extends UnaryLogExpression(math.log, "LOG")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the logarithm of x with base 2.",
- extended = "> SELECT _FUNC_(2);\n 1.0")
+ usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 2.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(2);
+ 1.0
+ """)
case class Log2(child: Expression)
extends UnaryLogExpression((x: Double) => math.log(x) / math.log(2), "LOG2") {
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -384,71 +464,127 @@ case class Log2(child: Expression)
}
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the logarithm of x with base 10.",
- extended = "> SELECT _FUNC_(10);\n 1.0")
+ usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 10.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(10);
+ 1.0
+ """)
case class Log10(child: Expression) extends UnaryLogExpression(math.log10, "LOG10")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns log(1 + x).",
- extended = "> SELECT _FUNC_(0);\n 0.0")
+ usage = "_FUNC_(expr) - Returns log(1 + `expr`).",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 0.0
+ """)
case class Log1p(child: Expression) extends UnaryLogExpression(math.log1p, "LOG1P") {
protected override val yAsymptote: Double = -1.0
}
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(x, d) - Return the rounded x at d decimal places.",
- extended = "> SELECT _FUNC_(12.3456, 1);\n 12.3")
+ usage = "_FUNC_(expr) - Returns the double value that is closest in value to the argument and is equal to a mathematical integer.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(12.3456);
+ 12.0
+ """)
+// scalastyle:on line.size.limit
case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND") {
override def funcName: String = "rint"
}
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the sign of x.",
- extended = "> SELECT _FUNC_(40);\n 1.0")
+ usage = "_FUNC_(expr) - Returns -1.0, 0.0 or 1.0 as `expr` is negative, 0 or positive.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(40);
+ 1.0
+ """)
case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "SIGNUM")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the sine of x.",
- extended = "> SELECT _FUNC_(0);\n 0.0")
+ usage = "_FUNC_(expr) - Returns the sine of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 0.0
+ """)
case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the hyperbolic sine of x.",
- extended = "> SELECT _FUNC_(0);\n 0.0")
+ usage = "_FUNC_(expr) - Returns the hyperbolic sine of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 0.0
+ """)
case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the square root of x.",
- extended = "> SELECT _FUNC_(4);\n 2.0")
+ usage = "_FUNC_(expr) - Returns the square root of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(4);
+ 2.0
+ """)
case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the tangent of x.",
- extended = "> SELECT _FUNC_(0);\n 0.0")
+ usage = "_FUNC_(expr) - Returns the tangent of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 0.0
+ """)
case class Tan(child: Expression) extends UnaryMathExpression(math.tan, "TAN")
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns the hyperbolic tangent of x.",
- extended = "> SELECT _FUNC_(0);\n 0.0")
+ usage = "_FUNC_(expr) - Returns the hyperbolic tangent of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0);
+ 0.0
+ """)
case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH")
@ExpressionDescription(
- usage = "_FUNC_(x) - Converts radians to degrees.",
- extended = "> SELECT _FUNC_(3.141592653589793);\n 180.0")
+ usage = "_FUNC_(expr) - Converts radians to degrees.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(3.141592653589793);
+ 180.0
+ """)
case class ToDegrees(child: Expression) extends UnaryMathExpression(math.toDegrees, "DEGREES") {
override def funcName: String = "toDegrees"
}
@ExpressionDescription(
- usage = "_FUNC_(x) - Converts degrees to radians.",
- extended = "> SELECT _FUNC_(180);\n 3.141592653589793")
+ usage = "_FUNC_(expr) - Converts degrees to radians.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(180);
+ 3.141592653589793
+ """)
case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadians, "RADIANS") {
override def funcName: String = "toRadians"
}
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(x) - Returns x in binary.",
- extended = "> SELECT _FUNC_(13);\n '1101'")
+ usage = "_FUNC_(expr) - Returns the string representation of the long value `expr` represented in binary.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(13);
+ 1101
+ > SELECT _FUNC_(-13);
+ 1111111111111111111111111111111111111111111111111111111111110011
+ > SELECT _FUNC_(13.3);
+ 1101
+ """)
+// scalastyle:on line.size.limit
case class Bin(child: Expression)
extends UnaryExpression with Serializable with ImplicitCastInputTypes {
@@ -541,8 +677,14 @@ object Hex {
* and returns the resulting STRING. Negative numbers would be treated as two's complement.
*/
@ExpressionDescription(
- usage = "_FUNC_(x) - Convert the argument to hexadecimal.",
- extended = "> SELECT _FUNC_(17);\n '11'\n> SELECT _FUNC_('Spark SQL');\n '537061726B2053514C'")
+ usage = "_FUNC_(expr) - Converts `expr` to hexadecimal.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(17);
+ 11
+ > SELECT _FUNC_('Spark SQL');
+ 537061726B2053514C
+ """)
case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def inputTypes: Seq[AbstractDataType] =
@@ -572,8 +714,12 @@ case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInput
* Resulting characters are returned as a byte array.
*/
@ExpressionDescription(
- usage = "_FUNC_(x) - Converts hexadecimal argument to binary.",
- extended = "> SELECT decode(_FUNC_('537061726B2053514C'),'UTF-8');\n 'Spark SQL'")
+ usage = "_FUNC_(expr) - Converts hexadecimal `expr` to binary.",
+ extended = """
+ Examples:
+ > SELECT decode(_FUNC_('537061726B2053514C'), 'UTF-8');
+ Spark SQL
+ """)
case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def inputTypes: Seq[AbstractDataType] = Seq(StringType)
@@ -602,9 +748,15 @@ case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInp
////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(x,y) - Returns the arc tangent2.",
- extended = "> SELECT _FUNC_(0, 0);\n 0.0")
+ usage = "_FUNC_(expr1, expr2) - Returns the angle in radians between the positive x-axis of a plane and the point given by the coordinates (`expr1`, `expr2`).",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0, 0);
+ 0.0
+ """)
+// scalastyle:on line.size.limit
case class Atan2(left: Expression, right: Expression)
extends BinaryMathExpression(math.atan2, "ATAN2") {
@@ -619,8 +771,12 @@ case class Atan2(left: Expression, right: Expression)
}
@ExpressionDescription(
- usage = "_FUNC_(x1, x2) - Raise x1 to the power of x2.",
- extended = "> SELECT _FUNC_(2, 3);\n 8.0")
+ usage = "_FUNC_(expr1, expr2) - Raises `expr1` to the power of `expr2`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(2, 3);
+ 8.0
+ """)
case class Pow(left: Expression, right: Expression)
extends BinaryMathExpression(math.pow, "POWER") {
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -636,8 +792,12 @@ case class Pow(left: Expression, right: Expression)
* @param right number of bits to left shift.
*/
@ExpressionDescription(
- usage = "_FUNC_(a, b) - Bitwise left shift.",
- extended = "> SELECT _FUNC_(2, 1);\n 4")
+ usage = "_FUNC_(base, expr) - Bitwise left shift.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(2, 1);
+ 4
+ """)
case class ShiftLeft(left: Expression, right: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -660,14 +820,18 @@ case class ShiftLeft(left: Expression, right: Expression)
/**
- * Bitwise right shift.
+ * Bitwise (signed) right shift.
*
* @param left the base number to shift.
* @param right number of bits to right shift.
*/
@ExpressionDescription(
- usage = "_FUNC_(a, b) - Bitwise right shift.",
- extended = "> SELECT _FUNC_(4, 1);\n 2")
+ usage = "_FUNC_(base, expr) - Bitwise (signed) right shift.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(4, 1);
+ 2
+ """)
case class ShiftRight(left: Expression, right: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -696,8 +860,12 @@ case class ShiftRight(left: Expression, right: Expression)
* @param right the number of bits to right shift.
*/
@ExpressionDescription(
- usage = "_FUNC_(a, b) - Bitwise unsigned right shift.",
- extended = "> SELECT _FUNC_(4, 1);\n 2")
+ usage = "_FUNC_(base, expr) - Bitwise unsigned right shift.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(4, 1);
+ 2
+ """)
case class ShiftRightUnsigned(left: Expression, right: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -719,8 +887,12 @@ case class ShiftRightUnsigned(left: Expression, right: Expression)
}
@ExpressionDescription(
- usage = "_FUNC_(a, b) - Returns sqrt(a**2 + b**2).",
- extended = "> SELECT _FUNC_(3, 4);\n 5.0")
+ usage = "_FUNC_(expr1, expr2) - Returns sqrt(`expr1`**2 + `expr2`**2).",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(3, 4);
+ 5.0
+ """)
case class Hypot(left: Expression, right: Expression)
extends BinaryMathExpression(math.hypot, "HYPOT")
@@ -732,8 +904,12 @@ case class Hypot(left: Expression, right: Expression)
* @param right the number to compute the logarithm of.
*/
@ExpressionDescription(
- usage = "_FUNC_(b, x) - Returns the logarithm of x with base b.",
- extended = "> SELECT _FUNC_(10, 100);\n 2.0")
+ usage = "_FUNC_(base, expr) - Returns the logarithm of `expr` with `base`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(10, 100);
+ 2.0
+ """)
case class Logarithm(left: Expression, right: Expression)
extends BinaryMathExpression((c1, c2) => math.log(c2) / math.log(c1), "LOG") {
@@ -956,9 +1132,15 @@ abstract class RoundBase(child: Expression, scale: Expression,
* Round an expression to d decimal places using HALF_UP rounding mode.
* round(2.5) == 3.0, round(3.5) == 4.0.
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(x, d) - Round x to d decimal places using HALF_UP rounding mode.",
- extended = "> SELECT _FUNC_(2.5, 0);\n 3.0")
+ usage = "_FUNC_(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_UP rounding mode.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(2.5, 0);
+ 3.0
+ """)
+// scalastyle:on line.size.limit
case class Round(child: Expression, scale: Expression)
extends RoundBase(child, scale, BigDecimal.RoundingMode.HALF_UP, "ROUND_HALF_UP")
with Serializable with ImplicitCastInputTypes {
@@ -970,9 +1152,15 @@ case class Round(child: Expression, scale: Expression)
* also known as Gaussian rounding or bankers' rounding.
* round(2.5) = 2.0, round(3.5) = 4.0.
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(x, d) - Round x to d decimal places using HALF_EVEN rounding mode.",
- extended = "> SELECT _FUNC_(2.5, 0);\n 2.0")
+ usage = "_FUNC_(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_EVEN rounding mode.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(2.5, 0);
+ 2.0
+ """)
+// scalastyle:on line.size.limit
case class BRound(child: Expression, scale: Expression)
extends RoundBase(child, scale, BigDecimal.RoundingMode.HALF_EVEN, "ROUND_HALF_EVEN")
with Serializable with ImplicitCastInputTypes {
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb2ca8e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index 5ead169..2ce10ef 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -38,8 +38,12 @@ import org.apache.spark.unsafe.Platform
* For input of type [[BinaryType]]
*/
@ExpressionDescription(
- usage = "_FUNC_(input) - Returns an MD5 128-bit checksum as a hex string of the input",
- extended = "> SELECT _FUNC_('Spark');\n '8cde774d6f7333752ed72cacddb05126'")
+ usage = "_FUNC_(expr) - Returns an MD5 128-bit checksum as a hex string of `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Spark');
+ 8cde774d6f7333752ed72cacddb05126
+ """)
case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = StringType
@@ -65,10 +69,15 @@ case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInput
*/
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = """_FUNC_(input, bitLength) - Returns a checksum of SHA-2 family as a hex string of the input.
- SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256.""",
- extended = """> SELECT _FUNC_('Spark', 0);
- '529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b'""")
+ usage = """
+ _FUNC_(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of `expr`.
+ SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256.
+ """,
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Spark', 256);
+ 529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b
+ """)
// scalastyle:on line.size.limit
case class Sha2(left: Expression, right: Expression)
extends BinaryExpression with Serializable with ImplicitCastInputTypes {
@@ -136,8 +145,12 @@ case class Sha2(left: Expression, right: Expression)
* For input of type [[BinaryType]] or [[StringType]]
*/
@ExpressionDescription(
- usage = "_FUNC_(input) - Returns a sha1 hash value as a hex string of the input",
- extended = "> SELECT _FUNC_('Spark');\n '85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c'")
+ usage = "_FUNC_(expr) - Returns a sha1 hash value as a hex string of the `expr`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Spark');
+ 85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c
+ """)
case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = StringType
@@ -159,8 +172,12 @@ case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInpu
* For input of type [[BinaryType]]
*/
@ExpressionDescription(
- usage = "_FUNC_(input) - Returns a cyclic redundancy check value as a bigint of the input",
- extended = "> SELECT _FUNC_('Spark');\n '1557323817'")
+ usage = "_FUNC_(expr) - Returns a cyclic redundancy check value of the `expr` as a bigint.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Spark');
+ 1557323817
+ """)
case class Crc32(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = LongType
@@ -490,7 +507,12 @@ abstract class InterpretedHashFunction {
* and bucketing have same data distribution.
*/
@ExpressionDescription(
- usage = "_FUNC_(a1, a2, ...) - Returns a hash value of the arguments.")
+ usage = "_FUNC_(expr1, expr2, ...) - Returns a hash value of the arguments.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Spark', array(123), 2);
+ -1321691492
+ """)
case class Murmur3Hash(children: Seq[Expression], seed: Int) extends HashExpression[Int] {
def this(arguments: Seq[Expression]) = this(arguments, 42)
@@ -544,7 +566,12 @@ case class PrintToStderr(child: Expression) extends UnaryExpression {
* A function throws an exception if 'condition' is not true.
*/
@ExpressionDescription(
- usage = "_FUNC_(condition) - Throw an exception if 'condition' is not true.")
+ usage = "_FUNC_(expr) - Throws an exception if `expr` is not true.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(0 < 1);
+ NULL
+ """)
case class AssertTrue(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def nullable: Boolean = true
@@ -613,7 +640,11 @@ object XxHash64Function extends InterpretedHashFunction {
*/
@ExpressionDescription(
usage = "_FUNC_() - Returns the current database.",
- extended = "> SELECT _FUNC_()")
+ extended = """
+ Examples:
+ > SELECT _FUNC_();
+ default
+ """)
case class CurrentDatabase() extends LeafExpression with Unevaluable {
override def dataType: DataType = StringType
override def foldable: Boolean = true
@@ -631,7 +662,7 @@ case class CurrentDatabase() extends LeafExpression with Unevaluable {
* TODO: Support Decimal and date related types
*/
@ExpressionDescription(
- usage = "_FUNC_(a1, a2, ...) - Returns a hash value of the arguments.")
+ usage = "_FUNC_(expr1, expr2, ...) - Returns a hash value of the arguments.")
case class HiveHash(children: Seq[Expression]) extends HashExpression[Int] {
override val seed = 0
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb2ca8e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
index 70862a8..8b2e8f3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
@@ -34,9 +34,15 @@ import org.apache.spark.sql.types._
* coalesce(null, null, null) => null
* }}}
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(a1, a2, ...) - Returns the first non-null argument if exists. Otherwise, NULL.",
- extended = "> SELECT _FUNC_(NULL, 1, NULL);\n 1")
+ usage = "_FUNC_(expr1, expr2, ...) - Returns the first non-null argument if exists. Otherwise, null.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(NULL, 1, NULL);
+ 1
+ """)
+// scalastyle:on line.size.limit
case class Coalesce(children: Seq[Expression]) extends Expression {
/** Coalesce is nullable if all of its children are nullable, or if it has no children. */
@@ -88,7 +94,13 @@ case class Coalesce(children: Seq[Expression]) extends Expression {
}
-@ExpressionDescription(usage = "_FUNC_(a,b) - Returns b if a is null, or a otherwise.")
+@ExpressionDescription(
+ usage = "_FUNC_(expr1, expr2) - Returns `expr2` if `expr1` is null, or `expr1` otherwise.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(NULL, array('2'));
+ ["2"]
+ """)
case class IfNull(left: Expression, right: Expression, child: Expression)
extends RuntimeReplaceable {
@@ -101,7 +113,13 @@ case class IfNull(left: Expression, right: Expression, child: Expression)
}
-@ExpressionDescription(usage = "_FUNC_(a,b) - Returns null if a equals to b, or a otherwise.")
+@ExpressionDescription(
+ usage = "_FUNC_(expr1, expr2) - Returns null if `expr1` equals to `expr2`, or `expr1` otherwise.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(2, 2);
+ NULL
+ """)
case class NullIf(left: Expression, right: Expression, child: Expression)
extends RuntimeReplaceable {
@@ -114,7 +132,13 @@ case class NullIf(left: Expression, right: Expression, child: Expression)
}
-@ExpressionDescription(usage = "_FUNC_(a,b) - Returns b if a is null, or a otherwise.")
+@ExpressionDescription(
+ usage = "_FUNC_(expr1, expr2) - Returns `expr2` if `expr1` is null, or `expr1` otherwise.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(NULL, array('2'));
+ ["2"]
+ """)
case class Nvl(left: Expression, right: Expression, child: Expression) extends RuntimeReplaceable {
def this(left: Expression, right: Expression) = {
@@ -126,7 +150,15 @@ case class Nvl(left: Expression, right: Expression, child: Expression) extends R
}
-@ExpressionDescription(usage = "_FUNC_(a,b,c) - Returns b if a is not null, or c otherwise.")
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = "_FUNC_(expr1, expr2, expr3) - Returns `expr2` if `expr1` is not null, or `expr3` otherwise.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(NULL, 2, 1);
+ 1
+ """)
+// scalastyle:on line.size.limit
case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, child: Expression)
extends RuntimeReplaceable {
@@ -143,7 +175,12 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, child:
* Evaluates to `true` iff it's NaN.
*/
@ExpressionDescription(
- usage = "_FUNC_(a) - Returns true if a is NaN and false otherwise.")
+ usage = "_FUNC_(expr) - Returns true if `expr` is NaN, or false otherwise.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(cast('NaN' as double));
+ true
+ """)
case class IsNaN(child: Expression) extends UnaryExpression
with Predicate with ImplicitCastInputTypes {
@@ -181,7 +218,12 @@ case class IsNaN(child: Expression) extends UnaryExpression
* This Expression is useful for mapping NaN values to null.
*/
@ExpressionDescription(
- usage = "_FUNC_(a,b) - Returns a iff it's not NaN, or b otherwise.")
+ usage = "_FUNC_(expr1, expr2) - Returns `expr1` if it's not NaN, or `expr2` otherwise.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(cast('NaN' as double), 123);
+ 123.0
+ """)
case class NaNvl(left: Expression, right: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -236,7 +278,12 @@ case class NaNvl(left: Expression, right: Expression)
* An expression that is evaluated to true if the input is null.
*/
@ExpressionDescription(
- usage = "_FUNC_(a) - Returns true if a is NULL and false otherwise.")
+ usage = "_FUNC_(expr) - Returns true if `expr` is null, or false otherwise.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(1);
+ false
+ """)
case class IsNull(child: Expression) extends UnaryExpression with Predicate {
override def nullable: Boolean = false
@@ -257,7 +304,12 @@ case class IsNull(child: Expression) extends UnaryExpression with Predicate {
* An expression that is evaluated to true if the input is not null.
*/
@ExpressionDescription(
- usage = "_FUNC_(a) - Returns true if a is not NULL and false otherwise.")
+ usage = "_FUNC_(expr) - Returns true if `expr` is not null, or false otherwise.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(1);
+ true
+ """)
case class IsNotNull(child: Expression) extends UnaryExpression with Predicate {
override def nullable: Boolean = false
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb2ca8e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index c941a57..7946c20 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -89,7 +89,7 @@ trait PredicateHelper {
}
@ExpressionDescription(
- usage = "_FUNC_ a - Logical not")
+ usage = "_FUNC_ expr - Logical not.")
case class Not(child: Expression)
extends UnaryExpression with Predicate with ImplicitCastInputTypes with NullIntolerant {
@@ -111,7 +111,7 @@ case class Not(child: Expression)
* Evaluates to `true` if `list` contains `value`.
*/
@ExpressionDescription(
- usage = "expr _FUNC_(val1, val2, ...) - Returns true if expr equals to any valN.")
+ usage = "expr1 _FUNC_(expr2, expr3, ...) - Returns true if `expr` equals to any valN.")
case class In(value: Expression, list: Seq[Expression]) extends Predicate
with ImplicitCastInputTypes {
@@ -248,7 +248,7 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
}
@ExpressionDescription(
- usage = "a _FUNC_ b - Logical AND.")
+ usage = "expr1 _FUNC_ expr2 - Logical AND.")
case class And(left: Expression, right: Expression) extends BinaryOperator with Predicate {
override def inputType: AbstractDataType = BooleanType
@@ -311,7 +311,7 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with
}
@ExpressionDescription(
- usage = "a _FUNC_ b - Logical OR.")
+ usage = "expr1 _FUNC_ expr2 - Logical OR.")
case class Or(left: Expression, right: Expression) extends BinaryOperator with Predicate {
override def inputType: AbstractDataType = BooleanType
@@ -406,7 +406,7 @@ object Equality {
}
@ExpressionDescription(
- usage = "a _FUNC_ b - Returns TRUE if a equals b and false otherwise.")
+ usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` equals `expr2`, or false otherwise.")
case class EqualTo(left: Expression, right: Expression)
extends BinaryComparison with NullIntolerant {
@@ -432,8 +432,10 @@ case class EqualTo(left: Expression, right: Expression)
}
@ExpressionDescription(
- usage = """a _FUNC_ b - Returns same result with EQUAL(=) operator for non-null operands,
- but returns TRUE if both are NULL, FALSE if one of the them is NULL.""")
+ usage = """
+ expr1 _FUNC_ expr2 - Returns same result as the EQUAL(=) operator for non-null operands,
+ but returns true if both are null, false if one of the them is null.
+ """)
case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComparison {
override def inputType: AbstractDataType = AnyDataType
@@ -473,7 +475,7 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp
}
@ExpressionDescription(
- usage = "a _FUNC_ b - Returns TRUE if a is less than b.")
+ usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is less than `expr2`.")
case class LessThan(left: Expression, right: Expression)
extends BinaryComparison with NullIntolerant {
@@ -487,7 +489,7 @@ case class LessThan(left: Expression, right: Expression)
}
@ExpressionDescription(
- usage = "a _FUNC_ b - Returns TRUE if a is not greater than b.")
+ usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is less than or equal to `expr2`.")
case class LessThanOrEqual(left: Expression, right: Expression)
extends BinaryComparison with NullIntolerant {
@@ -501,7 +503,7 @@ case class LessThanOrEqual(left: Expression, right: Expression)
}
@ExpressionDescription(
- usage = "a _FUNC_ b - Returns TRUE if a is greater than b.")
+ usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is greater than `expr2`.")
case class GreaterThan(left: Expression, right: Expression)
extends BinaryComparison with NullIntolerant {
@@ -515,7 +517,7 @@ case class GreaterThan(left: Expression, right: Expression)
}
@ExpressionDescription(
- usage = "a _FUNC_ b - Returns TRUE if a is not smaller than b.")
+ usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is greater than or equal to `expr2`.")
case class GreaterThanOrEqual(left: Expression, right: Expression)
extends BinaryComparison with NullIntolerant {
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb2ca8e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
index e09029f..a331a55 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
@@ -55,8 +55,17 @@ abstract class RDG extends LeafExpression with Nondeterministic {
}
/** Generate a random column with i.i.d. uniformly distributed values in [0, 1). */
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(a) - Returns a random column with i.i.d. uniformly distributed values in [0, 1).")
+ usage = "_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) uniformly distributed values in [0, 1).",
+ extended = """
+ Examples:
+ > SELECT _FUNC_();
+ 0.9629742951434543
+ > SELECT _FUNC_(0);
+ 0.8446490682263027
+ """)
+// scalastyle:on line.size.limit
case class Rand(seed: Long) extends RDG {
override protected def evalInternal(input: InternalRow): Double = rng.nextDouble()
@@ -78,9 +87,18 @@ case class Rand(seed: Long) extends RDG {
}
}
-/** Generate a random column with i.i.d. gaussian random distribution. */
+/** Generate a random column with i.i.d. values drawn from the standard normal distribution. */
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(a) - Returns a random column with i.i.d. gaussian random distribution.")
+ usage = "_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) values drawn from the standard normal distribution.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_();
+ -0.3254147983080288
+ > SELECT _FUNC_(0);
+ 1.1164209726833079
+ """)
+// scalastyle:on line.size.limit
case class Randn(seed: Long) extends RDG {
override protected def evalInternal(input: InternalRow): Double = rng.nextGaussian()
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb2ca8e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index d25da3f..5648ad6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -68,7 +68,7 @@ trait StringRegexExpression extends ImplicitCastInputTypes {
* Simple RegEx pattern matching function
*/
@ExpressionDescription(
- usage = "str _FUNC_ pattern - Returns true if str matches pattern and false otherwise.")
+ usage = "str _FUNC_ pattern - Returns true if `str` matches `pattern`, or false otherwise.")
case class Like(left: Expression, right: Expression)
extends BinaryExpression with StringRegexExpression {
@@ -121,7 +121,7 @@ case class Like(left: Expression, right: Expression)
}
@ExpressionDescription(
- usage = "str _FUNC_ regexp - Returns true if str matches regexp and false otherwise.")
+ usage = "str _FUNC_ regexp - Returns true if `str` matches `regexp`, or false otherwise.")
case class RLike(left: Expression, right: Expression)
extends BinaryExpression with StringRegexExpression {
@@ -175,8 +175,12 @@ case class RLike(left: Expression, right: Expression)
* Splits str around pat (pattern is a regular expression).
*/
@ExpressionDescription(
- usage = "_FUNC_(str, regex) - Splits str around occurrences that match regex",
- extended = "> SELECT _FUNC_('oneAtwoBthreeC', '[ABC]');\n ['one', 'two', 'three']")
+ usage = "_FUNC_(str, regex) - Splits `str` around occurrences that match `regex`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('oneAtwoBthreeC', '[ABC]');
+ ["one","two","three",""]
+ """)
case class StringSplit(str: Expression, pattern: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -206,9 +210,15 @@ case class StringSplit(str: Expression, pattern: Expression)
*
* NOTE: this expression is not THREAD-SAFE, as it has some internal mutable status.
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(str, regexp, rep) - replace all substrings of str that match regexp with rep.",
- extended = "> SELECT _FUNC_('100-200', '(\\d+)', 'num');\n 'num-num'")
+ usage = "_FUNC_(str, regexp, rep) - Replaces all substrings of `str` that match `regexp` with `rep`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('100-200', '(\d+)', 'num');
+ num-num
+ """)
+// scalastyle:on line.size.limit
case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -309,8 +319,12 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
* NOTE: this expression is not THREAD-SAFE, as it has some internal mutable status.
*/
@ExpressionDescription(
- usage = "_FUNC_(str, regexp[, idx]) - extracts a group that matches regexp.",
- extended = "> SELECT _FUNC_('100-200', '(\\d+)-(\\d+)', 1);\n '100'")
+ usage = "_FUNC_(str, regexp[, idx]) - Extracts a group that matches `regexp`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('100-200', '(\d+)-(\d+)', 1);
+ 100
+ """)
case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
def this(s: Expression, r: Expression) = this(s, r, Literal(1))
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb2ca8e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 25a5e3f..5f533fe 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -40,9 +40,15 @@ import org.apache.spark.unsafe.types.{ByteArray, UTF8String}
* An expression that concatenates multiple input strings into a single string.
* If any input is null, concat returns null.
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(str1, str2, ..., strN) - Returns the concatenation of str1, str2, ..., strN",
- extended = "> SELECT _FUNC_('Spark','SQL');\n 'SparkSQL'")
+ usage = "_FUNC_(str1, str2, ..., strN) - Returns the concatenation of `str1`, `str2`, ..., `strN`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Spark','SQL');
+ SparkSQL
+ """)
+// scalastyle:on line.size.limit
case class Concat(children: Seq[Expression]) extends Expression with ImplicitCastInputTypes {
override def inputTypes: Seq[AbstractDataType] = Seq.fill(children.size)(StringType)
@@ -78,10 +84,15 @@ case class Concat(children: Seq[Expression]) extends Expression with ImplicitCas
*
* Returns null if the separator is null. Otherwise, concat_ws skips all null values.
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage =
- "_FUNC_(sep, [str | array(str)]+) - Returns the concatenation of the strings separated by sep.",
- extended = "> SELECT _FUNC_(' ', Spark', 'SQL');\n 'Spark SQL'")
+ usage = "_FUNC_(sep, [str | array(str)]+) - Returns the concatenation of the strings separated by `sep`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(' ', Spark', 'SQL');
+ Spark SQL
+ """)
+// scalastyle:on line.size.limit
case class ConcatWs(children: Seq[Expression])
extends Expression with ImplicitCastInputTypes {
@@ -167,9 +178,15 @@ case class ConcatWs(children: Seq[Expression])
}
}
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(n, str1, str2, ...) - returns the n-th string, e.g. returns str2 when n is 2",
- extended = "> SELECT _FUNC_(1, 'scala', 'java') FROM src LIMIT 1;\n" + "'scala'")
+ usage = "_FUNC_(n, str1, str2, ...) - Returns the `n`-th string, e.g., returns `str2` when `n` is 2.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(1, 'scala', 'java');
+ scala
+ """)
+// scalastyle:on line.size.limit
case class Elt(children: Seq[Expression])
extends Expression with ImplicitCastInputTypes {
@@ -246,8 +263,12 @@ trait String2StringExpression extends ImplicitCastInputTypes {
* A function that converts the characters of a string to uppercase.
*/
@ExpressionDescription(
- usage = "_FUNC_(str) - Returns str with all characters changed to uppercase",
- extended = "> SELECT _FUNC_('SparkSql');\n 'SPARKSQL'")
+ usage = "_FUNC_(str) - Returns `str` with all characters changed to uppercase.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('SparkSql');
+ SPARKSQL
+ """)
case class Upper(child: Expression)
extends UnaryExpression with String2StringExpression {
@@ -262,8 +283,12 @@ case class Upper(child: Expression)
* A function that converts the characters of a string to lowercase.
*/
@ExpressionDescription(
- usage = "_FUNC_(str) - Returns str with all characters changed to lowercase",
- extended = "> SELECT _FUNC_('SparkSql');\n 'sparksql'")
+ usage = "_FUNC_(str) - Returns `str` with all characters changed to lowercase.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('SparkSql');
+ sparksql
+ """)
case class Lower(child: Expression) extends UnaryExpression with String2StringExpression {
override def convert(v: UTF8String): UTF8String = v.toLowerCase
@@ -347,8 +372,12 @@ object StringTranslate {
*/
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = """_FUNC_(input, from, to) - Translates the input string by replacing the characters present in the from string with the corresponding characters in the to string""",
- extended = "> SELECT _FUNC_('AaBbCc', 'abc', '123');\n 'A1B2C3'")
+ usage = "_FUNC_(input, from, to) - Translates the `input` string by replacing the characters present in the `from` string with the corresponding characters in the `to` string.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('AaBbCc', 'abc', '123');
+ A1B2C3
+ """)
// scalastyle:on line.size.limit
case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replaceExpr: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -407,9 +436,15 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac
*/
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = """_FUNC_(str, str_array) - Returns the index (1-based) of the given string (left) in the comma-delimited list (right).
- Returns 0, if the string wasn't found or if the given string (left) contains a comma.""",
- extended = "> SELECT _FUNC_('ab','abc,b,ab,c,def');\n 3")
+ usage = """
+ _FUNC_(str, str_array) - Returns the index (1-based) of the given string (`str`) in the comma-delimited list (`str_array`).
+ Returns 0, if the string was not found or if the given string (`str`) contains a comma.
+ """,
+ extended = """
+ Examples:
+ > SELECT _FUNC_('ab','abc,b,ab,c,def');
+ 3
+ """)
// scalastyle:on
case class FindInSet(left: Expression, right: Expression) extends BinaryExpression
with ImplicitCastInputTypes {
@@ -434,8 +469,12 @@ case class FindInSet(left: Expression, right: Expression) extends BinaryExpressi
* A function that trim the spaces from both ends for the specified string.
*/
@ExpressionDescription(
- usage = "_FUNC_(str) - Removes the leading and trailing space characters from str.",
- extended = "> SELECT _FUNC_(' SparkSQL ');\n 'SparkSQL'")
+ usage = "_FUNC_(str) - Removes the leading and trailing space characters from `str`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(' SparkSQL ');
+ SparkSQL
+ """)
case class StringTrim(child: Expression)
extends UnaryExpression with String2StringExpression {
@@ -452,8 +491,12 @@ case class StringTrim(child: Expression)
* A function that trim the spaces from left end for given string.
*/
@ExpressionDescription(
- usage = "_FUNC_(str) - Removes the leading space characters from str.",
- extended = "> SELECT _FUNC_(' SparkSQL ');\n 'SparkSQL '")
+ usage = "_FUNC_(str) - Removes the leading and trailing space characters from `str`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(' SparkSQL');
+ SparkSQL
+ """)
case class StringTrimLeft(child: Expression)
extends UnaryExpression with String2StringExpression {
@@ -470,8 +513,12 @@ case class StringTrimLeft(child: Expression)
* A function that trim the spaces from right end for given string.
*/
@ExpressionDescription(
- usage = "_FUNC_(str) - Removes the trailing space characters from str.",
- extended = "> SELECT _FUNC_(' SparkSQL ');\n ' SparkSQL'")
+ usage = "_FUNC_(str) - Removes the trailing space characters from `str`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(' SparkSQL ');
+ SparkSQL
+ """)
case class StringTrimRight(child: Expression)
extends UnaryExpression with String2StringExpression {
@@ -492,8 +539,12 @@ case class StringTrimRight(child: Expression)
* NOTE: that this is not zero based, but 1-based index. The first character in str has index 1.
*/
@ExpressionDescription(
- usage = "_FUNC_(str, substr) - Returns the (1-based) index of the first occurrence of substr in str.",
- extended = "> SELECT _FUNC_('SparkSQL', 'SQL');\n 6")
+ usage = "_FUNC_(str, substr) - Returns the (1-based) index of the first occurrence of `substr` in `str`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('SparkSQL', 'SQL');
+ 6
+ """)
case class StringInstr(str: Expression, substr: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -522,12 +573,18 @@ case class StringInstr(str: Expression, substr: Expression)
*/
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = """_FUNC_(str, delim, count) - Returns the substring from str before count occurrences of the delimiter delim.
- If count is positive, everything to the left of the final delimiter (counting from the
- left) is returned. If count is negative, everything to the right of the final delimiter
- (counting from the right) is returned. Substring_index performs a case-sensitive match
- when searching for delim.""",
- extended = "> SELECT _FUNC_('www.apache.org', '.', 2);\n 'www.apache'")
+ usage = """
+ _FUNC_(str, delim, count) - Returns the substring from `str` before `count` occurrences of the delimiter `delim`.
+ If `count` is positive, everything to the left of the final delimiter (counting from the
+ left) is returned. If `count` is negative, everything to the right of the final delimiter
+ (counting from the right) is returned. The function substring_index performs a case-sensitive match
+ when searching for `delim`.
+ """,
+ extended = """
+ Examples:
+ > SELECT _FUNC_('www.apache.org', '.', 2);
+ www.apache
+ """)
// scalastyle:on line.size.limit
case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -554,9 +611,15 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr:
*/
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = """_FUNC_(substr, str[, pos]) - Returns the position of the first occurrence of substr in str after position pos.
- The given pos and return value are 1-based.""",
- extended = "> SELECT _FUNC_('bar', 'foobarbar', 5);\n 7")
+ usage = """
+ _FUNC_(substr, str[, pos]) - Returns the position of the first occurrence of `substr` in `str` after position `pos`.
+ The given `pos` and return value are 1-based.
+ """,
+ extended = """
+ Examples:
+ > SELECT _FUNC_('bar', 'foobarbar', 5);
+ 7
+ """)
// scalastyle:on line.size.limit
case class StringLocate(substr: Expression, str: Expression, start: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -631,10 +694,17 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression)
* Returns str, left-padded with pad to a length of len.
*/
@ExpressionDescription(
- usage = """_FUNC_(str, len, pad) - Returns str, left-padded with pad to a length of len.
- If str is longer than len, the return value is shortened to len characters.""",
- extended = "> SELECT _FUNC_('hi', 5, '??');\n '???hi'\n" +
- "> SELECT _FUNC_('hi', 1, '??');\n 'h'")
+ usage = """
+ _FUNC_(str, len, pad) - Returns `str`, left-padded with `pad` to a length of `len`.
+ If `str` is longer than `len`, the return value is shortened to `len` characters.
+ """,
+ extended = """
+ Examples:
+ > SELECT _FUNC_('hi', 5, '??');
+ ???hi
+ > SELECT _FUNC_('hi', 1, '??');
+ h
+ """)
case class StringLPad(str: Expression, len: Expression, pad: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -657,10 +727,17 @@ case class StringLPad(str: Expression, len: Expression, pad: Expression)
* Returns str, right-padded with pad to a length of len.
*/
@ExpressionDescription(
- usage = """_FUNC_(str, len, pad) - Returns str, right-padded with pad to a length of len.
- If str is longer than len, the return value is shortened to len characters.""",
- extended = "> SELECT _FUNC_('hi', 5, '??');\n 'hi???'\n" +
- "> SELECT _FUNC_('hi', 1, '??');\n 'h'")
+ usage = """
+ _FUNC_(str, len, pad) - Returns `str`, right-padded with `pad` to a length of `len`.
+ If `str` is longer than `len`, the return value is shortened to `len` characters.
+ """,
+ extended = """
+ Examples:
+ > SELECT _FUNC_('hi', 5, '??');
+ hi???
+ > SELECT _FUNC_('hi', 1, '??');
+ h
+ """)
case class StringRPad(str: Expression, len: Expression, pad: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -696,16 +773,16 @@ object ParseUrl {
* Extracts a part from a URL
*/
@ExpressionDescription(
- usage = "_FUNC_(url, partToExtract[, key]) - extracts a part from a URL",
- extended = """Parts: HOST, PATH, QUERY, REF, PROTOCOL, AUTHORITY, FILE, USERINFO.
- Key specifies which query to extract.
+ usage = "_FUNC_(url, partToExtract[, key]) - Extracts a part from a URL.",
+ extended = """
Examples:
> SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST')
- 'spark.apache.org'
+ spark.apache.org
> SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY')
- 'query=1'
+ query=1
> SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY', 'query')
- '1'""")
+ 1
+ """)
case class ParseUrl(children: Seq[Expression])
extends Expression with ExpectsInputTypes with CodegenFallback {
@@ -851,8 +928,12 @@ case class ParseUrl(children: Seq[Expression])
*/
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(String format, Obj... args) - Returns a formatted string from printf-style format strings.",
- extended = "> SELECT _FUNC_(\"Hello World %d %s\", 100, \"days\");\n 'Hello World 100 days'")
+ usage = "_FUNC_(strfmt, obj, ...) - Returns a formatted string from printf-style format strings.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_("Hello World %d %s", 100, "days");
+ Hello World 100 days
+ """)
// scalastyle:on line.size.limit
case class FormatString(children: Expression*) extends Expression with ImplicitCastInputTypes {
@@ -923,10 +1004,15 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC
* Words are delimited by whitespace.
*/
@ExpressionDescription(
- usage =
- """_FUNC_(str) - Returns str with the first letter of each word in uppercase.
- All other letters are in lowercase. Words are delimited by white space.""",
- extended = "> SELECT initcap('sPark sql');\n 'Spark Sql'")
+ usage = """
+ _FUNC_(str) - Returns `str` with the first letter of each word in uppercase.
+ All other letters are in lowercase. Words are delimited by white space.
+ """,
+ extended = """
+ Examples:
+ > SELECT initcap('sPark sql');
+ Spark Sql
+ """)
case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def inputTypes: Seq[DataType] = Seq(StringType)
@@ -944,8 +1030,12 @@ case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastI
* Returns the string which repeat the given string value n times.
*/
@ExpressionDescription(
- usage = "_FUNC_(str, n) - Returns the string which repeat the given string value n times.",
- extended = "> SELECT _FUNC_('123', 2);\n '123123'")
+ usage = "_FUNC_(str, n) - Returns the string which repeats the given string value n times.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('123', 2);
+ 123123
+ """)
case class StringRepeat(str: Expression, times: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -970,7 +1060,11 @@ case class StringRepeat(str: Expression, times: Expression)
*/
@ExpressionDescription(
usage = "_FUNC_(str) - Returns the reversed given string.",
- extended = "> SELECT _FUNC_('Spark SQL');\n 'LQS krapS'")
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Spark SQL');
+ LQS krapS
+ """)
case class StringReverse(child: Expression) extends UnaryExpression with String2StringExpression {
override def convert(v: UTF8String): UTF8String = v.reverse()
@@ -982,11 +1076,15 @@ case class StringReverse(child: Expression) extends UnaryExpression with String2
}
/**
- * Returns a n spaces string.
+ * Returns a string consisting of n spaces.
*/
@ExpressionDescription(
- usage = "_FUNC_(n) - Returns a n spaces string.",
- extended = "> SELECT _FUNC_(2);\n ' '")
+ usage = "_FUNC_(n) - Returns a string consisting of `n` spaces.",
+ extended = """
+ Examples:
+ > SELECT concat(_FUNC_(2), '1');
+ 1
+ """)
case class StringSpace(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes {
@@ -1014,8 +1112,16 @@ case class StringSpace(child: Expression)
*/
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(str, pos[, len]) - Returns the substring of str that starts at pos and is of length len or the slice of byte array that starts at pos and is of length len.",
- extended = "> SELECT _FUNC_('Spark SQL', 5);\n 'k SQL'\n> SELECT _FUNC_('Spark SQL', -3);\n 'SQL'\n> SELECT _FUNC_('Spark SQL', 5, 1);\n 'k'")
+ usage = "_FUNC_(str, pos[, len]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Spark SQL', 5);
+ k SQL
+ > SELECT _FUNC_('Spark SQL', -3);
+ SQL
+ > SELECT _FUNC_('Spark SQL', 5, 1);
+ k
+ """)
// scalastyle:on line.size.limit
case class Substring(str: Expression, pos: Expression, len: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -1055,8 +1161,12 @@ case class Substring(str: Expression, pos: Expression, len: Expression)
* A function that return the length of the given string or binary expression.
*/
@ExpressionDescription(
- usage = "_FUNC_(str | binary) - Returns the length of str or number of bytes in binary data.",
- extended = "> SELECT _FUNC_('Spark SQL');\n 9")
+ usage = "_FUNC_(expr) - Returns the length of `expr` or number of bytes in binary data.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Spark SQL');
+ 9
+ """)
case class Length(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = IntegerType
override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(StringType, BinaryType))
@@ -1079,7 +1189,11 @@ case class Length(child: Expression) extends UnaryExpression with ImplicitCastIn
*/
@ExpressionDescription(
usage = "_FUNC_(str1, str2) - Returns the Levenshtein distance between the two given strings.",
- extended = "> SELECT _FUNC_('kitten', 'sitting');\n 3")
+ extended = """
+ Examples:
+ > SELECT _FUNC_('kitten', 'sitting');
+ 3
+ """)
case class Levenshtein(left: Expression, right: Expression) extends BinaryExpression
with ImplicitCastInputTypes {
@@ -1096,11 +1210,15 @@ case class Levenshtein(left: Expression, right: Expression) extends BinaryExpres
}
/**
- * A function that return soundex code of the given string expression.
+ * A function that return Soundex code of the given string expression.
*/
@ExpressionDescription(
- usage = "_FUNC_(str) - Returns soundex code of the string.",
- extended = "> SELECT _FUNC_('Miller');\n 'M460'")
+ usage = "_FUNC_(str) - Returns Soundex code of the string.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Miller');
+ M460
+ """)
case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputTypes {
override def dataType: DataType = StringType
@@ -1118,9 +1236,14 @@ case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputT
* Returns the numeric value of the first character of str.
*/
@ExpressionDescription(
- usage = "_FUNC_(str) - Returns the numeric value of the first character of str.",
- extended = "> SELECT _FUNC_('222');\n 50\n" +
- "> SELECT _FUNC_(2);\n 50")
+ usage = "_FUNC_(str) - Returns the numeric value of the first character of `str`.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('222');
+ 50
+ > SELECT _FUNC_(2);
+ 50
+ """)
case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = IntegerType
@@ -1153,7 +1276,12 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp
* Converts the argument from binary to a base 64 string.
*/
@ExpressionDescription(
- usage = "_FUNC_(bin) - Convert the argument from binary to a base 64 string.")
+ usage = "_FUNC_(bin) - Converts the argument from a binary `bin` to a base 64 string.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Spark SQL');
+ U3BhcmsgU1FM
+ """)
case class Base64(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = StringType
@@ -1177,7 +1305,12 @@ case class Base64(child: Expression) extends UnaryExpression with ImplicitCastIn
* Converts the argument from a base 64 string to BINARY.
*/
@ExpressionDescription(
- usage = "_FUNC_(str) - Convert the argument from a base 64 string to binary.")
+ usage = "_FUNC_(str) - Converts the argument from a base 64 string `str` to a binary.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('U3BhcmsgU1FM');
+ Spark SQL
+ """)
case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = BinaryType
@@ -1199,8 +1332,15 @@ case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCast
* (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').
* If either argument is null, the result will also be null.
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(bin, str) - Decode the first argument using the second argument character set.")
+ usage = "_FUNC_(bin, charset) - Decodes the first argument using the second argument character set.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_(encode('abc', 'utf-8'), 'utf-8');
+ abc
+ """)
+// scalastyle:on line.size.limit
case class Decode(bin: Expression, charset: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -1231,8 +1371,15 @@ case class Decode(bin: Expression, charset: Expression)
* (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').
* If either argument is null, the result will also be null.
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(str, str) - Encode the first argument using the second argument character set.")
+ usage = "_FUNC_(str, charset) - Encodes the first argument using the second argument character set.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('abc', 'utf-8');
+ abc
+ """)
+// scalastyle:on line.size.limit
case class Encode(value: Expression, charset: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -1263,10 +1410,16 @@ case class Encode(value: Expression, charset: Expression)
* fractional part.
*/
@ExpressionDescription(
- usage = """_FUNC_(X, D) - Formats the number X like '#,###,###.##', rounded to D decimal places.
- If D is 0, the result has no decimal point or fractional part.
- This is supposed to function like MySQL's FORMAT.""",
- extended = "> SELECT _FUNC_(12332.123456, 4);\n '12,332.1235'")
+ usage = """
+ _FUNC_(expr1, expr2) - Formats the number `expr1` like '#,###,###.##', rounded to `expr2`
+ decimal places. If `expr2` is 0, the result has no decimal point or fractional part.
+ This is supposed to function like MySQL's FORMAT.
+ """,
+ extended = """
+ Examples:
+ > SELECT _FUNC_(12332.123456, 4);
+ 12,332.1235
+ """)
case class FormatNumber(x: Expression, d: Expression)
extends BinaryExpression with ExpectsInputTypes {
@@ -1388,8 +1541,12 @@ case class FormatNumber(x: Expression, d: Expression)
* The 'lang' and 'country' arguments are optional, and if omitted, the default locale is used.
*/
@ExpressionDescription(
- usage = "_FUNC_(str[, lang, country]) - Splits str into an array of array of words.",
- extended = "> SELECT _FUNC_('Hi there! Good morning.');\n [['Hi','there'], ['Good','morning']]")
+ usage = "_FUNC_(str[, lang, country]) - Splits `str` into an array of array of words.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('Hi there! Good morning.');
+ [["Hi","there"],["Good","morning"]]
+ """)
case class Sentences(
str: Expression,
language: Expression = Literal(""),
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb2ca8e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
index b47486f..3cbbcdf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
@@ -321,7 +321,7 @@ abstract class OffsetWindowFunction
val input: Expression
/**
- * Default result value for the function when the 'offset'th row does not exist.
+ * Default result value for the function when the `offset`th row does not exist.
*/
val default: Expression
@@ -372,22 +372,23 @@ abstract class OffsetWindowFunction
}
/**
- * The Lead function returns the value of 'x' at the 'offset'th row after the current row in
+ * The Lead function returns the value of `input` at the `offset`th row after the current row in
* the window. Offsets start at 0, which is the current row. The offset must be constant
- * integer value. The default offset is 1. When the value of 'x' is null at the 'offset'th row,
- * null is returned. If there is no such offset row, the default expression is evaluated.
+ * integer value. The default offset is 1. When the value of `input` is null at the `offset`th row,
+ * null is returned. If there is no such offset row, the `default` expression is evaluated.
*
- * @param input expression to evaluate 'offset' rows after the current row.
+ * @param input expression to evaluate `offset` rows after the current row.
* @param offset rows to jump ahead in the partition.
* @param default to use when the offset is larger than the window. The default value is null.
*/
-@ExpressionDescription(usage =
- """_FUNC_(input, offset, default) - LEAD returns the value of 'x' at the 'offset'th row
- after the current row in the window.
- The default value of 'offset' is 1 and the default value of 'default' is null.
- If the value of 'x' at the 'offset'th row is null, null is returned.
- If there is no such offset row (e.g. when the offset is 1, the last row of the window
- does not have any subsequent row), 'default' is returned.""")
+@ExpressionDescription(
+ usage = """
+ _FUNC_(input[, offset[, default]]) - Returns the value of `input` at the `offset`th row
+ after the current row in the window. The default value of `offset` is 1 and the default
+ value of `default` is null. If the value of `input` at the `offset`th row is null,
+ null is returned. If there is no such an offset row (e.g., when the offset is 1, the last
+ row of the window does not have any subsequent row), `default` is returned.
+ """)
case class Lead(input: Expression, offset: Expression, default: Expression)
extends OffsetWindowFunction {
@@ -401,22 +402,23 @@ case class Lead(input: Expression, offset: Expression, default: Expression)
}
/**
- * The Lag function returns the value of 'x' at the 'offset'th row before the current row in
+ * The Lag function returns the value of `input` at the `offset`th row before the current row in
* the window. Offsets start at 0, which is the current row. The offset must be constant
- * integer value. The default offset is 1. When the value of 'x' is null at the 'offset'th row,
- * null is returned. If there is no such offset row, the default expression is evaluated.
+ * integer value. The default offset is 1. When the value of `input` is null at the `offset`th row,
+ * null is returned. If there is no such offset row, the `default` expression is evaluated.
*
- * @param input expression to evaluate 'offset' rows before the current row.
+ * @param input expression to evaluate `offset` rows before the current row.
* @param offset rows to jump back in the partition.
* @param default to use when the offset row does not exist.
*/
-@ExpressionDescription(usage =
- """_FUNC_(input, offset, default) - LAG returns the value of 'x' at the 'offset'th row
- before the current row in the window.
- The default value of 'offset' is 1 and the default value of 'default' is null.
- If the value of 'x' at the 'offset'th row is null, null is returned.
- If there is no such offset row (e.g. when the offset is 1, the first row of the window
- does not have any previous row), 'default' is returned.""")
+@ExpressionDescription(
+ usage = """
+ _FUNC_(input[, offset[, default]]) - Returns the value of `input` at the `offset`th row
+ before the current row in the window. The default value of `offset` is 1 and the default
+ value of `default` is null. If the value of `input` at the `offset`th row is null,
+ null is returned. If there is no such offset row (e.g., when the offset is 1, the first
+ row of the window does not have any previous row), `default` is returned.
+ """)
case class Lag(input: Expression, offset: Expression, default: Expression)
extends OffsetWindowFunction {
@@ -471,26 +473,28 @@ object SizeBasedWindowFunction {
*
* This documentation has been based upon similar documentation for the Hive and Presto projects.
*/
-@ExpressionDescription(usage =
- """_FUNC_() - The ROW_NUMBER() function assigns a unique, sequential number to
- each row, starting with one, according to the ordering of rows within
- the window partition.""")
+@ExpressionDescription(
+ usage = """
+ _FUNC_() - Assigns a unique, sequential number to each row, starting with one,
+ according to the ordering of rows within the window partition.
+ """)
case class RowNumber() extends RowNumberLike {
override val evaluateExpression = rowNumber
override def prettyName: String = "row_number"
}
/**
- * The CumeDist function computes the position of a value relative to a all values in the partition.
+ * The CumeDist function computes the position of a value relative to all values in the partition.
* The result is the number of rows preceding or equal to the current row in the ordering of the
* partition divided by the total number of rows in the window partition. Any tie values in the
* ordering will evaluate to the same position.
*
* This documentation has been based upon similar documentation for the Hive and Presto projects.
*/
-@ExpressionDescription(usage =
- """_FUNC_() - The CUME_DIST() function computes the position of a value relative to
- a all values in the partition.""")
+@ExpressionDescription(
+ usage = """
+ _FUNC_() - Computes the position of a value relative to all values in the partition.
+ """)
case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
override def dataType: DataType = DoubleType
// The frame for CUME_DIST is Range based instead of Row based, because CUME_DIST must
@@ -501,8 +505,8 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
}
/**
- * The NTile function divides the rows for each window partition into 'n' buckets ranging from 1 to
- * at most 'n'. Bucket values will differ by at most 1. If the number of rows in the partition does
+ * The NTile function divides the rows for each window partition into `n` buckets ranging from 1 to
+ * at most `n`. Bucket values will differ by at most 1. If the number of rows in the partition does
* not divide evenly into the number of buckets, then the remainder values are distributed one per
* bucket, starting with the first bucket.
*
@@ -521,9 +525,11 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
*
* @param buckets number of buckets to divide the rows in. Default value is 1.
*/
-@ExpressionDescription(usage =
- """_FUNC_(x) - The NTILE(n) function divides the rows for each window partition
- into 'n' buckets ranging from 1 to at most 'n'.""")
+@ExpressionDescription(
+ usage = """
+ _FUNC_(n) - Divides the rows for each window partition into `n` buckets ranging
+ from 1 to at most `n`.
+ """)
case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindowFunction {
def this() = this(Literal(1))
@@ -587,9 +593,9 @@ case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindow
/**
* A RankLike function is a WindowFunction that changes its value based on a change in the value of
- * the order of the window in which is processed. For instance, when the value of 'x' changes in a
- * window ordered by 'x' the rank function also changes. The size of the change of the rank function
- * is (typically) not dependent on the size of the change in 'x'.
+ * the order of the window in which is processed. For instance, when the value of `input` changes
+ * in a window ordered by `input` the rank function also changes. The size of the change of the
+ * rank function is (typically) not dependent on the size of the change in `input`.
*
* This documentation has been based upon similar documentation for the Hive and Presto projects.
*/
@@ -635,7 +641,7 @@ abstract class RankLike extends AggregateWindowFunction {
/**
* The Rank function computes the rank of a value in a group of values. The result is one plus the
- * number of rows preceding or equal to the current row in the ordering of the partition. Tie values
+ * number of rows preceding or equal to the current row in the ordering of the partition. The values
* will produce gaps in the sequence.
*
* This documentation has been based upon similar documentation for the Hive and Presto projects.
@@ -644,10 +650,12 @@ abstract class RankLike extends AggregateWindowFunction {
* change in rank. This is an internal parameter and will be assigned by the
* Analyser.
*/
-@ExpressionDescription(usage =
- """_FUNC_() - RANK() computes the rank of a value in a group of values. The result
- is one plus the number of rows preceding or equal to the current row in the
- ordering of the partition. Tie values will produce gaps in the sequence.""")
+@ExpressionDescription(
+ usage = """
+ _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the number
+ of rows preceding or equal to the current row in the ordering of the partition. The values
+ will produce gaps in the sequence.
+ """)
case class Rank(children: Seq[Expression]) extends RankLike {
def this() = this(Nil)
override def withOrder(order: Seq[Expression]): Rank = Rank(order)
@@ -655,8 +663,8 @@ case class Rank(children: Seq[Expression]) extends RankLike {
/**
* The DenseRank function computes the rank of a value in a group of values. The result is one plus
- * the previously assigned rank value. Unlike Rank, DenseRank will not produce gaps in the ranking
- * sequence.
+ * the previously assigned rank value. Unlike [[Rank]], [[DenseRank]] will not produce gaps in the
+ * ranking sequence.
*
* This documentation has been based upon similar documentation for the Hive and Presto projects.
*
@@ -664,10 +672,12 @@ case class Rank(children: Seq[Expression]) extends RankLike {
* change in rank. This is an internal parameter and will be assigned by the
* Analyser.
*/
-@ExpressionDescription(usage =
- """_FUNC_() - The DENSE_RANK() function computes the rank of a value in a group of
- values. The result is one plus the previously assigned rank value. Unlike Rank,
- DenseRank will not produce gaps in the ranking sequence.""")
+@ExpressionDescription(
+ usage = """
+ _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the
+ previously assigned rank value. Unlike the function rank, dense_rank will not produce gaps
+ in the ranking sequence.
+ """)
case class DenseRank(children: Seq[Expression]) extends RankLike {
def this() = this(Nil)
override def withOrder(order: Seq[Expression]): DenseRank = DenseRank(order)
@@ -692,9 +702,10 @@ case class DenseRank(children: Seq[Expression]) extends RankLike {
* change in rank. This is an internal parameter and will be assigned by the
* Analyser.
*/
-@ExpressionDescription(usage =
- """_FUNC_() - PERCENT_RANK() The PercentRank function computes the percentage
- ranking of a value in a group of values.""")
+@ExpressionDescription(
+ usage = """
+ _FUNC_() - Computes the percentage ranking of a value in a group of values.
+ """)
case class PercentRank(children: Seq[Expression]) extends RankLike with SizeBasedWindowFunction {
def this() = this(Nil)
override def withOrder(order: Seq[Expression]): PercentRank = PercentRank(order)
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb2ca8e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
index 47f039e..aa32804 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
@@ -55,9 +55,15 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with
def path: Expression
}
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(xml, xpath) - Evaluates a boolean xpath expression.",
- extended = "> SELECT _FUNC_('<a><b>1</b></a>','a/b');\ntrue")
+ usage = "_FUNC_(xml, xpath) - Returns true if the XPath expression evaluates to true, or if a matching node is found.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('<a><b>1</b></a>','a/b');
+ true
+ """)
+// scalastyle:on line.size.limit
case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract {
override def prettyName: String = "xpath_boolean"
@@ -68,11 +74,17 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract
}
}
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(xml, xpath) - Returns a short value that matches the xpath expression",
- extended = "> SELECT _FUNC_('<a><b>1</b><b>2</b></a>','sum(a/b)');\n3")
+ usage = "_FUNC_(xml, xpath) - Returns a short integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('<a><b>1</b><b>2</b></a>', 'sum(a/b)');
+ 3
+ """)
+// scalastyle:on line.size.limit
case class XPathShort(xml: Expression, path: Expression) extends XPathExtract {
- override def prettyName: String = "xpath_int"
+ override def prettyName: String = "xpath_short"
override def dataType: DataType = ShortType
override def nullSafeEval(xml: Any, path: Any): Any = {
@@ -81,9 +93,15 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract {
}
}
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(xml, xpath) - Returns an integer value that matches the xpath expression",
- extended = "> SELECT _FUNC_('<a><b>1</b><b>2</b></a>','sum(a/b)');\n3")
+ usage = "_FUNC_(xml, xpath) - Returns an integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('<a><b>1</b><b>2</b></a>', 'sum(a/b)');
+ 3
+ """)
+// scalastyle:on line.size.limit
case class XPathInt(xml: Expression, path: Expression) extends XPathExtract {
override def prettyName: String = "xpath_int"
override def dataType: DataType = IntegerType
@@ -94,9 +112,15 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract {
}
}
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(xml, xpath) - Returns a long value that matches the xpath expression",
- extended = "> SELECT _FUNC_('<a><b>1</b><b>2</b></a>','sum(a/b)');\n3")
+ usage = "_FUNC_(xml, xpath) - Returns a long integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('<a><b>1</b><b>2</b></a>', 'sum(a/b)');
+ 3
+ """)
+// scalastyle:on line.size.limit
case class XPathLong(xml: Expression, path: Expression) extends XPathExtract {
override def prettyName: String = "xpath_long"
override def dataType: DataType = LongType
@@ -107,9 +131,15 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract {
}
}
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(xml, xpath) - Returns a float value that matches the xpath expression",
- extended = "> SELECT _FUNC_('<a><b>1</b><b>2</b></a>','sum(a/b)');\n3.0")
+ usage = "_FUNC_(xml, xpath) - Returns a float value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('<a><b>1</b><b>2</b></a>', 'sum(a/b)');
+ 3.0
+ """)
+// scalastyle:on line.size.limit
case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract {
override def prettyName: String = "xpath_float"
override def dataType: DataType = FloatType
@@ -120,9 +150,15 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract {
}
}
+// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(xml, xpath) - Returns a double value that matches the xpath expression",
- extended = "> SELECT _FUNC_('<a><b>1</b><b>2</b></a>','sum(a/b)');\n3.0")
+ usage = "_FUNC_(xml, xpath) - Returns a double value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('<a><b>1</b><b>2</b></a>', 'sum(a/b)');
+ 3.0
+ """)
+// scalastyle:on line.size.limit
case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract {
override def prettyName: String = "xpath_float"
override def dataType: DataType = DoubleType
@@ -135,8 +171,12 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract {
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the xpath expression",
- extended = "> SELECT _FUNC_('<a><b>b</b><c>cc</c></a>','a/c');\ncc")
+ usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the XPath expression.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('<a><b>b</b><c>cc</c></a>','a/c');
+ cc
+ """)
// scalastyle:on line.size.limit
case class XPathString(xml: Expression, path: Expression) extends XPathExtract {
override def prettyName: String = "xpath_string"
@@ -150,8 +190,12 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract {
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(xml, xpath) - Returns a string array of values within xml nodes that match the xpath expression",
- extended = "> SELECT _FUNC_('<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>','a/b/text()');\n['b1','b2','b3']")
+ usage = "_FUNC_(xml, xpath) - Returns a string array of values within the nodes of xml that match the XPath expression.",
+ extended = """
+ Examples:
+ > SELECT _FUNC_('<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>','a/b/text()');
+ ['b1','b2','b3']
+ """)
// scalastyle:on line.size.limit
case class XPathList(xml: Expression, path: Expression) extends XPathExtract {
override def prettyName: String = "xpath"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org