You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2022/12/08 09:13:33 UTC
[GitHub] [spark] beliefer commented on a diff in pull request #38865: [SPARK-41232][SQL][PYTHON] Adding array_append function
beliefer commented on code in PR #38865:
URL: https://github.com/apache/spark/pull/38865#discussion_r1043086340
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala:
##########
@@ -4600,3 +4600,92 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): ArrayExcept = copy(left = newLeft, right = newRight)
}
+
+/**
+ * Given an array, and another element append the element at the end of the array.
+ */
+@ExpressionDescription(
+ usage = "_FUNC_(array, element) - Append the element",
+ examples =
+ """
+ Examples:
+ > SELECT _FUNC_(array('b', 'd', 'c', 'a'), 'd');
+ ["b","d","c","a","d"]
+
+ """,
+ since = "3.4.0",
+ group = "array_funcs")
+case class ArrayAppend(left: Expression, right: Expression)
+ extends BinaryExpression
+ with ImplicitCastInputTypes
+ with ComplexTypeMergingExpression
+ with QueryErrorsBase {
+ override def prettyName: String = "array_append"
+
+ override def inputTypes: Seq[AbstractDataType] = {
+ (left.dataType, right.dataType) match {
+ case (ArrayType(e1, hasNull), e2) =>
+ TypeCoercion.findTightestCommonType(e1, e2) match {
+ case Some(dt) => Seq(ArrayType(dt, hasNull), dt)
+ case _ => Seq.empty
+ }
+ case _ => Seq.empty
+ }
+ }
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ (left.dataType, right.dataType) match {
+ case (ArrayType(e1, _), e2) if e1.sameType(e2) => TypeCheckResult.TypeCheckSuccess
+ case (ArrayType(e1, _), e2) => DataTypeMismatch(
+ errorSubClass = "ARRAY_FUNCTION_DIFF_TYPES",
+ messageParameters = Map(
+ "functionName" -> toSQLId(prettyName),
+ "leftType" -> toSQLType(left.dataType),
+ "rightType" -> toSQLType(right.dataType),
+ "dataType" -> toSQLType(ArrayType)
+ ))
+ case _ =>
+ DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_INPUT_TYPE",
+ messageParameters = Map(
+ "paramIndex" -> "0",
+ "requiredType" -> toSQLType(ArrayType),
+ "inputSql" -> toSQLExpr(left),
+ "inputType" -> toSQLType(left.dataType)
+ )
+ )
+ }
+ }
+
+ protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): ArrayAppend =
+ copy(left = newLeft, right = newRight)
+
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+ val arrayData = input1.asInstanceOf[ArrayData]
+ val arrayElementType = dataType.asInstanceOf[ArrayType].elementType
Review Comment:
Please put elementType as the field of `ArrayAppend`
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala:
##########
@@ -4600,3 +4600,133 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): ArrayExcept = copy(left = newLeft, right = newRight)
}
+
+/**
+ * Given an array, and another element append the element at the end of the array.
+ */
+@ExpressionDescription(
+ usage = "_FUNC_(array, element) - Append the element",
Review Comment:
The usage will generated as document. So, `Append the element` lacks readability.
##########
sql/core/src/test/resources/sql-tests/results/ansi/array.sql.out:
##########
@@ -489,3 +489,43 @@ select get(array(1, 2, 3), -1)
struct<get(array(1, 2, 3), -1):int>
-- !query output
NULL
+
+
+-- !query
+select array_append(array(1, 2, 3), 4)
+-- !query schema
+struct<array_append(array(1, 2, 3), 4):array<int>>
+-- !query output
+[1,2,3,4]
+
+
+-- !query
+select array_append(array('a', 'b', 'c'), 'd')
+-- !query schema
+struct<array_append(array(a, b, c), d):array<string>>
+-- !query output
+["a","b","c","d"]
+
+
+-- !query
+select array_append(array(1, 2, 3), CAST(null AS INT))
+-- !query schema
+struct<array_append(array(1, 2, 3), CAST(NULL AS INT)):array<int>>
+-- !query output
+NULL
Review Comment:
The result is correct ?
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala:
##########
@@ -4600,3 +4600,92 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): ArrayExcept = copy(left = newLeft, right = newRight)
}
+
+/**
+ * Given an array, and another element append the element at the end of the array.
+ */
+@ExpressionDescription(
+ usage = "_FUNC_(array, element) - Append the element",
+ examples =
+ """
+ Examples:
+ > SELECT _FUNC_(array('b', 'd', 'c', 'a'), 'd');
+ ["b","d","c","a","d"]
+
+ """,
+ since = "3.4.0",
+ group = "array_funcs")
+case class ArrayAppend(left: Expression, right: Expression)
+ extends BinaryExpression
+ with ImplicitCastInputTypes
+ with ComplexTypeMergingExpression
+ with QueryErrorsBase {
+ override def prettyName: String = "array_append"
+
+ override def inputTypes: Seq[AbstractDataType] = {
+ (left.dataType, right.dataType) match {
+ case (ArrayType(e1, hasNull), e2) =>
+ TypeCoercion.findTightestCommonType(e1, e2) match {
+ case Some(dt) => Seq(ArrayType(dt, hasNull), dt)
+ case _ => Seq.empty
+ }
+ case _ => Seq.empty
+ }
+ }
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ (left.dataType, right.dataType) match {
+ case (ArrayType(e1, _), e2) if e1.sameType(e2) => TypeCheckResult.TypeCheckSuccess
+ case (ArrayType(e1, _), e2) => DataTypeMismatch(
+ errorSubClass = "ARRAY_FUNCTION_DIFF_TYPES",
+ messageParameters = Map(
+ "functionName" -> toSQLId(prettyName),
+ "leftType" -> toSQLType(left.dataType),
+ "rightType" -> toSQLType(right.dataType),
+ "dataType" -> toSQLType(ArrayType)
+ ))
+ case _ =>
+ DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_INPUT_TYPE",
+ messageParameters = Map(
+ "paramIndex" -> "0",
+ "requiredType" -> toSQLType(ArrayType),
+ "inputSql" -> toSQLExpr(left),
+ "inputType" -> toSQLType(left.dataType)
+ )
+ )
+ }
+ }
+
+ protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): ArrayAppend =
+ copy(left = newLeft, right = newRight)
+
+ override protected def nullSafeEval(input1: Any, input2: Any): Any = {
+ val arrayData = input1.asInstanceOf[ArrayData]
+ val arrayElementType = dataType.asInstanceOf[ArrayType].elementType
+ val elementData = input2
+ val numberOfElements = arrayData.numElements() + 1
+ if (numberOfElements > ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH) {
+ throw QueryExecutionErrors.concatArraysWithElementsExceedLimitError(numberOfElements)
+ }
+ val finalData = new Array[Any](numberOfElements)
+ arrayData.foreach(arrayElementType, finalData.update)
+ finalData.update(numberOfElements - 1, elementData)
+ new GenericArrayData(finalData)
+ }
+
+ override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+ nullSafeCodeGen(
+ ctx, ev, (left: String, right: String) => {
+ val expr = ctx.addReferenceObj("arraysAppendExpr", this)
+ s"${ev.value} = (ArrayData)$expr.nullSafeEval($left, $right);"
Review Comment:
Could you implement `doGenCode` by codegen? Please refer `ArrayRemove`.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala:
##########
@@ -4600,3 +4600,133 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): ArrayExcept = copy(left = newLeft, right = newRight)
}
+
+/**
+ * Given an array, and another element append the element at the end of the array.
+ */
+@ExpressionDescription(
+ usage = "_FUNC_(array, element) - Append the element",
+ examples =
+ """
Review Comment:
```suggestion
examples = """
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org