[GitHub] spark pull request #21073: [SPARK-23936][SQL] Implement map_concat

[ueshin <gi...@git.apache.org>]

Github user ueshin commented on a diff in the pull request:

    https://github.com/apache/spark/pull/21073#discussion_r192919160
  
    --- Diff: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala ---
    @@ -308,6 +308,170 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp
       override def prettyName: String = "map_entries"
     }
     
    +/**
    + * Returns the union of all the given maps.
    + */
    +@ExpressionDescription(
    +usage = "_FUNC_(map, ...) - Returns the union of all the given maps",
    +examples = """
    +    Examples:
    +      > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(2, 'c', 3, 'd'));
    +       [[1 -> "a"], [2 -> "c"], [3 -> "d"]
    +  """, since = "2.4.0")
    +case class MapConcat(children: Seq[Expression]) extends Expression {
    +
    +  override def checkInputDataTypes(): TypeCheckResult = {
    +    // check key types and value types separately to allow valueContainsNull to vary
    +    if (children.exists(!_.dataType.isInstanceOf[MapType])) {
    +      TypeCheckResult.TypeCheckFailure(
    +        s"The given input of function $prettyName should all be of type map, " +
    +          "but they are " + children.map(_.dataType.simpleString).mkString("[", ", ", "]"))
    +    } else if (children.map(_.dataType.asInstanceOf[MapType].keyType)
    +      .exists(_.isInstanceOf[MapType])) {
    +      // map_concat needs to pick a winner when multiple maps contain the same key. map_concat
    +      // can do that only if it can detect when two keys are the same. SPARK-9415 states "map type
    +      // should not support equality, hash". As a result, map_concat does not support a map type
    +      // as a key
    +      TypeCheckResult.TypeCheckFailure(
    +        s"The given input maps of function $prettyName cannot have a map type as a key")
    +    } else if (children.map(_.dataType.asInstanceOf[MapType].keyType).distinct.length > 1) {
    +      TypeCheckResult.TypeCheckFailure(
    +        s"The given input maps of function $prettyName should all be the same type, " +
    +          "but they are " + children.map(_.dataType.simpleString).mkString("[", ", ", "]"))
    +    } else if (children.map(_.dataType.asInstanceOf[MapType].valueType).distinct.length > 1) {
    +      TypeCheckResult.TypeCheckFailure(
    +        s"The given input maps of function $prettyName should all be the same type, " +
    +          "but they are " + children.map(_.dataType.simpleString).mkString("[", ", ", "]"))
    +    } else {
    +      TypeCheckResult.TypeCheckSuccess
    +    }
    +  }
    +
    +  override def dataType: MapType = {
    +    MapType(
    --- End diff --
    
    How about doing `children.map(_.dataType.asInstanceOf[MapType])` first?


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org