[jira] [Commented] (SPARK-34184) Remove redundant repartition nodes in the optimizer

["Anton Okolnychyi (Jira)" <ji...@apache.org>]

    [ https://issues.apache.org/jira/browse/SPARK-34184?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17271632#comment-17271632 ] 

Anton Okolnychyi commented on SPARK-34184:
------------------------------------------


{code:java}
  ignore("ordered distribution and sort with manual repartition: append") {
    checkOrderedDistributionAndSortWithManualRepartition("append")
  }

  ignore("ordered distribution and sort with manual repartition: overwrite") {
    checkOrderedDistributionAndSortWithManualRepartition("overwrite")
  }

  ignore("ordered distribution and sort with manual repartition: overwriteDynamic") {
    checkOrderedDistributionAndSortWithManualRepartition("overwriteDynamic")
  }

  private def checkOrderedDistributionAndSortWithManualRepartition(command: String): Unit = {
    val tableOrdering = Array[SortOrder](
      sort(FieldReference("data"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST),
      sort(FieldReference("id"), SortDirection.ASCENDING, NullOrdering.NULLS_FIRST)
    )
    val tableDistribution = Distributions.ordered(tableOrdering)

    val writeOrdering = Seq(
      catalyst.expressions.SortOrder(
        attr("data"),
        catalyst.expressions.Ascending,
        catalyst.expressions.NullsFirst,
        Seq.empty
      ),
      catalyst.expressions.SortOrder(
        attr("id"),
        catalyst.expressions.Ascending,
        catalyst.expressions.NullsFirst,
        Seq.empty
      )
    )
    val writePartitioning = RangePartitioning(writeOrdering, conf.numShufflePartitions)

    checkWriteRequirements(
      tableDistribution,
      tableOrdering,
      expectedWritePartitioning = writePartitioning,
      expectedWriteOrdering = writeOrdering,
      writeTransform = df => df.repartitionByRange(df("data"), df("id")),
      writeCommand = command)
  }

{code}


> Remove redundant repartition nodes in the optimizer
> ---------------------------------------------------
>
>                 Key: SPARK-34184
>                 URL: https://issues.apache.org/jira/browse/SPARK-34184
>             Project: Spark
>          Issue Type: Improvement
>          Components: SQL
>    Affects Versions: 3.2.0
>            Reporter: Anton Okolnychyi
>            Priority: Major
>
> We should consider optimizing redundant repartition nodes. See [here| https://github.com/apache/spark/pull/31083#discussion_r553252442]



--
This message was sent by Atlassian Jira
(v8.3.4#803005)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org