You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Tanel Kiis (Jira)" <ji...@apache.org> on 2022/02/22 07:05:00 UTC

[jira] [Updated] (SPARK-38282) Avoid duplicating complex partitioning expressions

     [ https://issues.apache.org/jira/browse/SPARK-38282?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Tanel Kiis updated SPARK-38282:
-------------------------------
    Description: 

{code}
  test("SPARK-XXXXX: Avoid duplicating complex partitioning expressions") {
    val group = functions.col("id") % 2
    val min = functions.min("id").over(Window.partitionBy(group))
    val max = functions.max("id").over(Window.partitionBy(group))

    val df1 = spark.range(1, 4)
      .withColumn("ratio", max / min)

    val df2 = spark.range(1, 4)
      .withColumn("min", min)
      .withColumn("max", max)
      .select(col("id"), (col("max") / col("min")).as("ratio"))

    Seq(df1, df2).foreach { df =>
      checkAnswer(
        df,
        Seq(Row(1L, 3.0), Row(2L, 1.0), Row(3L, 3.0)))

      val windows = collect(df.queryExecution.executedPlan) {
        case w: WindowExec => w
      }
      assert(windows.size == 1)
    }
  }
{code}

  was:

{code}

  test("SPARK-XXXXX: Avoid duplicating complex partitioning expressions") {
    val group = functions.col("id") % 2
    val min = functions.min("id").over(Window.partitionBy(group))
    val max = functions.max("id").over(Window.partitionBy(group))

    val df1 = spark.range(1, 4)
      .withColumn("ratio", max / min)

    val df2 = spark.range(1, 4)
      .withColumn("min", min)
      .withColumn("max", max)
      .select(col("id"), (col("max") / col("min")).as("ratio"))

    Seq(df1, df2).foreach { df =>
      checkAnswer(
        df,
        Seq(Row(1L, 3.0), Row(2L, 1.0), Row(3L, 3.0)))

      val windows = collect(df.queryExecution.executedPlan) {
        case w: WindowExec => w
      }
      assert(windows.size == 1)
    }
  }
{code}


> Avoid duplicating complex partitioning expressions
> --------------------------------------------------
>
>                 Key: SPARK-38282
>                 URL: https://issues.apache.org/jira/browse/SPARK-38282
>             Project: Spark
>          Issue Type: Improvement
>          Components: SQL
>    Affects Versions: 3.3.0
>            Reporter: Tanel Kiis
>            Priority: Major
>
> {code}
>   test("SPARK-XXXXX: Avoid duplicating complex partitioning expressions") {
>     val group = functions.col("id") % 2
>     val min = functions.min("id").over(Window.partitionBy(group))
>     val max = functions.max("id").over(Window.partitionBy(group))
>     val df1 = spark.range(1, 4)
>       .withColumn("ratio", max / min)
>     val df2 = spark.range(1, 4)
>       .withColumn("min", min)
>       .withColumn("max", max)
>       .select(col("id"), (col("max") / col("min")).as("ratio"))
>     Seq(df1, df2).foreach { df =>
>       checkAnswer(
>         df,
>         Seq(Row(1L, 3.0), Row(2L, 1.0), Row(3L, 3.0)))
>       val windows = collect(df.queryExecution.executedPlan) {
>         case w: WindowExec => w
>       }
>       assert(windows.size == 1)
>     }
>   }
> {code}



--
This message was sent by Atlassian Jira
(v8.20.1#820001)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org