You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ph...@apache.org on 2024/11/27 03:24:28 UTC
(incubator-gluten) branch main updated: [GLUTEN-7631][VL] Fall back lead/lag if input is foldable (#8038)
This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new bfafa576f6 [GLUTEN-7631][VL] Fall back lead/lag if input is foldable (#8038)
bfafa576f6 is described below
commit bfafa576f6f7a333301788bffb434b3a5cc5ccf4
Author: Mingliang Zhu <zh...@gmail.com>
AuthorDate: Wed Nov 27 11:24:23 2024 +0800
[GLUTEN-7631][VL] Fall back lead/lag if input is foldable (#8038)
---
.../main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala | 4 +++-
.../test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 3 ---
.../test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 3 ---
.../test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 3 ---
.../test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 3 ---
5 files changed, 3 insertions(+), 13 deletions(-)
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
index e05fd92e32..b1245687b0 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -400,7 +400,9 @@ object VeloxBackendSettings extends BackendSettingsApi {
}
windowExpression.windowFunction match {
case _: RowNumber | _: Rank | _: CumeDist | _: DenseRank | _: PercentRank |
- _: NthValue | _: NTile | _: Lag | _: Lead =>
+ _: NthValue | _: NTile =>
+ case l: Lag if !l.input.foldable =>
+ case l: Lead if !l.input.foldable =>
case aggrExpr: AggregateExpression
if !aggrExpr.aggregateFunction.isInstanceOf[ApproximatePercentile]
&& !aggrExpr.aggregateFunction.isInstanceOf[Percentile] =>
diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 363f9c85ed..276a8a0f7a 100644
--- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -318,9 +318,6 @@ class VeloxTestSettings extends BackendTestSettings {
// spill not supported yet.
enableSuite[GlutenSQLWindowFunctionSuite]
.exclude("test with low buffer spill threshold")
- // https://github.com/apache/incubator-gluten/issues/7631
- .exclude(
- "SPARK-16633: lead/lag should return the default value if the offset row does not exist")
enableSuite[GlutenSortSuite]
// Sort spill is not supported.
.exclude("sorting does not crash for large inputs")
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 1fc64577d2..08fab69c84 100644
--- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -886,9 +886,6 @@ class VeloxTestSettings extends BackendTestSettings {
// spill not supported yet.
enableSuite[GlutenSQLWindowFunctionSuite]
.exclude("test with low buffer spill threshold")
- // https://github.com/apache/incubator-gluten/issues/7631
- .exclude(
- "SPARK-16633: lead/lag should return the default value if the offset row does not exist")
enableSuite[GlutenTakeOrderedAndProjectSuite]
enableSuite[GlutenSessionExtensionSuite]
enableSuite[TestFileSourceScanExecTransformer]
diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 8c65afd1a3..df79b663bc 100644
--- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -886,9 +886,6 @@ class VeloxTestSettings extends BackendTestSettings {
// spill not supported yet.enableSuite[GlutenSQLWindowFunctionSuite]
enableSuite[GlutenSQLWindowFunctionSuite]
.exclude("test with low buffer spill threshold")
- // https://github.com/apache/incubator-gluten/issues/7631
- .exclude(
- "SPARK-16633: lead/lag should return the default value if the offset row does not exist")
enableSuite[GlutenTakeOrderedAndProjectSuite]
enableSuite[GlutenSessionExtensionSuite]
enableSuite[TestFileSourceScanExecTransformer]
diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 18c46a3042..cb2151fe46 100644
--- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -900,9 +900,6 @@ class VeloxTestSettings extends BackendTestSettings {
// spill not supported yet.
enableSuite[GlutenSQLWindowFunctionSuite]
.exclude("test with low buffer spill threshold")
- // https://github.com/apache/incubator-gluten/issues/7631
- .exclude(
- "SPARK-16633: lead/lag should return the default value if the offset row does not exist")
enableSuite[GlutenTakeOrderedAndProjectSuite]
enableSuite[GlutenSessionExtensionSuite]
enableSuite[TestFileSourceScanExecTransformer]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org