You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ho...@apache.org on 2024/06/05 03:41:40 UTC
(incubator-gluten) branch main updated: [VL] Quick fix for Uniffle CI error (#5986)
This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 59aaa1cc3 [VL] Quick fix for Uniffle CI error (#5986)
59aaa1cc3 is described below
commit 59aaa1cc36686b34900b44b920679ed335f94302
Author: Hongze Zhang <ho...@intel.com>
AuthorDate: Wed Jun 5 11:41:36 2024 +0800
[VL] Quick fix for Uniffle CI error (#5986)
---
.../org/apache/gluten/integration/BaseMixin.java | 2 +-
.../org/apache/gluten/integration/Suite.scala | 22 ++++++++++++----------
2 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
index dc1691e50..93c82a6fa 100644
--- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
+++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
@@ -75,7 +75,7 @@ public class BaseMixin {
@CommandLine.Option(names = {"--shuffle-partitions"}, description = "Shuffle partition number", defaultValue = "100")
private int shufflePartitions;
- @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan partition number. This is an approximate value, so the actual scan partition number might vary around this value", defaultValue = "100")
+ @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan partition number. This is an approximate value, so the actual scan partition number might vary around this value. -1 for letting Spark choose an appropriate number.", defaultValue = "-1")
private int scanPartitions;
@CommandLine.Option(names = {"--extra-conf"}, description = "Extra Spark config entries applying to generated Spark session. E.g. --extra-conf=k1=v1 --extra-conf=k2=v2")
diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
index bb5cb1889..51e1777e2 100644
--- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
+++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
@@ -103,16 +103,18 @@ abstract class Suite(
sessionSwitcher.defaultConf().setWarningOnOverriding("spark.sql.codegen.wholeStage", "false")
}
- // Scan partition number.
- sessionSwitcher
- .defaultConf()
- .setWarningOnOverriding("spark.sql.files.maxPartitionBytes", s"${ByteUnit.PiB.toBytes(1L)}")
- sessionSwitcher
- .defaultConf()
- .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0")
- sessionSwitcher
- .defaultConf()
- .setWarningOnOverriding("spark.sql.files.minPartitionNum", s"${(scanPartitions - 1) max 1}")
+ if (scanPartitions != -1) {
+ // Scan partition number.
+ sessionSwitcher
+ .defaultConf()
+ .setWarningOnOverriding("spark.sql.files.maxPartitionBytes", s"${ByteUnit.PiB.toBytes(1L)}")
+ sessionSwitcher
+ .defaultConf()
+ .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0")
+ sessionSwitcher
+ .defaultConf()
+ .setWarningOnOverriding("spark.sql.files.minPartitionNum", s"${(scanPartitions - 1) max 1}")
+ }
extraSparkConf.toStream.foreach { kv =>
sessionSwitcher.defaultConf().setWarningOnOverriding(kv._1, kv._2)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org