You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ho...@apache.org on 2024/06/05 03:41:40 UTC

(incubator-gluten) branch main updated: [VL] Quick fix for Uniffle CI error (#5986)

This is an automated email from the ASF dual-hosted git repository.

hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 59aaa1cc3 [VL] Quick fix for Uniffle CI error (#5986)
59aaa1cc3 is described below

commit 59aaa1cc36686b34900b44b920679ed335f94302
Author: Hongze Zhang <ho...@intel.com>
AuthorDate: Wed Jun 5 11:41:36 2024 +0800

    [VL] Quick fix for Uniffle CI error (#5986)
---
 .../org/apache/gluten/integration/BaseMixin.java   |  2 +-
 .../org/apache/gluten/integration/Suite.scala      | 22 ++++++++++++----------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
index dc1691e50..93c82a6fa 100644
--- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
+++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
@@ -75,7 +75,7 @@ public class BaseMixin {
   @CommandLine.Option(names = {"--shuffle-partitions"}, description = "Shuffle partition number", defaultValue = "100")
   private int shufflePartitions;
 
-  @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan partition number. This is an approximate value, so the actual scan partition number might vary around this value", defaultValue = "100")
+  @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan partition number. This is an approximate value, so the actual scan partition number might vary around this value. -1 for letting Spark choose an appropriate number.", defaultValue = "-1")
   private int scanPartitions;
 
   @CommandLine.Option(names = {"--extra-conf"}, description = "Extra Spark config entries applying to generated Spark session. E.g. --extra-conf=k1=v1 --extra-conf=k2=v2")
diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
index bb5cb1889..51e1777e2 100644
--- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
+++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
@@ -103,16 +103,18 @@ abstract class Suite(
     sessionSwitcher.defaultConf().setWarningOnOverriding("spark.sql.codegen.wholeStage", "false")
   }
 
-  // Scan partition number.
-  sessionSwitcher
-    .defaultConf()
-    .setWarningOnOverriding("spark.sql.files.maxPartitionBytes", s"${ByteUnit.PiB.toBytes(1L)}")
-  sessionSwitcher
-    .defaultConf()
-    .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0")
-  sessionSwitcher
-    .defaultConf()
-    .setWarningOnOverriding("spark.sql.files.minPartitionNum", s"${(scanPartitions - 1) max 1}")
+  if (scanPartitions != -1) {
+    // Scan partition number.
+    sessionSwitcher
+      .defaultConf()
+      .setWarningOnOverriding("spark.sql.files.maxPartitionBytes", s"${ByteUnit.PiB.toBytes(1L)}")
+    sessionSwitcher
+      .defaultConf()
+      .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0")
+    sessionSwitcher
+      .defaultConf()
+      .setWarningOnOverriding("spark.sql.files.minPartitionNum", s"${(scanPartitions - 1) max 1}")
+  }
 
   extraSparkConf.toStream.foreach { kv =>
     sessionSwitcher.defaultConf().setWarningOnOverriding(kv._1, kv._2)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org