You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ho...@apache.org on 2025/10/13 07:27:18 UTC

(incubator-gluten) branch main updated: [GLUTEN-10707] Improve SparkDirectoryUtil by reduce synchronized (#10708)

This is an automated email from the ASF dual-hosted git repository.

hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 1030678a97 [GLUTEN-10707] Improve SparkDirectoryUtil by reduce synchronized (#10708)
1030678a97 is described below

commit 1030678a97acb10e88ccd99257dc88d0d28126f1
Author: Jiaan Geng <be...@163.com>
AuthorDate: Mon Oct 13 15:27:14 2025 +0800

    [GLUTEN-10707] Improve SparkDirectoryUtil by reduce synchronized (#10708)
---
 .../org/apache/spark/util/SparkDirectoryUtil.scala | 25 +++++++++++-----------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala b/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala
index 24379853d3..8e62ad7ae7 100644
--- a/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala
@@ -62,30 +62,31 @@ class SparkDirectoryUtil private (val roots: Array[String]) extends Logging {
 }
 
 object SparkDirectoryUtil extends Logging {
-  private var INSTANCE: SparkDirectoryUtil = _
+  @volatile private var roots: Array[String] = _
+  private lazy val INSTANCE: SparkDirectoryUtil = {
+    if (this.roots == null) {
+      throw new IllegalStateException("SparkDirectoryUtil not initialized")
+    }
+    new SparkDirectoryUtil(this.roots)
+  }
 
-  def init(conf: SparkConf): Unit = synchronized {
+  def init(conf: SparkConf): Unit = {
     val roots = Utils.getConfiguredLocalDirs(conf)
     init(roots)
   }
 
   private def init(roots: Array[String]): Unit = synchronized {
-    if (INSTANCE == null) {
-      INSTANCE = new SparkDirectoryUtil(roots)
-      return
-    }
-    if (INSTANCE.roots.toSet != roots.toSet) {
+    if (this.roots == null) {
+      this.roots = roots
+    } else if (this.roots.toSet != roots.toSet) {
       throw new IllegalArgumentException(
-        s"Reinitialize SparkDirectoryUtil with different root dirs: old: ${INSTANCE.ROOTS
+        s"Reinitialize SparkDirectoryUtil with different root dirs: old: ${this.roots
             .mkString("Array(", ", ", ")")}, new: ${roots.mkString("Array(", ", ", ")")}"
       )
     }
   }
 
-  def get(): SparkDirectoryUtil = synchronized {
-    assert(INSTANCE != null, "Default instance of SparkDirectoryUtil was not set yet")
-    INSTANCE
-  }
+  def get(): SparkDirectoryUtil = INSTANCE
 }
 
 class Namespace(private val parents: Array[File], private val name: String) {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org