(incubator-gluten) branch main updated: [CORE] Fix the java.nio.file.NoSuchFileException: default in spark 3.5 (#6175)

[ke...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kejia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 9cceba681 [CORE] Fix the java.nio.file.NoSuchFileException: default in spark 3.5 (#6175)
9cceba681 is described below

commit 9cceba6812d4de9883695f20bc506648083715f9
Author: Xiduo You <ul...@gmail.com>
AuthorDate: Mon Jun 24 09:07:09 2024 +0800

    [CORE] Fix the java.nio.file.NoSuchFileException: default in spark 3.5 (#6175)
---
 gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala
index 9756837d9..04272517e 100644
--- a/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala
@@ -41,8 +41,8 @@ object HdfsConfGenerator extends Logging {
     addFileMethod.invoke(sc, path, Boolean.box(false), Boolean.box(true), Boolean.box(false))
     // Overwrite the spark internal config `spark.app.initial.file.urls`,
     // so that the file can be available before initializing executor plugin.
-    assert(sc.addedFiles.nonEmpty)
-    sc.conf.set("spark.app.initial.file.urls", sc.addedFiles.keys.toSeq.mkString(","))
+    assert(sc.listFiles.nonEmpty)
+    sc.conf.set("spark.app.initial.file.urls", sc.listFiles().mkString(","))
   }
 
   private def ignoreKey(key: String): Boolean = {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org