You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ch...@apache.org on 2026/04/13 09:37:19 UTC

(spark) branch master updated: [SPARK-56353][BUILD][TESTS][FOLLOWUP] Fix resource loading to work when resources are inside JARs

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 87e38619e56f [SPARK-56353][BUILD][TESTS][FOLLOWUP] Fix resource loading to work when resources are inside JARs
87e38619e56f is described below

commit 87e38619e56f0ea1c84713d4e35a4c582d5d5d54
Author: Kousuke Saruta <sa...@amazon.co.jp>
AuthorDate: Mon Apr 13 17:37:07 2026 +0800

    [SPARK-56353][BUILD][TESTS][FOLLOWUP] Fix resource loading to work when resources are inside JARs
    
    ### What changes were proposed in this pull request?
    
    This PR fixes `FileSystemNotFoundException` in `TestSpark21101Jar`, `TestUDTFJar`, and `TestHiveUdfsJar` when running tests with Maven.
    
    These files used `Paths.get(url.toURI)` to read Java source files from classpath resources. This works with SBT where resources are extracted to `target/test-classes/` as regular files, but fails with Maven where resources may be inside JAR files, producing a `jar:file:...!/path` URI that `Paths.get()` cannot handle without a ZipFileSystem.
    
    The fix replaces `Files.readAllBytes(Paths.get(url.toURI))` with `url.openStream().readAllBytes()`, which works regardless of whether the resource is a regular file or inside a JAR.
    
    ### Why are the changes needed?
    
    CI runs with Maven and hits `FileSystemNotFoundException` when initializing `TestUDTFJar`, causing `CliSuite` and `HiveThriftBinaryServerSuite` to fail:
    
    ```
    Cause: java.lang.ExceptionInInitializerError:
      Exception java.nio.file.FileSystemNotFoundException
      at jdk.zipfs/jdk.nio.zipfs.ZipFileSystemProvider.getFileSystem(ZipFileSystemProvider.java:156)
      at java.base/java.nio.file.Paths.get(Paths.java:98)
      at org.apache.spark.sql.hive.test.TestUDTFJar$.loadResource(TestUDTFJar.scala:37)
    ```
    
    Reported in https://github.com/apache/spark/pull/55272#issuecomment-4225700808
    
    ### Does this PR introduce _any_ user-facing change?
    No.
    
    ### How was this patch tested?
    Verified with Maven (the environment where the failure occurs):
    - CliSuite: 44/44 passed
    - HiveThriftBinaryServerSuite: 30/30 passed
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Kiro CLI / Opus 4.6
    
    Closes #55299 from sarutak/fix-resource-loading.
    
    Authored-by: Kousuke Saruta <sa...@amazon.co.jp>
    Signed-off-by: Cheng Pan <ch...@apache.org>
---
 .../test/scala/org/apache/spark/sql/hive/test/TestHiveUdfsJar.scala  | 5 +++--
 .../scala/org/apache/spark/sql/hive/test/TestSpark21101Jar.scala     | 5 +++--
 .../src/test/scala/org/apache/spark/sql/hive/test/TestUDTFJar.scala  | 5 +++--
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHiveUdfsJar.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHiveUdfsJar.scala
index 73d88c392f10..3885d0df56d6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHiveUdfsJar.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHiveUdfsJar.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.hive.test
 import java.io.File
 import java.lang.management.ManagementFactory
 import java.nio.charset.StandardCharsets
-import java.nio.file.{Files, Paths}
 
 import org.apache.spark.util.{SparkTestUtils, Utils}
 
@@ -64,6 +63,8 @@ object TestHiveUdfsJar {
   private def readResource(name: String): String = {
     val url = Thread.currentThread().getContextClassLoader.getResource(name)
     assert(url != null, s"Resource not found: $name")
-    new String(Files.readAllBytes(Paths.get(url.toURI)), StandardCharsets.UTF_8)
+    Utils.tryWithResource(url.openStream()) { is =>
+      new String(is.readAllBytes(), StandardCharsets.UTF_8)
+    }
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestSpark21101Jar.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestSpark21101Jar.scala
index 0263afd8e488..bd4c4eeddf8a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestSpark21101Jar.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestSpark21101Jar.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.hive.test
 import java.io.File
 import java.lang.management.ManagementFactory
 import java.nio.charset.StandardCharsets
-import java.nio.file.{Files, Paths}
 
 import org.apache.spark.util.{SparkTestUtils, Utils}
 
@@ -52,6 +51,8 @@ object TestSpark21101Jar {
   private def readResource(name: String): String = {
     val url = Thread.currentThread().getContextClassLoader.getResource(name)
     assert(url != null, s"Resource not found: $name")
-    new String(Files.readAllBytes(Paths.get(url.toURI)), StandardCharsets.UTF_8)
+    Utils.tryWithResource(url.openStream()) { is =>
+      new String(is.readAllBytes(), StandardCharsets.UTF_8)
+    }
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestUDTFJar.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestUDTFJar.scala
index b3fb0aca1977..a2eee6097ba8 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestUDTFJar.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestUDTFJar.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.hive.test
 import java.io.File
 import java.lang.management.ManagementFactory
 import java.nio.charset.StandardCharsets
-import java.nio.file.{Files, Paths}
 
 import org.apache.spark.TestUtils
 import org.apache.spark.util.Utils
@@ -34,7 +33,9 @@ object TestUDTFJar {
   private def loadResource(name: String): String = {
     val url = Thread.currentThread().getContextClassLoader.getResource(name)
     assert(url != null, s"Resource not found: $name")
-    new String(Files.readAllBytes(Paths.get(url.toURI)), StandardCharsets.UTF_8)
+    Utils.tryWithResource(url.openStream()) { is =>
+      new String(is.readAllBytes(), StandardCharsets.UTF_8)
+    }
   }
 
   private val source = Map(


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org