[hudi] 04/04: [HUDI-5632] Fix failure launching Spark jobs from hudi-cli-bundle (#7790)

[yi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-0.13.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 8ffc7ce3d8fe352490c376583c518c837343a71b
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Mon Jan 30 07:54:40 2023 -0800

    [HUDI-5632] Fix failure launching Spark jobs from hudi-cli-bundle (#7790)
    
    - Ensures that Hudi CLI commands which require launching Spark can be executed with hudi-cli-bundle
---
 .../src/main/java/org/apache/hudi/cli/utils/SparkUtil.java   | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
index 1cffdf941f9..fd09a27271a 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
@@ -57,10 +57,18 @@ public class SparkUtil {
     }
 
     File libDirectory = new File(new File(currentJar).getParent(), "lib");
-    // This lib directory may be not required, such as providing libraries through a bundle jar
     if (libDirectory.exists()) {
+      // When directly using hudi-cli module, the jars under the lib directory
+      // generated by the compilation is required
       Arrays.stream(libDirectory.list()).forEach(library ->
-              sparkLauncher.addJar(new File(libDirectory, library).getAbsolutePath()));
+          sparkLauncher.addJar(new File(libDirectory, library).getAbsolutePath()));
+    } else {
+      // When using hudi-cli-bundle, we also need to add the hudi-spark*-bundle
+      // so that the Hudi Spark job can be launched
+      String sparkBundleJarPath = System.getenv("SPARK_BUNDLE_JAR");
+      if (!StringUtils.isNullOrEmpty(sparkBundleJarPath)) {
+        sparkLauncher.addJar(sparkBundleJarPath);
+      }
     }
     return sparkLauncher;
   }