(celeborn) branch branch-0.4 updated: [CELEBORN-1331] Remove third-party dependencies in shaded clients' pom

[ch...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch branch-0.4
in repository https://gitbox.apache.org/repos/asf/celeborn.git


The following commit(s) were added to refs/heads/branch-0.4 by this push:
     new b394da8b5 [CELEBORN-1331] Remove third-party dependencies in shaded clients' pom
b394da8b5 is described below

commit b394da8b51d5662f688e73b149cf6fa4c131d2aa
Author: xiyu.zk <xi...@alibaba-inc.com>
AuthorDate: Mon Apr 8 13:51:30 2024 +0800

    [CELEBORN-1331] Remove third-party dependencies in shaded clients' pom
    
    ### What changes were proposed in this pull request?
    celeborn-client-spark-3-shaded_2.12 should not expose the shipped transitive deps
    
    ### Why are the changes needed?
    maven dependency
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    CI
    
    Closes #2391 from kerwin-zk/issue-1331.
    
    Authored-by: xiyu.zk <xi...@alibaba-inc.com>
    Signed-off-by: Cheng Pan <ch...@apache.org>
    (cherry picked from commit 6ce8b8894d7f754995d8638b67396f4c69fe5215)
    Signed-off-by: Cheng Pan <ch...@apache.org>
---
 project/CelebornBuild.scala | 41 +++++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index e9d8615b3..5b050b067 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -20,6 +20,8 @@ import java.nio.charset.StandardCharsets.UTF_8
 import java.util.Locale
 
 import scala.util.Properties
+import scala.xml._
+import scala.xml.transform._
 
 import sbtassembly.AssemblyPlugin.autoImport._
 import sbtprotoc.ProtocPlugin.autoImport._
@@ -350,6 +352,21 @@ object Utils {
     require(ALL_SCALA_VERSIONS.contains(v), s"found not allow scala version: $v")
     v
   }
+
+  /**
+   * The deps for shaded clients are already packaged in the jar,
+   * so we should not expose the shipped transitive deps.
+   */
+  def removeDependenciesTransformer: xml.Node => xml.Node = { node =>
+    new RuleTransformer(new RewriteRule {
+      override def transform(n: xml.Node): Seq[xml.Node] = n match {
+        case e: Elem if e.label == "dependencies" =>
+          Nil
+        case _ =>
+          n
+      }
+    }).transform(node).head
+  }
 }
 
 object CelebornCommon {
@@ -709,9 +726,14 @@ trait SparkClientProjects {
   }
 
   def sparkClientShade: Project = {
-    val p = Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
+    var p = Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
       .dependsOn(sparkClient)
-      .disablePlugins(AddMetaInfLicenseFiles)
+
+    if (includeColumnarShuffle) {
+      p = p.dependsOn(sparkColumnarShuffle)
+    }
+
+    p = p.disablePlugins(AddMetaInfLicenseFiles)
       .settings (
         commonSettings,
         releaseSettings,
@@ -772,13 +794,10 @@ trait SparkClientProjects {
           case _ => MergeStrategy.first
         },
 
-        Compile / packageBin := assembly.value
+        Compile / packageBin := assembly.value,
+        pomPostProcess := removeDependenciesTransformer
       )
-    if (includeColumnarShuffle) {
-        p.dependsOn(sparkColumnarShuffle)
-    } else {
-        p
-    }
+    p
   }
 }
 
@@ -976,7 +995,8 @@ trait FlinkClientProjects {
           case _ => MergeStrategy.first
         },
 
-        Compile / packageBin := assembly.value
+        Compile / packageBin := assembly.value,
+        pomPostProcess := removeDependenciesTransformer
       )
   }
 }
@@ -1087,7 +1107,8 @@ object MRClientProjects {
           case _ => MergeStrategy.first
         },
 
-        Compile / packageBin := assembly.value
+        Compile / packageBin := assembly.value,
+        pomPostProcess := removeDependenciesTransformer
       )
   }