[GitHub] [incubator-celeborn] FMX commented on a diff in pull request #1930: [CELEBORN-1002] add SBT MRClientProject

["FMX (via GitHub)" <gi...@apache.org>]

FMX commented on code in PR #1930:
URL: https://github.com/apache/incubator-celeborn/pull/1930#discussion_r1335696045


##########
project/CelebornBuild.scala:
##########
@@ -872,3 +882,82 @@ trait FlinkClientProjects {
       )
   }
 }
+
+////////////////////////////////////////////////////////
+//                   MR Client                        //
+////////////////////////////////////////////////////////
+object MRClientProjects {
+
+  def mrClient: Project = {
+    Project("celeborn-client-mr", file("client-mr/mr"))
+      .dependsOn(CelebornCommon.common % "test->test;compile->compile")
+      .dependsOn(CelebornClient.client % "test->test;compile->compile")
+      .settings(
+        commonSettings,
+        libraryDependencies ++= Seq(
+          Dependencies.hadoopClientApi,
+          Dependencies.hadoopClientRuntime,
+          Dependencies.hadoopMapreduceClientApp
+        ) ++ commonUnitTestDependencies
+      )
+  }
+
+  def mrClientShade: Project = {
+    Project("celeborn-client-mr-shaded", file("client-mr/mr-shaded"))
+      .dependsOn(mrClient)
+      .settings(
+        commonSettings,
+
+        // align final shaded jar name with maven.
+        (assembly / assemblyJarName) := {
+          val extension = artifact.value.extension
+          s"${moduleName.value}_${scalaBinaryVersion.value}-${version.value}.$extension"
+        },
+
+        (assembly / test) := {},
+
+        (assembly / logLevel) := Level.Info,
+
+        // Exclude `scala-library` from assembly.
+        (assembly / assemblyPackageScala / assembleArtifact) := false,
+
+        (assembly / assemblyExcludedJars) := {
+          val cp = (assembly / fullClasspath).value
+          cp filter { v =>
+            val name = v.data.getName
+            !(name.startsWith("celeborn-") ||
+              name.startsWith("protobuf-java-") ||
+              name.startsWith("guava-") ||
+              name.startsWith("netty-") ||
+              name.startsWith("commons-lang3-") ||
+              name.startsWith("RoaringBitmap-"))

Review Comment:
   I think Hadoop has `zstd-jni` already. I'll make a test to verify this later.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@celeborn.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org