(incubator-celeborn) branch main updated: [CELEBORN-1255] Fix MR UT

[zh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new 5358bd168 [CELEBORN-1255] Fix MR UT
5358bd168 is described below

commit 5358bd16842c82d30ba8ce35d1f6ccc48f318650
Author: [cxzl25 <sy...@ctrip.com>
AuthorDate: Tue Jan 30 09:58:29 2024 +0800

    [CELEBORN-1255] Fix MR UT
    
    ### What changes were proposed in this pull request?
    To fix MR ut caused by incorrect classpath.
    
    ### Why are the changes needed?
    The error is the same as below:
    ![ea](https://github.com/apache/incubator-celeborn/assets/4150993/4defd3fa-57ed-40e9-a935-e33aba73f28c)
    
    ### Does this PR introduce _any_ user-facing change?
    NO.
    
    ### How was this patch tested?
    GA.
    
    Closes #2266 from FMX/B1255.
    
    Lead-authored-by: [cxzl25 <sy...@ctrip.com>]
    Co-authored-by: cxzl25 <38...@users.noreply.github.com>
    Co-authored-by: mingji <fe...@alibaba-inc.com>
    Signed-off-by: zky.zhoukeyong <zk...@alibaba-inc.com>
---
 .../test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala   | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/mr-it/src/test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala b/tests/mr-it/src/test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala
index d7c0ca41f..18c7fd676 100644
--- a/tests/mr-it/src/test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala
+++ b/tests/mr-it/src/test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala
@@ -77,6 +77,7 @@ class WordCountTest extends AnyFunSuite with Logging with MiniClusterFeature
       classOf[Service])
 
     yarnCluster = new MiniYARNCluster("MiniClusterWordCount", 1, 1, 1)
+    logInfo(s"Test working dir ${yarnCluster.getTestWorkDir.getAbsolutePath}")
     yarnCluster.init(hadoopConf)
     yarnCluster.start()
   }
@@ -142,12 +143,17 @@ class WordCountTest extends AnyFunSuite with Logging with MiniClusterFeature
           (Utils.getCodeSourceLocation(getClass).split("/").dropRight(1) ++ Array(
             "mapreduce_lib")).mkString("/")
         val excludeJarList =
-          Seq("hadoop-client-api", "hadoop-client-runtime", "hadoop-client-minicluster")
+          Seq(
+            "hadoop-client-api",
+            "hadoop-client-runtime",
+            "hadoop-client-minicluster",
+            "celeborn-client-mr-shaded")
         Files.list(Paths.get(mapreduceLibPath)).iterator().asScala.foreach(path => {
           if (!excludeJarList.exists(path.toFile.getPath.contains(_))) {
             job.addFileToClassPath(new Path(path.toString))
           }
         })
+        logInfo(s"Job class path ${job.getFileClassPaths.map(_.toString).mkString(",")}")
 
         exitCode = job.waitForCompletion(true)
         if (exitCode) {