[incubator-celeborn] 01/01: add UT about hdfs path validation.

[et...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ethanfeng pushed a commit to branch renovate-hdfs-regex
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git

commit 9b8897e03304b030a25d6fa00c1215b02f2e9bd2
Author: Ethan Feng <et...@apache.org>
AuthorDate: Thu Mar 23 10:18:20 2023 +0800

    add UT about hdfs path validation.
---
 .../apache/celeborn/common/util/UtilsSuite.scala   | 28 ++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala b/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
index 21da12353..a1130d348 100644
--- a/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
+++ b/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
@@ -105,6 +105,34 @@ class UtilsSuite extends CelebornFunSuite {
     assert(mapperEnd == mapperEndTrans)
   }
 
+  test("validate hdfs compatible fs") {
+    val hdfsPath = "hdfs://xxx:9000/xxxx/xx-xx/x-x-x"
+    val simpleHdfsPath = "hdfs:///xxxx/xx-xx/x-x-x"
+    val sortedHdfsPath = "hdfs://xxx:9000/xxxx/xx-xx/x-x-x.sorted"
+    val indexHdfsPath = "hdfs://xxx:9000/xxxx/xx-xx/x-x-x.index"
+    assert(true == Utils.isHdfsPath(hdfsPath))
+    assert(true == Utils.isHdfsPath(sortedHdfsPath))
+    assert(true == Utils.isHdfsPath(indexHdfsPath))
+    assert(true == Utils.isHdfsPath(simpleHdfsPath))
+
+    val juicePath = "jfs://xxxx/xx-xx/x-x-x"
+    val sortedJuicePath = "jfs://xxxx/xx-xx/x-x-x.sorted"
+    val indexJuicePath = "jfs://xxxx/xx-xx/x-x-x.index"
+    assert(true == Utils.isHdfsPath(juicePath))
+    assert(true == Utils.isHdfsPath(sortedJuicePath))
+    assert(true == Utils.isHdfsPath(indexJuicePath))
+
+    val ossPath = "oss://xxxx/xx-xx/x-x-x"
+    val sortedOssPath = "oss://xxxx/xx-xx/x-x-x.sorted"
+    val indexOssPath = "oss://xxxx/xx-xx/x-x-x.index"
+    assert(true == Utils.isHdfsPath(ossPath))
+    assert(true == Utils.isHdfsPath(sortedOssPath))
+    assert(true == Utils.isHdfsPath(indexOssPath))
+
+    val localPath = "/xxx/xxx/xx-xx/x-x-x"
+    assert(false == Utils.isHdfsPath(localPath))
+  }
+
   test("GetReducerFileGroupResponse class convert with pb") {
     val fileGroup = new util.HashMap[Integer, util.Set[PartitionLocation]]
     fileGroup.put(0, partitionLocation(0))