[incubator-celeborn] branch renovate-hdfs-regex created (now 9b8897e03)

[et...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ethanfeng pushed a change to branch renovate-hdfs-regex
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


      at 9b8897e03 add UT about hdfs path validation.

This branch includes the following new commits:

     new 9b8897e03 add UT about hdfs path validation.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[incubator-celeborn] 01/01: add UT about hdfs path validation.

[et...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ethanfeng pushed a commit to branch renovate-hdfs-regex
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git

commit 9b8897e03304b030a25d6fa00c1215b02f2e9bd2
Author: Ethan Feng <et...@apache.org>
AuthorDate: Thu Mar 23 10:18:20 2023 +0800

    add UT about hdfs path validation.
---
 .../apache/celeborn/common/util/UtilsSuite.scala   | 28 ++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala b/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
index 21da12353..a1130d348 100644
--- a/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
+++ b/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
@@ -105,6 +105,34 @@ class UtilsSuite extends CelebornFunSuite {
     assert(mapperEnd == mapperEndTrans)
   }
 
+  test("validate hdfs compatible fs") {
+    val hdfsPath = "hdfs://xxx:9000/xxxx/xx-xx/x-x-x"
+    val simpleHdfsPath = "hdfs:///xxxx/xx-xx/x-x-x"
+    val sortedHdfsPath = "hdfs://xxx:9000/xxxx/xx-xx/x-x-x.sorted"
+    val indexHdfsPath = "hdfs://xxx:9000/xxxx/xx-xx/x-x-x.index"
+    assert(true == Utils.isHdfsPath(hdfsPath))
+    assert(true == Utils.isHdfsPath(sortedHdfsPath))
+    assert(true == Utils.isHdfsPath(indexHdfsPath))
+    assert(true == Utils.isHdfsPath(simpleHdfsPath))
+
+    val juicePath = "jfs://xxxx/xx-xx/x-x-x"
+    val sortedJuicePath = "jfs://xxxx/xx-xx/x-x-x.sorted"
+    val indexJuicePath = "jfs://xxxx/xx-xx/x-x-x.index"
+    assert(true == Utils.isHdfsPath(juicePath))
+    assert(true == Utils.isHdfsPath(sortedJuicePath))
+    assert(true == Utils.isHdfsPath(indexJuicePath))
+
+    val ossPath = "oss://xxxx/xx-xx/x-x-x"
+    val sortedOssPath = "oss://xxxx/xx-xx/x-x-x.sorted"
+    val indexOssPath = "oss://xxxx/xx-xx/x-x-x.index"
+    assert(true == Utils.isHdfsPath(ossPath))
+    assert(true == Utils.isHdfsPath(sortedOssPath))
+    assert(true == Utils.isHdfsPath(indexOssPath))
+
+    val localPath = "/xxx/xxx/xx-xx/x-x-x"
+    assert(false == Utils.isHdfsPath(localPath))
+  }
+
   test("GetReducerFileGroupResponse class convert with pb") {
     val fileGroup = new util.HashMap[Integer, util.Set[PartitionLocation]]
     fileGroup.put(0, partitionLocation(0))