[hudi] branch master updated: [HUDI-4495] Fix handling of S3 paths incompatible with java URI standards (#6237)

[yi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new c39e88dcf0 [HUDI-4495] Fix handling of S3 paths incompatible with java URI standards (#6237)
c39e88dcf0 is described below

commit c39e88dcf0c196bca08aced6e16d47fc72f5e83e
Author: Udit Mehrotra <ud...@amazon.com>
AuthorDate: Thu Jul 28 20:04:14 2022 -0700

    [HUDI-4495] Fix handling of S3 paths incompatible with java URI standards (#6237)
---
 .../apache/hudi/common/fs/HoodieWrapperFileSystem.java  |  7 +++++--
 .../org/apache/hudi/common/fs/TestStorageSchemes.java   | 17 +++++++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java
index bceea8e367..2979696be7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java
@@ -145,8 +145,11 @@ public class HoodieWrapperFileSystem extends FileSystem {
     URI oldURI = oldPath.toUri();
     URI newURI;
     try {
-      newURI = new URI(newScheme, oldURI.getUserInfo(), oldURI.getHost(), oldURI.getPort(), oldURI.getPath(),
-          oldURI.getQuery(), oldURI.getFragment());
+      newURI = new URI(newScheme,
+                oldURI.getAuthority(),
+                oldURI.getPath(),
+                oldURI.getQuery(),
+                oldURI.getFragment());
       return new CachingPath(newURI);
     } catch (URISyntaxException e) {
       // TODO - Better Exception handling
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java
index 85f3ce65ec..9b173254ac 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java
@@ -18,8 +18,10 @@
 
 package org.apache.hudi.common.fs;
 
+import org.apache.hadoop.fs.Path;
 import org.junit.jupiter.api.Test;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -54,4 +56,19 @@ public class TestStorageSchemes {
       StorageSchemes.isAppendSupported("s2");
     }, "Should throw exception for unsupported schemes");
   }
+
+  @Test
+  public void testConversionToNewSchema() {
+    Path s3TablePath1 = new Path("s3://test.1234/table1");
+    assertEquals(s3TablePath1, HoodieWrapperFileSystem.convertPathWithScheme(s3TablePath1, "s3"));
+
+    Path s3TablePath2 = new Path("s3://1234.test/table1");
+    assertEquals(s3TablePath2, HoodieWrapperFileSystem.convertPathWithScheme(s3TablePath2, "s3"));
+
+    Path s3TablePath3 = new Path("s3://test1234/table1");
+    assertEquals(s3TablePath3, HoodieWrapperFileSystem.convertPathWithScheme(s3TablePath3, "s3"));
+
+    Path hdfsTablePath = new Path("hdfs://sandbox.foo.com:8020/test.1234/table1");
+    System.out.println(HoodieWrapperFileSystem.convertPathWithScheme(hdfsTablePath, "hdfs"));
+  }
 }