[druid] branch master updated: Fix GCS based ingestion if bucket name contains underscores (#12445)

[kf...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git


The following commit(s) were added to refs/heads/master by this push:
     new 177e1856cd Fix GCS based ingestion if bucket name contains underscores (#12445)
177e1856cd is described below

commit 177e1856cdaf3e5aa7a5754d129fc1945b9f7c72
Author: Tejaswini Bandlamudi <96...@users.noreply.github.com>
AuthorDate: Thu Apr 21 09:22:35 2022 +0530

    Fix GCS based ingestion if bucket name contains underscores (#12445)
    
    GCP allows bucket names to contain underscores. When a location in such a bucket
    is mapped to `java.net.URI`, `URI.getHost()` returns null. `URI.getHost()` is used as
    the bucket name in `CloudObjectLocation`, leading to an NPE.
    
    This commit uses `URI.getAuthority()` as the bucket name if `URI.getHost()` is null.
---
 .../druid/data/input/impl/CloudObjectLocation.java |  2 +-
 .../data/input/impl/CloudObjectLocationTest.java   | 28 ++++++++++++++++------
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/apache/druid/data/input/impl/CloudObjectLocation.java b/core/src/main/java/org/apache/druid/data/input/impl/CloudObjectLocation.java
index ccc6916633..c2cdd07e77 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/CloudObjectLocation.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/CloudObjectLocation.java
@@ -71,7 +71,7 @@ public class CloudObjectLocation
 
   public CloudObjectLocation(URI uri)
   {
-    this(uri.getHost(), uri.getPath());
+    this(uri.getHost() != null ? uri.getHost() : uri.getAuthority(), uri.getPath());
   }
 
   /**
diff --git a/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectLocationTest.java b/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectLocationTest.java
index c7cd9957f4..f79b3d5b70 100644
--- a/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectLocationTest.java
+++ b/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectLocationTest.java
@@ -115,13 +115,27 @@ public class CloudObjectLocationTest
   }
 
   @Test
-  public void testInvalidBucketName()
+  public void testBucketNameWithoutUnderscores()
   {
-    expectedException.expect(NullPointerException.class);
-    expectedException.expectMessage("bucket name cannot be null. Please verify if bucket name adheres to naming rules");
-    // Underscore(_) character is not valid for bucket names
-    CloudObjectLocation invalidBucket1 = new CloudObjectLocation("test_bucket", "path/to/path");
-    CloudObjectLocation invalidBucket2 = new CloudObjectLocation(invalidBucket1.toUri(SCHEME));
-    Assert.assertEquals("test_bucket", new CloudObjectLocation(invalidBucket2.toUri(SCHEME)));
+    CloudObjectLocation gsValidBucket = new CloudObjectLocation(URI.create("gs://1test.bucket-value/path/to/path"));
+    Assert.assertEquals("1test.bucket-value", gsValidBucket.getBucket());
+    Assert.assertEquals("path/to/path", gsValidBucket.getPath());
+
+    CloudObjectLocation s3ValidBucket = new CloudObjectLocation(URI.create("s3://2test.bucket-value/path/to/path"));
+    Assert.assertEquals("2test.bucket-value", s3ValidBucket.getBucket());
+    Assert.assertEquals("path/to/path", s3ValidBucket.getPath());
+  }
+
+  @Test
+  public void testBucketNameWithUnderscores()
+  {
+    // Underscore(_) character is allowed for bucket names by GCP
+    CloudObjectLocation gsValidBucket = new CloudObjectLocation(URI.create("gs://test_bucket/path/to/path"));
+    Assert.assertEquals("test_bucket", gsValidBucket.getBucket());
+    Assert.assertEquals("path/to/path", gsValidBucket.getPath());
+
+    CloudObjectLocation s3ValidBucket = new CloudObjectLocation(URI.create("s3://test_bucket/path/to/path"));
+    Assert.assertEquals("test_bucket", s3ValidBucket.getBucket());
+    Assert.assertEquals("path/to/path", s3ValidBucket.getPath());
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org