You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by da...@apache.org on 2019/10/17 05:08:12 UTC
[hadoop] branch branch-2 updated: HADOOP-16640. WASB: Override
getCanonicalServiceName() to return URI
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 7f71473 HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI
7f71473 is described below
commit 7f71473888f1c06b22083038e6c398252fdf42d0
Author: Da Zhou <da...@apache.org>
AuthorDate: Wed Oct 16 13:14:15 2019 -0700
HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI
(cherry picked from commit 9a8edb0aeddd7787b2654f6e2a8465c325e048a2)
---
.../hadoop/fs/azure/NativeAzureFileSystem.java | 22 ++++++++
.../fs/azure/ITestWasbUriAndConfiguration.java | 61 ++++++++++++++++------
2 files changed, 67 insertions(+), 16 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 11a110b..41b1306 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -640,6 +640,20 @@ public class NativeAzureFileSystem extends FileSystem {
return "wasb";
}
+ /**
+ * If fs.azure.override.canonical.service.name is set as true, return URI of
+ * the WASB filesystem, otherwise use the default implementation.
+ *
+ * @return a service string that uniquely identifies this file system
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ if (returnUriAsCanonicalServiceName) {
+ return getUri().toString();
+ }
+ return super.getCanonicalServiceName();
+ }
+
/**
* <p>
@@ -723,6 +737,11 @@ public class NativeAzureFileSystem extends FileSystem {
*/
public static final String APPEND_SUPPORT_ENABLE_PROPERTY_NAME = "fs.azure.enable.append.support";
+ /*
+ * Property to override canonical service name with filesystem's URI.
+ */
+ public static final String RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME = "fs.azure.override.canonical.service.name";
+
/**
* The configuration property to set number of threads to be used for rename operation.
*/
@@ -1198,6 +1217,7 @@ public class NativeAzureFileSystem extends FileSystem {
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
private boolean appendSupportEnabled = false;
+ private boolean returnUriAsCanonicalServiceName = false;
private DelegationTokenAuthenticatedURL authURL;
private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
private String credServiceUrl;
@@ -1395,6 +1415,8 @@ public class NativeAzureFileSystem extends FileSystem {
if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
this.wasbDelegationTokenManager = new RemoteWasbDelegationTokenManager(conf);
}
+
+ this.returnUriAsCanonicalServiceName = conf.getBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, false);
}
@Override
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
index 7783684..62a751d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
@@ -18,30 +18,16 @@
package org.apache.hadoop.fs.azure;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
-import static org.junit.Assume.assumeNotNull;
-
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.Date;
import java.util.EnumSet;
-import java.io.File;
-import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
-import org.apache.hadoop.security.ProviderUtils;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.AbstractFileSystem;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
@@ -49,10 +35,24 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.CloudBlockBlob;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
+import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+
+import static org.junit.Assume.assumeNotNull;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.fs.azure.NativeAzureFileSystem.RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME;
+
public class ITestWasbUriAndConfiguration extends AbstractWasbTestWithTimeout {
private static final int FILE_SIZE = 4096;
@@ -581,4 +581,33 @@ public class ITestWasbUriAndConfiguration extends AbstractWasbTestWithTimeout {
FileSystem.closeAll();
}
}
+
+ @Test
+ public void testCanonicalServiceName() throws Exception {
+ AzureBlobStorageTestAccount testAccount = AzureBlobStorageTestAccount.createMock();
+ Configuration conf = testAccount.getFileSystem().getConf();
+ String authority = testAccount.getFileSystem().getUri().getAuthority();
+ URI defaultUri = new URI("wasbs", authority, null, null, null);
+ conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
+ try (FileSystem fs = FileSystem.get(conf)){
+ // Default getCanonicalServiceName() will try to resolve the host to IP,
+ // because the mock container does not exist, this call is expected to fail.
+ fs.getCanonicalServiceName();
+ Assert.assertTrue("Excepting exception", false);
+ } catch (IllegalArgumentException ex) {
+ if (!ex.getMessage().contains("java.net.UnknownHostException")) {
+ throw ex;
+ }
+ }
+ conf.setBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, true);
+
+ try (FileSystem fs = FileSystem.newInstance(defaultUri, conf)) {
+ Assert.assertEquals("getCanonicalServiceName() should return URI",
+ fs.getUri().toString(), fs.getCanonicalServiceName());
+ } finally {
+ testAccount.cleanup();
+ FileSystem.closeAll();
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org