You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2024/09/27 14:47:04 UTC
(hadoop) branch branch-3.4 updated: HADOOP-19284: [ABFS] Allow "fs.azure.account.hns.enabled" to be set as Account Specific Config (#7062) (#7076)
This is an automated email from the ASF dual-hosted git repository.
surendralilhore pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new cf699efcaa7 HADOOP-19284: [ABFS] Allow "fs.azure.account.hns.enabled" to be set as Account Specific Config (#7062) (#7076)
cf699efcaa7 is described below
commit cf699efcaa7f8c0aab88c9465eedb09972e2cfef
Author: Anuj Modi <12...@users.noreply.github.com>
AuthorDate: Fri Sep 27 20:16:55 2024 +0530
HADOOP-19284: [ABFS] Allow "fs.azure.account.hns.enabled" to be set as Account Specific Config (#7062) (#7076)
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 3 +-
.../fs/azurebfs/ITestAzureBlobFileSystemE2E.java | 4 ++
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 65 ++++++++++++++++++++++
3 files changed, 71 insertions(+), 1 deletion(-)
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 43923f758f9..0f77024fa61 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -414,7 +414,8 @@ public class AbfsConfiguration{
}
public Trilean getIsNamespaceEnabledAccount() {
- return Trilean.getTrilean(isNamespaceEnabledAccount);
+ return Trilean.getTrilean(
+ getString(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount));
}
/**
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
index c7a1fa91a98..00e9fc5b614 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
@@ -40,6 +40,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_HT
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_HTTP_READ_TIMEOUT;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_TOLERATE_CONCURRENT_APPEND;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -259,6 +260,9 @@ public class ITestAzureBlobFileSystemE2E extends AbstractAbfsIntegrationTest {
public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs)
throws Exception {
+ // This is to make sure File System creation goes through before network calls start failing.
+ assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
+
Configuration conf = this.getRawConfiguration();
// set to small values that will cause timeouts
conf.setInt(AZURE_HTTP_CONNECTION_TIMEOUT, connectionTimeoutMs);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
index d168ed38844..cc540e93185 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
@@ -40,6 +40,8 @@ import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doReturn;
@@ -125,6 +127,8 @@ public class ITestGetNameSpaceEnabled extends AbstractAbfsIntegrationTest {
Configuration rawConfig = new Configuration();
rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME);
rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount);
+ rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED,
+ this.getAccountName()), isNamespaceEnabledAccount);
rawConfig
.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
@@ -265,4 +269,65 @@ public class ITestGetNameSpaceEnabled extends AbstractAbfsIntegrationTest {
Mockito.verify(mockClient, times(1))
.getAclStatus(anyString(), any(TracingContext.class));
}
+
+ @Test
+ public void testAccountSpecificConfig() throws Exception {
+ Configuration rawConfig = new Configuration();
+ rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME);
+ rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
+ rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED,
+ this.getAccountName()));
+ String testAccountName = "testAccount.dfs.core.windows.net";
+ String otherAccountName = "otherAccount.dfs.core.windows.net";
+ String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName);
+ String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName);
+
+ // Set both account specific and account agnostic config for test account
+ rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
+ rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR);
+ // Assert that account specific config takes precedence
+ rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
+ assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
+ // Assert that other account still uses account agnostic config
+ rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, otherUri);
+ assertFileSystemInitWithExpectedHNSSettings(rawConfig, true);
+
+ // Set only the account specific config for test account
+ rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
+ rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
+ // Assert that only account specific config is enough for test account
+ rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
+ assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
+
+ // Set only account agnostic config
+ rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, FALSE_STR);
+ rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName));
+ rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
+ assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
+
+ // Unset both account specific and account agnostic config
+ rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
+ rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName));
+ rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
+ rawConfig.set(AZURE_MAX_IO_RETRIES, "0");
+ // Assert that file system init fails with UnknownHost exception as getAcl() is needed.
+ try {
+ assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
+ } catch (Exception e) {
+ Assertions.assertThat(e.getCause().getMessage())
+ .describedAs("getAcl() to determine HNS Nature of account should"
+ + "fail with Unknown Host Exception").contains("UnknownHostException");
+ }
+ }
+
+ private void assertFileSystemInitWithExpectedHNSSettings(
+ Configuration configuration, boolean expectedIsHnsEnabledValue) throws IOException {
+ try (AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration)) {
+ Assertions.assertThat(getIsNamespaceEnabled(fs)).describedAs(
+ "getIsNamespaceEnabled should return true when the "
+ + "account specific config is not set").isEqualTo(expectedIsHnsEnabledValue);
+ } catch (Exception e) {
+ throw e;
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org