You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2020/07/16 17:52:04 UTC
[hadoop] branch branch-3.3 updated (9e7266d -> 187b161)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 9e7266d HADOOP-17099. Replace Guava Predicate with Java8+ Predicate
new 0d88ed2 HADOOP-17129. Validating storage keys in ABFS correctly (#2141)
new 187b161 HADOOP-17130. Configuration.getValByRegex() shouldn't be updating the results while fetching. (#2142)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/hadoop/conf/Configuration.java | 6 ++++--
.../apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 14 --------------
.../fs/azurebfs/services/SimpleKeyProvider.java | 20 +++++++++++++++++++-
.../TestAbfsConfigurationFieldsValidation.java | 4 ++--
4 files changed, 25 insertions(+), 19 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HADOOP-17130. Configuration.getValByRegex()
shouldn't be updating the results while fetching. (#2142)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 187b161c2063ac5713215bac8395a466e9e86501
Author: Mukund Thakur <mt...@cloudera.com>
AuthorDate: Thu Jul 16 22:39:59 2020 +0530
HADOOP-17130. Configuration.getValByRegex() shouldn't be updating the results while fetching. (#2142)
Contributed by Mukund Thakur
Change-Id: Ic0c9c44a1b14b4bfa962cede3f782a98fdfd7513
---
.../src/main/java/org/apache/hadoop/conf/Configuration.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 9a8841e7..d821638 100755
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -3865,6 +3865,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
Pattern p = Pattern.compile(regex);
Map<String,String> result = new HashMap<String,String>();
+ List<String> resultKeys = new ArrayList<>();
Matcher m;
for(Map.Entry<Object,Object> item: getProps().entrySet()) {
@@ -3872,11 +3873,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
item.getValue() instanceof String) {
m = p.matcher((String)item.getKey());
if(m.find()) { // match
- result.put((String) item.getKey(),
- substituteVars(getProps().getProperty((String) item.getKey())));
+ resultKeys.add((String) item.getKey());
}
}
}
+ resultKeys.forEach(item ->
+ result.put(item, substituteVars(getProps().getProperty(item))));
return result;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HADOOP-17129. Validating storage keys in ABFS
correctly (#2141)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 0d88ed279425da14c8876be9d7c0a1181fb0d5c7
Author: Mehakmeet Singh <me...@gmail.com>
AuthorDate: Thu Jul 16 21:59:37 2020 +0530
HADOOP-17129. Validating storage keys in ABFS correctly (#2141)
Contributed by Mehakmeet Singh
Change-Id: I8016ee2f9ffbc86ea867f4a3d960b134e507d099
---
.../apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 14 --------------
.../fs/azurebfs/services/SimpleKeyProvider.java | 20 +++++++++++++++++++-
.../TestAbfsConfigurationFieldsValidation.java | 4 ++--
3 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 354176f..7996ec1 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs.azurebfs;
import java.io.IOException;
import java.lang.reflect.Field;
-import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -215,8 +214,6 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS)
private long sasTokenRenewPeriodForStreamsInSeconds;
- private Map<String, String> storageAccountKeys;
-
public AbfsConfiguration(final Configuration rawConfig, String accountName)
throws IllegalAccessException, InvalidConfigurationValueException, IOException {
this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders(
@@ -224,7 +221,6 @@ public class AbfsConfiguration{
this.accountName = accountName;
this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false);
- validateStorageAccountKeys();
Field[] fields = this.getClass().getDeclaredFields();
for (Field field : fields) {
field.setAccessible(true);
@@ -665,16 +661,6 @@ public class AbfsConfiguration{
}
}
- void validateStorageAccountKeys() throws InvalidConfigurationValueException {
- Base64StringConfigurationBasicValidator validator = new Base64StringConfigurationBasicValidator(
- FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, "", true);
- this.storageAccountKeys = rawConfig.getValByRegex(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX);
-
- for (Map.Entry<String, String> account : storageAccountKeys.entrySet()) {
- validator.validate(account.getValue());
- }
- }
-
int validateInt(Field field) throws IllegalAccessException, InvalidConfigurationValueException {
IntegerConfigurationValidatorAnnotation validator = field.getAnnotation(IntegerConfigurationValidatorAnnotation.class);
String value = get(validator.ConfigurationKey());
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
index 727e1b3..bb1ec9e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException;
+import org.apache.hadoop.fs.azurebfs.diagnostics.Base64StringConfigurationBasicValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +44,10 @@ public class SimpleKeyProvider implements KeyProvider {
try {
AbfsConfiguration abfsConfig = new AbfsConfiguration(rawConfig, accountName);
key = abfsConfig.getPasswordString(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME);
- } catch(IllegalAccessException | InvalidConfigurationValueException e) {
+
+ // Validating the key.
+ validateStorageAccountKey(key);
+ } catch (IllegalAccessException | InvalidConfigurationValueException e) {
throw new KeyProviderException("Failure to initialize configuration", e);
} catch(IOException ioe) {
LOG.warn("Unable to get key from credential providers. {}", ioe);
@@ -51,4 +55,18 @@ public class SimpleKeyProvider implements KeyProvider {
return key;
}
+
+ /**
+ * A method to validate the storage key.
+ *
+ * @param key the key to be validated.
+ * @throws InvalidConfigurationValueException
+ */
+ private void validateStorageAccountKey(String key)
+ throws InvalidConfigurationValueException {
+ Base64StringConfigurationBasicValidator validator = new Base64StringConfigurationBasicValidator(
+ ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, "", true);
+
+ validator.validate(key);
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
index 0f550d8..898667f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidati
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.LongConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.Base64StringConfigurationValidatorAnnotation;
-import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConfigurationPropertyNotFoundException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException;
import org.apache.hadoop.fs.azurebfs.utils.Base64;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SSL_CHANNEL_MODE_KEY;
@@ -155,7 +155,7 @@ public class TestAbfsConfigurationFieldsValidation {
assertEquals(this.encodedAccountKey, accountKey);
}
- @Test(expected = ConfigurationPropertyNotFoundException.class)
+ @Test(expected = KeyProviderException.class)
public void testGetAccountKeyWithNonExistingAccountName() throws Exception {
Configuration configuration = new Configuration();
configuration.addResource(TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org