You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/03/24 04:32:25 UTC
hadoop git commit: HADOOP-15295. Remove redundant logging related to
tags from Configuration. Contributed by Ajay Kumar.
Repository: hadoop
Updated Branches:
refs/heads/trunk 24f75e097 -> 28790b81e
HADOOP-15295. Remove redundant logging related to tags from Configuration. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28790b81
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28790b81
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28790b81
Branch: refs/heads/trunk
Commit: 28790b81ecb719ac26a01ad0c400ee5f1c29ccf8
Parents: 24f75e0
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Mar 23 21:17:18 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Fri Mar 23 21:17:18 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 41 ++++++++++----------
.../apache/hadoop/conf/TestConfiguration.java | 34 ++++------------
2 files changed, 27 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28790b81/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index e0a5866..25fd656a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -229,7 +229,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
private static final Logger LOG_DEPRECATION =
LoggerFactory.getLogger(
"org.apache.hadoop.conf.Configuration.deprecation");
- private static final Set<String> TAGS = new HashSet<>();
+ private static final Set<String> TAGS = ConcurrentHashMap.newKeySet();
private boolean quietmode = true;
@@ -2935,7 +2935,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
resources.set(i, ret);
}
}
- this.removeUndeclaredTags(properties);
+ this.addTags(properties);
}
private Resource loadResource(Properties properties,
@@ -3183,29 +3183,28 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
/**
- * Removes undeclared tags and related properties from propertyTagsMap.
- * Its required because ordering of properties in xml config files is not
- * guaranteed.
+ * Add tags defined in HADOOP_SYSTEM_TAGS, HADOOP_CUSTOM_TAGS.
* @param prop
*/
- private void removeUndeclaredTags(Properties prop) {
+ public void addTags(Properties prop) {
// Get all system tags
- if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)){
- String systemTags = prop.getProperty(CommonConfigurationKeys
- .HADOOP_SYSTEM_TAGS);
- Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag));
- }
- // Get all custom tags
- if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) {
- String customTags = prop.getProperty(CommonConfigurationKeys
- .HADOOP_CUSTOM_TAGS);
- Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag));
- }
+ try {
+ if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)) {
+ String systemTags = prop.getProperty(CommonConfigurationKeys
+ .HADOOP_SYSTEM_TAGS);
+ Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag));
+ }
+ // Get all custom tags
+ if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) {
+ String customTags = prop.getProperty(CommonConfigurationKeys
+ .HADOOP_CUSTOM_TAGS);
+ Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag));
+ }
- Set undeclaredTags = propertyTagsMap.keySet();
- if (undeclaredTags.retainAll(TAGS)) {
- LOG.info("Removed undeclared tags:");
+ } catch (Exception ex) {
+ LOG.trace("Error adding tags in configuration", ex);
}
+
}
/**
@@ -3219,8 +3218,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
private void readTagFromConfig(String attributeValue, String confName, String
confValue, List<String> confSource) {
for (String tagStr : attributeValue.split(",")) {
- tagStr = tagStr.trim();
try {
+ tagStr = tagStr.trim();
// Handle property with no/null value
if (confValue == null) {
confValue = "";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28790b81/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 8f54464..b0bb0d7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -27,7 +27,6 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
-import java.io.PrintStream;
import java.io.StringWriter;
import java.net.InetAddress;
import java.net.InetSocketAddress;
@@ -2392,33 +2391,14 @@ public class TestConfiguration {
@Test
public void testInvalidTags() throws Exception {
- PrintStream output = System.out;
- try {
- ByteArrayOutputStream bytes = new ByteArrayOutputStream();
- System.setOut(new PrintStream(bytes));
-
- out = new BufferedWriter(new FileWriter(CONFIG));
- startConfig();
- appendPropertyByTag("dfs.cblock.trace.io", "false", "MYOWNTAG,TAG2");
- endConfig();
-
- Path fileResource = new Path(CONFIG);
- conf.addResource(fileResource);
- conf.getProps();
-
- List<String> tagList = new ArrayList<>();
- tagList.add("REQUIRED");
- tagList.add("MYOWNTAG");
- tagList.add("TAG2");
+ Path fileResource = new Path(CONFIG);
+ conf.addResource(fileResource);
+ conf.getProps();
- Properties properties = conf.getAllPropertiesByTags(tagList);
- assertEq(0, properties.size());
- assertFalse(properties.containsKey("dfs.cblock.trace.io"));
- assertFalse(bytes.toString().contains("Invalid tag "));
- assertFalse(bytes.toString().contains("Tag"));
- } finally {
- System.setOut(output);
- }
+ assertFalse(conf.isPropertyTag("BADTAG"));
+ assertFalse(conf.isPropertyTag("CUSTOM_TAG"));
+ assertTrue(conf.isPropertyTag("DEBUG"));
+ assertTrue(conf.isPropertyTag("HDFS"));
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org