You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/07/02 20:32:51 UTC
[34/45] hadoop git commit: HADOOP-15554. Improve JIT performance for
Configuration parsing. Contributed by Todd Lipcon.
HADOOP-15554. Improve JIT performance for Configuration parsing. Contributed by Todd Lipcon.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f51da9c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f51da9c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f51da9c4
Branch: refs/heads/HDDS-4
Commit: f51da9c4d1423c2ac92eb4f40e973264e7e968cc
Parents: 5d748bd
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Jul 2 18:31:21 2018 +0200
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Jul 2 18:31:21 2018 +0200
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 458 +++++++++++--------
1 file changed, 276 insertions(+), 182 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f51da9c4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index b1125e5..a78e311 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -41,6 +41,7 @@ import java.io.Writer;
import java.lang.ref.WeakReference;
import java.net.InetSocketAddress;
import java.net.JarURLConnection;
+import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
@@ -2981,187 +2982,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
if(returnCachedProperties) {
toAddTo = new Properties();
}
- DeprecationContext deprecations = deprecationContext.get();
- StringBuilder token = new StringBuilder();
- String confName = null;
- String confValue = null;
- String confInclude = null;
- String confTag = null;
- boolean confFinal = false;
- boolean fallbackAllowed = false;
- boolean fallbackEntered = false;
- boolean parseToken = false;
- LinkedList<String> confSource = new LinkedList<String>();
-
- while (reader.hasNext()) {
- switch (reader.next()) {
- case XMLStreamConstants.START_ELEMENT:
- switch (reader.getLocalName()) {
- case "property":
- confName = null;
- confValue = null;
- confFinal = false;
- confTag = null;
- confSource.clear();
-
- // First test for short format configuration
- int attrCount = reader.getAttributeCount();
- for (int i = 0; i < attrCount; i++) {
- String propertyAttr = reader.getAttributeLocalName(i);
- if ("name".equals(propertyAttr)) {
- confName = StringInterner.weakIntern(
- reader.getAttributeValue(i));
- } else if ("value".equals(propertyAttr)) {
- confValue = StringInterner.weakIntern(
- reader.getAttributeValue(i));
- } else if ("final".equals(propertyAttr)) {
- confFinal = "true".equals(reader.getAttributeValue(i));
- } else if ("source".equals(propertyAttr)) {
- confSource.add(StringInterner.weakIntern(
- reader.getAttributeValue(i)));
- } else if ("tag".equals(propertyAttr)) {
- confTag = StringInterner
- .weakIntern(reader.getAttributeValue(i));
- }
- }
- break;
- case "name":
- case "value":
- case "final":
- case "source":
- case "tag":
- parseToken = true;
- token.setLength(0);
- break;
- case "include":
- // Determine href for xi:include
- confInclude = null;
- attrCount = reader.getAttributeCount();
- for (int i = 0; i < attrCount; i++) {
- String attrName = reader.getAttributeLocalName(i);
- if ("href".equals(attrName)) {
- confInclude = reader.getAttributeValue(i);
- }
- }
- if (confInclude == null) {
- break;
- }
- if (isRestricted) {
- throw new RuntimeException("Error parsing resource " + wrapper
- + ": XInclude is not supported for restricted resources");
- }
- // Determine if the included resource is a classpath resource
- // otherwise fallback to a file resource
- // xi:include are treated as inline and retain current source
- URL include = getResource(confInclude);
- if (include != null) {
- Resource classpathResource = new Resource(include, name,
- wrapper.isParserRestricted());
- loadResource(properties, classpathResource, quiet);
- } else {
- URL url;
- try {
- url = new URL(confInclude);
- url.openConnection().connect();
- } catch (IOException ioe) {
- File href = new File(confInclude);
- if (!href.isAbsolute()) {
- // Included resources are relative to the current resource
- File baseFile = new File(name).getParentFile();
- href = new File(baseFile, href.getPath());
- }
- if (!href.exists()) {
- // Resource errors are non-fatal iff there is 1 xi:fallback
- fallbackAllowed = true;
- break;
- }
- url = href.toURI().toURL();
- }
- Resource uriResource = new Resource(url, name,
- wrapper.isParserRestricted());
- loadResource(properties, uriResource, quiet);
- }
- break;
- case "fallback":
- fallbackEntered = true;
- break;
- case "configuration":
- break;
- default:
- break;
- }
- break;
-
- case XMLStreamConstants.CHARACTERS:
- if (parseToken) {
- char[] text = reader.getTextCharacters();
- token.append(text, reader.getTextStart(), reader.getTextLength());
- }
- break;
-
- case XMLStreamConstants.END_ELEMENT:
- switch (reader.getLocalName()) {
- case "name":
- if (token.length() > 0) {
- confName = StringInterner.weakIntern(token.toString().trim());
- }
- break;
- case "value":
- if (token.length() > 0) {
- confValue = StringInterner.weakIntern(token.toString());
- }
- break;
- case "final":
- confFinal = "true".equals(token.toString());
- break;
- case "source":
- confSource.add(StringInterner.weakIntern(token.toString()));
- break;
- case "tag":
- if (token.length() > 0) {
- confTag = StringInterner.weakIntern(token.toString());
- }
- break;
- case "include":
- if (fallbackAllowed && !fallbackEntered) {
- throw new IOException("Fetch fail on include for '"
- + confInclude + "' with no fallback while loading '"
- + name + "'");
- }
- fallbackAllowed = false;
- fallbackEntered = false;
- break;
- case "property":
- if (confName == null || (!fallbackAllowed && fallbackEntered)) {
- break;
- }
- confSource.add(name);
- // Read tags and put them in propertyTagsMap
- if (confTag != null) {
- readTagFromConfig(confTag, confName, confValue, confSource);
- }
-
- DeprecatedKeyInfo keyInfo =
- deprecations.getDeprecatedKeyMap().get(confName);
- if (keyInfo != null) {
- keyInfo.clearAccessed();
- for (String key : keyInfo.newKeys) {
- // update new keys with deprecated key's value
- loadProperty(toAddTo, name, key, confValue, confFinal,
- confSource.toArray(new String[confSource.size()]));
- }
- } else {
- loadProperty(toAddTo, name, confName, confValue, confFinal,
- confSource.toArray(new String[confSource.size()]));
- }
- break;
- default:
- break;
- }
- default:
- break;
- }
+ List<ParsedItem> items = new Parser(reader, wrapper, quiet).parse();
+ for (ParsedItem item : items) {
+ loadProperty(toAddTo, item.name, item.key, item.value,
+ item.isFinal, item.sources);
}
reader.close();
@@ -3179,6 +3004,275 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
}
+ private static class ParsedItem {
+ String name;
+ String key;
+ String value;
+ boolean isFinal;
+ String[] sources;
+
+ ParsedItem(String name, String key, String value,
+ boolean isFinal, String[] sources) {
+ this.name = name;
+ this.key = key;
+ this.value = value;
+ this.isFinal = isFinal;
+ this.sources = sources;
+ }
+ }
+
+ /**
+ * Parser to consume SAX stream of XML elements from a Configuration.
+ */
+ private class Parser {
+ private final XMLStreamReader2 reader;
+ private final Resource wrapper;
+ private final String name;
+ private final String[] nameSingletonArray;
+ private final boolean isRestricted;
+ private final boolean quiet;
+
+ DeprecationContext deprecations = deprecationContext.get();
+
+ private StringBuilder token = new StringBuilder();
+ private String confName = null;
+ private String confValue = null;
+ private String confInclude = null;
+ private String confTag = null;
+ private boolean confFinal = false;
+ private boolean fallbackAllowed = false;
+ private boolean fallbackEntered = false;
+ private boolean parseToken = false;
+ private List<String> confSource = new ArrayList<>();
+ private List<ParsedItem> results = new ArrayList<>();
+
+ Parser(XMLStreamReader2 reader,
+ Resource wrapper,
+ boolean quiet) {
+ this.reader = reader;
+ this.wrapper = wrapper;
+ this.name = wrapper.getName();
+ this.nameSingletonArray = new String[]{ name };
+ this.isRestricted = wrapper.isParserRestricted();
+ this.quiet = quiet;
+
+ }
+
+ List<ParsedItem> parse() throws IOException, XMLStreamException {
+ while (reader.hasNext()) {
+ parseNext();
+ }
+ return results;
+ }
+
+ private void handleStartElement() throws MalformedURLException {
+ switch (reader.getLocalName()) {
+ case "property":
+ handleStartProperty();
+ break;
+
+ case "name":
+ case "value":
+ case "final":
+ case "source":
+ case "tag":
+ parseToken = true;
+ token.setLength(0);
+ break;
+ case "include":
+ handleInclude();
+ break;
+ case "fallback":
+ fallbackEntered = true;
+ break;
+ case "configuration":
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void handleStartProperty() {
+ confName = null;
+ confValue = null;
+ confFinal = false;
+ confTag = null;
+ confSource.clear();
+
+ // First test for short format configuration
+ int attrCount = reader.getAttributeCount();
+ for (int i = 0; i < attrCount; i++) {
+ String propertyAttr = reader.getAttributeLocalName(i);
+ if ("name".equals(propertyAttr)) {
+ confName = StringInterner.weakIntern(
+ reader.getAttributeValue(i));
+ } else if ("value".equals(propertyAttr)) {
+ confValue = StringInterner.weakIntern(
+ reader.getAttributeValue(i));
+ } else if ("final".equals(propertyAttr)) {
+ confFinal = "true".equals(reader.getAttributeValue(i));
+ } else if ("source".equals(propertyAttr)) {
+ confSource.add(StringInterner.weakIntern(
+ reader.getAttributeValue(i)));
+ } else if ("tag".equals(propertyAttr)) {
+ confTag = StringInterner
+ .weakIntern(reader.getAttributeValue(i));
+ }
+ }
+ }
+
+ private void handleInclude() throws MalformedURLException {
+ // Determine href for xi:include
+ confInclude = null;
+ int attrCount = reader.getAttributeCount();
+ for (int i = 0; i < attrCount; i++) {
+ String attrName = reader.getAttributeLocalName(i);
+ if ("href".equals(attrName)) {
+ confInclude = reader.getAttributeValue(i);
+ }
+ }
+ if (confInclude == null) {
+ return;
+ }
+ if (isRestricted) {
+ throw new RuntimeException("Error parsing resource " + wrapper
+ + ": XInclude is not supported for restricted resources");
+ }
+ // Determine if the included resource is a classpath resource
+ // otherwise fallback to a file resource
+ // xi:include are treated as inline and retain current source
+ URL include = getResource(confInclude);
+ if (include != null) {
+ Resource classpathResource = new Resource(include, name,
+ wrapper.isParserRestricted());
+ // This is only called recursively while the lock is already held
+ // by this thread, but synchronizing avoids a findbugs warning.
+ synchronized (Configuration.this) {
+ loadResource(properties, classpathResource, quiet);
+ }
+ } else {
+ URL url;
+ try {
+ url = new URL(confInclude);
+ url.openConnection().connect();
+ } catch (IOException ioe) {
+ File href = new File(confInclude);
+ if (!href.isAbsolute()) {
+ // Included resources are relative to the current resource
+ File baseFile = new File(name).getParentFile();
+ href = new File(baseFile, href.getPath());
+ }
+ if (!href.exists()) {
+ // Resource errors are non-fatal iff there is 1 xi:fallback
+ fallbackAllowed = true;
+ return;
+ }
+ url = href.toURI().toURL();
+ }
+ Resource uriResource = new Resource(url, name,
+ wrapper.isParserRestricted());
+ // This is only called recursively while the lock is already held
+ // by this thread, but synchronizing avoids a findbugs warning.
+ synchronized (Configuration.this) {
+ loadResource(properties, uriResource, quiet);
+ }
+ }
+ }
+
+ void handleEndElement() throws IOException {
+ String tokenStr = token.toString();
+ switch (reader.getLocalName()) {
+ case "name":
+ if (token.length() > 0) {
+ confName = StringInterner.weakIntern(tokenStr.trim());
+ }
+ break;
+ case "value":
+ if (token.length() > 0) {
+ confValue = StringInterner.weakIntern(tokenStr);
+ }
+ break;
+ case "final":
+ confFinal = "true".equals(tokenStr);
+ break;
+ case "source":
+ confSource.add(StringInterner.weakIntern(tokenStr));
+ break;
+ case "tag":
+ if (token.length() > 0) {
+ confTag = StringInterner.weakIntern(tokenStr);
+ }
+ break;
+ case "include":
+ if (fallbackAllowed && !fallbackEntered) {
+ throw new IOException("Fetch fail on include for '"
+ + confInclude + "' with no fallback while loading '"
+ + name + "'");
+ }
+ fallbackAllowed = false;
+ fallbackEntered = false;
+ break;
+ case "property":
+ handleEndProperty();
+ break;
+ default:
+ break;
+ }
+ }
+
+ void handleEndProperty() {
+ if (confName == null || (!fallbackAllowed && fallbackEntered)) {
+ return;
+ }
+ String[] confSourceArray;
+ if (confSource.isEmpty()) {
+ confSourceArray = nameSingletonArray;
+ } else {
+ confSource.add(name);
+ confSourceArray = confSource.toArray(new String[confSource.size()]);
+ }
+
+ // Read tags and put them in propertyTagsMap
+ if (confTag != null) {
+ readTagFromConfig(confTag, confName, confValue, confSourceArray);
+ }
+
+ DeprecatedKeyInfo keyInfo =
+ deprecations.getDeprecatedKeyMap().get(confName);
+
+ if (keyInfo != null) {
+ keyInfo.clearAccessed();
+ for (String key : keyInfo.newKeys) {
+ // update new keys with deprecated key's value
+ results.add(new ParsedItem(
+ name, key, confValue, confFinal, confSourceArray));
+ }
+ } else {
+ results.add(new ParsedItem(name, confName, confValue, confFinal,
+ confSourceArray));
+ }
+ }
+
+ void parseNext() throws IOException, XMLStreamException {
+ switch (reader.next()) {
+ case XMLStreamConstants.START_ELEMENT:
+ handleStartElement();
+ break;
+ case XMLStreamConstants.CHARACTERS:
+ if (parseToken) {
+ char[] text = reader.getTextCharacters();
+ token.append(text, reader.getTextStart(), reader.getTextLength());
+ }
+ break;
+ case XMLStreamConstants.END_ELEMENT:
+ handleEndElement();
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
/**
* Add tags defined in HADOOP_TAGS_SYSTEM, HADOOP_TAGS_CUSTOM.
* @param prop
@@ -3225,7 +3319,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @param confSource
*/
private void readTagFromConfig(String attributeValue, String confName, String
- confValue, List<String> confSource) {
+ confValue, String[] confSource) {
for (String tagStr : attributeValue.split(",")) {
try {
tagStr = tagStr.trim();
@@ -3243,7 +3337,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
} catch (Exception ex) {
// Log the exception at trace level.
LOG.trace("Tag '{}' for property:{} Source:{}", tagStr, confName,
- Arrays.toString(confSource.toArray()), ex);
+ confSource, ex);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org