You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jz...@apache.org on 2017/08/11 18:00:49 UTC
[1/3] hadoop git commit: HADOOP-13588. ConfServlet should respect
Accept request header. Contributed by Weiwei Yang
Repository: hadoop
Updated Branches:
refs/heads/branch-2 9c6e8ece9 -> e208a4e7e
HADOOP-13588. ConfServlet should respect Accept request header. Contributed by Weiwei Yang
(cherry picked from commit 59d59667a8b1d3fb4a744a41774b2397fd91cbb3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ba3268f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ba3268f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ba3268f
Branch: refs/heads/branch-2
Commit: 9ba3268fa3ad4fec5222361058a2086d2cb8c3e0
Parents: 9c6e8ec
Author: John Zhuge <jz...@apache.org>
Authored: Fri Aug 11 10:54:25 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Fri Aug 11 10:54:25 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/ConfServlet.java | 21 ++++++++++-------
.../org/apache/hadoop/conf/TestConfServlet.java | 24 ++++++++++++++++++++
2 files changed, 37 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ba3268f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
index d4b34e9..f9a2535 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
@@ -25,11 +25,14 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.HttpHeaders;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.http.HttpServer2;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* A servlet to print out the running configuration data.
*/
@@ -38,9 +41,8 @@ import org.apache.hadoop.http.HttpServer2;
public class ConfServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
- private static final String FORMAT_JSON = "json";
- private static final String FORMAT_XML = "xml";
- private static final String FORMAT_PARAM = "format";
+ protected static final String FORMAT_JSON = "json";
+ protected static final String FORMAT_XML = "xml";
/**
* Return the Configuration of the daemon hosting this servlet.
@@ -67,11 +69,7 @@ public class ConfServlet extends HttpServlet {
return;
}
- String format = request.getParameter(FORMAT_PARAM);
- if (null == format) {
- format = FORMAT_XML;
- }
-
+ String format = parseAccecptHeader(request);
if (FORMAT_XML.equals(format)) {
response.setContentType("text/xml; charset=utf-8");
} else if (FORMAT_JSON.equals(format)) {
@@ -87,6 +85,13 @@ public class ConfServlet extends HttpServlet {
out.close();
}
+ @VisibleForTesting
+ static String parseAccecptHeader(HttpServletRequest request) {
+ String format = request.getHeader(HttpHeaders.ACCEPT);
+ return format != null && format.contains(FORMAT_JSON) ?
+ FORMAT_JSON : FORMAT_XML;
+ }
+
/**
* Guts of the servlet - extracted for easy testing.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ba3268f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
index 1c22ee6..73d4992 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
@@ -19,7 +19,11 @@ package org.apache.hadoop.conf;
import java.io.StringWriter;
import java.io.StringReader;
+import java.util.HashMap;
import java.util.Map;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.core.HttpHeaders;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -32,6 +36,7 @@ import org.xml.sax.InputSource;
import junit.framework.TestCase;
import org.junit.Test;
+import org.mockito.Mockito;
/**
* Basic test case that the ConfServlet can write configuration
@@ -48,6 +53,25 @@ public class TestConfServlet extends TestCase {
}
@Test
+ public void testParseHeaders() throws Exception {
+ HashMap<String, String> verifyMap = new HashMap<String, String>();
+ verifyMap.put("text/plain", ConfServlet.FORMAT_XML);
+ verifyMap.put(null, ConfServlet.FORMAT_XML);
+ verifyMap.put("text/xml", ConfServlet.FORMAT_XML);
+ verifyMap.put("application/xml", ConfServlet.FORMAT_XML);
+ verifyMap.put("application/json", ConfServlet.FORMAT_JSON);
+
+ HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+ for(String contentTypeExpected : verifyMap.keySet()) {
+ String contenTypeActual = verifyMap.get(contentTypeExpected);
+ Mockito.when(request.getHeader(HttpHeaders.ACCEPT))
+ .thenReturn(contentTypeExpected);
+ assertEquals(contenTypeActual,
+ ConfServlet.parseAccecptHeader(request));
+ }
+ }
+
+ @Test
@SuppressWarnings("unchecked")
public void testWriteJson() throws Exception {
StringWriter sw = new StringWriter();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/3] hadoop git commit: HADOOP-14260.
Configuration.dumpConfiguration should redact sensitive information.
Contributed by John Zhuge.
Posted by jz...@apache.org.
HADOOP-14260. Configuration.dumpConfiguration should redact sensitive information. Contributed by John Zhuge.
(cherry picked from commit 582648befaf9908159f937d2cc8f549583a3483e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e208a4e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e208a4e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e208a4e7
Branch: refs/heads/branch-2
Commit: e208a4e7ee5725b910a7e799066000cacf855218
Parents: 76c4aee
Author: John Zhuge <jz...@apache.org>
Authored: Fri Aug 11 10:54:41 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Fri Aug 11 10:54:41 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 15 +++---
.../apache/hadoop/conf/TestConfiguration.java | 48 ++++++++++++++++++--
2 files changed, 53 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e208a4e7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 89714bf..42aa8f6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -3062,7 +3062,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("property");
- appendJSONProperty(dumpGenerator, config, propertyName);
+ appendJSONProperty(dumpGenerator, config, propertyName,
+ new ConfigRedactor(config));
dumpGenerator.writeEndObject();
dumpGenerator.flush();
}
@@ -3102,11 +3103,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
dumpGenerator.writeFieldName("properties");
dumpGenerator.writeStartArray();
dumpGenerator.flush();
+ ConfigRedactor redactor = new ConfigRedactor(config);
synchronized (config) {
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
- appendJSONProperty(dumpGenerator,
- config,
- item.getKey().toString());
+ appendJSONProperty(dumpGenerator, config, item.getKey().toString(),
+ redactor);
}
}
dumpGenerator.writeEndArray();
@@ -3124,12 +3125,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @throws IOException
*/
private static void appendJSONProperty(JsonGenerator jsonGen,
- Configuration config, String name) throws IOException {
+ Configuration config, String name, ConfigRedactor redactor)
+ throws IOException {
// skip writing if given property name is empty or null
if(!Strings.isNullOrEmpty(name) && jsonGen != null) {
jsonGen.writeStartObject();
jsonGen.writeStringField("key", name);
- jsonGen.writeStringField("value", config.get(name));
+ jsonGen.writeStringField("value",
+ redactor.redact(name, config.get(name)));
jsonGen.writeBooleanField("isFinal",
config.finalParameters.contains(name));
String[] resources = config.updatingResource.get(name);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e208a4e7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 24a49ad..66f717d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -48,6 +48,7 @@ import static org.junit.Assert.assertArrayEquals;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
@@ -83,6 +84,11 @@ public class TestConfiguration extends TestCase {
/** Four apostrophes. */
public static final String ESCAPED = "''''";
+ private static final String SENSITIVE_CONFIG_KEYS =
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
+
+ private BufferedWriter out;
+
@Override
protected void setUp() throws Exception {
super.setUp();
@@ -91,6 +97,9 @@ public class TestConfiguration extends TestCase {
@Override
protected void tearDown() throws Exception {
+ if(out != null) {
+ out.close();
+ }
super.tearDown();
new File(CONFIG).delete();
new File(CONFIG2).delete();
@@ -791,8 +800,6 @@ public class TestConfiguration extends TestCase {
new File(new File(relConfig).getParent()).delete();
}
- BufferedWriter out;
-
public void testIntegerRanges() {
Configuration conf = new Configuration();
conf.set("first", "-100");
@@ -1654,8 +1661,41 @@ public class TestConfiguration extends TestCase {
assertEquals(fileResource.toString(),prop.getResource());
}
}
-
-
+
+ public void testDumpSensitiveProperty() throws IOException {
+ final String myPassword = "ThisIsMyPassword";
+ Configuration testConf = new Configuration(false);
+ out = new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.password", myPassword);
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ testConf.addResource(fileResource);
+
+ try (StringWriter outWriter = new StringWriter()) {
+ testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
+ Configuration.dumpConfiguration(testConf, "test.password", outWriter);
+ assertFalse(outWriter.toString().contains(myPassword));
+ }
+ }
+
+ public void testDumpSensitiveConfiguration() throws IOException {
+ final String myPassword = "ThisIsMyPassword";
+ Configuration testConf = new Configuration(false);
+ out = new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.password", myPassword);
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ testConf.addResource(fileResource);
+
+ try (StringWriter outWriter = new StringWriter()) {
+ testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
+ Configuration.dumpConfiguration(testConf, outWriter);
+ assertFalse(outWriter.toString().contains(myPassword));
+ }
+ }
+
public void testGetValByRegex() {
Configuration conf = new Configuration();
String key1 = "t.abc.key1";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/3] hadoop git commit: HADOOP-13628. Support to retrieve specific
property from configuration via REST API. Contributed by Weiwei Yang
Posted by jz...@apache.org.
HADOOP-13628. Support to retrieve specific property from configuration via REST API. Contributed by Weiwei Yang
(cherry picked from commit 00160f71b6d98244fcb1cb58b2db9fc24f1cd672)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76c4aee2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76c4aee2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76c4aee2
Branch: refs/heads/branch-2
Commit: 76c4aee2e3fd66750df94efd4e5b7504a6c1e6fa
Parents: 9ba3268
Author: John Zhuge <jz...@apache.org>
Authored: Fri Aug 11 10:54:30 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Fri Aug 11 10:54:30 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/ConfServlet.java | 19 +-
.../org/apache/hadoop/conf/Configuration.java | 284 +++++++++++++++----
.../org/apache/hadoop/conf/TestConfServlet.java | 122 +++++++-
.../apache/hadoop/conf/TestConfiguration.java | 139 ++++++++-
4 files changed, 491 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76c4aee2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
index f9a2535..cfd7b97 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
@@ -76,11 +76,14 @@ public class ConfServlet extends HttpServlet {
response.setContentType("application/json; charset=utf-8");
}
+ String name = request.getParameter("name");
Writer out = response.getWriter();
try {
- writeResponse(getConfFromContext(), out, format);
+ writeResponse(getConfFromContext(), out, format, name);
} catch (BadFormatException bfe) {
response.sendError(HttpServletResponse.SC_BAD_REQUEST, bfe.getMessage());
+ } catch (IllegalArgumentException iae) {
+ response.sendError(HttpServletResponse.SC_NOT_FOUND, iae.getMessage());
}
out.close();
}
@@ -95,17 +98,23 @@ public class ConfServlet extends HttpServlet {
/**
* Guts of the servlet - extracted for easy testing.
*/
- static void writeResponse(Configuration conf, Writer out, String format)
- throws IOException, BadFormatException {
+ static void writeResponse(Configuration conf,
+ Writer out, String format, String propertyName)
+ throws IOException, IllegalArgumentException, BadFormatException {
if (FORMAT_JSON.equals(format)) {
- Configuration.dumpConfiguration(conf, out);
+ Configuration.dumpConfiguration(conf, propertyName, out);
} else if (FORMAT_XML.equals(format)) {
- conf.writeXml(out);
+ conf.writeXml(propertyName, out);
} else {
throw new BadFormatException("Bad format: " + format);
}
}
+ static void writeResponse(Configuration conf, Writer out, String format)
+ throws IOException, BadFormatException {
+ writeResponse(conf, out, format, null);
+ }
+
public static class BadFormatException extends Exception {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76c4aee2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 40509c1..89714bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -103,8 +103,9 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
-/**
+/**
* Provides access to configuration parameters.
*
* <h4 id="Resources">Resources</h4>
@@ -2866,14 +2867,37 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
writeXml(new OutputStreamWriter(out, "UTF-8"));
}
- /**
- * Write out the non-default properties in this configuration to the given
- * {@link Writer}.
- *
+ public void writeXml(Writer out) throws IOException {
+ writeXml(null, out);
+ }
+
+ /**
+ * Write out the non-default properties in this configuration to the
+ * given {@link Writer}.
+ *
+ * <li>
+ * When property name is not empty and the property exists in the
+ * configuration, this method writes the property and its attributes
+ * to the {@link Writer}.
+ * </li>
+ * <p>
+ *
+ * <li>
+ * When property name is null or empty, this method writes all the
+ * configuration properties and their attributes to the {@link Writer}.
+ * </li>
+ * <p>
+ *
+ * <li>
+ * When property name is not empty but the property doesn't exist in
+ * the configuration, this method throws an {@link IllegalArgumentException}.
+ * </li>
+ * <p>
* @param out the writer to write to.
*/
- public void writeXml(Writer out) throws IOException {
- Document doc = asXmlDocument();
+ public void writeXml(String propertyName, Writer out)
+ throws IOException, IllegalArgumentException {
+ Document doc = asXmlDocument(propertyName);
try {
DOMSource source = new DOMSource(doc);
@@ -2893,62 +2917,180 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/**
* Return the XML DOM corresponding to this Configuration.
*/
- private synchronized Document asXmlDocument() throws IOException {
+ private synchronized Document asXmlDocument(String propertyName)
+ throws IOException, IllegalArgumentException {
Document doc;
try {
- doc =
- DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
+ doc = DocumentBuilderFactory
+ .newInstance()
+ .newDocumentBuilder()
+ .newDocument();
} catch (ParserConfigurationException pe) {
throw new IOException(pe);
}
+
Element conf = doc.createElement("configuration");
doc.appendChild(conf);
conf.appendChild(doc.createTextNode("\n"));
handleDeprecation(); //ensure properties is set and deprecation is handled
- for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
- String name = (String)e.nextElement();
- Object object = properties.get(name);
- String value = null;
- if (object instanceof String) {
- value = (String) object;
- }else {
- continue;
- }
- Element propNode = doc.createElement("property");
- conf.appendChild(propNode);
-
- Element nameNode = doc.createElement("name");
- nameNode.appendChild(doc.createTextNode(name));
- propNode.appendChild(nameNode);
-
- Element valueNode = doc.createElement("value");
- valueNode.appendChild(doc.createTextNode(value));
- propNode.appendChild(valueNode);
-
- if (updatingResource != null) {
- String[] sources = updatingResource.get(name);
- if(sources != null) {
- for(String s : sources) {
- Element sourceNode = doc.createElement("source");
- sourceNode.appendChild(doc.createTextNode(s));
- propNode.appendChild(sourceNode);
+
+ if(!Strings.isNullOrEmpty(propertyName)) {
+ if (!properties.containsKey(propertyName)) {
+ // given property not found, illegal argument
+ throw new IllegalArgumentException("Property " +
+ propertyName + " not found");
+ } else {
+ // given property is found, write single property
+ appendXMLProperty(doc, conf, propertyName);
+ conf.appendChild(doc.createTextNode("\n"));
+ }
+ } else {
+ // append all elements
+ for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
+ appendXMLProperty(doc, conf, (String)e.nextElement());
+ conf.appendChild(doc.createTextNode("\n"));
+ }
+ }
+ return doc;
+ }
+
+ /**
+ * Append a property with its attributes to a given {#link Document}
+ * if the property is found in configuration.
+ *
+ * @param doc
+ * @param conf
+ * @param propertyName
+ */
+ private synchronized void appendXMLProperty(Document doc, Element conf,
+ String propertyName) {
+ // skip writing if given property name is empty or null
+ if (!Strings.isNullOrEmpty(propertyName)) {
+ String value = properties.getProperty(propertyName);
+ if (value != null) {
+ Element propNode = doc.createElement("property");
+ conf.appendChild(propNode);
+
+ Element nameNode = doc.createElement("name");
+ nameNode.appendChild(doc.createTextNode(propertyName));
+ propNode.appendChild(nameNode);
+
+ Element valueNode = doc.createElement("value");
+ valueNode.appendChild(doc.createTextNode(
+ properties.getProperty(propertyName)));
+ propNode.appendChild(valueNode);
+
+ Element finalNode = doc.createElement("final");
+ finalNode.appendChild(doc.createTextNode(
+ String.valueOf(finalParameters.contains(propertyName))));
+ propNode.appendChild(finalNode);
+
+ if (updatingResource != null) {
+ String[] sources = updatingResource.get(propertyName);
+ if(sources != null) {
+ for(String s : sources) {
+ Element sourceNode = doc.createElement("source");
+ sourceNode.appendChild(doc.createTextNode(s));
+ propNode.appendChild(sourceNode);
+ }
}
}
}
-
- conf.appendChild(doc.createTextNode("\n"));
}
- return doc;
}
/**
- * Writes out all the parameters and their properties (final and resource) to
- * the given {@link Writer}
- * The format of the output would be
- * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
- * key2.isFinal,key2.resource}... ] }
- * It does not output the parameters of the configuration object which is
- * loaded from an input stream.
+ * Writes properties and their attributes (final and resource)
+ * to the given {@link Writer}.
+ *
+ * <li>
+ * When propertyName is not empty, and the property exists
+ * in the configuration, the format of the output would be,
+ * <pre>
+ * {
+ * "property": {
+ * "key" : "key1",
+ * "value" : "value1",
+ * "isFinal" : "key1.isFinal",
+ * "resource" : "key1.resource"
+ * }
+ * }
+ * </pre>
+ * </li>
+ *
+ * <li>
+ * When propertyName is null or empty, it behaves same as
+ * {@link #dumpConfiguration(Configuration, Writer)}, the
+ * output would be,
+ * <pre>
+ * { "properties" :
+ * [ { key : "key1",
+ * value : "value1",
+ * isFinal : "key1.isFinal",
+ * resource : "key1.resource" },
+ * { key : "key2",
+ * value : "value2",
+ * isFinal : "ke2.isFinal",
+ * resource : "key2.resource" }
+ * ]
+ * }
+ * </pre>
+ * </li>
+ *
+ * <li>
+ * When propertyName is not empty, and the property is not
+ * found in the configuration, this method will throw an
+ * {@link IllegalArgumentException}.
+ * </li>
+ * <p>
+ * @param config the configuration
+ * @param propertyName property name
+ * @param out the Writer to write to
+ * @throws IOException
+ * @throws IllegalArgumentException when property name is not
+ * empty and the property is not found in configuration
+ **/
+ public static void dumpConfiguration(Configuration config,
+ String propertyName, Writer out) throws IOException {
+ if(Strings.isNullOrEmpty(propertyName)) {
+ dumpConfiguration(config, out);
+ } else if (Strings.isNullOrEmpty(config.get(propertyName))) {
+ throw new IllegalArgumentException("Property " +
+ propertyName + " not found");
+ } else {
+ JsonFactory dumpFactory = new JsonFactory();
+ JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
+ dumpGenerator.writeStartObject();
+ dumpGenerator.writeFieldName("property");
+ appendJSONProperty(dumpGenerator, config, propertyName);
+ dumpGenerator.writeEndObject();
+ dumpGenerator.flush();
+ }
+ }
+
+ /**
+ * Writes out all properties and their attributes (final and resource) to
+ * the given {@link Writer}, the format of the output would be,
+ *
+ * <pre>
+ * { "properties" :
+ * [ { key : "key1",
+ * value : "value1",
+ * isFinal : "key1.isFinal",
+ * resource : "key1.resource" },
+ * { key : "key2",
+ * value : "value2",
+ * isFinal : "ke2.isFinal",
+ * resource : "key2.resource" }
+ * ]
+ * }
+ * </pre>
+ *
+ * It does not output the properties of the configuration object which
+ * is loaded from an input stream.
+ * <p>
+ *
+ * @param config the configuration
* @param out the Writer to write to
* @throws IOException
*/
@@ -2962,29 +3104,47 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
dumpGenerator.flush();
synchronized (config) {
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
- dumpGenerator.writeStartObject();
- dumpGenerator.writeStringField("key", (String) item.getKey());
- dumpGenerator.writeStringField("value",
- config.get((String) item.getKey()));
- dumpGenerator.writeBooleanField("isFinal",
- config.finalParameters.contains(item.getKey()));
- String[] resources = config.updatingResource.get(item.getKey());
- String resource = UNKNOWN_RESOURCE;
- if(resources != null && resources.length > 0) {
- resource = resources[0];
- }
- dumpGenerator.writeStringField("resource", resource);
- dumpGenerator.writeEndObject();
+ appendJSONProperty(dumpGenerator,
+ config,
+ item.getKey().toString());
}
}
dumpGenerator.writeEndArray();
dumpGenerator.writeEndObject();
dumpGenerator.flush();
}
-
+
+ /**
+ * Write property and its attributes as json format to given
+ * {@link JsonGenerator}.
+ *
+ * @param jsonGen json writer
+ * @param config configuration
+ * @param name property name
+ * @throws IOException
+ */
+ private static void appendJSONProperty(JsonGenerator jsonGen,
+ Configuration config, String name) throws IOException {
+ // skip writing if given property name is empty or null
+ if(!Strings.isNullOrEmpty(name) && jsonGen != null) {
+ jsonGen.writeStartObject();
+ jsonGen.writeStringField("key", name);
+ jsonGen.writeStringField("value", config.get(name));
+ jsonGen.writeBooleanField("isFinal",
+ config.finalParameters.contains(name));
+ String[] resources = config.updatingResource.get(name);
+ String resource = UNKNOWN_RESOURCE;
+ if(resources != null && resources.length > 0) {
+ resource = resources[0];
+ }
+ jsonGen.writeStringField("resource", resource);
+ jsonGen.writeEndObject();
+ }
+ }
+
/**
* Get the {@link ClassLoader} for this job.
- *
+ *
* @return the correct class loader.
*/
public ClassLoader getClassLoader() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76c4aee2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
index 73d4992..68b2586 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
@@ -18,11 +18,15 @@
package org.apache.hadoop.conf;
import java.io.StringWriter;
+import java.io.PrintWriter;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletContext;
import javax.ws.rs.core.HttpHeaders;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -34,17 +38,36 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
-import junit.framework.TestCase;
+import com.google.common.base.Strings;
+
+import org.apache.hadoop.http.HttpServer2;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.mock;
+import static org.junit.Assert.*;
/**
* Basic test case that the ConfServlet can write configuration
* to its output in XML and JSON format.
*/
-public class TestConfServlet extends TestCase {
+public class TestConfServlet {
private static final String TEST_KEY = "testconfservlet.key";
private static final String TEST_VAL = "testval";
+ private static final Map<String, String> TEST_PROPERTIES =
+ new HashMap<String, String>();
+ private static final Map<String, String> TEST_FORMATS =
+ new HashMap<String, String>();
+
+ @BeforeClass
+ public static void initTestProperties() {
+ TEST_PROPERTIES.put("test.key1", "value1");
+ TEST_PROPERTIES.put("test.key2", "value2");
+ TEST_PROPERTIES.put("test.key3", "value3");
+ TEST_FORMATS.put(ConfServlet.FORMAT_XML, "application/xml");
+ TEST_FORMATS.put(ConfServlet.FORMAT_JSON, "application/json");
+ }
private Configuration getTestConf() {
Configuration testConf = new Configuration();
@@ -52,6 +75,14 @@ public class TestConfServlet extends TestCase {
return testConf;
}
+ private Configuration getMultiPropertiesConf() {
+ Configuration testConf = new Configuration(false);
+ for(String key : TEST_PROPERTIES.keySet()) {
+ testConf.set(key, TEST_PROPERTIES.get(key));
+ }
+ return testConf;
+ }
+
@Test
public void testParseHeaders() throws Exception {
HashMap<String, String> verifyMap = new HashMap<String, String>();
@@ -71,6 +102,92 @@ public class TestConfServlet extends TestCase {
}
}
+ private void verifyGetProperty(Configuration conf, String format,
+ String propertyName) throws Exception {
+ StringWriter sw = null;
+ PrintWriter pw = null;
+ ConfServlet service = null;
+ try {
+ service = new ConfServlet();
+ ServletConfig servletConf = mock(ServletConfig.class);
+ ServletContext context = mock(ServletContext.class);
+ service.init(servletConf);
+ when(context.getAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE))
+ .thenReturn(conf);
+ when(service.getServletContext())
+ .thenReturn(context);
+
+ HttpServletRequest request = mock(HttpServletRequest.class);
+ when(request.getHeader(HttpHeaders.ACCEPT))
+ .thenReturn(TEST_FORMATS.get(format));
+ when(request.getParameter("name"))
+ .thenReturn(propertyName);
+
+ HttpServletResponse response = mock(HttpServletResponse.class);
+ sw = new StringWriter();
+ pw = new PrintWriter(sw);
+ when(response.getWriter()).thenReturn(pw);
+
+ // response request
+ service.doGet(request, response);
+ String result = sw.toString().trim();
+
+ // if property name is null or empty, expect all properties
+ // in the response
+ if (Strings.isNullOrEmpty(propertyName)) {
+ for(String key : TEST_PROPERTIES.keySet()) {
+ assertTrue(result.contains(key) &&
+ result.contains(TEST_PROPERTIES.get(key)));
+ }
+ } else {
+ if(conf.get(propertyName) != null) {
+ // if property name is not empty and property is found
+ assertTrue(result.contains(propertyName));
+ for(String key : TEST_PROPERTIES.keySet()) {
+ if(!key.equals(propertyName)) {
+ assertFalse(result.contains(key));
+ }
+ }
+ } else {
+ // if property name is not empty, and it's not in configuration
+ // expect proper error code and error message is set to the response
+ Mockito.verify(response).sendError(
+ Mockito.eq(HttpServletResponse.SC_NOT_FOUND),
+ Mockito.eq("Property " + propertyName + " not found"));
+ }
+ }
+ } finally {
+ if (sw != null) {
+ sw.close();
+ }
+ if (pw != null) {
+ pw.close();
+ }
+ if (service != null) {
+ service.destroy();
+ }
+ }
+ }
+
+ @Test
+ public void testGetProperty() throws Exception {
+ Configuration configurations = getMultiPropertiesConf();
+ // list various of property names
+ String[] testKeys = new String[] {
+ "test.key1",
+ "test.unknown.key",
+ "",
+ "test.key2",
+ null
+ };
+
+ for(String format : TEST_FORMATS.keySet()) {
+ for(String key : testKeys) {
+ verifyGetProperty(configurations, format, key);
+ }
+ }
+ }
+
@Test
@SuppressWarnings("unchecked")
public void testWriteJson() throws Exception {
@@ -109,7 +226,6 @@ public class TestConfServlet extends TestCase {
for (int i = 0; i < nameNodes.getLength(); i++) {
Node nameNode = nameNodes.item(i);
String key = nameNode.getTextContent();
- System.err.println("xml key: " + key);
if (TEST_KEY.equals(key)) {
foundSetting = true;
Element propertyElem = (Element)nameNode.getParentNode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76c4aee2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 15a175a..24a49ad 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -1349,7 +1349,19 @@ public class TestConfiguration extends TestCase {
this.properties = properties;
}
}
-
+
+ static class SingleJsonConfiguration {
+ private JsonProperty property;
+
+ public JsonProperty getProperty() {
+ return property;
+ }
+
+ public void setProperty(JsonProperty property) {
+ this.property = property;
+ }
+ }
+
static class JsonProperty {
String key;
public String getKey() {
@@ -1380,7 +1392,14 @@ public class TestConfiguration extends TestCase {
boolean isFinal;
String resource;
}
-
+
+ private Configuration getActualConf(String xmlStr) {
+ Configuration ac = new Configuration(false);
+ InputStream in = new ByteArrayInputStream(xmlStr.getBytes());
+ ac.addResource(in);
+ return ac;
+ }
+
public void testGetSetTrimmedNames() throws IOException {
Configuration conf = new Configuration(false);
conf.set(" name", "value");
@@ -1389,7 +1408,121 @@ public class TestConfiguration extends TestCase {
assertEquals("value", conf.getRaw(" name "));
}
- public void testDumpConfiguration () throws IOException {
+ public void testDumpProperty() throws IOException {
+ StringWriter outWriter = new StringWriter();
+ ObjectMapper mapper = new ObjectMapper();
+ String jsonStr = null;
+ String xmlStr = null;
+ try {
+ Configuration testConf = new Configuration(false);
+ out = new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.key1", "value1");
+ appendProperty("test.key2", "value2", true);
+ appendProperty("test.key3", "value3");
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ testConf.addResource(fileResource);
+ out.close();
+
+ // case 1: dump an existing property
+ // test json format
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(testConf, "test.key2", outWriter);
+ jsonStr = outWriter.toString();
+ outWriter.close();
+ mapper = new ObjectMapper();
+ SingleJsonConfiguration jconf1 =
+ mapper.readValue(jsonStr, SingleJsonConfiguration.class);
+ JsonProperty jp1 = jconf1.getProperty();
+ assertEquals("test.key2", jp1.getKey());
+ assertEquals("value2", jp1.getValue());
+ assertEquals(true, jp1.isFinal);
+ assertEquals(fileResource.toUri().getPath(), jp1.getResource());
+
+ // test xml format
+ outWriter = new StringWriter();
+ testConf.writeXml("test.key2", outWriter);
+ xmlStr = outWriter.toString();
+ outWriter.close();
+ Configuration actualConf1 = getActualConf(xmlStr);
+ assertEquals(1, actualConf1.size());
+ assertEquals("value2", actualConf1.get("test.key2"));
+ assertTrue(actualConf1.getFinalParameters().contains("test.key2"));
+ assertEquals(fileResource.toUri().getPath(),
+ actualConf1.getPropertySources("test.key2")[0]);
+
+ // case 2: dump an non existing property
+ // test json format
+ try {
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(testConf,
+ "test.unknown.key", outWriter);
+ outWriter.close();
+ } catch (Exception e) {
+ assertTrue(e instanceof IllegalArgumentException);
+ assertTrue(e.getMessage().contains("test.unknown.key") &&
+ e.getMessage().contains("not found"));
+ }
+ // test xml format
+ try {
+ outWriter = new StringWriter();
+ testConf.writeXml("test.unknown.key", outWriter);
+ outWriter.close();
+ } catch (Exception e) {
+ assertTrue(e instanceof IllegalArgumentException);
+ assertTrue(e.getMessage().contains("test.unknown.key") &&
+ e.getMessage().contains("not found"));
+ }
+
+ // case 3: specify a null property, ensure all configurations are dumped
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(testConf, null, outWriter);
+ jsonStr = outWriter.toString();
+ mapper = new ObjectMapper();
+ JsonConfiguration jconf3 =
+ mapper.readValue(jsonStr, JsonConfiguration.class);
+ assertEquals(3, jconf3.getProperties().length);
+
+ outWriter = new StringWriter();
+ testConf.writeXml(null, outWriter);
+ xmlStr = outWriter.toString();
+ outWriter.close();
+ Configuration actualConf3 = getActualConf(xmlStr);
+ assertEquals(3, actualConf3.size());
+ assertTrue(actualConf3.getProps().containsKey("test.key1") &&
+ actualConf3.getProps().containsKey("test.key2") &&
+ actualConf3.getProps().containsKey("test.key3"));
+
+ // case 4: specify an empty property, ensure all configurations are dumped
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(testConf, "", outWriter);
+ jsonStr = outWriter.toString();
+ mapper = new ObjectMapper();
+ JsonConfiguration jconf4 =
+ mapper.readValue(jsonStr, JsonConfiguration.class);
+ assertEquals(3, jconf4.getProperties().length);
+
+ outWriter = new StringWriter();
+ testConf.writeXml("", outWriter);
+ xmlStr = outWriter.toString();
+ outWriter.close();
+ Configuration actualConf4 = getActualConf(xmlStr);
+ assertEquals(3, actualConf4.size());
+ assertTrue(actualConf4.getProps().containsKey("test.key1") &&
+ actualConf4.getProps().containsKey("test.key2") &&
+ actualConf4.getProps().containsKey("test.key3"));
+ } finally {
+ if(outWriter != null) {
+ outWriter.close();
+ }
+ if(out != null) {
+ out.close();
+ }
+ }
+ }
+
+ public void testDumpConfiguration() throws IOException {
StringWriter outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
String jsonStr = outWriter.toString();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org