You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yh...@apache.org on 2009/08/24 13:21:19 UTC
svn commit: r807149 - in /hadoop/common/trunk: CHANGES.txt
src/java/org/apache/hadoop/conf/Configuration.java
src/test/core/org/apache/hadoop/conf/TestConfiguration.java
Author: yhemanth
Date: Mon Aug 24 11:21:18 2009
New Revision: 807149
URL: http://svn.apache.org/viewvc?rev=807149&view=rev
Log:
HADOOP-6184. Provide an API to dump Configuration in a JSON format. Contributed by V.V.Chaitanya Krishna.
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/conf/TestConfiguration.java
Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=807149&r1=807148&r2=807149&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Mon Aug 24 11:21:18 2009
@@ -504,6 +504,9 @@
HADOOP-6173. Change src/native/packageNativeHadoop.sh to package all
native library files. (Hong Tang via szetszwo)
+
+ HADOOP-6184. Provide an API to dump Configuration in a JSON format.
+ (V.V.Chaitanya Krishna via yhemanth)
OPTIMIZATIONS
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/conf/Configuration.java?rev=807149&r1=807148&r2=807149&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/conf/Configuration.java Mon Aug 24 11:21:18 2009
@@ -28,6 +28,7 @@
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
+import java.io.Writer;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
@@ -62,6 +63,8 @@
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@@ -155,7 +158,7 @@
private boolean loadDefaults = true;
/**
- * Configurtion objects
+ * Configuration objects
*/
private static final WeakHashMap<Configuration,Object> REGISTRY =
new WeakHashMap<Configuration,Object>();
@@ -167,6 +170,18 @@
private static final ArrayList<String> defaultResources =
new ArrayList<String>();
+ /**
+ * Flag to indicate if the storage of resource which updates a key needs
+ * to be stored for each key
+ */
+ private boolean storeResource;
+
+ /**
+ * Stores the mapping of key to the resource which modifies or loads
+ * the key most recently
+ */
+ private HashMap<String, String> updatingResource;
+
static{
//print deprecation warning if hadoop-site.xml is found in classpath
ClassLoader cL = Thread.currentThread().getContextClassLoader();
@@ -214,6 +229,23 @@
synchronized(Configuration.class) {
REGISTRY.put(this, null);
}
+ this.storeResource = false;
+ }
+
+ /**
+ * A new configuration with the same settings and additional facility for
+ * storage of resource to each key which loads or updates
+ * the key most recently
+ * @param other the configuration from which to clone settings
+ * @param storeResource flag to indicate if the storage of resource to
+ * each key is to be stored
+ */
+ private Configuration(Configuration other, boolean storeResource) {
+ this(other);
+ this.storeResource = storeResource;
+ if (storeResource) {
+ updatingResource = new HashMap<String, String>();
+ }
}
/**
@@ -1081,8 +1113,14 @@
if (properties == null) {
properties = new Properties();
loadResources(properties, resources, quietmode);
- if (overlay!= null)
+ if (overlay!= null) {
properties.putAll(overlay);
+ if (storeResource) {
+ for (Map.Entry<Object,Object> item: overlay.entrySet()) {
+ updatingResource.put((String) item.getKey(), "Unknown");
+ }
+ }
+ }
}
return properties;
}
@@ -1251,6 +1289,9 @@
if (attr != null && value != null) {
if (!finalParameters.contains(attr)) {
properties.setProperty(attr, value);
+ if (storeResource) {
+ updatingResource.put(attr, name.toString());
+ }
if (finalParameter)
finalParameters.add(attr);
} else {
@@ -1323,6 +1364,43 @@
}
/**
+ * Writes out all the parameters and their properties (final and resource) to
+ * the given {@link Writer}
+ * The format of the output would be
+ * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
+ * key2.isFinal,key2.resource}... ] }
+ * It does not output the parameters of the configuration object which is
+ * loaded from an input stream.
+ * @param out the Writer to write to
+ * @throws IOException
+ */
+ public static void dumpConfiguration(Configuration conf,
+ Writer out) throws IOException {
+ Configuration config = new Configuration(conf,true);
+ config.reloadConfiguration();
+ JsonFactory dumpFactory = new JsonFactory();
+ JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
+ dumpGenerator.writeStartObject();
+ dumpGenerator.writeFieldName("properties");
+ dumpGenerator.writeStartArray();
+ dumpGenerator.flush();
+ for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
+ dumpGenerator.writeStartObject();
+ dumpGenerator.writeStringField("key", (String) item.getKey());
+ dumpGenerator.writeStringField("value",
+ config.get((String) item.getKey()));
+ dumpGenerator.writeBooleanField("isFinal",
+ config.finalParameters.contains(item.getKey()));
+ dumpGenerator.writeStringField("resource",
+ config.updatingResource.get(item.getKey()));
+ dumpGenerator.writeEndObject();
+ }
+ dumpGenerator.writeEndArray();
+ dumpGenerator.writeEndObject();
+ dumpGenerator.flush();
+ }
+
+ /**
* Get the {@link ClassLoader} for this job.
*
* @return the correct class loader.
Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/conf/TestConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/conf/TestConfiguration.java?rev=807149&r1=807148&r2=807149&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/conf/TestConfiguration.java (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/conf/TestConfiguration.java Mon Aug 24 11:21:18 2009
@@ -21,12 +21,15 @@
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.StringWriter;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Random;
import junit.framework.TestCase;
import org.apache.hadoop.fs.Path;
+import org.codehaus.jackson.map.ObjectMapper;
public class TestConfiguration extends TestCase {
@@ -408,6 +411,182 @@
assertTrue(other.getClassLoader() instanceof Fake_ClassLoader);
}
+ static class JsonConfiguration {
+ JsonProperty[] properties;
+
+ public JsonProperty[] getProperties() {
+ return properties;
+ }
+
+ public void setProperties(JsonProperty[] properties) {
+ this.properties = properties;
+ }
+ }
+
+ static class JsonProperty {
+ String key;
+ public String getKey() {
+ return key;
+ }
+ public void setKey(String key) {
+ this.key = key;
+ }
+ public String getValue() {
+ return value;
+ }
+ public void setValue(String value) {
+ this.value = value;
+ }
+ public boolean getIsFinal() {
+ return isFinal;
+ }
+ public void setIsFinal(boolean isFinal) {
+ this.isFinal = isFinal;
+ }
+ public String getResource() {
+ return resource;
+ }
+ public void setResource(String resource) {
+ this.resource = resource;
+ }
+ String value;
+ boolean isFinal;
+ String resource;
+ }
+
+ public void testDumpConfiguration () throws IOException {
+ StringWriter outWriter = new StringWriter();
+ Configuration.dumpConfiguration(conf, outWriter);
+ String jsonStr = outWriter.toString();
+ ObjectMapper mapper = new ObjectMapper();
+ JsonConfiguration jconf =
+ mapper.readValue(jsonStr, JsonConfiguration.class);
+ int defaultLength = jconf.getProperties().length;
+
+ // add 3 keys to the existing configuration properties
+ out=new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.key1", "value1");
+ appendProperty("test.key2", "value2",true);
+ appendProperty("test.key3", "value3");
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ conf.addResource(fileResource);
+ out.close();
+
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(conf, outWriter);
+ jsonStr = outWriter.toString();
+ mapper = new ObjectMapper();
+ jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+ int length = jconf.getProperties().length;
+ // check for consistency in the number of properties parsed in Json format.
+ assertEquals(length, defaultLength+3);
+
+ //change few keys in another resource file
+ out=new BufferedWriter(new FileWriter(CONFIG2));
+ startConfig();
+ appendProperty("test.key1", "newValue1");
+ appendProperty("test.key2", "newValue2");
+ endConfig();
+ Path fileResource1 = new Path(CONFIG2);
+ conf.addResource(fileResource1);
+ out.close();
+
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(conf, outWriter);
+ jsonStr = outWriter.toString();
+ mapper = new ObjectMapper();
+ jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+
+ // put the keys and their corresponding attributes into a hashmap for their
+ // efficient retrieval
+ HashMap<String,JsonProperty> confDump = new HashMap<String,JsonProperty>();
+ for(JsonProperty prop : jconf.getProperties()) {
+ confDump.put(prop.getKey(), prop);
+ }
+ // check if the value and resource of test.key1 is changed
+ assertEquals("newValue1", confDump.get("test.key1").getValue());
+ assertEquals(false, confDump.get("test.key1").getIsFinal());
+ assertEquals(fileResource1.toString(),
+ confDump.get("test.key1").getResource());
+ // check if final parameter test.key2 is not changed, since it is first
+ // loaded as final parameter
+ assertEquals("value2", confDump.get("test.key2").getValue());
+ assertEquals(true, confDump.get("test.key2").getIsFinal());
+ assertEquals(fileResource.toString(),
+ confDump.get("test.key2").getResource());
+ // check for other keys which are not modified later
+ assertEquals("value3", confDump.get("test.key3").getValue());
+ assertEquals(false, confDump.get("test.key3").getIsFinal());
+ assertEquals(fileResource.toString(),
+ confDump.get("test.key3").getResource());
+ // check for resource to be "Unknown" for keys which are loaded using 'set'
+ // and expansion of properties
+ conf.set("test.key4", "value4");
+ conf.set("test.key5", "value5");
+ conf.set("test.key6", "${test.key5}");
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(conf, outWriter);
+ jsonStr = outWriter.toString();
+ mapper = new ObjectMapper();
+ jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+ confDump = new HashMap<String, JsonProperty>();
+ for(JsonProperty prop : jconf.getProperties()) {
+ confDump.put(prop.getKey(), prop);
+ }
+ assertEquals("value5",confDump.get("test.key6").getValue());
+ assertEquals("Unknown", confDump.get("test.key4").getResource());
+ outWriter.close();
+ }
+
+ public void testDumpConfiguratioWithoutDefaults() throws IOException {
+ // check for case when default resources are not loaded
+ Configuration config = new Configuration(false);
+ StringWriter outWriter = new StringWriter();
+ Configuration.dumpConfiguration(config, outWriter);
+ String jsonStr = outWriter.toString();
+ ObjectMapper mapper = new ObjectMapper();
+ JsonConfiguration jconf =
+ mapper.readValue(jsonStr, JsonConfiguration.class);
+
+ //ensure that no properties are loaded.
+ assertEquals(0, jconf.getProperties().length);
+
+ // add 2 keys
+ out=new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.key1", "value1");
+ appendProperty("test.key2", "value2",true);
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ config.addResource(fileResource);
+ out.close();
+
+ outWriter = new StringWriter();
+ Configuration.dumpConfiguration(config, outWriter);
+ jsonStr = outWriter.toString();
+ mapper = new ObjectMapper();
+ jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
+
+ HashMap<String, JsonProperty>confDump = new HashMap<String, JsonProperty>();
+ for (JsonProperty prop : jconf.getProperties()) {
+ confDump.put(prop.getKey(), prop);
+ }
+ //ensure only 2 keys are loaded
+ assertEquals(2,jconf.getProperties().length);
+ //ensure the values are consistent
+ assertEquals(confDump.get("test.key1").getValue(),"value1");
+ assertEquals(confDump.get("test.key2").getValue(),"value2");
+ //check the final tag
+ assertEquals(false, confDump.get("test.key1").getIsFinal());
+ assertEquals(true, confDump.get("test.key2").getIsFinal());
+ //check the resource for each property
+ for (JsonProperty prop : jconf.getProperties()) {
+ assertEquals(fileResource.toString(),prop.getResource());
+ }
+ }
+
public static void main(String[] argv) throws Exception {
junit.textui.TestRunner.main(new String[]{
TestConfiguration.class.getName()