You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/02/03 23:17:06 UTC
svn commit: r374760 - in /lucene/hadoop/trunk: ./ conf/
src/java/org/apache/hadoop/conf/Configuration.java
src/java/org/apache/hadoop/mapred/JobConf.java
src/java/org/apache/hadoop/mapred/TaskTracker.java
Author: cutting
Date: Fri Feb 3 14:17:03 2006
New Revision: 374760
URL: http://svn.apache.org/viewcvs?rev=374760&view=rev
Log:
Permit multiple default and final Configuration resources.
Modified:
lucene/hadoop/trunk/ (props changed)
lucene/hadoop/trunk/conf/ (props changed)
lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
Propchange: lucene/hadoop/trunk/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Fri Feb 3 14:17:03 2006
@@ -0,0 +1 @@
+build
Propchange: lucene/hadoop/trunk/conf/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Fri Feb 3 14:17:03 2006
@@ -0,0 +1,2 @@
+hadoop-site.xml
+mapred-default.xml
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java?rev=374760&r1=374759&r2=374760&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java Fri Feb 3 14:17:03 2006
@@ -31,59 +31,83 @@
import org.apache.hadoop.util.LogFormatter;
-/** Provides access to configuration parameters.
- * <p>An ordered list of configuration parameter files with
- * default and always-overrides site parameters.
- * <p>Default values for all parameters are specified in a file named
- * <tt>hadoop-default.xml</tt> located on the classpath. Overrides for these
- * defaults should be in an optional file named <tt>hadoop-site.xml</tt>, also
- * located on the classpath. Typically these files reside in the
- * <tt>conf/</tt> subdirectory at the top-level of a Hadoop installation.
- * <p>The resource files are read upon first access of values (set, get,
- * or write) after {@link #addConfResource(String)} or
- * {@link #addConfResource(File)}.
+/** Provides access to configuration parameters. Configurations are specified
+ * by resources. A resource contains a set of name/value pairs.
+ *
+ * <p>Each resources is named by either a String or by a File. If named by a
+ * String, then the classpath is examined for a file with that name. If a
+ * File, then the filesystem is examined directly, without referring to the
+ * CLASSPATH.
+ *
+ * <p>Configuration resources are of three types: default, application and
+ * final. Default values are loaded first. Final values are loaded last, and
+ * thus override all other types. Application values are loaded after defaults
+ * and before finals, and can thus override default values but not final
+ * values.
+ *
+ * <p>Hadoop's default resource is the String "hadoop-default.xml" and its
+ * final resource is the String "hadoop-site.xml". Other tools built on Hadoop
+ * may specify additional default resources.
*/
public class Configuration {
private static final Logger LOG =
LogFormatter.getLogger("org.apache.hadoop.conf.Configuration");
- private ArrayList resourceNames = new ArrayList();
+ private ArrayList defaultResources = new ArrayList();
+ private ArrayList appResources = new ArrayList();
+ private ArrayList finalResources = new ArrayList();
+
private Properties properties;
private ClassLoader classLoader =
Thread.currentThread().getContextClassLoader();
/** A new configuration. */
public Configuration() {
- resourceNames.add("hadoop-default.xml");
- resourceNames.add("hadoop-site.xml");
+ defaultResources.add("hadoop-default.xml");
+ finalResources.add("hadoop-site.xml");
}
/** A new configuration with the same settings cloned from another. */
public Configuration(Configuration other) {
- this.resourceNames = (ArrayList)other.resourceNames.clone();
+ this.defaultResources = (ArrayList)other.defaultResources.clone();
+ this.appResources = (ArrayList)other.appResources.clone();
+ this.finalResources = (ArrayList)other.finalResources.clone();
if (other.properties != null)
this.properties = (Properties)other.properties.clone();
}
- /** Adds a resource name to the chain of resources read. Such resources are
- * located on the CLASSPATH. The first resource is always
- * <tt>hadoop-default.xml</tt>, and the last is always
- * <tt>hadoop-site.xml</tt>. New resources are inserted between these, so
- * they can override defaults, but not site-specifics. */
- public synchronized void addConfResource(String name) {
- addConfResourceInternal(name);
- }
-
- /** Adds a file to the chain of resources read. The first resource is always
- * <tt>hadoop-default.xml</tt>, and the last is always
- * <tt>hadoop-site.xml</tt>. New resources are inserted between these, so
- * they can override defaults, but not site-specifics. */
- public synchronized void addConfResource(File file) {
- addConfResourceInternal(file);
+ /** Add an application resource. */
+ public void addAppResource(String name) {
+ addResource(appResources, name);
+ }
+
+ /** Add an application resource. */
+ public void addAppResource(File file) {
+ addResource(appResources, file);
+ }
+
+ /** Add a default resource. */
+ public void addDefaultResource(String name) {
+ addResource(defaultResources, name);
+ }
+
+ /** Add a default resource. */
+ public void addDefaultResource(File file) {
+ addResource(defaultResources, file);
+ }
+
+ /** Add a final resource. */
+ public void addFinalResource(String name) {
+ addResource(finalResources, name);
+ }
+
+ /** Add a final resource. */
+ public void addFinalResource(File file) {
+ addResource(finalResources, file);
}
- private synchronized void addConfResourceInternal(Object name) {
- resourceNames.add(resourceNames.size()-1, name); // add second to last
+ private synchronized void addResource(ArrayList resources, Object resource) {
+ resources.add(resource); // add to resources
properties = null; // trigger reload
}
@@ -299,25 +323,26 @@
private synchronized Properties getProps() {
if (properties == null) {
- Properties defaults = new Properties();
+ Properties defaults = new Properties(); // keep defaults separate
Properties newProps = new Properties(defaults);
- ListIterator i = resourceNames.listIterator();
- while (i.hasNext()) {
- if (i.nextIndex() == 0) { // load defaults
- loadResource(defaults, i.next(), false);
- } else if (i.nextIndex()==resourceNames.size()-1) { // load site
- loadResource(newProps, i.next(), true);
- } else { // load intermediate
- loadResource(newProps, i.next(), false);
- }
- }
+ loadResources(defaults, defaultResources, false);
+ loadResources(newProps, appResources, false);
+ loadResources(newProps, finalResources, true);
properties = newProps;
}
return properties;
}
- private void loadResource(Properties properties,
- Object name, boolean quietFail) {
+ private void loadResources(Properties props,
+ ArrayList resources,
+ boolean reverse) {
+ ListIterator i = resources.listIterator(reverse ? resources.size() : 0);
+ while (reverse ? i.hasPrevious() : i.hasNext()) {
+ loadResource(props, reverse ? i.previous() : i.next());
+ }
+ }
+
+ private void loadResource(Properties properties, Object name) {
try {
DocumentBuilder builder =
DocumentBuilderFactory.newInstance().newDocumentBuilder();
@@ -338,8 +363,6 @@
}
if (doc == null) {
- if (quietFail)
- return;
throw new RuntimeException(name + " not found");
}
@@ -420,11 +443,20 @@
}
}
-
public String toString() {
- StringBuffer sb = new StringBuffer(resourceNames.size()*30);
+ StringBuffer sb = new StringBuffer();
sb.append("Configuration: ");
- ListIterator i = resourceNames.listIterator();
+ sb.append("defaults: ");
+ toString(defaultResources, sb);
+ sb.append("app: ");
+ toString(appResources, sb);
+ sb.append("final: ");
+ toString(finalResources, sb);
+ return sb.toString();
+ }
+
+ private void toString(ArrayList resources, StringBuffer sb) {
+ ListIterator i = resources.listIterator();
while (i.hasNext()) {
if (i.nextIndex() != 0) {
sb.append(" , ");
@@ -436,7 +468,6 @@
sb.append((String)obj);
}
}
- return sb.toString();
}
/** For debugging. List non-default properties to the terminal and exit. */
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=374760&r1=374759&r2=374760&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Fri Feb 3 14:17:03 2006
@@ -63,7 +63,7 @@
*/
public JobConf(Configuration conf) {
super(conf);
- addConfResource("mapred-default.xml");
+ addAppResource("mapred-default.xml");
}
@@ -81,8 +81,8 @@
*/
public JobConf(File config) {
super();
- addConfResource("mapred-default.xml");
- addConfResource(config);
+ addAppResource("mapred-default.xml");
+ addAppResource(config);
}
public String getJar() { return get("mapred.jar"); }
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=374760&r1=374759&r2=374760&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Fri Feb 3 14:17:03 2006
@@ -629,7 +629,7 @@
Task task = umbilical.getTask(taskid);
JobConf job = new JobConf(task.getJobFile());
- conf.addConfResource(new File(task.getJobFile()));
+ conf.addAppResource(new File(task.getJobFile()));
startPinging(umbilical, taskid); // start pinging parent