You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/05/13 18:35:44 UTC
svn commit: r655940 - in /incubator/pig/trunk: CHANGES.txt
src/org/apache/pig/Main.java
src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
src/org/apache/pig/impl/util/PropertiesUtil.java
Author: gates
Date: Tue May 13 09:35:44 2008
New Revision: 655940
URL: http://svn.apache.org/viewvc?rev=655940&view=rev
Log:
PIG-236: Fix properties so that values specified via the command line (-D) are not ignored.
Modified:
incubator/pig/trunk/CHANGES.txt
incubator/pig/trunk/src/org/apache/pig/Main.java
incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
incubator/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java
Modified: incubator/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=655940&r1=655939&r2=655940&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Tue May 13 09:35:44 2008
@@ -293,3 +293,5 @@
PIG-232: let valid cache specifications through (acmurthy via olgan)
PIG-237: validation of the output directory (pi_song via olgan)
+
+ PIG-236: Fix properties so that values specified via the command line (-D) are not ignored (pkamath via gates).
Modified: incubator/pig/trunk/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/Main.java?rev=655940&r1=655939&r2=655940&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/Main.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/Main.java Tue May 13 09:35:44 2008
@@ -206,8 +206,6 @@
}
}
}
- // set the cluster
- properties.setProperty(HExecutionEngine.JOB_TRACKER_LOCATION, cluster);
// configure logging
configureLog4J(properties);
// create the context with the parameter
Modified: incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=655940&r1=655939&r2=655940&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Tue May 13 09:35:44 2008
@@ -111,8 +111,9 @@
setSSHFactory();
String hodServer = properties.getProperty(HOD_SERVER);
- String cluster = properties.getProperty(JOB_TRACKER_LOCATION);
- String nameNode = properties.getProperty( FILE_SYSTEM_LOCATION);
+ String cluster = null;
+ String nameNode = null;
+ Configuration configuration = null;
if (hodServer != null && hodServer.length() > 0) {
String hdfsAndMapred[] = doHod(hodServer, properties);
@@ -120,6 +121,24 @@
properties.setProperty(JOB_TRACKER_LOCATION, hdfsAndMapred[1]);
}
else {
+
+ // We need to build a configuration object first in the manner described below
+ // and then get back a properties object to inspect the JOB_TRACKER_LOCATION
+ // and FILE_SYSTEM_LOCATION. The reason to do this is if we looked only at
+ // the existing properties object, we may not get the right settings. So we want
+ // to read the configurations in the order specified below and only then look
+ // for JOB_TRACKER_LOCATION and FILE_SYSTEM_LOCATION.
+
+ // Hadoop by default specifies two resources, loaded in-order from the classpath:
+ // 1. hadoop-default.xml : Read-only defaults for hadoop.
+ // 2. hadoop-site.xml: Site-specific configuration for a given hadoop installation.
+ // Now add the settings from "properties" object to override any existing properties
+ // All of the above is accomplished in the method call below
+ configuration = ConfigurationUtil.toConfiguration(properties);
+ properties = ConfigurationUtil.toProperties(configuration);
+ cluster = properties.getProperty(JOB_TRACKER_LOCATION);
+ nameNode = properties.getProperty(FILE_SYSTEM_LOCATION);
+
if (cluster != null && cluster.length() > 0) {
if(!cluster.contains(":") && !cluster.equalsIgnoreCase(LOCAL)) {
cluster = cluster + ":50020";
@@ -137,8 +156,13 @@
log.info("Connecting to hadoop file system at: " + (nameNode==null? LOCAL: nameNode) ) ;
ds = new HDataStorage(properties);
+
+ // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
+ // So we need to reconstruct the configuration object for the non HOD case
+ // In the HOD case, this is the first time the configuration object will be created
+ configuration = ConfigurationUtil.toConfiguration(properties);
+
- Configuration configuration = ConfigurationUtil.toConfiguration(properties);
if(cluster != null && !cluster.equalsIgnoreCase(LOCAL)){
log.info("Connecting to map-reduce job tracker at: " + properties.get(JOB_TRACKER_LOCATION));
if (!LOCAL.equalsIgnoreCase(cluster)) {
Modified: incubator/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java?rev=655940&r1=655939&r2=655940&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java Tue May 13 09:35:44 2008
@@ -66,6 +66,10 @@
}
}
+ // Add System properties which include command line overrides
+ // Any existing keys will be overridden
+ properties.putAll(System.getProperties());
+
// For telling error fast when there are problems
ConfigurationValidator.validatePigProperties(properties) ;
}