You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by am...@apache.org on 2011/04/28 07:43:21 UTC
svn commit: r1097325 - in /hadoop/mapreduce/trunk: ./
src/test/mapred/org/apache/hadoop/tools/rumen/
src/test/tools/data/rumen/small-trace-test/
src/tools/org/apache/hadoop/tools/rumen/
Author: amarrk
Date: Thu Apr 28 05:43:20 2011
New Revision: 1097325
URL: http://svn.apache.org/viewvc?rev=1097325&view=rev
Log:
Bring in more job configuration properties in to the trace file.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz
hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz
hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/truncated-trace-output
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Apr 28 05:43:20 2011
@@ -7,6 +7,10 @@ Trunk (unreleased changes)
NEW FEATURES
IMPROVEMENTS
+
+ MAPREDUCE-2153. Bring in more job configuration properties in to the trace
+ file. (Rajesh Balamohan via amarrk)
+
MAPREDUCE-1461. Feature to instruct rumen-folder utility to skip jobs worth
of specific duration. (Rajesh Balamohan via amarrk)
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java Thu Apr 28 05:43:20 2011
@@ -18,6 +18,7 @@
package org.apache.hadoop.tools.rumen;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
@@ -446,37 +447,12 @@ public class TestRumenJobTraces {
String[] oldProps1 = { "mapred.job.queue.name", "mapred.job.name",
"mapred.child.java.opts" };
- String[] oldProps2 = { "mapred.job.queue.name", "mapred.child.java.opts" };
-
- validateJobConfParser(oldProps1, oldProps2, "sample-conf.file.xml", false);
-
- // Validate parser with new mapreduce config properties from
- // sample-conf-file.new.xml
- String[] newProps1 = { MRJobConfig.QUEUE_NAME, MRJobConfig.JOB_NAME,
- MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.REDUCE_JAVA_OPTS };
-
- String[] newProps2 = { MRJobConfig.QUEUE_NAME,
- MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.REDUCE_JAVA_OPTS };
-
- validateJobConfParser(newProps1, newProps2,
- "sample-conf.file.new.xml", true);
+ validateJobConfParser("sample-conf.file.xml", false);
+ validateJobConfParser("sample-conf.file.new.xml", true);
}
- private void validateJobConfParser(String[] list1, String[] list2,
- String confFile, boolean newConfig)
+ private void validateJobConfParser(String confFile, boolean newConfig)
throws Exception {
- List<String> interested1 = new ArrayList<String>();
- for (String interested : list1) {
- interested1.add(interested);
- }
-
- List<String> interested2 = new ArrayList<String>();
- for (String interested : list2) {
- interested2.add(interested);
- }
-
- JobConfigurationParser jcp1 = new JobConfigurationParser(interested1);
- JobConfigurationParser jcp2 = new JobConfigurationParser(interested2);
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
@@ -494,43 +470,20 @@ public class TestRumenJobTraces {
new PossiblyDecompressedInputStream(inputPath, conf);
try {
- Properties props1 = jcp1.parse(inputConfStream);
+ Properties props = JobConfigurationParser.parse(inputConfStream);
inputConfStream.close();
- inputConfStream = new PossiblyDecompressedInputStream(inputPath, conf);
- Properties props2 = jcp2.parse(inputConfStream);
-
- assertEquals("testJobConfigurationParser: wrong number of properties",
- list1.length, props1.size());
- assertEquals("testJobConfigurationParser: wrong number of properties",
- list2.length, props2.size());
-
- // Make sure that parser puts the interested properties into props1 and
- // props2 as defined by list1 and list2.
String oldOrNew = newConfig ? "New" : "Old";
assertEquals(oldOrNew + " config property for job queue name is not "
+ " extracted properly.", "TheQueue",
- JobBuilder.extract(props1, JobConfPropertyNames.QUEUE_NAMES
+ JobBuilder.extract(props, JobConfPropertyNames.QUEUE_NAMES
.getCandidates(), null));
assertEquals(oldOrNew + " config property for job name is not "
+ " extracted properly.", "MyMRJob",
- JobBuilder.extract(props1, JobConfPropertyNames.JOB_NAMES
- .getCandidates(), null));
-
- assertEquals(oldOrNew + " config property for job queue name is not "
- + " extracted properly.", "TheQueue",
- JobBuilder.extract(props2, JobConfPropertyNames.QUEUE_NAMES
- .getCandidates(), null));
-
- // This config property is not interested for props2. So props should not
- // contain this.
- assertNull("Uninterested " + oldOrNew + " config property for job name "
- + " is extracted.",
- JobBuilder.extract(props2, JobConfPropertyNames.JOB_NAMES
+ JobBuilder.extract(props, JobConfPropertyNames.JOB_NAMES
.getCandidates(), null));
- validateChildJavaOpts(newConfig, props1);
- validateChildJavaOpts(newConfig, props2);
+ validateChildJavaOpts(newConfig, props);
} finally {
inputConfStream.close();
@@ -575,6 +528,64 @@ public class TestRumenJobTraces {
.getCandidates(), null));
}
}
+
+ /**
+ * Test if the {@link JobConfigurationParser} can correctly extract out
+ * key-value pairs from the job configuration.
+ */
+ @Test
+ public void testJobConfigurationParsing() throws Exception {
+ final FileSystem lfs = FileSystem.getLocal(new Configuration());
+
+ final Path rootTempDir =
+ new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(
+ lfs.getUri(), lfs.getWorkingDirectory());
+
+ final Path tempDir = new Path(rootTempDir, "TestJobConfigurationParser");
+ lfs.delete(tempDir, true);
+
+ // Add some configuration parameters to the conf
+ JobConf jConf = new JobConf(false);
+ String key = "test.data";
+ String value = "hello world";
+ jConf.set(key, value);
+
+ // create the job conf file
+ Path jobConfPath = new Path(tempDir.toString(), "job.xml");
+ lfs.delete(jobConfPath, false);
+ DataOutputStream jobConfStream = lfs.create(jobConfPath);
+ jConf.writeXml(jobConfStream);
+ jobConfStream.close();
+
+ // now read the job conf file using the job configuration parser
+ Properties properties =
+ JobConfigurationParser.parse(lfs.open(jobConfPath));
+
+ // check if the required parameter is loaded
+ assertEquals("Total number of extracted properties (" + properties.size()
+ + ") doesn't match the expected size of 1 ["
+ + "JobConfigurationParser]",
+ 1, properties.size());
+ // check if the key is present in the extracted configuration
+ assertTrue("Key " + key + " is missing in the configuration extracted "
+ + "[JobConfigurationParser]",
+ properties.keySet().contains(key));
+ // check if the desired property has the correct value
+ assertEquals("JobConfigurationParser couldn't recover the parameters"
+ + " correctly",
+ value, properties.get(key));
+
+ // Test ZombieJob
+ LoggedJob job = new LoggedJob();
+ job.setJobProperties(properties);
+
+ ZombieJob zjob = new ZombieJob(job, null);
+ Configuration zconf = zjob.getJobConf();
+ // check if the required parameter is loaded
+ assertEquals("ZombieJob couldn't recover the parameters correctly",
+ value, zconf.get(key));
+ }
+
@Test
public void testTopologyBuilder() throws Exception {
Modified: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
Binary files - no diff available.
Modified: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
Binary files - no diff available.
Modified: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
Binary files - no diff available.
Modified: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/truncated-trace-output
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/truncated-trace-output?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/truncated-trace-output (original)
+++ hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/truncated-trace-output Thu Apr 28 05:43:20 2011
@@ -3,6 +3,9 @@
"user" : "hadoopqa",
"jobName" : null,
"jobID" : "job_200904211745_0002",
+ "jobProperties" : {
+ "mapred.child.java.opts" : "-server -Xmx640m -Djava.net.preferIPv4Stack=true"
+ },
"mapTasks" : [ {
"startTime" : 1240336753705,
"attempts" : [ {
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java Thu Apr 28 05:43:20 2011
@@ -1594,6 +1594,8 @@ public class HadoopLogsAnalyzer extends
jobBeingTraced.setJobMapMB(jobconf.jobMapMB);
jobBeingTraced.setJobReduceMB(jobconf.jobReduceMB);
+ jobBeingTraced.setJobProperties(jobconf.properties);
+
jobconf = null;
finalizeJob();
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java Thu Apr 28 05:43:20 2011
@@ -96,6 +96,8 @@ public class JobBuilder {
private static final Pattern heapPattern =
Pattern.compile("-Xmx([0-9]+[kKmMgGtT])");
+ private Properties jobConfigurationParameters = null;
+
public JobBuilder(String jobID) {
this.jobID = jobID;
}
@@ -228,6 +230,7 @@ public class JobBuilder {
"JobBuilder.process(Properties conf) called after LoggedJob built");
}
+ //TODO remove this once the deprecate APIs in LoggedJob are removed
result.setQueue(extract(conf, JobConfPropertyNames.QUEUE_NAMES
.getCandidates(), "default"));
result.setJobName(extract(conf, JobConfPropertyNames.JOB_NAMES
@@ -239,6 +242,8 @@ public class JobBuilder {
JobConfPropertyNames.MAP_JAVA_OPTS_S.getCandidates()));
maybeSetJobReduceMB(extractMegabytes(conf,
JobConfPropertyNames.REDUCE_JAVA_OPTS_S.getCandidates()));
+
+ this.jobConfigurationParameters = conf;
}
/**
@@ -248,9 +253,12 @@ public class JobBuilder {
* @return Parsed {@link LoggedJob} object.
*/
public LoggedJob build() {
- // The main job here is to build CDFs
+ // The main job here is to build CDFs and manage the conf
finalized = true;
+ // set the conf
+ result.setJobProperties(jobConfigurationParameters);
+
// initialize all the per-job statistics gathering places
Histogram[] successfulMapAttemptTimes =
new Histogram[ParsedHost.numberOfDistances() + 1];
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java Thu Apr 28 05:43:20 2011
@@ -17,13 +17,9 @@
*/
package org.apache.hadoop.tools.rumen;
-import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.HashSet;
-import java.util.List;
import java.util.Properties;
-import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -38,22 +34,11 @@ import org.xml.sax.SAXException;
/**
* {@link JobConfigurationParser} parses the job configuration xml file, and
- * extracts various framework specific properties. It parses the file using a
+ * extracts configuration properties. It parses the file using a
* stream-parser and thus is more memory efficient. [This optimization may be
* postponed for a future release]
*/
public class JobConfigurationParser {
- final private Set<String> interested;
-
- /**
- * Constructor
- *
- * @param interested
- * properties we should extract from the job configuration xml.
- */
- public JobConfigurationParser(List<String> interested) {
- this.interested = new HashSet<String>(interested);
- }
/**
* Parse the job configuration file (as an input stream) and return a
@@ -66,7 +51,7 @@ public class JobConfigurationParser {
* configuration xml.
* @throws IOException
*/
- Properties parse(InputStream input) throws IOException {
+ static Properties parse(InputStream input) throws IOException {
Properties result = new Properties();
try {
@@ -117,7 +102,7 @@ public class JobConfigurationParser {
}
}
- if (interested.contains(attr) && value != null) {
+ if (attr != null && value != null) {
result.put(attr, value);
}
}
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java Thu Apr 28 05:43:20 2011
@@ -22,6 +22,8 @@ package org.apache.hadoop.tools.rumen;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
@@ -92,6 +94,8 @@ public class LoggedJob implements DeepCo
double[] mapperTriesToSucceed;
double failedMapperFraction; // !!!!!
+ private Properties jobProperties = new Properties();
+
LoggedJob() {
}
@@ -102,6 +106,20 @@ public class LoggedJob implements DeepCo
setJobID(jobID);
}
+ /**
+ * Set the configuration properties of the job.
+ */
+ void setJobProperties(Properties conf) {
+ this.jobProperties = conf;
+ }
+
+ /**
+ * Get the configuration properties of the job.
+ */
+ public Properties getJobProperties() {
+ return jobProperties;
+ }
+
void adjustTimes(long adjustment) {
submitTime += adjustment;
launchTime += adjustment;
@@ -537,6 +555,35 @@ public class LoggedJob implements DeepCo
}
}
+ private void compareJobProperties(Properties prop1, Properties prop2,
+ TreePath loc, String eltname)
+ throws DeepInequalityException {
+ if (prop1 == null && prop2 == null) {
+ return;
+ }
+
+ if (prop1 == null || prop2 == null) {
+ throw new DeepInequalityException(eltname + " miscompared [null]",
+ new TreePath(loc, eltname));
+ }
+
+ if (prop1.size() != prop2.size()) {
+ throw new DeepInequalityException(eltname + " miscompared [size]",
+ new TreePath(loc, eltname));
+ }
+
+ for (Map.Entry<Object, Object> entry : prop1.entrySet()) {
+ Object v1 = entry.getValue();
+ Object v2 = prop2.get(entry.getKey());
+ if (v1 == null || v2 == null || !v1.equals(v2)) {
+ throw new DeepInequalityException(
+ eltname + " miscompared for value of key : "
+ + entry.getKey().toString(),
+ new TreePath(loc, eltname));
+ }
+ }
+ }
+
public void deepCompare(DeepCompare comparand, TreePath loc)
throws DeepInequalityException {
if (!(comparand instanceof LoggedJob)) {
@@ -600,5 +647,9 @@ public class LoggedJob implements DeepCo
compare1(clusterReduceMB, other.clusterReduceMB, loc, "clusterReduceMB");
compare1(jobMapMB, other.jobMapMB, loc, "jobMapMB");
compare1(jobReduceMB, other.jobReduceMB, loc, "jobReduceMB");
+
+ // compare the job configuration parameters
+ compareJobProperties(jobProperties, other.getJobProperties(), loc,
+ "JobProperties");
}
}
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java Thu Apr 28 05:43:20 2011
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.tools.rumen;
+import java.util.Properties;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
@@ -57,6 +58,8 @@ class ParsedConfigFile {
final String jobID;
final boolean valid;
+
+ final Properties properties = new Properties();
private int maybeGetIntValue(String propName, String attr, String value,
int oldValue) {
@@ -145,6 +148,8 @@ class ParsedConfigFile {
"true".equals(((Text) field.getFirstChild()).getData());
}
}
+
+ properties.setProperty(attr, value);
if ("mapred.child.java.opts".equals(attr) && value != null) {
Matcher matcher = heapPattern.matcher(value);
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java Thu Apr 28 05:43:20 2011
@@ -50,7 +50,6 @@ public class TraceBuilder extends Config
static final int RUN_METHOD_FAILED_EXIT_CODE = 3;
TopologyBuilder topologyBuilder = new TopologyBuilder();
- JobConfigurationParser jobConfParser;
Outputter<LoggedJob> traceWriter;
Outputter<LoggedNetworkTopology> topologyWriter;
@@ -191,25 +190,11 @@ public class TraceBuilder extends Config
return jobId != null;
}
- private void addInterestedProperties(List<String> interestedProperties,
- String[] names) {
- for (String name : names) {
- interestedProperties.add(name);
- }
- }
@SuppressWarnings("unchecked")
@Override
public int run(String[] args) throws Exception {
MyOptions options = new MyOptions(args, getConf());
- List<String> interestedProperties = new ArrayList<String>();
- {
- for (JobConfPropertyNames candidateSet : JobConfPropertyNames.values()) {
- addInterestedProperties(interestedProperties, candidateSet
- .getCandidates());
- }
- }
- jobConfParser = new JobConfigurationParser(interestedProperties);
traceWriter = options.clazzTraceOutputter.newInstance();
traceWriter.init(options.traceOutput, getConf());
topologyWriter = new DefaultOutputter<LoggedNetworkTopology>();
@@ -254,7 +239,7 @@ public class TraceBuilder extends Config
}
if (isJobConfXml(filePair.first(), ris)) {
- processJobConf(jobConfParser.parse(ris.rewind()), jobBuilder);
+ processJobConf(JobConfigurationParser.parse(ris.rewind()), jobBuilder);
} else {
parser = JobHistoryParserFactory.getParser(ris);
if (parser == null) {
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java Thu Apr 28 05:43:20 2011
@@ -120,8 +120,20 @@ public class ZombieJob implements JobSto
@Override
public synchronized JobConf getJobConf() {
if (jobConf == null) {
- // TODO : add more to jobConf ?
jobConf = new JobConf();
+
+ // Add parameters from the configuration in the job trace
+ //
+ // The reason why the job configuration parameters, as seen in the jobconf
+ // file, are added first because the specialized values obtained from
+ // Rumen should override the job conf values.
+ //
+ for (Map.Entry<Object, Object> entry : job.getJobProperties().entrySet()) {
+ jobConf.set(entry.getKey().toString(), entry.getValue().toString());
+ }
+
+ //TODO Eliminate parameters that are already copied from the job's
+ // configuration file.
jobConf.setJobName(getName());
jobConf.setUser(getUser());
jobConf.setNumMapTasks(getNumberMaps());