You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by am...@apache.org on 2010/08/03 07:46:20 UTC
svn commit: r981745 - in /hadoop/mapreduce/trunk: ./
src/test/mapred/org/apache/hadoop/tools/rumen/
src/test/tools/data/rumen/small-trace-test/
src/tools/org/apache/hadoop/tools/rumen/
Author: amareshwari
Date: Tue Aug 3 05:46:19 2010
New Revision: 981745
URL: http://svn.apache.org/viewvc?rev=981745&view=rev
Log:
MAPREDUCE-1982. Fixes Rumen's TraceBuilder to extract job name from either of configuration properties mapreduce.job.name and mapred.job.name. Contributed by Ravi Gummadi
Added:
hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=981745&r1=981744&r2=981745&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Aug 3 05:46:19 2010
@@ -216,6 +216,10 @@ Trunk (unreleased changes)
the owner of the localized file in the mapping from cache URIs to
CacheStatus objects. (ddas)
+ MAPREDUCE-1982. Fixes Rumen's TraceBuilder to extract job name from either
+ of configuration properties "mapreduce.job.name" and "mapred.job.name".
+ (Ravi Gummadi via amareshwari)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=981745&r1=981744&r2=981745&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java Tue Aug 3 05:46:19 2010
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.MapReduceTestUtil;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
@@ -380,15 +381,34 @@ public class TestRumenJobTraces {
lfs.delete(tempDir, true);
}
}
-
+
@Test
public void testJobConfigurationParser() throws Exception {
- String[] list1 =
- { "mapred.job.queue.name", "mapreduce.job.name",
- "mapred.child.java.opts" };
- String[] list2 = { "mapred.job.queue.name", "mapred.child.java.opts" };
+ // Validate parser with old mapred config properties from
+ // sample-conf-file.xml
+ String[] oldProps1 = { "mapred.job.queue.name", "mapred.job.name",
+ "mapred.child.java.opts" };
+
+ String[] oldProps2 = { "mapred.job.queue.name", "mapred.child.java.opts" };
+
+ validateJobConfParser(oldProps1, oldProps2, "sample-conf.file.xml", false);
+
+ // Validate parser with new mapreduce config properties from
+ // sample-conf-file.new.xml
+ String[] newProps1 = { MRJobConfig.QUEUE_NAME, MRJobConfig.JOB_NAME,
+ MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.REDUCE_JAVA_OPTS };
+
+ String[] newProps2 = { MRJobConfig.QUEUE_NAME,
+ MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.REDUCE_JAVA_OPTS };
+
+ validateJobConfParser(newProps1, newProps2,
+ "sample-conf.file.new.xml", true);
+ }
+ private void validateJobConfParser(String[] list1, String[] list2,
+ String confFile, boolean newConfig)
+ throws Exception {
List<String> interested1 = new ArrayList<String>();
for (String interested : list1) {
interested1.add(interested);
@@ -412,7 +432,7 @@ public class TestRumenJobTraces {
final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test");
- final Path inputPath = new Path(rootInputPath, "sample-conf.file.xml");
+ final Path inputPath = new Path(rootInputPath, confFile);
InputStream inputConfStream =
new PossiblyDecompressedInputStream(inputPath, conf);
@@ -424,28 +444,82 @@ public class TestRumenJobTraces {
inputConfStream = new PossiblyDecompressedInputStream(inputPath, conf);
Properties props2 = jcp2.parse(inputConfStream);
- assertEquals("testJobConfigurationParser: wrong number of properties", 3,
- props1.size());
- assertEquals("testJobConfigurationParser: wrong number of properties", 2,
- props2.size());
-
- assertEquals("prop test 1", "TheQueue", props1
- .get("mapred.job.queue.name"));
- assertEquals("prop test 2", "job_0001", props1.get("mapreduce.job.name"));
- assertEquals("prop test 3",
- "-server -Xmx640m -Djava.net.preferIPv4Stack=true", props1
- .get("mapred.child.java.opts"));
- assertEquals("prop test 4", "TheQueue", props2
- .get("mapred.job.queue.name"));
- assertEquals("prop test 5",
- "-server -Xmx640m -Djava.net.preferIPv4Stack=true", props2
- .get("mapred.child.java.opts"));
+ assertEquals("testJobConfigurationParser: wrong number of properties",
+ list1.length, props1.size());
+ assertEquals("testJobConfigurationParser: wrong number of properties",
+ list2.length, props2.size());
+
+ // Make sure that parser puts the interested properties into props1 and
+ // props2 as defined by list1 and list2.
+ String oldOrNew = newConfig ? "New" : "Old";
+ assertEquals(oldOrNew + " config property for job queue name is not "
+ + " extracted properly.", "TheQueue",
+ JobBuilder.extract(props1, JobConfPropertyNames.QUEUE_NAMES
+ .getCandidates(), null));
+ assertEquals(oldOrNew + " config property for job name is not "
+ + " extracted properly.", "MyMRJob",
+ JobBuilder.extract(props1, JobConfPropertyNames.JOB_NAMES
+ .getCandidates(), null));
+
+ assertEquals(oldOrNew + " config property for job queue name is not "
+ + " extracted properly.", "TheQueue",
+ JobBuilder.extract(props2, JobConfPropertyNames.QUEUE_NAMES
+ .getCandidates(), null));
+
+ // This config property is not interested for props2. So props should not
+ // contain this.
+ assertNull("Uninterested " + oldOrNew + " config property for job name "
+ + " is extracted.",
+ JobBuilder.extract(props2, JobConfPropertyNames.JOB_NAMES
+ .getCandidates(), null));
+
+ validateChildJavaOpts(newConfig, props1);
+ validateChildJavaOpts(newConfig, props2);
} finally {
inputConfStream.close();
}
}
+ // Validate child java opts in properties.
+ // newConfigProperties: boolean that specifies if the config properties to be
+ // validated are new OR old.
+ private void validateChildJavaOpts(boolean newConfigProperties,
+ Properties props) {
+ if (newConfigProperties) {
+ assertEquals("New config property " + MRJobConfig.MAP_JAVA_OPTS
+ + " is not extracted properly.",
+ "-server -Xmx640m -Djava.net.preferIPv4Stack=true",
+ JobBuilder.extract(props, JobConfPropertyNames.MAP_JAVA_OPTS_S
+ .getCandidates(), null));
+ assertEquals("New config property " + MRJobConfig.REDUCE_JAVA_OPTS
+ + " is not extracted properly.",
+ "-server -Xmx650m -Djava.net.preferIPv4Stack=true",
+ JobBuilder.extract(props, JobConfPropertyNames.REDUCE_JAVA_OPTS_S
+ .getCandidates(), null));
+ }
+ else {
+ // if old property mapred.child.java.opts is set, then extraction of all
+ // the following 3 properties should give that value.
+ assertEquals("mapred.child.java.opts is not extracted properly.",
+ "-server -Xmx640m -Djava.net.preferIPv4Stack=true",
+ JobBuilder.extract(props, JobConfPropertyNames.TASK_JAVA_OPTS_S
+ .getCandidates(), null));
+ assertEquals("New config property " + MRJobConfig.MAP_JAVA_OPTS
+ + " is not extracted properly when the old config property "
+ + "mapred.child.java.opts is set.",
+ "-server -Xmx640m -Djava.net.preferIPv4Stack=true",
+ JobBuilder.extract(props, JobConfPropertyNames.MAP_JAVA_OPTS_S
+ .getCandidates(), null));
+ assertEquals("New config property " + MRJobConfig.REDUCE_JAVA_OPTS
+ + " is not extracted properly when the old config property "
+ + "mapred.child.java.opts is set.",
+ "-server -Xmx640m -Djava.net.preferIPv4Stack=true",
+ JobBuilder.extract(props, JobConfPropertyNames.REDUCE_JAVA_OPTS_S
+ .getCandidates(), null));
+ }
+ }
+
@Test
public void testTopologyBuilder() throws Exception {
final TopologyBuilder subject = new TopologyBuilder();
Added: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml?rev=981745&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml (added)
+++ hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml Tue Aug 3 05:46:19 2010
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+<!--
+New mapreduce config properties
+-->
+ <property>
+ <name>mapreduce.job.queuename</name><value>TheQueue</value>
+ </property>
+ <property>
+ <name>mapreduce.job.name</name><value>MyMRJob</value>
+ </property>
+ <property>
+ <name>maproduce.uninteresting.property</name><value>abcdef</value>
+ </property>
+ <property><name>mapreduce.map.java.opts</name><value>-server -Xmx640m -Djava.net.preferIPv4Stack=true</value></property>
+ <property><name>mapreduce.reduce.java.opts</name><value>-server -Xmx650m -Djava.net.preferIPv4Stack=true</value></property>
+</configuration>
Modified: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml?rev=981745&r1=981744&r2=981745&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml (original)
+++ hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml Tue Aug 3 05:46:19 2010
@@ -19,11 +19,14 @@
*/
-->
<configuration>
+<!--
+Old mapred config properties
+-->
<property>
<name>mapred.job.queue.name</name><value>TheQueue</value>
</property>
<property>
- <name>mapreduce.job.name</name><value>job_0001</value>
+ <name>mapred.job.name</name><value>MyMRJob</value>
</property>
<property>
<name>maproduce.uninteresting.property</name><value>abcdef</value>
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=981745&r1=981744&r2=981745&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java Tue Aug 3 05:46:19 2010
@@ -164,7 +164,7 @@ public class JobBuilder {
"JobBuilder.process(HistoryEvent): unknown event type");
}
- private String extract(Properties conf, String[] names, String defaultValue) {
+ static String extract(Properties conf, String[] names, String defaultValue) {
for (String name : names) {
String result = conf.getProperty(name);
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java?rev=981745&r1=981744&r2=981745&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java Tue Aug 3 05:46:19 2010
@@ -20,8 +20,9 @@ package org.apache.hadoop.tools.rumen;
import org.apache.hadoop.mapreduce.MRJobConfig;
public enum JobConfPropertyNames {
- QUEUE_NAMES("mapred.job.queue.name", MRJobConfig.QUEUE_NAME), JOB_NAMES(
- MRJobConfig.JOB_NAME), TASK_JAVA_OPTS_S("mapred.child.java.opts"),
+ QUEUE_NAMES("mapred.job.queue.name", MRJobConfig.QUEUE_NAME),
+ JOB_NAMES("mapred.job.name", MRJobConfig.JOB_NAME),
+ TASK_JAVA_OPTS_S("mapred.child.java.opts"),
MAP_JAVA_OPTS_S("mapred.child.java.opts", MRJobConfig.MAP_JAVA_OPTS),
REDUCE_JAVA_OPTS_S("mapred.child.java.opts", MRJobConfig.REDUCE_JAVA_OPTS);