You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by am...@apache.org on 2011/11/09 10:47:50 UTC
svn commit: r1199678 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/
src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/
src/contrib/gridmix/src/test/system/org/apa...
Author: amarrk
Date: Wed Nov 9 09:47:50 2011
New Revision: 1199678
URL: http://svn.apache.org/viewvc?rev=1199678&view=rev
Log:
MAPREDUCE-2733. [Gridmix] Gridmix3 cpu emulation system tests. (Vinay Thota via amarrk)
Added:
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithCustomInterval.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithDefaultInterval.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithCustomInterval.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithDefaultInterval.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/cpu_emul_case1.json.gz (with props)
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/cpu_emul_case2.json.gz (with props)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1199678&r1=1199677&r2=1199678&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Nov 9 09:47:50 2011
@@ -10,6 +10,9 @@ Trunk (unreleased changes)
(Plamen Jeliazkov via shv)
IMPROVEMENTS
+ MAPREDUCE-2733. [Gridmix] Gridmix3 cpu emulation system tests.
+ (Vinay Thota via amarrk)
+
MAPREDUCE-3008. Improvements to cumulative CPU emulation for short running
tasks in Gridmix. (amarrk)
Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java?rev=1199678&r1=1199677&r2=1199678&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java Wed Nov 9 09:47:50 2011
@@ -123,7 +123,7 @@ public class Gridmix extends Configured
summarizer = new Summarizer(args);
}
- Gridmix() {
+ public Gridmix() {
summarizer = new Summarizer();
}
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithCustomInterval.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithCustomInterval.java?rev=1199678&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithCustomInterval.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithCustomInterval.java Wed Nov 9 09:47:50 2011
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test cpu emulation with default interval for gridmix jobs
+ * against different input data, submission policies and user resolvers.
+ * Verify the cpu resource metrics of both maps and reduces phase of
+ * Gridmix jobs with their corresponding original job in the input trace.
+ */
+public class TestCPUEmulationForMapsAndReducesWithCustomInterval
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestCPUEmulationWithUncompressedInput.class");
+ int execMode = GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue();
+
+ /**
+ * Generate compressed input and run {@link Gridmix} by turning on the
+ * cpu emulation feature with default setting. The {@link Gridmix}
+ * should use the following runtime parameters.
+ * Submission Policy : STRESS, UserResovler: RoundRobinUserResolver.
+ * Once the {@link Gridmix} run is complete, verify cpu resource metrics of
+ * {@link Gridmix} jobs with their corresponding original job in a trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testCPUEmulationForMapsAndReducesWithCompressedInputCase7()
+ throws Exception {
+ final long inputSizeInMB = 1024 * 7;
+ String tracePath = getTraceFile("cpu_emul_case2");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridMixConfig.GRIDMIX_CPU_CUSTOM_INTERVAL + "=0.35F",
+ "-D", GridMixConfig.GRIDMIX_CPU_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode);
+ }
+
+ /**
+ * Generate uncompressed input and run {@link Gridmix} by turning on the
+ * cpu emulation feature with default setting. The {@link Gridmix}
+ * should use the following runtime parameters.
+ * Submission Policy : SERIAL, UserResovler: SubmitterUserResolver
+ * Once the {@link Gridmix} run is complete, verify cpu resource metrics of
+ * {@link Gridmix} jobs with their corresponding original job in a trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testCPUEmulatonForMapsAndReducesWithUncompressedInputCase8()
+ throws Exception {
+ final long inputSizeInMB = cSize * 300;
+ String tracePath = getTraceFile("cpu_emul_case2");
+ Assert.assertNotNull("Trace file not found.", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "SERIAL",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridMixConfig.GRIDMIX_CPU_CUSTOM_INTERVAL + "=0.4F",
+ "-D", GridMixConfig.GRIDMIX_CPU_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN };
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode);
+ }
+}
+
+
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithDefaultInterval.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithDefaultInterval.java?rev=1199678&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithDefaultInterval.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsAndReducesWithDefaultInterval.java Wed Nov 9 09:47:50 2011
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test cpu emulation with default interval for gridmix jobs
+ * against different input data, submission policies and user resolvers.
+ * Verify the cpu resource metrics for both maps and reduces of
+ * Gridmix jobs with their corresponding original job in the input trace.
+ */
+public class TestCPUEmulationForMapsAndReducesWithDefaultInterval
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog(
+ "TestCPUEmulationForMapsAndReducesWithDefaultInterval.class");
+ int execMode = GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue();
+
+ /**
+ * Generate compressed input and run {@link Gridmix} by turning on the
+ * cpu emulation feature with default setting. The {@link Gridmix}
+ * should use the following runtime parameters.
+ * Submission Policy : REPLAY, UserResovler: RoundRobinUserResolver.
+ * Once the {@link Gridmix} run is complete, verify cpu resource metrics of
+ * {@link Gridmix} jobs with their corresponding original jobs in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testCPUEmulationForMapsAndReducesWithCompressedInputCase5()
+ throws Exception {
+ final long inputSizeInMB = 7168;
+ String tracePath = getTraceFile("cpu_emul_case2");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridMixConfig.GRIDMIX_CPU_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode);
+ }
+
+ /**
+ * Generate uncompressed input and run {@link Gridmix} by turning on the
+ * cpu emulation feature with default settings. The {@link Gridmix}
+ * should use the following runtime parameters.
+ * Submission Policy : STRESS, UserResovler: SubmitterUserResolver
+ * Once the Gridmix run is complete, verify cpu resource metrics of
+ * {@link Gridmix} jobs with their corresponding original jobs in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testCPUEmulatonForMapsAndReducesWithUncompressedInputCase6()
+ throws Exception {
+ final long inputSizeInMB = cSize * 400;
+ String tracePath = getTraceFile("cpu_emul_case2");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridMixConfig.GRIDMIX_CPU_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN };
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode);
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithCustomInterval.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithCustomInterval.java?rev=1199678&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithCustomInterval.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithCustomInterval.java Wed Nov 9 09:47:50 2011
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} cpu emulation with custom interval for
+ * gridmix jobs against different input data, submission policies and
+ * user resolvers. Verify the map phase cpu metrics of gridmix jobs
+ * against their original job in the trace.
+ */
+public class TestCPUEmulationForMapsWithCustomInterval
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestCPUEmulationForMapsWithCustomInterval.class");
+ int execMode = GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue();
+
+ /**
+ * Generate compressed input and run {@link Gridmix} by turning on
+ * cpu emulation feature with custom setting. The {@link Gridmix} should
+ * use the following runtime parameters while running gridmix jobs.
+ * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+ * Once {@link Gridmix} run is complete, verify maps phase cpu resource
+ * metrics of {@link Gridmix} jobs with their corresponding original
+ * in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testCPUEmulatonForMapsWithCompressedInputCase3()
+ throws Exception {
+ final long inputSizeInMB = 1024 * 7;
+ String tracePath = getTraceFile("cpu_emul_case1");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_CPU_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_CPU_CUSTOM_INTERVAL + "=0.25F"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode);
+ }
+
+ /**
+ * Generate uncompressed input and run {@link Gridmix} by turning on
+ * cpu emulation feature with custom settings. The {@link Gridmix}
+ * should use the following runtime paramters while running gridmix jobs.
+ * Submission Policy: REPLAY User Resolver Mode: RoundRobinUserResolver
+ * Once {@link Gridmix} run is complete, verify the map phase cpu resource
+ * metrics of {@link Gridmix} jobs with their corresponding jobs
+ * in the original trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testCPUEmulatonForMapsUnCompressedInputCase4()
+ throws Exception {
+ final long inputSizeInMB = cSize * 200;
+ String tracePath = getTraceFile("cpu_emul_case1");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_CPU_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_CPU_CUSTOM_INTERVAL + "=0.35F"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode);
+ }
+}
+
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithDefaultInterval.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithDefaultInterval.java?rev=1199678&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithDefaultInterval.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestCPUEmulationForMapsWithDefaultInterval.java Wed Nov 9 09:47:50 2011
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} cpu emulation with default settings for
+ * gridmix jobs against different input data, submission policies and
+ * user resolvers. Verify the map phase cpu metrics of gridmix jobs
+ * against their original jobs in the trace.
+ */
+public class TestCPUEmulationForMapsWithDefaultInterval
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestCPUEmulationForMapsWithDefaultInterval.class");
+ int execMode = GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue();
+
+ /**
+ * Generate compressed input and run {@link Gridmix} by turning on cpu
+ * emulation feature with default settings. The {@link Gridmix} should
+ * use the following runtime parameters while running the gridmix jobs.
+ * Submission Policy: STRESS, UserResolver: SubmitterUserResolver.
+ * Once the {@link Gridmix} run is complete, verify map phase cpu metrics of
+ * {@link Gridmix} jobs with their corresponding original job in a trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testCPUEmulatonForMapsWithCompressedInputCase1()
+ throws Exception {
+ final long inputSizeInMB = 1024 * 6;
+ String tracePath = getTraceFile("cpu_emul_case1");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues = { "LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_CPU_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_HIGH_RAM_JOB_ENABLE + "=false"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath, execMode);
+ }
+
+ /**
+ * Generate uncompressed input and run {@link Gridmix} by turning on
+ * cpu emulation feature with default settings. The {@link Gridmix}
+ * should use the following runtime parameters while running Gridmix jobs.
+ * Submission Policy: REPLAY, UserResolver: RoundRobinUserResolver
+ * Once the Gridmix run is complete, verify cpu resource metrics of
+ * {@link Gridmix} jobs with their corresponding original job in a trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testCPUEmulatonForMapsWithUnCompressedInputCase2()
+ throws Exception {
+ final long inputSizeInMB = cSize * 200;
+ String tracePath = getTraceFile("cpu_emul_case1");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_HIGH_RAM_JOB_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_CPU_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java?rev=1199678&r1=1199677&r2=1199678&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java Wed Nov 9 09:47:50 2011
@@ -23,6 +23,7 @@ import org.apache.hadoop.mapred.gridmix.
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.*;
/**
* Gridmix system tests configurations.
@@ -218,4 +219,33 @@ public class GridMixConfig {
*/
public static final String CLUSTER_MAX_REDUCE_MEMORY =
JTConfig.JT_MAX_REDUCEMEMORY_MB;
+
+ /**
+ * Gridmix cpu emulation.
+ */
+ public static final String GRIDMIX_CPU_EMULATON =
+ ResourceUsageMatcher.RESOURCE_USAGE_EMULATION_PLUGINS;
+
+ /**
+ * Gridmix cpu usage emulation plugin.
+ */
+ public static final String GRIDMIX_CPU_USAGE_PLUGIN =
+ CumulativeCpuUsageEmulatorPlugin.class.getName();
+
+ /**
+ * Gridmix cpu emulation custom interval.
+ */
+ public static final String GRIDMIX_CPU_CUSTOM_INTERVAL =
+ CumulativeCpuUsageEmulatorPlugin.CPU_EMULATION_PROGRESS_INTERVAL;
+
+ /**
+ * Gridmix cpu emulation lower limit.
+ */
+ public static int GRIDMIX_CPU_EMULATION_LOWER_LIMIT = 55;
+
+ /**
+ * Gridmix cpu emulation upper limit.
+ */
+ public static int GRIDMIX_CPU_EMULATION_UPPER_LIMIT = 130;
+
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java?rev=1199678&r1=1199677&r2=1199678&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java Wed Nov 9 09:47:50 2011
@@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.CounterGroup;
@@ -105,7 +106,7 @@ public class GridmixJobVerification {
* @throws ParseException - if an parse error occurs.
*/
public void verifyGridmixJobsWithJobStories(List<JobID> jobids)
- throws IOException, ParseException {
+ throws Exception {
SortedMap <Long, String> origSubmissionTime = new TreeMap <Long, String>();
SortedMap <Long, String> simuSubmissionTime = new TreeMap<Long, String>();
@@ -147,6 +148,7 @@ public class GridmixJobVerification {
setJobDistributedCacheInfo(simuJobId.toString(), simuJobConf,
zombieJob.getJobConf());
verifyHighRamMemoryJobs(zombieJob, simuJobConf);
+ verifyCPUEmulationOfJobs(zombieJob, jhInfo, simuJobConf);
LOG.info("Done.");
}
verifyDistributedCacheBetweenJobs(simuAndOrigJobsInfo);
@@ -353,6 +355,119 @@ public class GridmixJobVerification {
fs.close();
}
}
+
+ /**
+ * It verifies the cpu resource usage of a gridmix job against
+ * their original job.
+ * @param origJobHistory - Original job history.
+ * @param simuJobHistoryInfo - Simulated job history.
+ * @param simuJobConf - simulated job configuration.
+ */
+ public void verifyCPUEmulationOfJobs(ZombieJob origJobHistory,
+ JobHistoryParser.JobInfo simuJobHistoryInfo,
+ JobConf simuJobConf) throws Exception {
+
+ if (simuJobConf.get(GridMixConfig.GRIDMIX_CPU_EMULATON) != null) {
+ Map<String,Long> origJobMetrics =
+ getOriginalJobCPUMetrics(origJobHistory);
+ Map<String,Long> simuJobMetrics =
+ getSimulatedJobCPUMetrics(simuJobHistoryInfo);
+
+ long origMapUsage = origJobMetrics.get("MAP");
+ LOG.info("Maps cpu usage of original job:" + origMapUsage);
+
+ long origReduceUsage = origJobMetrics.get("REDUCE");
+ LOG.info("Reduces cpu usage of original job:" + origReduceUsage);
+
+ long simuMapUsage = simuJobMetrics.get("MAP");
+ LOG.info("Maps cpu usage of simulated job:" + simuMapUsage);
+
+ long simuReduceUsage = simuJobMetrics.get("REDUCE");
+ LOG.info("Reduces cpu usage of simulated job:"+ simuReduceUsage);
+
+ long mapCount = simuJobHistoryInfo.getTotalMaps();
+ long reduceCount = simuJobHistoryInfo.getTotalReduces();
+
+ if (mapCount > 0) {
+ double mapEmulFactor = (simuMapUsage * 100) / origMapUsage;
+ long mapEmulAccuracy = Math.round(mapEmulFactor);
+ LOG.info("CPU emulation accuracy for maps in job " +
+ simuJobHistoryInfo.getJobId() +
+ ":"+ mapEmulAccuracy + "%");
+ Assert.assertTrue("Map-side cpu emulaiton inaccurate!" +
+ " Actual cpu usage: " + simuMapUsage +
+ " Expected cpu usage: " + origMapUsage, mapEmulAccuracy
+ >= GridMixConfig.GRIDMIX_CPU_EMULATION_LOWER_LIMIT
+ && mapEmulAccuracy
+ <= GridMixConfig.GRIDMIX_CPU_EMULATION_UPPER_LIMIT);
+ }
+
+ if (reduceCount >0) {
+ double reduceEmulFactor = (simuReduceUsage * 100) / origReduceUsage;
+ long reduceEmulAccuracy = Math.round(reduceEmulFactor);
+ LOG.info("CPU emulation accuracy for reduces in job " +
+ simuJobHistoryInfo.getJobId() +
+ ": " + reduceEmulAccuracy + "%");
+ Assert.assertTrue("Reduce side cpu emulaiton inaccurate!" +
+ " Actual cpu usage:" + simuReduceUsage +
+ "Expected cpu usage: " + origReduceUsage,
+ reduceEmulAccuracy
+ >= GridMixConfig.GRIDMIX_CPU_EMULATION_LOWER_LIMIT
+ && reduceEmulAccuracy
+ <= GridMixConfig.GRIDMIX_CPU_EMULATION_UPPER_LIMIT);
+ }
+ }
+ }
+
+ /**
+ * Get the simulated job cpu metrics.
+ * @param jhInfo - Simulated job history
+ * @return - cpu metrics as a map.
+ * @throws Exception - if an error occurs.
+ */
+ private Map<String,Long> getSimulatedJobCPUMetrics(
+ JobHistoryParser.JobInfo jhInfo) throws Exception {
+ Map<String, Long> resourceMetrics = new HashMap<String, Long>();
+ long mapCPUUsage =
+ getCounterValue(jhInfo.getMapCounters(),
+ TaskCounter.CPU_MILLISECONDS.toString());
+ resourceMetrics.put("MAP", mapCPUUsage);
+ long reduceCPUUsage =
+ getCounterValue(jhInfo.getReduceCounters(),
+ TaskCounter.CPU_MILLISECONDS.toString());
+ resourceMetrics.put("REDUCE", reduceCPUUsage);
+ return resourceMetrics;
+ }
+
+ /**
+ * Get the original job cpu metrics.
+ * @param zombieJob - original job history.
+ * @return - cpu metrics as map.
+ */
+ private Map<String, Long> getOriginalJobCPUMetrics(ZombieJob zombieJob) {
+ long mapTotalCPUUsage = 0;
+ long reduceTotalCPUUsage = 0;
+ Map<String,Long> resourceMetrics = new HashMap<String,Long>();
+
+ for (int index = 0; index < zombieJob.getNumberMaps(); index ++) {
+ TaskInfo mapTask = zombieJob.getTaskInfo(TaskType.MAP, index);
+ if (mapTask.getResourceUsageMetrics().getCumulativeCpuUsage() > 0) {
+ mapTotalCPUUsage +=
+ mapTask.getResourceUsageMetrics().getCumulativeCpuUsage();
+ }
+ }
+ resourceMetrics.put("MAP", mapTotalCPUUsage);
+
+ for (int index = 0; index < zombieJob.getNumberReduces(); index ++) {
+ TaskInfo reduceTask = zombieJob.getTaskInfo(TaskType.REDUCE, index);
+ if (reduceTask.getResourceUsageMetrics().getCumulativeCpuUsage() > 0) {
+ reduceTotalCPUUsage +=
+ reduceTask.getResourceUsageMetrics().getCumulativeCpuUsage();
+ }
+ }
+ resourceMetrics.put("REDUCE", reduceTotalCPUUsage);
+ return resourceMetrics;
+ }
/**
* Get the user resolver of a job.
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/cpu_emul_case1.json.gz
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/cpu_emul_case1.json.gz?rev=1199678&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/cpu_emul_case1.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/cpu_emul_case2.json.gz
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/cpu_emul_case2.json.gz?rev=1199678&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/cpu_emul_case2.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream