You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by am...@apache.org on 2011/11/15 02:48:43 UTC
svn commit: r1201997 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/
src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/
src/contrib/gridmix/src/...
Author: amarrk
Date: Tue Nov 15 01:48:42 2011
New Revision: 1201997
URL: http://svn.apache.org/viewvc?rev=1201997&view=rev
Log:
MAPREDUCE-3375. [Gridmix] Memory Emulation system tests. (Vinay Thota via amarrk)
Added:
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithCustomIntrvl.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithDefaultIntrvl.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomHeapMemoryRatio.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomIntrvl.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithDefaultIntrvl.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case1.json.gz (with props)
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case2.json.gz (with props)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1201997&r1=1201996&r2=1201997&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Nov 15 01:48:42 2011
@@ -10,6 +10,9 @@ Trunk (unreleased changes)
(Plamen Jeliazkov via shv)
IMPROVEMENTS
+ MAPREDUCE-3375. [Gridmix] Memory Emulation system tests.
+ (Vinay Thota via amarrk)
+
MAPREDUCE-2733. [Gridmix] Gridmix3 cpu emulation system tests.
(Vinay Thota via amarrk)
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithCustomIntrvl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithCustomIntrvl.java?rev=1201997&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithCustomIntrvl.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithCustomIntrvl.java Tue Nov 15 01:48:42 2011
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} memory emulation feature for the jobs with
+ * custom progress interval, different input data, submission policies
+ * and user resolver modes. Verify the total heap usage of map and reduce
+ * tasks of the jobs with corresponding original job in the trace.
+ */
+public class TestMemEmulForMapsAndReducesWithCustomIntrvl
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestMemEmulForMapsAndReducesWithCustomIntrvl.class");
+ /**
+ * Generate compressed input and run {@link Gridmix} by turning on the
+ * memory emulation with custom progress interval. The {@link Gridmix}
+ * should use the following runtime parameters while running the jobs.
+ * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+ * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs
+ * with corresponding original job in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testMemoryEmulationForReducesWithCompressedInputCase7()
+ throws Exception {
+ final long inputSizeInMB = 1024 * 7;
+ String tracePath = getTraceFile("mem_emul_case2");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.3F",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+
+ /**
+ * Generate uncompressed input and run {@link Gridmix} by turning on the
+ * memory emulation with custom progress interval. The {@link Gridmix}
+ * should use the following runtime parameters while running the jobs.
+ * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+ * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs
+ * with corresponding original job in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testMemoryEmulationForReducesWithUncompressedInputCase8()
+ throws Exception {
+ final long inputSizeInMB = cSize * 300;
+ String tracePath = getTraceFile("mem_emul_case2");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.2F",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithDefaultIntrvl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithDefaultIntrvl.java?rev=1201997&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithDefaultIntrvl.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsAndReducesWithDefaultIntrvl.java Tue Nov 15 01:48:42 2011
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} memory emulation feature for gridmix jobs
+ * with default progress interval, different input data, submission
+ * policies and user resolver modes. Verify the total heap usage of
+ * map and reduce tasks of the jobs with corresponding original
+ * job in the trace.
+ */
+public class TestMemEmulForMapsAndReducesWithDefaultIntrvl
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestMemEmulForMapsAndReducesWithDefaultIntrvl.class");
+
+ /**
+ * Generate compressed input and run {@link Gridmix} by turning on the
+ * memory emulation with default progress interval. The {@link Gridmix}
+ * should use the following runtime parameters while running the jobs.
+ * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+ * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs
+ * with corresponding original job in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testMemoryEmulationForReducesWithCompressedInputCase5()
+ throws Exception {
+ final long inputSizeInMB = 1024 * 7;
+ String tracePath = getTraceFile("mem_emul_case2");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+
+ /**
+ * Generate uncompressed input and run {@link Gridmix} by turning on the
+ * memory emulation with default progress interval. The {@link Gridmix}
+ * should use the following runtime parameters while running the jobs.
+ * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+ * Verify maps and reduces total heap memory usage of {@link Gridmix} jobs
+ * with corresponding original job in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testMemoryEmulationForReducesWithUncompressedInputCase6()
+ throws Exception {
+ final long inputSizeInMB = cSize * 300;
+ String tracePath = getTraceFile("mem_emul_case2");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomHeapMemoryRatio.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomHeapMemoryRatio.java?rev=1201997&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomHeapMemoryRatio.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomHeapMemoryRatio.java Tue Nov 15 01:48:42 2011
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs
+ * with default progress interval, custom heap memory ratio, different input
+ * data, submission policies and user resolver modes. Verify the total heap
+ * usage of map and reduce tasks of the jobs with corresponding the original job
+ * in the trace.
+ */
+public class TestMemEmulForMapsWithCustomHeapMemoryRatio
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestMemEmulForMapsWithCustomHeapMemoryRatio.class");
+
+ /**
+ * Generate compressed input and run {@link Gridmix} by turning on the
+ * memory emulation. The {@link Gridmix} should use the following runtime
+ * parameters while running the jobs.
+ * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+ * Verify total heap memory usage of the tasks of {@link Gridmix} jobs with
+ * corresponding original job in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testMemoryEmulationForMapsWithCompressedInputCase1()
+ throws Exception {
+ final long inputSizeInMB = 1024 * 7;
+ String tracePath = getTraceFile("mem_emul_case2");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO + "=0.5F"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+
+ /**
+ * Generate uncompressed input and run {@link Gridmix} by turning on the
+ * memory emulation. The {@link Gridmix} should use the following runtime
+ * parameters while running the jobs.
+ * Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver
+ * Verify total heap memory usage of tasks of {@link Gridmix} jobs with
+ * corresponding original job in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testMemoryEmulationForMapsWithUncompressedInputCase2()
+ throws Exception {
+ final long inputSizeInMB = cSize * 300;
+ String tracePath = getTraceFile("mem_emul_case2");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO + "=0.4F"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomIntrvl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomIntrvl.java?rev=1201997&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomIntrvl.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithCustomIntrvl.java Tue Nov 15 01:48:42 2011
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs
+ * with custom progress interval, different input data, submission policies
+ * and user resolver modes. Verify the total heap usage of map tasks of
+ * the jobs with corresponding the original job in the trace.
+ */
+public class TestMemEmulForMapsWithCustomIntrvl extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestMemEmulForMapsWithCustomIntrvl.class");
+
+ /**
+ * Generate compressed input and run {@link Gridmix} by turning on the
+ * memory emulation with custom progress interval. The {@link Gridmix}
+ * should use the following runtime parameters while running the jobs.
+ * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+ * Verify maps total heap memory usage of {@link Gridmix} jobs with
+ * corresponding original job in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testMemoryEmulationForMapsWithCompressedInputCase3()
+ throws Exception {
+ final long inputSizeInMB = 1024 * 7;
+ String tracePath = getTraceFile("mem_emul_case1");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.2F",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+
+ /**
+ * Generate uncompressed input and run {@link Gridmix} by turning on the
+ * memory emulation with custom progress interval. The {@link Gridmix}
+ * should use the following runtime parameters while running the jobs.
+ * Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver
+ * Verify maps total heap memory usage of {@link Gridmix} jobs with
+ * corresponding original job in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testMemoryEmulationForMapsWithUncompressedInputCase4()
+ throws Exception {
+ final long inputSizeInMB = cSize * 300;
+ String tracePath = getTraceFile("mem_emul_case1");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL + "=0.3F",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithDefaultIntrvl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithDefaultIntrvl.java?rev=1201997&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithDefaultIntrvl.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestMemEmulForMapsWithDefaultIntrvl.java Tue Nov 15 01:48:42 2011
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Test the {@link Gridmix} memory emulation feature for {@link Gridmix} jobs
+ * with default progress interval, different input data, submission policies
+ * and user resolver modes. Verify the total heap usage of map tasks of the
+ * jobs with corresponding original job in the trace.
+ */
+public class TestMemEmulForMapsWithDefaultIntrvl extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestMemEmulForMapsWithDefaultIntrvl.class");
+
+ /**
+ * Generate compressed input and run {@link Gridmix} by turning on the
+ * memory emulation with default progress interval. The {@link Gridmix}
+ * should use the following runtime parameters while running the jobs.
+ * Submission Policy : STRESS, User Resolver Mode : SumitterUserResolver
+ * Verify maps total heap memory usage of {@link Gridmix} jobs with
+ * corresponding original job in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testMemoryEmulationForMapsWithCompressedInputCase1()
+ throws Exception {
+ final long inputSizeInMB = 1024 * 7;
+ String tracePath = getTraceFile("mem_emul_case1");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+
+ /**
+ * Generate uncompressed input and run {@link Gridmix} by turning on the
+ * memory emulation with default progress interval. The {@link Gridmix}
+ * should use the following runtime parameters while running the jobs.
+ * Submission Policy : STRESS, User Resolver Mode : RoundRobinUserResolver
+ * Verify maps total heap memory usage of {@link Gridmix} jobs with
+ * corresponding original job in the trace.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testMemoryEmulationForMapsWithUncompressedInputCase2()
+ throws Exception {
+ final long inputSizeInMB = cSize * 300;
+ String tracePath = getTraceFile("mem_emul_case1");
+ Assert.assertNotNull("Trace file not found!", tracePath);
+ String [] runtimeValues =
+ { "LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_MEMORY_EMULATON + "=" +
+ GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN + "=false"};
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java?rev=1201997&r1=1201996&r2=1201997&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java Tue Nov 15 01:48:42 2011
@@ -248,4 +248,38 @@ public class GridMixConfig {
*/
public static int GRIDMIX_CPU_EMULATION_UPPER_LIMIT = 130;
+ /**
+ * Gridmix heap memory custom interval
+ */
+ public static final String GRIDMIX_HEAP_MEMORY_CUSTOM_INTRVL =
+ TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL;
+
+ /**
+ * Gridmix heap free memory ratio
+ */
+ public static final String GRIDMIX_HEAP_FREE_MEMORY_RATIO =
+ TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO;
+
+ /**
+ * Gridmix memory emulation plugin
+ */
+ public static final String GRIDMIX_MEMORY_EMULATION_PLUGIN =
+ TotalHeapUsageEmulatorPlugin.class.getName();
+
+ /**
+ * Gridmix memory emulation
+ */
+ public static final String GRIDMIX_MEMORY_EMULATON =
+ ResourceUsageMatcher.RESOURCE_USAGE_EMULATION_PLUGINS;
+
+ /**
+ * Gridmix memory emulation lower limit.
+ */
+ public static int GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT = 55;
+
+ /**
+ * Gridmix memory emulation upper limit.
+ */
+ public static int GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT = 130;
+
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java?rev=1201997&r1=1201996&r2=1201997&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobVerification.java Tue Nov 15 01:48:42 2011
@@ -149,6 +149,7 @@ public class GridmixJobVerification {
zombieJob.getJobConf());
verifyHighRamMemoryJobs(zombieJob, simuJobConf);
verifyCPUEmulationOfJobs(zombieJob, jhInfo, simuJobConf);
+ verifyMemoryEmulationOfJobs(zombieJob, jhInfo, simuJobConf);
LOG.info("Done.");
}
verifyDistributedCacheBetweenJobs(simuAndOrigJobsInfo);
@@ -357,6 +358,109 @@ public class GridmixJobVerification {
}
/**
+ * It verifies the heap memory resource usage of gridmix jobs with
+ * corresponding original job in the trace.
+ * @param zombieJob - Original job history.
+ * @param jhInfo - Simulated job history.
+ * @param simuJobConf - simulated job configuration.
+ */
+ public void verifyMemoryEmulationOfJobs(ZombieJob zombieJob,
+ JobHistoryParser.JobInfo jhInfo,
+ JobConf simuJobConf) throws Exception {
+ long origJobMapsTHU = 0;
+ long origJobReducesTHU = 0;
+ long simuJobMapsTHU = 0;
+ long simuJobReducesTHU = 0;
+ boolean isMemEmulOn = false;
+ if (simuJobConf.get(GridMixConfig.GRIDMIX_MEMORY_EMULATON) != null) {
+ isMemEmulOn =
+ simuJobConf.get(GridMixConfig.GRIDMIX_MEMORY_EMULATON).
+ contains(GridMixConfig.GRIDMIX_MEMORY_EMULATION_PLUGIN);
+ }
+
+ if (isMemEmulOn) {
+ for (int index = 0; index < zombieJob.getNumberMaps(); index ++) {
+ TaskInfo mapTask = zombieJob.getTaskInfo(TaskType.MAP, index);
+ if (mapTask.getResourceUsageMetrics().getHeapUsage() > 0) {
+ origJobMapsTHU +=
+ mapTask.getResourceUsageMetrics().getHeapUsage();
+ }
+ }
+ LOG.info("Original Job Maps Total Heap Usage: " + origJobMapsTHU);
+
+ for (int index = 0; index < zombieJob.getNumberReduces(); index ++) {
+ TaskInfo reduceTask = zombieJob.getTaskInfo(TaskType.REDUCE, index);
+ if (reduceTask.getResourceUsageMetrics().getHeapUsage() > 0) {
+ origJobReducesTHU +=
+ reduceTask.getResourceUsageMetrics().getHeapUsage();
+ }
+ }
+ LOG.info("Original Job Reduces Total Heap Usage: " + origJobReducesTHU);
+
+ simuJobMapsTHU =
+ getCounterValue(jhInfo.getMapCounters(),
+ TaskCounter.COMMITTED_HEAP_BYTES.toString());
+ LOG.info("Simulated Job Maps Total Heap Usage: " + simuJobMapsTHU);
+
+ simuJobReducesTHU =
+ getCounterValue(jhInfo.getReduceCounters(),
+ TaskCounter.COMMITTED_HEAP_BYTES.toString());
+ LOG.info("Simulated Jobs Reduces Total Heap Usage: " + simuJobReducesTHU);
+
+ long mapCount = jhInfo.getTotalMaps();
+ long reduceCount = jhInfo.getTotalReduces();
+
+ String strHeapRatio =
+ simuJobConf.get(GridMixConfig.GRIDMIX_HEAP_FREE_MEMORY_RATIO);
+ if (strHeapRatio == null) {
+ strHeapRatio = "0.3F";
+ }
+
+ if (mapCount > 0) {
+ double mapEmulFactor = (simuJobMapsTHU * 100) / origJobMapsTHU;
+ long mapEmulAccuracy = Math.round(mapEmulFactor);
+ LOG.info("Maps memory emulation accuracy of a job:"
+ + mapEmulAccuracy + "%");
+ Assert.assertTrue("Map phase total memory emulation had crossed the "
+ + "configured max limit.", mapEmulAccuracy
+ <= GridMixConfig.GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT);
+ Assert.assertTrue("Map phase total memory emulation had not crossed "
+ + "the configured min limit.", mapEmulAccuracy
+ >= GridMixConfig.GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT);
+ double expHeapRatio = Double.parseDouble(strHeapRatio);
+ LOG.info("expHeapRatio for maps:" + expHeapRatio);
+ double actHeapRatio =
+ ((double)Math.abs(origJobMapsTHU - simuJobMapsTHU)) ;
+ actHeapRatio /= origJobMapsTHU;
+ LOG.info("actHeapRatio for maps:" + actHeapRatio);
+ Assert.assertTrue("Simulate job maps heap ratio not matched.",
+ actHeapRatio <= expHeapRatio);
+ }
+
+ if (reduceCount >0) {
+ double reduceEmulFactor = (simuJobReducesTHU * 100) / origJobReducesTHU;
+ long reduceEmulAccuracy = Math.round(reduceEmulFactor);
+ LOG.info("Reduces memory emulation accuracy of a job:"
+ + reduceEmulAccuracy + "%");
+ Assert.assertTrue("Reduce phase total memory emulation had crossed "
+ + "configured max limit.", reduceEmulAccuracy
+ <= GridMixConfig.GRIDMIX_MEMORY_EMULATION_UPPER_LIMIT);
+ Assert.assertTrue("Reduce phase total memory emulation had not "
+ + "crosssed configured min limit.", reduceEmulAccuracy
+ >= GridMixConfig.GRIDMIX_MEMORY_EMULATION_LOWER_LIMIT);
+ double expHeapRatio = Double.parseDouble(strHeapRatio);
+ LOG.info("expHeapRatio for reduces:" + expHeapRatio);
+ double actHeapRatio =
+ ((double)Math.abs(origJobReducesTHU - simuJobReducesTHU));
+ actHeapRatio /= origJobReducesTHU;
+ LOG.info("actHeapRatio for reduces:" + actHeapRatio);
+ Assert.assertTrue("Simulate job reduces heap ratio not matched.",
+ actHeapRatio <= expHeapRatio);
+ }
+ }
+ }
+
+ /**
* It verifies the cpu resource usage of a gridmix job against
* their original job.
* @param origJobHistory - Original job history.
@@ -367,7 +471,14 @@ public class GridmixJobVerification {
JobHistoryParser.JobInfo simuJobHistoryInfo,
JobConf simuJobConf) throws Exception {
+ boolean isCpuEmulOn = false;
if (simuJobConf.get(GridMixConfig.GRIDMIX_CPU_EMULATON) != null) {
+ isCpuEmulOn =
+ simuJobConf.get(GridMixConfig.GRIDMIX_CPU_EMULATON).
+ contains(GridMixConfig.GRIDMIX_CPU_USAGE_PLUGIN);
+ }
+
+ if (isCpuEmulOn) {
Map<String,Long> origJobMetrics =
getOriginalJobCPUMetrics(origJobHistory);
Map<String,Long> simuJobMetrics =
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case1.json.gz
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case1.json.gz?rev=1201997&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case1.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case2.json.gz
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case2.json.gz?rev=1201997&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/system/resources/mem_emul_case2.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream