You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by am...@apache.org on 2011/12/23 06:38:56 UTC
svn commit: r1222572 [2/5] - in /hadoop/common/branches/branch-1: ./ ivy/
src/contrib/ src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/
src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/
src/contrib/gridmix/src/test/sys...
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHighRamAndNormalMRJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHighRamAndNormalMRJobs.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHighRamAndNormalMRJobs.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHighRamAndNormalMRJobs.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Run the {@link Gridmix} with combination of high ram and normal jobs of
+ * trace and verify whether high ram jobs{@link Gridmix} are honoring or not.
+ * Normal MR jobs should not honors the high ram emulation.
+ */
+public class TestEmulationOfHighRamAndNormalMRJobs
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestEmulationOfHighRamAndNormalMRJobs.class");
+
+ /**
+ * Generate input data and run the combination normal and high ram
+ * {@link Gridmix} jobs as load job and STRESS submission policy
+ * in a SubmitterUserResolver mode. Verify whether each {@link Gridmix}
+ * job honors the high ram or not after completion of execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testEmulationOfHighRamForReducersOfMRJobs()
+ throws Exception {
+ final long inputSizeInMB = cSize * 250;
+ String tracePath = getTraceFile("highram_mr_jobs_case4");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ String [] runtimeArgs = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "SERIAL",
+ inputSizeInMB + "m",
+ tracePath};
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_HIGH_RAM_JOB_ENABLE + "=true"};
+
+ validateTaskMemoryParamters(tracePath, true);
+ runGridmixAndVerify(runtimeArgs, otherArgs, tracePath);
+ }
+}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the emulation of local FS distributed cache files.
+ *
+ */
+public class TestEmulationOfLocalFSDCFiles extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestEmulationOfLocalFSDCFiles.class");
+
+ /**
+ * Generate the input data and distributer cache files.Verify the
+ * gridmix emulation of local file system distributed cache files
+ * in RoundRobinUserResolver mode with REPLAY submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateInputAndEmulateLocalFSDCFile()
+ throws Exception {
+ final long inputSizeInMB = 1024 * 6;
+ final String tracePath = getTraceFile("distcache_case7_trace");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ final String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ final String [] otherArgs = {
+ "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+
+ /**
+ * Use existing input and local distributed cache files and verify
+ * the gridmix emulation of local file system distributed cache
+ * files in SubmitterUserResolver mode with STRESS
+ * Submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testEmulationOfLocalFSDCFile()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case7_trace");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ final String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ tracePath};
+
+ final String [] otherArgs = {
+ "-D",JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX.getValue());
+ }
+}
Modified: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java?rev=1222572&r1=1222571&r2=1222572&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java (original)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java Fri Dec 23 05:38:53 2011
@@ -27,6 +27,9 @@ import org.apache.hadoop.mapred.JobStatu
import org.apache.hadoop.mapred.gridmix.RoundRobinUserResolver;
import org.apache.hadoop.mapred.gridmix.EchoUserResolver;
import org.apache.hadoop.mapred.gridmix.SubmitterUserResolver;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
@@ -55,15 +58,14 @@ public class TestGridMixDataGeneration {
@BeforeClass
public static void before() throws Exception {
String [] excludeExpList = {"java.net.ConnectException",
- "java.io.IOException"};
+ "java.io.IOException"};
cluster = MRCluster.createCluster(conf);
cluster.setExcludeExpList(excludeExpList);
cluster.setUp();
cSize = cluster.getTTClients().size();
jtClient = cluster.getJTClient();
rtClient = jtClient.getProxy();
- gridmixDir = new Path("hdfs:///user/" + UtilsForGridmix.getUserName()
- + "/herriot-gridmix");
+ gridmixDir = new Path("herriot-gridmix");
UtilsForGridmix.createDirs(gridmixDir, rtClient.getDaemonConf());
}
@@ -82,17 +84,24 @@ public class TestGridMixDataGeneration {
@Test
public void testGenerateDataWithSTRESSSubmission() throws Exception {
conf = rtClient.getDaemonConf();
- final long inputSize = cSize * 128;
- String [] runtimeValues ={"LOADJOB",
- SubmitterUserResolver.class.getName(),
- "STRESS",
- inputSize+"m",
- "file:///dev/null"};
+ final long inputSizeInMB = cSize * 128;
+ String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file:///dev/null"};
- int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
- conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+ };
+ int exitCode =
+ UtilsForGridmix.runGridmixJob(gridmixDir, conf,
+ GridMixRunMode.DATA_GENERATION.getValue(),
+ runtimeValues, otherArgs);
Assert.assertEquals("Data generation has failed.", 0 , exitCode);
- checkGeneratedDataAndJobStatus(inputSize);
+ checkGeneratedDataAndJobStatus(inputSizeInMB);
}
/**
@@ -104,18 +113,27 @@ public class TestGridMixDataGeneration {
@Test
public void testGenerateDataWithREPLAYSubmission() throws Exception {
conf = rtClient.getDaemonConf();
- final long inputSize = cSize * 300;
- String [] runtimeValues ={"LOADJOB",
- RoundRobinUserResolver.class.getName(),
- "REPLAY",
- inputSize +"m",
- "file://" + UtilsForGridmix.getProxyUsersFile(cluster.getHadoopProxyUsers()),
- "file:///dev/null"};
+ final long inputSizeInMB = cSize * 300;
+ String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB +"m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ "file:///dev/null"};
- int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
- conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+ };
+
+ int exitCode =
+ UtilsForGridmix.runGridmixJob(gridmixDir, conf,
+ GridMixRunMode.DATA_GENERATION.getValue(),
+ runtimeValues, otherArgs);
Assert.assertEquals("Data generation has failed.", 0 , exitCode);
- checkGeneratedDataAndJobStatus(inputSize);
+ checkGeneratedDataAndJobStatus(inputSizeInMB);
}
/**
@@ -128,72 +146,81 @@ public class TestGridMixDataGeneration {
@Test
public void testGenerateDataWithSERIALSubmission() throws Exception {
conf = rtClient.getDaemonConf();
- int perNodeSize = 500; // 500 mb per node data
- final long inputSize = cSize * perNodeSize;
- String [] runtimeValues ={"LOADJOB",
- EchoUserResolver.class.getName(),
- "SERIAL",
- inputSize + "m",
- "file:///dev/null"};
- int bytesPerFile = 200; // 200 mb per file of data
+ long perNodeSizeInMB = 500; // 500 mb per node data
+ final long inputSizeInMB = cSize * perNodeSizeInMB;
+ String [] runtimeValues ={"LOADJOB",
+ EchoUserResolver.class.getName(),
+ "SERIAL",
+ inputSizeInMB + "m",
+ "file:///dev/null"};
+ long bytesPerFile = 200 * 1024 * 1024; // 200 mb per file of data
String [] otherArgs = {
- "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE +
- "=" + (bytesPerFile * 1024 * 1024)
+ "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile,
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};
- int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
- conf,GridMixRunMode.DATA_GENERATION, runtimeValues,otherArgs);
+ int exitCode =
+ UtilsForGridmix.runGridmixJob(gridmixDir, conf,
+ GridMixRunMode.DATA_GENERATION.getValue(),
+ runtimeValues, otherArgs);
Assert.assertEquals("Data generation has failed.", 0 , exitCode);
LOG.info("Verify the eache file size in a generate data.");
- verifyEachNodeSize(new Path(gridmixDir,"input"));
- verifyNumOfFilesGeneratedInEachNode(new Path(gridmixDir,"input"),
- perNodeSize, bytesPerFile);
- checkGeneratedDataAndJobStatus(inputSize);
+ verifyEachNodeSize(new Path(gridmixDir, "input"), perNodeSizeInMB);
+ verifyNumOfFilesGeneratedInEachNode(new Path(gridmixDir, "input"),
+ perNodeSizeInMB, bytesPerFile);
+ checkGeneratedDataAndJobStatus(inputSizeInMB);
}
private void checkGeneratedDataAndJobStatus(long inputSize)
- throws IOException {
+ throws IOException {
LOG.info("Verify the generated data size.");
- long dataSize = getDataSize(new Path(gridmixDir,"input"));
+ long dataSizeInMB = getDataSizeInMB(new Path(gridmixDir,"input"));
Assert.assertTrue("Generate data has not matched with given size",
- dataSize + 0.1 > inputSize || dataSize - 0.1 < inputSize);
+ dataSizeInMB + 0.1 > inputSize || dataSizeInMB - 0.1 < inputSize);
JobClient jobClient = jtClient.getClient();
+ int len = jobClient.getAllJobs().length;
LOG.info("Verify the job status after completion of job.");
Assert.assertEquals("Job has not succeeded.", JobStatus.SUCCEEDED,
- jobClient.getAllJobs()[0].getRunState());
+ jobClient.getAllJobs()[len-1].getRunState());
}
- private void verifyEachNodeSize(Path inputDir) throws IOException {
+ private void verifyEachNodeSize(Path inputDir, long dataSizePerNode)
+ throws IOException {
FileSystem fs = inputDir.getFileSystem(conf);
FileStatus [] fstatus = fs.listStatus(inputDir);
for (FileStatus fstat : fstatus) {
if ( fstat.isDir()) {
- long fileSize = getDataSize(fstat.getPath());
- Assert.assertTrue("The Size has not " +
- " matched with given per node file size(500mb)",
- fileSize + 0.1 > 500 || fileSize - 0.1 < 500);
+ long fileSize = getDataSizeInMB(fstat.getPath());
+ Assert.assertTrue("The Size has not matched with given "
+ + "per node file size(" + dataSizePerNode +"MB)",
+ fileSize + 0.1 > dataSizePerNode
+ || fileSize - 0.1 < dataSizePerNode);
}
}
}
private void verifyNumOfFilesGeneratedInEachNode(Path inputDir,
- int nodeSize, int fileSize) throws IOException {
- int expFileCount = Math.round(nodeSize/fileSize) +
- ((nodeSize%fileSize != 0)? 1:0);
+ long nodeSize, long fileSize) throws IOException {
+ long fileCount = (nodeSize * 1024 * 1024)/fileSize;
+ long expFileCount = Math.round(fileCount);
+ expFileCount = expFileCount + ((nodeSize%fileSize != 0)? 1:0);
FileSystem fs = inputDir.getFileSystem(conf);
FileStatus [] fstatus = fs.listStatus(inputDir);
for (FileStatus fstat : fstatus) {
- if ( fstat.isDir()) {
+ if (fstat.isDir()) {
FileSystem nodeFs = fstat.getPath().getFileSystem(conf);
- long actFileCount = nodeFs.getContentSummary(fstat.getPath())
- .getFileCount();
- Assert.assertEquals("File count has not matched.",
- expFileCount, actFileCount);
+ LOG.info("getPath():" + fstat.getPath().toString());
+ long actFileCount = nodeFs.getContentSummary(
+ fstat.getPath()).getFileCount();
+ Assert.assertEquals("File count has not matched.", expFileCount,
+ actFileCount);
}
}
}
- private static long getDataSize(Path inputDir) throws IOException {
+ private static long getDataSizeInMB(Path inputDir) throws IOException {
FileSystem fs = inputDir.getFileSystem(conf);
ContentSummary csmry = fs.getContentSummary(inputDir);
long dataSize = csmry.getLength();
Modified: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java?rev=1222572&r1=1222571&r2=1222572&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java (original)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java Fri Dec 23 05:38:53 2011
@@ -23,6 +23,9 @@ import org.apache.hadoop.mapreduce.test.
import org.apache.hadoop.mapreduce.test.system.JTClient;
import org.apache.hadoop.mapreduce.test.system.JTProtocol;
import org.apache.hadoop.mapred.gridmix.FilePool;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.junit.Assert;
@@ -33,8 +36,8 @@ import java.io.IOException;
import java.util.ArrayList;
public class TestGridMixFilePool {
- private static final Log LOG = LogFactory
- .getLog(TestGridMixFilePool.class);
+ private static final Log LOG =
+ LogFactory.getLog(TestGridMixFilePool.class);
private static Configuration conf = new Configuration();
private static MRCluster cluster;
private static JTProtocol remoteClient;
@@ -45,15 +48,14 @@ public class TestGridMixFilePool {
@BeforeClass
public static void before() throws Exception {
String [] excludeExpList = {"java.net.ConnectException",
- "java.io.IOException"};
+ "java.io.IOException"};
cluster = MRCluster.createCluster(conf);
cluster.setExcludeExpList(excludeExpList);
cluster.setUp();
jtClient = cluster.getJTClient();
remoteClient = jtClient.getProxy();
clusterSize = cluster.getTTClients().size();
- gridmixDir = new Path("hdfs:///user/" + UtilsForGridmix.getUserName()
- + "/herriot-gridmix");
+ gridmixDir = new Path("herriot-gridmix");
UtilsForGridmix.createDirs(gridmixDir, remoteClient.getDaemonConf());
}
@@ -66,50 +68,61 @@ public class TestGridMixFilePool {
@Test
public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
conf = remoteClient.getDaemonConf();
- final long inputSize = clusterSize * 200;
+ final long inputSizeInMB = clusterSize * 200;
int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
long targetSize = Long.MAX_VALUE;
- final int expFileCount = 13;
+ final int expFileCount = clusterSize + 4;
String [] runtimeValues ={"LOADJOB",
- SubmitterUserResolver.class.getName(),
- "STRESS",
- inputSize+"m",
- "file:///dev/null"};
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file:///dev/null"};
- int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
- conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+ };
+
+ // Generate the input data by using gridmix framework.
+ int exitCode =
+ UtilsForGridmix.runGridmixJob(gridmixDir, conf,
+ GridMixRunMode.DATA_GENERATION.getValue(),
+ runtimeValues, otherArgs);
Assert.assertEquals("Data generation has failed.", 0 , exitCode);
- // create files for given sizes.
- createFiles(new Path(gridmixDir,"input"),fileSizesInMB);
+ // Create the files without using gridmix input generation with
+ // above mentioned sizes in a array.
+ createFiles(new Path(gridmixDir, "input"), fileSizesInMB);
conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
- FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input"));
+ FilePool fpool = new FilePool(conf, new Path(gridmixDir, "input"));
fpool.refresh();
- verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool);
+ verifyFilesSizeAndCountForSpecifiedPool(expFileCount, targetSize, fpool);
}
private void createFiles(Path inputDir, int [] fileSizes)
- throws Exception {
+ throws Exception {
for (int size : fileSizes) {
UtilsForGridmix.createFile(size, inputDir, conf);
}
}
private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount,
- long minFileSize, FilePool pool) throws IOException {
+ long minFileSize, FilePool pool) throws IOException {
final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
- long actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024);
- long expFilesSize = 3100 ;
- Assert.assertEquals("Files Size has not matched for specified pool.",
- expFilesSize, actFilesSize);
- int actFileCount = files.size();
- Assert.assertEquals("File count has not matched.",
- expFileCount, actFileCount);
+ long filesSizeInBytes = pool.getInputFiles(minFileSize, files);
+ long actFilesSizeInMB = filesSizeInBytes / (1024 * 1024);
+ long expFilesSizeInMB = (clusterSize * 200) + 1300;
+ Assert.assertEquals("Files Size has not matched for specified pool.",
+ expFilesSizeInMB, actFilesSizeInMB);
+ int actFileCount = files.size();
+ Assert.assertEquals("File count has not matched.", expFileCount,
+ actFileCount);
int count = 0;
for (FileStatus fstat : files) {
String fp = fstat.getPath().toString();
count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
}
Assert.assertEquals("Total folders are not matched with cluster size",
- clusterSize, count);
+ clusterSize, count);
}
}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressedInputGeneration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressedInputGeneration.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressedInputGeneration.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressedInputGeneration.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.gridmix.Gridmix;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the Gridmix generated input if compression emulation turn on.
+ */
+public class TestGridmixCompressedInputGeneration
+ extends GridmixSystemTestCase {
+
+ private static final Log LOG =
+ LogFactory.getLog("TestGridmixCompressedInputGeneration.class");
+
+ /**
+ * Generate input data and verify whether input files are compressed
+ * or not.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixCompressionInputGeneration() throws Exception {
+ final long inputSizeInMB = 1024 * 7;
+ final String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file:///dev/null"};
+ final String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true"
+ };
+ LOG.info("Verify the generated compressed input data.");
+ runAndVerify(true, inputSizeInMB, runtimeValues, otherArgs);
+ }
+
+ /**
+ * Disable compression emulation and verify whether input files are
+ * compressed or not.
+ * @throws Exception
+ */
+ @Test
+ public void testGridmixInputGenerationWithoutCompressionEnable()
+ throws Exception {
+ UtilsForGridmix.cleanup(gridmixDir, rtClient.getDaemonConf());
+ final long inputSizeInMB = 1024 * 6;
+ final String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file:///dev/null"};
+ final String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+ };
+
+ LOG.info("Verify the generated uncompressed input data.");
+ runAndVerify(false, inputSizeInMB, runtimeValues, otherArgs);
+ }
+
+ private void runAndVerify(boolean isCompressed, long INPUT_SIZE,
+ String [] runtimeValues, String [] otherArgs) throws Exception {
+ int exitCode =
+ UtilsForGridmix.runGridmixJob(gridmixDir, conf,
+ GridMixRunMode.DATA_GENERATION.getValue(),
+ runtimeValues,otherArgs);
+ Assert.assertEquals("Data generation has failed.", 0, exitCode);
+ verifyJobStatus();
+ verifyInputDataSize(INPUT_SIZE);
+ verifyInputFiles(isCompressed);
+ }
+
+ private void verifyInputFiles(boolean isCompressed) throws IOException {
+ List<String> inputFiles =
+ getInputFiles(conf, Gridmix.getGridmixInputDataPath(gridmixDir));
+ for (String inputFile: inputFiles) {
+ boolean fileStatus = (inputFile.contains(".gz")
+ || inputFile.contains(".tgz")
+ || inputFile.contains(".deflate"))? true : false;
+ if (isCompressed) {
+ Assert.assertTrue("Compressed input split file was not found.",
+ fileStatus);
+ } else {
+ Assert.assertFalse("Uncompressed input split file was not found.",
+ fileStatus);
+ }
+ }
+ }
+
+ private void verifyInputDataSize(long INPUT_SIZE) throws IOException {
+ long actDataSize =
+ getInputDataSizeInMB(conf, Gridmix.getGridmixInputDataPath(gridmixDir));
+ double ratio = ((double)actDataSize)/INPUT_SIZE;
+ long expDataSize = (long)(INPUT_SIZE * ratio);
+ Assert.assertEquals("Generated data has not matched with given size.",
+ expDataSize, actDataSize);
+ }
+
+ private void verifyJobStatus() throws IOException {
+ JobClient jobClient = jtClient.getClient();
+ int len = jobClient.getAllJobs().length;
+ LOG.info("Verify the job status after completion of job...");
+ Assert.assertEquals("Job has not succeeded.", JobStatus.SUCCEEDED,
+ jobClient.getAllJobs()[len -1].getRunState());
+ }
+
+ private long getInputDataSizeInMB(Configuration conf, Path inputDir)
+ throws IOException {
+ FileSystem fs = inputDir.getFileSystem(conf);
+ ContentSummary csmry = fs.getContentSummary(inputDir);
+ long dataSize = csmry.getLength();
+ dataSize = dataSize/(1024 * 1024);
+ return dataSize;
+ }
+
+ private List<String> getInputFiles(Configuration conf, Path inputDir)
+ throws IOException {
+ FileSystem fs = inputDir.getFileSystem(conf);
+ FileStatus [] listStatus = fs.listStatus(inputDir);
+ List<String> files = new ArrayList<String>();
+ for (FileStatus fileStat : listStatus) {
+ files.add(getInputFile(fileStat, conf));
+ }
+ return files;
+ }
+
+ private String getInputFile(FileStatus fstatus, Configuration conf)
+ throws IOException {
+ String fileName = null;
+ if (!fstatus.isDir()) {
+ fileName = fstatus.getPath().getName();
+ LOG.info("fileName++++:" + fileName);
+ } else {
+ FileSystem fs = fstatus.getPath().getFileSystem(conf);
+ FileStatus [] listStatus = fs.listStatus(fstatus.getPath());
+ for (FileStatus fileStat : listStatus) {
+ return getInputFile(fileStat, conf);
+ }
+ }
+ return fileName;
+ }
+}
+
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixCompressionEmulationWithCompressInput.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the gridmix jobs compression ratios of map input,
+ * map output and reduce output with default and user specified
+ * compression ratios.
+ *
+ */
+public class TestGridmixCompressionEmulationWithCompressInput
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog(
+ "TestGridmixCompressionEmulationWithCompressInput.class");
+ final long inputSizeInMB = 1024 * 6;
+
+ /**
+ * Generate compressed input data and verify the map input,
+ * map output and reduce output compression ratios of gridmix jobs
+ * against the default compression ratios.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixCompressionRatiosAgainstDefaultCompressionRatio()
+ throws Exception {
+ final String tracePath = getTraceFile("compression_case1_trace");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+
+ final String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ final String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+
+ /**
+ * Verify map input, map output and reduce output compression ratios of
+ * gridmix jobs against user specified compression ratios.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixOuputCompressionRatiosAgainstCustomRatios()
+ throws Exception {
+ final String tracePath = getTraceFile("compression_case1_trace");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ UtilsForGridmix.cleanup(gridmixDir, rtClient.getDaemonConf());
+
+ final String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ final String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=true",
+ "-D", GridMixConfig.GRIDMIX_INPUT_DECOMPRESS_ENABLE + "=true",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_INPUT_COMPRESS_RATIO + "=0.68",
+ "-D", GridMixConfig.GRIDMIX_INTERMEDIATE_COMPRESSION_RATIO + "=0.35",
+ "-D", GridMixConfig.GRIDMIX_OUTPUT_COMPRESSION_RATIO + "=0.40"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+}
+
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the Gridmix emulation of HDFS private distributed cache file.
+ */
+public class TestGridmixEmulationOfHDFSPrivateDCFile
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestGridmixEmulationOfHDFSPrivateDCFile.class");
+ /**
+ * Generate input data and single HDFS private distributed cache
+ * file based on given input trace.Verify the Gridmix emulation of
+ * single private HDFS distributed cache file in RoundRobinUserResolver
+ * mode with STRESS submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateAndEmulateOfHDFSPrivateDCFile()
+ throws Exception {
+ final long inputSizeInMB = 8192;
+ final String tracePath = getTraceFile("distcache_case3_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ final String [] otherArgs = {
+ "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+ /**
+ * Verify the Gridmix emulation of single HDFS private distributed
+ * cache file in SubmitterUserResolver mode with REPLAY submission
+ * policy by using the existing input data and HDFS private
+ * distributed cache file.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixEmulationOfHDFSPrivateDCFile()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case3_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "REPLAY",
+ tracePath};
+ final String [] otherArgs = {
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX.getValue());
+ }
+}
+
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+//import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the Gridmix emulation of HDFS public distributed cache file.
+ */
+public class TestGridmixEmulationOfHDFSPublicDCFile
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestGridmixEmulationOfHDFSPublicDCFile.class");
+
+ /**
+ * Generate the input data and HDFS distributed cache file based
+ * on given input trace. Verify the Gridmix emulation of single HDFS
+ * public distributed cache file in SubmitterUserResolver mode with
+ * STRESS submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateAndEmulationOfSingleHDFSDCFile()
+ throws Exception {
+ final long inputSizeInMB = 7168;
+ final String tracePath = getTraceFile("distcache_case1_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ tracePath};
+
+ final String [] otherArgs = {
+ "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+
+ /**
+ * Verify the Gridmix emulation of Single HDFS public distributed cache
+ * file in RoundRobinUserResolver mode with REPLAY submission policy
+ * by using the existing input data and HDFS public distributed cache file.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixEmulationOfSingleHDFSPublicDCFile()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case1_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues =
+ { "LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ final String [] otherArgs = {
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX.getValue());
+ }
+}
+
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase1.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase1.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase1.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase1.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.GridmixJob;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Run the {@link Gridmix} with a high ram jobs trace and
+ * verify each {@link Gridmix} job whether it honors the high ram or not.
+ * In the trace the jobs should use the high ram for both maps and reduces.
+ */
+public class TestGridmixEmulationOfHighRamJobsCase1
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestGridmixEmulationOfHighRamJobsCase1.class");
+
+ /**
+ * Generate input data and run {@link Gridmix} with a high ram jobs trace
+ * as a load job and STRESS submission policy in a SubmitterUserResolver
+ * mode. Verify each {@link Gridmix} job whether it honors the high ram or not
+ * after completion of execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testEmulationOfHighRamForMapsAndReducesOfMRJobs()
+ throws Exception {
+ final long inputSizeInMB = cSize * 400;
+ String tracePath = getTraceFile("highram_mr_jobs_case1");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=true"};
+
+ validateTaskMemoryParamters(tracePath, true);
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase2.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase2.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase2.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase2.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.GridmixJob;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Run the {@link Gridmix} with a high ram jobs trace and
+ * verify each {@link Gridmix} job whether it honors the high ram or not.
+ * In the trace the jobs should use the high ram only for maps.
+ */
+public class TestGridmixEmulationOfHighRamJobsCase2
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestGridmixEmulationOfHighRamJobsCase2.class");
+
+ /**
+ * Generate input data and run {@link Gridmix} with a high ram jobs trace
+ * as a load job and REPALY submission policy in a RoundRobinUserResolver
+ * mode. Verify each {@link Gridmix} job whether it honors the high ram or not
+ * after completion of execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testEmulationOfHighRamForMapsOfMRJobs()
+ throws Exception {
+ final long inputSizeInMB = cSize * 300;
+ String tracePath = getTraceFile("highram_mr_jobs_case2");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=true"};
+
+ validateTaskMemoryParamters(tracePath, true);
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase3.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase3.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase3.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHighRamJobsCase3.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.GridmixJob;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Run the {@link Gridmix} with a high ram jobs trace and
+ * verify each {@link Gridmix} job whether it honors the high ram or not.
+ * In the trace the jobs should use the high ram only for reducers.
+ */
+public class TestGridmixEmulationOfHighRamJobsCase3
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog(TestGridmixEmulationOfHighRamJobsCase3.class);
+
+ /**
+ * Generate input data and run {@link Gridmix} with a high ram jobs trace
+ * as a load job and SERIAL submission policy in a SubmitterUserResolver
+ * mode. Verify each {@link Gridmix} job whether it honors the
+ * high ram or not after completion of execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testEmulationOfHighRamForReducersOfMRJobs()
+ throws Exception {
+ final long inputSizeInMB = cSize * 250;
+ String tracePath = getTraceFile("highram_mr_jobs_case3");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "SERIAL",
+ inputSizeInMB + "m",
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=true"};
+
+ validateTaskMemoryParamters(tracePath, true);
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+//import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the Gridmix emulation of Multiple HDFS private distributed
+ * cache files.
+ */
+public class TestGridmixEmulationOfMultipleHDFSPrivateDCFiles
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog(
+ "TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.class");
+
+ /**
+ * Generate input data and multiple HDFS private distributed cache
+ * files based on given input trace.Verify the Gridmix emulation of
+ * multiple private HDFS distributed cache files in RoundRobinUserResolver
+ * mode with SERIAL submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateAndEmulationOfMultipleHDFSPrivateDCFiles()
+ throws Exception {
+ final long inputSize = 6144;
+ final String tracePath = getTraceFile("distcache_case4_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "SERIAL",
+ inputSize+"m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+ final String [] otherArgs = {
+ "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+
+ /**
+ * Verify the Gridmix emulation of multiple HDFS private distributed
+ * cache files in SubmitterUserResolver mode with STRESS submission
+ * policy by using the existing input data and HDFS private
+ * distributed cache files.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixEmulationOfMultipleHDFSPrivateDCFiles()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case4_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ tracePath};
+ final String [] otherArgs = {
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX.getValue());
+ }
+}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+//import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.junit.Assert;
+import org.junit.Test;
+import java.io.IOException;
+
+/**
+ * Verify the Gridmix emulation of Multiple HDFS public distributed
+ * cache files.
+ */
+public class TestGridmixEmulationOfMultipleHDFSPublicDCFiles
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog(
+ "TestGridmixEmulationOfMultipleHDFSPublicDCFiles.class");
+
+ /**
+ * Generate the compressed input data and dist cache files based
+ * on input trace. Verify the Gridmix emulation of
+ * multiple HDFS public distributed cache file.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateAndEmulationOfMultipleHDFSDCFiles()
+ throws Exception {
+ final long inputSizeInMB = 7168;
+ final String tracePath = getTraceFile("distcache_case2_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ final String [] otherArgs = {
+ "-D", JobContext.JOB_CANCEL_DELEGATION_TOKEN + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX.getValue());
+ }
+
+ /**
+ * Verify the Gridmix emulation of Single HDFS public distributed cache file
+ * by using an existing input compressed data and HDFS dist cache file.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixEmulationOfMulitpleHDFSPublicDCFile()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case2_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "SERIAL",
+ tracePath};
+
+ final String [] otherArgs = {
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX.getValue());
+ }
+}
+
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith10minTrace.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.junit.Test;
+
+/**
+ * Run the Gridmix with 10 minutes MR jobs trace and
+ * verify each job history against the corresponding job story
+ * in a given trace file.
+ */
+public class TestGridmixWith10minTrace extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog(TestGridmixWith10minTrace.class);
+
+ /**
+ * Generate data and run gridmix by sleep jobs with STRESS submission
+ * policy in a RoundRobinUserResolver mode against 10 minutes trace file.
+ * Verify each Gridmix job history with a corresponding job story
+ * in a trace file after completion of all the jobs execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixWith10minTrace() throws Exception {
+ final long inputSizeInMB = cSize * 250;
+ final long minFileSize = 200 * 1024 * 1024;
+ String [] runtimeValues =
+ {"SLEEPJOB",
+ RoundRobinUserResolver.class.getName(),
+ "SERIAL",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ map.get("10m")};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize,
+ "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=false",
+ "-D", GridMixConfig.GRIDMIX_SLEEPJOB_MAPTASK_ONLY + "=true",
+ "-D", GridMixConfig.GRIDMIX_SLEEP_MAP_MAX_TIME + "=10"
+ };
+ String tracePath = map.get("10m");
+ runGridmixAndVerify(runtimeValues, otherArgs,tracePath);
+ }
+}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith12minTrace.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.junit.Test;
+
+/**
+ * Run the Gridmix with 12 minutes MR job traces and
+ * verify each job history against the corresponding job story
+ * in a given trace file.
+ */
+public class TestGridmixWith12minTrace extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog(TestGridmixWith12minTrace.class);
+
+ /**
+ * Generate data and run gridmix sleep jobs with REPLAY submission
+ * policy in a SubmitterUserResolver mode against 12 minutes trace file.
+ * Verify each Gridmix job history with a corresponding job story
+ * in a trace file after completion of all the jobs execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixWith12minTrace() throws Exception {
+ final long inputSizeInMB = cSize * 150;
+ String [] runtimeValues = {"SLEEPJOB",
+ SubmitterUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB + "m",
+ map.get("12m")};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_SLEEP_MAP_MAX_TIME + "=10",
+ "-D", GridMixConfig.GRIDMIX_SLEEP_REDUCE_MAX_TIME + "=5"
+ };
+
+ String tracePath = map.get("12m");
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith1minTrace.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.junit.Test;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+
+/**
+ * Run the Gridmix with 1 minute MR jobs trace and
+ * verify each job history against the corresponding job story
+ * in a given trace file.
+ */
+public class TestGridmixWith1minTrace extends GridmixSystemTestCase{
+ private static final Log LOG =
+ LogFactory.getLog(TestGridmixWith1minTrace.class);
+
+ /**
+ * Generate data and run gridmix by load job with STRESS submission policy
+ * in a SubmitterUserResolver mode against 1 minute trace file.
+ * Verify each Gridmix job history with a corresponding job story in the
+ * trace after completion of all the jobs execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixWith1minTrace() throws Exception {
+ final long inputSizeInMB = cSize * 400;
+ String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ map.get("1m")};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+ };
+
+ String tracePath = map.get("1m");
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Run the Gridmix with 2 minutes job trace which has been generated with
+ * streaming jobs histories and verify each job history against
+ * the corresponding job story in a given trace file.
+ */
+public class TestGridmixWith2minStreamingJobTrace
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestGridmixWith2minStreamingJobTrace.class");
+
+ /**
+ * Generate input data and run Gridmix by load job with STRESS submission
+ * policy in a SubmitterUserResolver mode against 2 minutes job
+ * trace file of streaming jobs. Verify each Gridmix job history with
+ * a corresponding job story in a trace file after completion of all
+ * the jobs execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixWith2minStreamJobTrace() throws Exception {
+ final long inputSizeInMB = cSize * 250;
+ final long minFileSize = 150 * 1024 * 1024;
+ String tracePath = getTraceFile("2m_stream");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSizeInMB + "m",
+ tracePath};
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=true",
+ "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Run the Gridmix with 3 minutes job trace which has been generated with
+ * streaming jobs histories and verify each job history against
+ * corresponding job story in a given trace file.
+ */
+public class TestGridmixWith3minStreamingJobTrace
+ extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog("TestGridmixWith3minStreamingJobTrace.class");
+
+ /**
+ * Generate input data and run gridmix by load job with REPLAY submission
+ * policy in a RoundRobinUserResolver mode against 3 minutes job trace file
+ * of streaming job. Verify each gridmix job history with a corresponding
+ * job story in a trace file after completion of all the jobs execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixWith3minStreamJobTrace() throws Exception {
+ final long inputSizeInMB = cSize * 200;
+ final long bytesPerFile = 150 * 1024 * 1024;
+ String tracePath = getTraceFile("3m_stream");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=true",
+ "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+ };
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}
Added: hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java?rev=1222572&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java (added)
+++ hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minTrace.java Fri Dec 23 05:38:53 2011
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.junit.Test;
+
+/**
+ * Run the Gridmix with 3 minutes MR jobs trace and
+ * verify each job history against the corresponding job story
+ * in a given trace file.
+ */
+public class TestGridmixWith3minTrace extends GridmixSystemTestCase {
+ private static final Log LOG =
+ LogFactory.getLog(TestGridmixWith3minTrace.class);
+
+ /**
+ * Generate data and run gridmix by load job with REPLAY submission
+ * policy in a RoundRobinUserResolver mode by using 3 minutes trace file.
+ * Verify each Gridmix job history with a corresponding job story in
+ * a trace after completion of all the jobs execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixWith3minTrace() throws Exception {
+ final long inputSizeInMB = cSize * 200;
+ String [] runtimeValues =
+ {"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ inputSizeInMB + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ map.get("3m")};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
+ "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
+ };
+
+ String tracePath = map.get("3m");
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}