You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:43:05 UTC
svn commit: r1077676 -
/hadoop/common/branches/branch-0.20-security-patches/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
Author: omalley
Date: Fri Mar 4 04:43:05 2011
New Revision: 1077676
URL: http://svn.apache.org/viewvc?rev=1077676&view=rev
Log:
commit 09c2d459762c4c793b4887c48555373e7de4fbb5
Author: Vinay Kumar Thota <vi...@yahoo-inc.com>
Date: Tue Sep 14 04:50:31 2010 +0000
3957917 from
Added:
hadoop/common/branches/branch-0.20-security-patches/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
Added: hadoop/common/branches/branch-0.20-security-patches/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java?rev=1077676&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java Fri Mar 4 04:43:05 2011
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.test.system.MRCluster;
+import org.apache.hadoop.mapreduce.test.system.JTClient;
+import org.apache.hadoop.mapreduce.test.system.JTProtocol;
+import org.apache.hadoop.mapred.gridmix.FilePool;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import org.junit.Test;
+import java.io.IOException;
+import java.util.ArrayList;
+
+public class TestGridMixFilePool {
+ private static final Log LOG = LogFactory
+ .getLog(TestGridMixFilePool.class);
+ private static Configuration conf = new Configuration();
+ private static MRCluster cluster;
+ private static JTProtocol remoteClient;
+ private static JTClient jtClient;
+ private static Path gridmixDir;
+ private static int clusterSize;
+
+ @BeforeClass
+ public static void before() throws Exception {
+ String [] excludeExpList = {"java.net.ConnectException",
+ "java.io.IOException"};
+ cluster = MRCluster.createCluster(conf);
+ cluster.setExcludeExpList(excludeExpList);
+ cluster.setUp();
+ jtClient = cluster.getJTClient();
+ remoteClient = jtClient.getProxy();
+ clusterSize = cluster.getTTClients().size();
+ gridmixDir = new Path("hdfs:///user/" + UtilsForGridmix.getUserName()
+ + "/herriot-gridmix");
+ UtilsForGridmix.createDirs(gridmixDir, remoteClient.getDaemonConf());
+ }
+
+ @AfterClass
+ public static void after() throws Exception {
+ UtilsForGridmix.cleanup(gridmixDir, conf);
+ cluster.tearDown();
+ }
+
+ @Test
+ public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
+ conf = remoteClient.getDaemonConf();
+ final long inputSize = clusterSize * 200;
+ int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
+ long targetSize = Long.MAX_VALUE;
+ final int expFileCount = 13;
+ String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSize+"m",
+ "file:///dev/null"};
+
+ int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
+ conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
+ Assert.assertEquals("Data generation has failed.", 0 , exitCode);
+ // create files for given sizes.
+ createFiles(new Path(gridmixDir,"input"),fileSizesInMB);
+ conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
+ FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input"));
+ fpool.refresh();
+ verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool);
+ }
+
+ private void createFiles(Path inputDir, int [] fileSizes)
+ throws Exception {
+ for (int size : fileSizes) {
+ UtilsForGridmix.createFile(size, inputDir, conf);
+ }
+ }
+
+ private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount,
+ long minFileSize, FilePool pool) throws IOException {
+ final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
+ long actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024);
+ long expFilesSize = 3100 ;
+ Assert.assertEquals("Files Size has not matched for specified pool.",
+ expFilesSize, actFilesSize);
+ int actFileCount = files.size();
+ Assert.assertEquals("File count has not matched.",
+ expFileCount, actFileCount);
+ int count = 0;
+ for (FileStatus fstat : files) {
+ String fp = fstat.getPath().toString();
+ count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
+ }
+ Assert.assertEquals("Total folders are not matched with cluster size",
+ clusterSize, count);
+ }
+}