You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2014/02/20 04:21:50 UTC
git commit: BIGTOP-1208. Add DFSIO test into longevity suite
Repository: bigtop
Updated Branches:
refs/heads/master 158e74608 -> 54d45491f
BIGTOP-1208. Add DFSIO test into longevity suite
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/54d45491
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/54d45491
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/54d45491
Branch: refs/heads/master
Commit: 54d45491fb56ae29df0fee8507cb96582727f70c
Parents: 158e746
Author: Dasha Boudnik <da...@wandisco.com>
Authored: Wed Feb 19 19:21:12 2014 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Wed Feb 19 19:21:12 2014 -0800
----------------------------------------------------------------------
.../bigtop/itest/iolongevity/TestDFSIO.groovy | 97 ++++++++++++++++++++
bigtop-tests/test-execution/README | 4 +
2 files changed, 101 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/54d45491/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy b/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy
new file mode 100644
index 0000000..e32d56e
--- /dev/null
+++ b/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.iolongevity
+
+import org.apache.bigtop.itest.JarContent
+import org.apache.bigtop.itest.shell.Shell
+import static org.apache.bigtop.itest.LogErrorsUtils.logError
+import static org.junit.Assert.assertNotNull
+import static org.junit.Assert.assertTrue
+import org.junit.BeforeClass
+import org.junit.Before
+import org.junit.Test
+
+public class TestDFSIO {
+ static Shell sh = new Shell("/bin/bash -s");
+
+ private static final String WRITE_CMD = "-write";
+ private static final String APPEND_CMD = "-append";
+ private static final String READ_CMD = "-read";
+
+ private static final String BENCHMARKS_DIR = "/benchmarks/TestDFSIO";
+ private static final String HADOOP_MAPRED_HOME = System.getenv('HADOOP_MAPRED_HOME');
+
+ private final String numFiles = System.getProperty("numFiles", "10");
+ private final String sizeFile = System.getProperty("sizeFile", "10MB");
+ private final int numOfIterations = Integer.getInteger("numOfIterations", 1);
+ private final int TIMEOUT = 5000;
+
+ private String DFSIO_TEMPLATE;
+ private String result = "TestDFSIO_Results.log";
+
+ final String hadoopExamplesJar =
+ JarContent.getJarName(HADOOP_MAPRED_HOME, 'hadoop.*jobclient.*tests.*.jar');
+
+ final String hadoopTestJar = HADOOP_MAPRED_HOME + '/' + hadoopExamplesJar;
+
+ @BeforeClass
+ static void setUp() throws IOException {
+ assertNotNull("HADOOP_MAPRED_HOME has to be set to run this test", HADOOP_MAPRED_HOME)
+ }
+
+ @Before
+ public void cleanup() {
+ sh.exec("hadoop fs -rm -r " + BENCHMARKS_DIR);
+ Thread.sleep(TIMEOUT);
+ sh.exec("hadoop fs -mkdir -p " + BENCHMARKS_DIR);
+ assertTrue("Failed to create " + BENCHMARKS_DIR, sh.getRet() == 0);
+ }
+
+ @Test
+ public void testDFSIO() {
+ DFSIO_TEMPLATE = "hadoop jar " + hadoopTestJar + " TestDFSIO %s ";
+ final String DFSIO_ARGS_TEMPLATE = "-nrFiles %s -fileSize %s -resFile %s";
+
+ final String writeSuccess = BENCHMARKS_DIR + "/io_write/_SUCCESS";
+ final String appendSuccess = BENCHMARKS_DIR + "/io_append/_SUCCESS";
+ final String readSuccess = BENCHMARKS_DIR + "/io_read/_SUCCESS";
+ String argStr = String.format(DFSIO_ARGS_TEMPLATE, numFiles, sizeFile, result);
+
+ for (int counter = 0; counter < numOfIterations; counter++) {
+ executeCmd(String.format(DFSIO_TEMPLATE + argStr, WRITE_CMD), writeSuccess);
+ executeCmd(String.format(DFSIO_TEMPLATE + argStr, APPEND_CMD), appendSuccess);
+ executeCmd(String.format(DFSIO_TEMPLATE + argStr, READ_CMD), readSuccess);
+ argStr = String.format(DFSIO_ARGS_TEMPLATE, numFiles, sizeFile, result + "." + counter);
+ }
+ }
+
+ private void executeCmd(String cmd, String expectedFile){
+ sh.exec(cmd);
+ logError(sh);
+ assertTrue("Command " + cmd + " is unsuccessful", sh.getRet() == 0);
+ sh.exec("hadoop fs -ls " + expectedFile);
+ boolean success = false;
+ sh.getOut().each { str ->
+ if (str.contains(expectedFile)) {
+ success = true;
+ }
+ }
+ assertTrue("File " + expectedFile + " was not found", success);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/bigtop/blob/54d45491/bigtop-tests/test-execution/README
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-execution/README b/bigtop-tests/test-execution/README
index c3e5bba..136cbbf 100644
--- a/bigtop-tests/test-execution/README
+++ b/bigtop-tests/test-execution/README
@@ -16,3 +16,7 @@
Repository contains the collection of starter test suites for different kinds
of Hadoop stack validations using iTest framework
+To run the DFSIO longevity test, you can parametrize the number of files (-nrFiles),
+size of files (-fileSize), and number of cycles using system properties
+numFiles, sizeFile, and numOfIterations, respectively.
+User must make sure that the user home directory exists prior to running the test.
\ No newline at end of file