You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2013/02/10 07:04:01 UTC
[24/50] [abbrv] git commit: BIGTOP-793 add crunch integration tests
BIGTOP-793 add crunch integration tests
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4a307414
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4a307414
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4a307414
Branch: refs/heads/RCs
Commit: 4a3074149297ecb188d55bf05d8b4a751a7b92d1
Parents: 57fbb28
Author: Johnny Zhang <xi...@cloudera.com>
Authored: Thu Nov 29 19:53:37 2012 -0500
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Tue Dec 4 13:59:46 2012 -0800
----------------------------------------------------------------------
bigtop-tests/test-artifacts/crunch/pom.xml | 32 +
.../itest/crunchsmoke/TestCrunchSmoke.groovy | 92 +
.../resources/crunch-examples/access_log/000000 |13501 +++++++++++++++
.../main/resources/crunch-examples/text/pg11.txt | 3735 ++++
.../crunch-examples/text/secondary_sort_input.txt | 8 +
bigtop-tests/test-execution/smokes/crunch/pom.xml | 98 +
bigtop-tests/test-execution/smokes/pom.xml | 2 +
7 files changed, 17468 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a307414/bigtop-tests/test-artifacts/crunch/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/crunch/pom.xml b/bigtop-tests/test-artifacts/crunch/pom.xml
new file mode 100644
index 0000000..80b069e
--- /dev/null
+++ b/bigtop-tests/test-artifacts/crunch/pom.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>bigtop-smokes</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>crunch-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <name>crunchsmoke</name>
+</project>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a307414/bigtop-tests/test-artifacts/crunch/src/main/groovy/org/apache/bigtop/itest/crunchsmoke/TestCrunchSmoke.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/crunch/src/main/groovy/org/apache/bigtop/itest/crunchsmoke/TestCrunchSmoke.groovy b/bigtop-tests/test-artifacts/crunch/src/main/groovy/org/apache/bigtop/itest/crunchsmoke/TestCrunchSmoke.groovy
new file mode 100644
index 0000000..5a7a931
--- /dev/null
+++ b/bigtop-tests/test-artifacts/crunch/src/main/groovy/org/apache/bigtop/itest/crunchsmoke/TestCrunchSmoke.groovy
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.itest.crunch.smoke
+
+import org.junit.BeforeClass
+import org.junit.Test
+import static org.junit.Assert.assertEquals
+import static org.junit.Assert.assertTrue
+
+import org.apache.bigtop.itest.shell.Shell
+
+public class TestCrunchSmoke {
+ static String runnerScript = "hadoop jar"
+
+ static String crunchJar = "/usr/lib/crunch/crunch-examples-*job.jar";
+
+ static Shell sh = new Shell("/bin/bash -s");
+ private static final String EXAMPLES = "crunch-examples";
+ private static final String EXAMPLES_OUT = "crunch-examples-output";
+
+ @BeforeClass
+ static void setUp() {
+ sh.exec("hadoop fs -test -e $EXAMPLES");
+ if (sh.getRet() == 0) {
+ sh.exec("hadoop fs -rmr -skipTrash $EXAMPLES");
+ assertTrue("Deletion of previous $EXAMPLES from HDFS failed",
+ sh.getRet() == 0);
+ }
+ sh.exec("hadoop fs -test -e $EXAMPLES_OUT");
+ if (sh.getRet() == 0) {
+ sh.exec("hadoop fs -rmr -skipTrash $EXAMPLES_OUT");
+ assertTrue("Deletion of previous examples output from HDFS failed",
+ sh.getRet() == 0);
+ }
+
+ // copy test files to HDFS
+ sh.exec("hadoop fs -put $EXAMPLES $EXAMPLES",
+ "hadoop fs -mkdir $EXAMPLES_OUT");
+ assertTrue("Could not create output directory", sh.getRet() == 0);
+ }
+
+ static Map examples =
+ [
+ WordCount : "${EXAMPLES}/text/pg11.txt $EXAMPLES_OUT",
+ SecondarySortExample : "${EXAMPLES}/text/secondary_sort_input.txt ${EXAMPLES_OUT}",
+ AverageBytesByIP : "${EXAMPLES}/access_log/000000 ${EXAMPLES_OUT}",
+ TotalBytesByIP : "${EXAMPLES}/access_log/000000 ${EXAMPLES_OUT}"
+ ];
+
+ private void _runExampleJobs(String algorithm) {
+ sh.exec("${runnerScript} ${crunchJar}"
+ + " org.apache.crunch.examples.${algorithm}"
+ + " ${examples.get(algorithm)}"
+ );
+ assertEquals("running Crunch example failed", sh.getRet(), 0);
+ }
+
+ @Test(timeout=300000L)
+ public void testWordCount() {
+ _runExampleJobs("WordCount");
+ }
+
+ @Test(timeout=300000L)
+ public void testSecondarySort() {
+ _runExampleJobs("SecondarySortExample");
+ }
+
+ @Test(timeout=300000L)
+ public void testAverageBytesByIP() {
+ _runExampleJobs("AverageBytesByIP");
+ }
+
+ @Test(timeout=300000L)
+ public void testTotalBytesByIP() {
+ _runExampleJobs("TotalBytesByIP");
+ }
+}