You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2013/05/01 23:52:25 UTC
[4/6] git commit: BIGTOP-956. fix small issues with mahout,
hue and oozie tests
BIGTOP-956. fix small issues with mahout, hue and oozie tests
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a4eb310b
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a4eb310b
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a4eb310b
Branch: refs/heads/master
Commit: a4eb310b236314379aed4a82036d01fa8b51a992
Parents: 3414748
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Wed May 1 11:05:51 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Wed May 1 14:50:38 2013 -0700
----------------------------------------------------------------------
.../bigtop/itest/huesmoke/TestHueSmoke.groovy | 4 +-
.../itest/mahout/smoke/TestMahoutExamples.groovy | 69 ++++++++-------
.../bigtop/itest/ooziesmoke/TestOozieSmoke.groovy | 10 --
bigtop-tests/test-execution/smokes/hue/pom.xml | 34 +++++++-
4 files changed, 72 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy b/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy
index da12d50..50c0a46 100644
--- a/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy
+++ b/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.bigtop.itest.giraph.smoke
+package org.apache.bigtop.itest.hue.smoke
import static org.junit.Assert.assertEquals
import static org.junit.Assert.assertTrue
@@ -25,7 +25,7 @@ import org.apache.bigtop.itest.shell.Shell
import org.junit.Test
public class TestHueSmoke {
- final static String hueServer = "http://localhost:8888"
+ final static String hueServer = System.getProperty("org.apache.bigtop.itest.hue_url", "http://localhost:8888");
final static String loginURL = "${hueServer}/accounts/login/";
final static String checkURL = "${hueServer}/debug/check_config";
final static String creds = "username=admin&password=admin";
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy b/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy
index 586792c..8cc8efc 100644
--- a/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy
+++ b/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy
@@ -102,8 +102,12 @@ public class TestMahoutExamples {
}
}
- @Test(timeout=1200000L)
+ @Test(timeout=12000000L)
public void factorizeMovieLensRatings() {
+ // convert ratings
+ sh.exec("cat ${TEMP_DIR}/movielens/ml-1m/ratings.dat |sed -e s/::/,/g| cut -d, -f1,2,3 > ${TEMP_DIR}/movielens/ratings.csv");
+ assertEquals("Unexpected error from converting ratings", 0, sh.getRet());
+
// put ratings in hdfs
sh.exec("hadoop fs -mkdir ${WORK_DIR}/movielens",
"hadoop fs -put ${TEMP_DIR}/movielens/ratings.csv ${WORK_DIR}/movielens/ratings.csv");
@@ -218,40 +222,43 @@ public class TestMahoutExamples {
@Test(timeout=9000000L)
public void testBayesNewsgroupClassifier() {
- sh.exec("""mahout org.apache.mahout.classifier.bayes.PrepareTwentyNewsgroups \
- -p ${TEMP_DIR}/20news-bydate/20news-bydate-train \
- -o ${TEMP_DIR}/20news-bydate/bayes-train-input \
- -a org.apache.mahout.vectorizer.DefaultAnalyzer \
- -c UTF-8""");
+ // put bayes-train-input and bayes-test-input in hdfs
+ sh.exec("hadoop fs -mkdir ${WORK_DIR}/20news-vectors");
+ sh.exec("hadoop fs -put ${TEMP_DIR}/20news-all ${WORK_DIR}/20news-all");
+ assertEquals("Unable to put bayes-train-input in hdfs", 0, sh.getRet());
+
+ sh.exec("mahout seqdirectory -i ${WORK_DIR}/20news-all -o ${WORK_DIR}/20news-seq");
assertEquals("Unexpected error from running mahout", 0, sh.getRet());
- sh.exec("""mahout org.apache.mahout.classifier.bayes.PrepareTwentyNewsgroups \
- -p ${TEMP_DIR}/20news-bydate/20news-bydate-test \
- -o ${TEMP_DIR}/20news-bydate/bayes-test-input \
- -a org.apache.mahout.vectorizer.DefaultAnalyzer \
- -c UTF-8""");
+
+ sh.exec("mahout seq2sparse -i ${WORK_DIR}/20news-seq -o ${WORK_DIR}/20news-vectors -lnorm -nv -wt tfidf");
assertEquals("Unexpected error from running mahout", 0, sh.getRet());
- // put bayes-train-input and bayes-test-input in hdfs
- sh.exec("hadoop fs -mkdir ${WORK_DIR}/20news-bydate");
- sh.exec("hadoop fs -put ${TEMP_DIR}/20news-bydate/bayes-train-input ${WORK_DIR}/20news-bydate/bayes-train-input");
- assertEquals("Unable to put bayes-train-input in hdfs", 0, sh.getRet());
- sh.exec("hadoop fs -put ${TEMP_DIR}/20news-bydate/bayes-test-input ${WORK_DIR}/20news-bydate/bayes-test-input");
- assertEquals("Unable to put bayes-test-input in hdfs", 0, sh.getRet());
-
- sh.exec("""mahout trainclassifier \
--i ${WORK_DIR}/20news-bydate/bayes-train-input \
--o ${WORK_DIR}/20news-bydate/bayes-model \
--type bayes \
--ng 1 \
--source hdfs""");
+ sh.exec("""mahout split \
+-i ${WORK_DIR}/20news-vectors/tfidf-vectors \
+--trainingOutput ${WORK_DIR}/20news-train-vectors \
+--testOutput ${WORK_DIR}/20news-test-vectors \
+--randomSelectionPct 40 --overwrite --sequenceFiles -xm sequential""");
+ assertEquals("Unexpected error from running mahout", 0, sh.getRet());
+
+ sh.exec("""mahout trainnb \
+-i ${WORK_DIR}/20news-train-vectors -el \
+-o ${WORK_DIR}/model \
+-li ${WORK_DIR}/labelindex \
+-ow""");
assertEquals("Unexpected error from running mahout", 0, sh.getRet());
- sh.exec("""mahout testclassifier \
--m ${WORK_DIR}/20news-bydate/bayes-model \
--d ${WORK_DIR}/20news-bydate/bayes-test-input \
--type bayes \
--ng 1 \
--source hdfs \
--method mapreduce""");
+
+ sh.exec("""mahout testnb \
+-i ${WORK_DIR}/20news-train-vectors \
+-m ${WORK_DIR}/model \
+-l ${WORK_DIR}/labelindex \
+-ow -o ${WORK_DIR}/20news-testing""");
+ assertEquals("Unexpected error from running mahout", 0, sh.getRet());
+
+ sh.exec("""mahout testnb \
+-i ${WORK_DIR}/20news-test-vectors \
+-m ${WORK_DIR}/model \
+-l ${WORK_DIR}/labelindex \
+-ow -o ${WORK_DIR}/20news-testing""");
assertEquals("Unexpected error from running mahout", 0, sh.getRet());
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
index 23a00cb..7d1a90c 100644
--- a/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
+++ b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
@@ -125,16 +125,6 @@ class TestOozieSmoke {
}
@Test(timeout=300000L)
- public void testSqoop() {
- testOozieExamplesCommon("sqoop");
- }
-
- @Test(timeout=300000L)
- public void testSqoopFreeform() {
- testOozieExamplesCommon("sqoop-freeform");
- }
-
- @Test(timeout=300000L)
public void testSubwf() {
testOozieExamplesCommon("subwf");
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-execution/smokes/hue/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-execution/smokes/hue/pom.xml b/bigtop-tests/test-execution/smokes/hue/pom.xml
index ed96ae7..19e27a6 100644
--- a/bigtop-tests/test-execution/smokes/hue/pom.xml
+++ b/bigtop-tests/test-execution/smokes/hue/pom.xml
@@ -30,9 +30,11 @@
<groupId>org.apache.bigtop.itest</groupId>
<artifactId>hue-smoke-execution</artifactId>
<version>0.6.0-SNAPSHOT</version>
- <name>Giraph smoke test execution</name>
+ <name>Hue smoke test execution</name>
<properties>
+ <HUE_URL>${env.HUE_URL}</HUE_URL>
+
<org.apache.maven-dependency-plugin.groupId>org.apache.bigtop.itest</org.apache.maven-dependency-plugin.groupId>
<org.apache.maven-dependency-plugin.artifactId>hue-smoke</org.apache.maven-dependency-plugin.artifactId>
<org.apache.maven-dependency-plugin.version>${hue-smoke.version}</org.apache.maven-dependency-plugin.version>
@@ -55,8 +57,36 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <systemPropertyVariables>
+ <org.apache.bigtop.itest.hue_url>${HUE_URL}</org.apache.bigtop.itest.hue_url>
+ </systemPropertyVariables>
+ <forkMode>always</forkMode>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>enforce-property-oozie</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireProperty>
+ <property>HUE_URL</property>
+ <message>HUE_URL env. variable has to be set</message>
+ </requireProperty>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
-
</project>