You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2019/02/15 17:42:53 UTC
[bigtop] branch master updated: BIGTOP-3137: Giraph smoke tests
implementation
This is an automated email from the ASF dual-hosted git repository.
evansye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new f1500c1 BIGTOP-3137: Giraph smoke tests implementation
f1500c1 is described below
commit f1500c13124759897105ceb5270de5d242b2b5eb
Author: Yuqi Gu <yu...@arm.com>
AuthorDate: Thu Jan 17 02:45:36 2019 +0000
BIGTOP-3137: Giraph smoke tests implementation
Add smoke tests for Giraph
Change-Id: I80ec1fbc916d5f616e92a16a15db777c4f4d3979
Signed-off-by: Yuqi Gu <yu...@arm.com>
Signed-off-by: Evans Ye <ev...@apache.org>
---
.../modules/hadoop/templates/mapred-site.xml | 5 ++
.../src/common/giraph/install_giraph.sh | 4 +-
bigtop-tests/smoke-tests/giraph/build.gradle | 37 ++++++++++++
.../itest/giraphsmoke/TestGiraphSmoke.groovy | 68 +++++-----------------
provisioner/utils/smoke-tests.sh | 1 +
5 files changed, 61 insertions(+), 54 deletions(-)
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
index a25f66b..b7f2bba 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
@@ -77,6 +77,11 @@
<% end %>
<% if @mapreduce_jobhistory_host %>
<property>
+ <name>mapred.job.tracker</name>
+ <value>BIGTOP-3137: Required by Giraph checkLocalJobRunnerConfiguration</value>
+ </property>
+
+ <property>
<name>mapreduce.jobhistory.address</name>
<value><%= @mapreduce_jobhistory_host %>:<%= @mapreduce_jobhistory_port %></value>
</property>
diff --git a/bigtop-packages/src/common/giraph/install_giraph.sh b/bigtop-packages/src/common/giraph/install_giraph.sh
index 0cc9d94..79b7c37 100644
--- a/bigtop-packages/src/common/giraph/install_giraph.sh
+++ b/bigtop-packages/src/common/giraph/install_giraph.sh
@@ -110,11 +110,12 @@ install -d -m 0755 ${PREFIX}/${LIB_DIR}
# Installing Giraph core
install -d -m 0755 ${PREFIX}/${LIB_DIR}
cp -r $BUILD_DIR/giraph-dist/target/giraph*-bin/*/* ${PREFIX}/${LIB_DIR}
+cp $BUILD_DIR/giraph-examples/target/giraph-examples*.jar ${PREFIX}/${LIB_DIR}
# Installing docs and examples
install -d -m 0755 $PREFIX/${DOC_DIR}
# cp -r $BUILD_DIR/target/staging/* $PREFIX/${DOC_DIR}
-mv ${PREFIX}/${LIB_DIR}/giraph-examples*.jar $PREFIX/${DOC_DIR}
+cp ${PREFIX}/${LIB_DIR}/giraph-examples*.jar $PREFIX/${DOC_DIR}
# Install executable wrappers
install -d -m 0755 $PREFIX/usr/bin
@@ -134,6 +135,7 @@ for i in giraph ; do
# Workaround for GIRAPH-199
export HADOOP_HOME=\${HADOOP_HOME:-/usr/lib/hadoop}
export HADOOP_CONF_DIR=\${HADOOP_CONF_DIR:-/etc/hadoop/conf}
+export HADOOP_CLASSPATH=\${HADOOP_CLASSPATH:-/etc/giraph/conf}
export GIRAPH_HOME=$LIB_DIR
exec $BIN_DIR/$i "\$@"
diff --git a/bigtop-tests/smoke-tests/giraph/build.gradle b/bigtop-tests/smoke-tests/giraph/build.gradle
new file mode 100644
index 0000000..06d47cf
--- /dev/null
+++ b/bigtop-tests/smoke-tests/giraph/build.gradle
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+def tests_to_include() {
+ return [
+ "TestGiraphSmoke.groovy"
+ ];
+}
+
+sourceSets {
+ test {
+ groovy {
+ srcDirs = [
+ "${BIGTOP_HOME}/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/"
+ ]
+ exclude { FileTreeElement elem -> (doExclude(elem.getName())) }
+ }
+ }
+}
+
+test.doFirst {
+ checkEnv(["GIRAPH_HOME"])
+}
diff --git a/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/TestGiraphSmoke.groovy b/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/TestGiraphSmoke.groovy
index 39a150e..7afbee1 100644
--- a/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/TestGiraphSmoke.groovy
+++ b/bigtop-tests/test-artifacts/giraph/src/main/groovy/org/apache/bigtop/itest/giraphsmoke/TestGiraphSmoke.groovy
@@ -24,17 +24,12 @@ import org.apache.bigtop.itest.shell.Shell
import org.junit.Test
public class TestGiraphSmoke {
- static String runnerScript = "HADOOP_CLASSPATH=/etc/giraph/conf hadoop jar"
static String testDir = "test.giraphsmoke." + (new Date().getTime())
-
- static String giraphHome = System.getProperty('GIRAPH_HOME', '/usr/lib/giraph')
- static String giraphJar = "${giraphHome}/giraph-jar-with-dependencies.jar"
-
static Shell sh = new Shell("/bin/bash -s")
@Test(timeout = 300000L)
public void testPageRankBenchmark() {
- sh.exec("${runnerScript} ${giraphJar}"
+ sh.exec("hadoop jar /usr/lib/giraph/giraph-examples*with-dependencies.jar"
+ " org.apache.giraph.benchmark.PageRankBenchmark"
+ " -v " // verbose
+ " -e 1" // edges per vertex
@@ -47,7 +42,7 @@ public class TestGiraphSmoke {
@Test(timeout = 300000L)
public void testRandomMessageBenchmark() {
- sh.exec("${runnerScript} ${giraphJar}"
+ sh.exec("hadoop jar /usr/lib/giraph/giraph-examples*with-dependencies.jar"
+ " org.apache.giraph.benchmark.RandomMessageBenchmark"
+ " -v " // verbose
+ " -e 1" // edges per vertex
@@ -61,55 +56,22 @@ public class TestGiraphSmoke {
}
@Test(timeout = 300000L)
- public void testSimpleCheckpointVertex() {
- sh.exec("hadoop fs -rmr ${testDir}");
- sh.exec("${runnerScript} ${giraphJar}"
- + " org.apache.giraph.examples.SimpleCheckpointVertex"
- + " -v " // verbose
- + " -s 3" // number of supersteps
- + " -w 3" // workers
- + " -o ${testDir}"
- )
- assertEquals("running SimpleCheckpointVertex failed", 0, sh.getRet());
- }
-
- @Test(timeout = 300000L)
- public void testSimpleVertexWithWorkerContext() {
- sh.exec("hadoop fs -rmr ${testDir}");
- sh.exec("${runnerScript} ${giraphJar}"
- + " org.apache.giraph.examples.SimpleVertexWithWorkerContext"
- + " ${testDir} 3"
- )
- assertEquals("running SimpleCheckpointVertex failed", 0, sh.getRet());
- }
-
- @Test(timeout = 300000L)
- public void testSimpleShortestPathsVertex() {
- // A graph definition:
- // [vertex id, vertex value, [[edge1, value1], .. [edgeN, valueN]]]
- List graphDescription = [[0, 0, [[1, 1], [2, 2]]],
- [1, 1, [[2, 2], [3, 3]]],
- [2, 2, [[3, 3], [4, 4]]],
- [3, 3, [[4, 4], [5, 5]]],
- [4, 4, [[5, 5], [0, 0]]],
- [5, 5, [[0, 0], [1, 1]]]];
- int partitionSize = 2;
+ public void testSimpleShortestPathsComputation() {
+ sh.exec("echo -e '[0,0,[[1,1],[3,3]]]\n[1,0,[[0,1],[2,2],[3,1]]]\n[2,0,[[1,2],[4,4]]]\n[3,0,[[0,3],[1,1],[4,4]]]\n[4,0,[[3,4],[2,4]]]' > ./giraphTest.txt");
- sh.exec("hadoop fs -rmr ${testDir}",
- "hadoop fs -mkdir ${testDir}/input");
+ sh.exec("hadoop fs -mkdir ${testDir}");
- for (int i = 0; i < graphDescription.size(); i += partitionSize) {
- String part = graphDescription[i..(i + partitionSize - 1)].join("\n");
- int partId = i / partitionSize;
- sh.exec("hadoop fs -put <(echo '${part}') ${testDir}/input/part-m-${partId}");
- }
+ sh.exec("hadoop fs -copyFromLocal ./giraphTest.txt ${testDir}/giraphTest.txt");
- sh.exec("${runnerScript} ${giraphJar}"
- + " org.apache.giraph.examples.SimpleShortestPathsVertex"
- + " ${testDir}/input"
- + " ${testDir}/output"
- + " 0 ${graphDescription.size() / partitionSize}"
+ sh.exec("hadoop jar /usr/lib/giraph/giraph-examples*with-dependencies.jar"
+ + " org.apache.giraph.GiraphRunner"
+ + " org.apache.giraph.examples.SimpleShortestPathsComputation"
+ + " -vif org.apache.giraph.io.formats.JsonLongDoubleFloatDoubleVertexInputFormat"
+ + " -vip ${testDir}/giraphTest.txt"
+ + " -vof org.apache.giraph.io.formats.IdWithValueTextOutputFormat"
+ + " -op ${testDir}/shortestpaths"
+ + " -w 3"
)
- assertEquals("running SimpleShortestPathsVertex failed", 0, sh.getRet());
+ assertEquals("running SimpleShortestPathsComputation failed", 0, sh.getRet());
}
}
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
index 6250cba..a28f799 100755
--- a/provisioner/utils/smoke-tests.sh
+++ b/provisioner/utils/smoke-tests.sh
@@ -47,6 +47,7 @@ export MAHOUT_HOME=${MAHOUT_HOME:-/usr/lib/mahout}
export SPARK_HOME=${SPARK_HOME:-/usr/lib/spark}
export SQOOP_HOME=${SQOOP_HOME:-/usr/lib/sqoop}
export ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/lib/zookeeper}
+export GIRAPH_HOME=${GIRAPH_HOME:-/usr/lib/giraph}
echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"