You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2016/07/29 17:02:53 UTC
[12/51] [abbrv] bigtop git commit: BIGTOP-2441. spark smoke test
doesn't work relying on yarn-client submittion
BIGTOP-2441. spark smoke test doesn't work relying on yarn-client submittion
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6e81da1e
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6e81da1e
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6e81da1e
Branch: refs/heads/BIGTOP-2253
Commit: 6e81da1e896338927af4160854794da489718a67
Parents: b6f50e6
Author: Konstantin Boudnik <co...@apache.org>
Authored: Fri May 20 18:00:13 2016 -0700
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Wed May 25 10:57:16 2016 -0700
----------------------------------------------------------------------
bigtop-tests/smoke-tests/README | 9 +++++++++
bigtop-tests/smoke-tests/spark/TestSpark.groovy | 18 ++++++++++++++++--
2 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6e81da1e/bigtop-tests/smoke-tests/README
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/README b/bigtop-tests/smoke-tests/README
index 54d060d..759f53d 100644
--- a/bigtop-tests/smoke-tests/README
+++ b/bigtop-tests/smoke-tests/README
@@ -31,6 +31,15 @@ export TAJO_HOME="/usr/lib/tajo"
export OOZIE_TAR_HOME="/usr/lib/oozie/doc"
```
+Fallback mode for shark-shell submission is 'yarn-client'. For a good reason,
+we aren't using YARN to provision cluster services. Hence, in order to execute
+the test against a standalone Spark cluster, like we deploy would require two
+more environment variables:
+`
+export SPARK_MASTER_IP=bigtop1.docker
+export SPARK_MASTER_PORT=7077
+`
+
Then, simply invoke the tests you want to run, as described below.
Each directory is a gradle "subproject" mapping to an ecosystem component.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6e81da1e/bigtop-tests/smoke-tests/spark/TestSpark.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/spark/TestSpark.groovy b/bigtop-tests/smoke-tests/spark/TestSpark.groovy
index 4a715b3..8908c3d 100644
--- a/bigtop-tests/smoke-tests/spark/TestSpark.groovy
+++ b/bigtop-tests/smoke-tests/spark/TestSpark.groovy
@@ -40,7 +40,6 @@ class TestSpark {
static Shell sh = new Shell("/bin/bash -s")
static final String SPARK_HOME = System.getenv("SPARK_HOME")
- static final String SPARK_SHELL = SPARK_HOME + "/bin/spark-shell --master yarn-client"
static final String TEST_SPARKSQL_LOG = "/tmp/TestSpark_testSparkSQL.log"
@BeforeClass
@@ -69,7 +68,22 @@ class TestSpark {
@Test
void testSparkSQL() {
- sh.exec(SPARK_SHELL + " --class org.apache.spark.examples.sql.JavaSparkSQL " + " --jars " + SPARK_HOME + "/lib/spark-examples*.jar > " + TEST_SPARKSQL_LOG + " 2>&1")
+ // Let's figure out the proper mode for the submission
+ // If SPARK_MASTER_IP nor SPARK_MASTER_PORT are set, we'll assume
+ // 'yarn-client' mode
+ String masterMode = 'yarn-client'
+ if (System.env.SPARK_MASTER_IP != null && System.env.SPARK_MASTER_PORT != null)
+ masterMode = "spark://$MASTER_IP:$MASTER_PORT"
+ else
+ println("SPARK_MASTER isn't set. yarn-client submission will be used. " +
+ "Refer to smoke-tests/README If this isn't what you you expect.")
+
+ final String SPARK_SHELL = SPARK_HOME + "/bin/spark-shell --master $masterMode"
+ // Let's use time, 'cause the test has one job
+ sh.exec("timeout 120 " + SPARK_SHELL +
+ " --class org.apache.spark.examples.sql.JavaSparkSQL " +
+ " --jars " + SPARK_HOME + "/lib/spark-examples*.jar > " +
+ TEST_SPARKSQL_LOG + " 2>&1")
logError(sh)
assertTrue("Failed ...", sh.getRet() == 0);
}