You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2016/12/11 18:24:15 UTC
zeppelin git commit: ZEPPELIN-1607. Refactor Livy Interpreter to
adapt scope mode
Repository: zeppelin
Updated Branches:
refs/heads/master a10545ba1 -> 04c62e41f
ZEPPELIN-1607. Refactor Livy Interpreter to adapt scope mode
### What is this PR for?
ZEPPELIN-1606 will make scoped mode as the default interpreter mode of livy. This PR is to refactor livy interpreter to adapt the scoped mode. Besides, it also fix several bugs of livy interpreter. like ZEPPELIN-1516. Besides that I also enable livy integration test in travis to make sure the refactoring correct.
### What type of PR is it?
[Refactoring]
### Todos
* Move session creation to open()
* Other refactoring in livy interpreter, especially LivyHelper.java
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1607
### How should this be tested?
Travis is also enabled for livy integration test.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #1612 from zjffdu/ZEPPELIN-1607 and squashes the following commits:
980b796 [Jeff Zhang] run livy test separately
2dd73e4 [Jeff Zhang] change version from 2.3 to 2.6 in .travis.yml
6c9795f [Jeff Zhang] update test
e99be8d [Jeff Zhang] update .travis.yml
2ce1532 [Jeff Zhang] add licence header
995ddf3 [Jeff Zhang] enable travis for livy integration test
798de1b [Jeff Zhang] ZEPPELIN-1607. Refactor Livy Interpreter to adapt scope mode
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/04c62e41
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/04c62e41
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/04c62e41
Branch: refs/heads/master
Commit: 04c62e41fdf15f4a6b556a49ef57c4f81776d2c4
Parents: a10545b
Author: Jeff Zhang <zj...@apache.org>
Authored: Wed Dec 7 16:32:08 2016 +0800
Committer: Lee moon soo <mo...@apache.org>
Committed: Sun Dec 11 13:24:10 2016 -0500
----------------------------------------------------------------------
.travis.yml | 29 +-
livy/pom.xml | 35 +-
.../zeppelin/livy/BaseLivyInterprereter.java | 143 ++++++++
.../org/apache/zeppelin/livy/LivyHelper.java | 40 ++-
.../zeppelin/livy/LivyPySparkInterpreter.java | 73 +----
.../zeppelin/livy/LivySparkInterpreter.java | 124 +------
.../zeppelin/livy/LivySparkRInterpreter.java | 73 +----
.../zeppelin/livy/LivySparkSQLInterpreter.java | 95 +++---
.../apache/zeppelin/livy/LivyHelperTest.java | 10 +-
.../zeppelin/livy/LivyIntegrationTest.java | 249 --------------
.../apache/zeppelin/livy/LivyInterpreterIT.java | 328 +++++++++++++++++++
.../zeppelin/livy/LivyInterpreterTest.java | 86 -----
testing/downloadLivy.sh | 82 +++++
13 files changed, 679 insertions(+), 688 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index cc3bf61..b82c387 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,39 +40,43 @@ matrix:
include:
# Test License compliance using RAT tool
- jdk: "oraclejdk7"
- env: SCALA_VER="2.11" SPARK_VER="2.0.1" HADOOP_VER="2.3" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
+ env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Test all modules with spark 2.0.2 and scala 2.11
- jdk: "oraclejdk7"
- env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
+ env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.10
- jdk: "oraclejdk7"
- env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pbeam -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
+ env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Pbeam -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.11
- jdk: "oraclejdk7"
- env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
+ env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
- env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+ env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.6" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.6 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
- env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+ env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.6 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.3
- jdk: "oraclejdk7"
- env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
+ env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
# Test python/pyspark with python 2
- jdk: "oraclejdk7"
- env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
+ env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
# Test python/pyspark with python 3
- jdk: "oraclejdk7"
- env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
+ env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
+
+ # Test livy with spark 1.5.2 and hadoop 2.6
+ - jdk: "oraclejdk7"
+ env: SCALA_VER="2.10" $LIVY_VER="0.2.0" SPARK_VER="1.5.2" HADOOP_VER="2.6" PROFILE="-Pspark-1.5 -Phadoop-2.6" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,livy -DfailIfNoTests=false"
before_install:
- echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
@@ -88,7 +92,10 @@ install:
before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
+ - if [[ -n $LIVY_VER ]]; then travis_retry ./testing/downloadLivy.sh $LIVY_VER; fi
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
+ - if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-server-$LIVY_VER; fi
+ - if [[ -n $LIVY_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi
- tail conf/zeppelin-env.sh
script:
@@ -105,3 +112,7 @@ after_failure:
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
- cat zeppelin-web/npm-debug.log
- cat spark-*/logs/*
+ - cat livy/target/tmp/*/output.log
+ - ls -R livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*
+ - cat livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stdout
+ - cat livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/pom.xml
----------------------------------------------------------------------
diff --git a/livy/pom.xml b/livy/pom.xml
index f5c0277..c173e9e 100644
--- a/livy/pom.xml
+++ b/livy/pom.xml
@@ -286,18 +286,28 @@
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-yarn_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
+ <artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
@@ -305,6 +315,14 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
+ <classifier>tests</classifier>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
@@ -312,6 +330,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
+ <classifier>tests</classifier>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
@@ -332,7 +351,15 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
+ <classifier>tests</classifier>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
new file mode 100644
index 0000000..d0167c2
--- /dev/null
+++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.livy;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+
+/**
+ * Base class for livy interpreters.
+ */
+public abstract class BaseLivyInterprereter extends Interpreter {
+
+ protected static final Logger LOGGER = LoggerFactory.getLogger(BaseLivyInterprereter.class);
+
+ // -1 means session is not created yet, valid sessionId start from 0
+ protected int sessionId = -1;
+ protected String appId;
+ protected String webUIAddress;
+ protected boolean displayAppInfo;
+ protected LivyOutputStream out;
+ protected LivyHelper livyHelper;
+
+ public BaseLivyInterprereter(Properties property) {
+ super(property);
+ this.out = new LivyOutputStream();
+ this.livyHelper = new LivyHelper(property);
+ }
+
+ public abstract String getSessionKind();
+
+ @Override
+ public void open() {
+ // TODO(zjffdu) move session creation here.
+ }
+
+ @Override
+ public void close() {
+ if (sessionId != -1) {
+ livyHelper.closeSession(sessionId);
+ // reset sessionId to -1
+ sessionId = -1;
+ }
+ }
+
+ protected void createSession(InterpreterContext context) throws Exception {
+ sessionId = livyHelper.createSession(context, getSessionKind());
+ if (displayAppInfo) {
+ this.appId = extractStatementResult(
+ livyHelper.interpret("sc.applicationId", context, sessionId).message().get(0).getData());
+ livyHelper.interpret(
+ "val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get",
+ context, sessionId);
+ this.webUIAddress = extractStatementResult(
+ livyHelper.interpret(
+ "webui.getClass.getMethod(\"appUIAddress\").invoke(webui)",
+ context, sessionId).message().get(0).getData());
+ LOGGER.info("Create livy session with sessionId: {}, appId: {}, webUI: {}",
+ sessionId, appId, webUIAddress);
+ }
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ try {
+ // add synchronized, because LivySparkSQLInterperter will use ParallelScheduler
+ synchronized (this) {
+ if (sessionId == -1) {
+ try {
+ createSession(context);
+ } catch (Exception e) {
+ LOGGER.error("Exception while creating livy session", e);
+ return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
+ }
+ }
+ }
+ if (StringUtils.isEmpty(st)) {
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
+ }
+
+ return livyHelper.interpretInput(st, context, sessionId, out,
+ appId, webUIAddress, displayAppInfo);
+ } catch (Exception e) {
+ LOGGER.error("Exception in LivyInterpreter.", e);
+ return new InterpreterResult(InterpreterResult.Code.ERROR,
+ InterpreterUtils.getMostRelevantMessage(e));
+ }
+ }
+
+ /**
+ * Extract the eval result of spark shell, e.g. extract application_1473129941656_0048
+ * from following:
+ * res0: String = application_1473129941656_0048
+ *
+ * @param result
+ * @return
+ */
+ private static String extractStatementResult(String result) {
+ int pos = -1;
+ if ((pos = result.indexOf("=")) >= 0) {
+ return result.substring(pos + 1).trim();
+ } else {
+ throw new RuntimeException("No result can be extracted from '" + result + "', " +
+ "something must be wrong");
+ }
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+ livyHelper.cancelHTTP(context.getParagraphId());
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.SIMPLE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
index 40bedf8..76f5167 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
@@ -133,7 +133,7 @@ public class LivyHelper {
public InterpreterResult interpretInput(String stringLines,
final InterpreterContext context,
- final Map<String, Integer> userSessionMap,
+ int sessionId,
LivyOutputStream out,
String appId,
String webUI,
@@ -185,7 +185,7 @@ public class LivyHelper {
InterpreterResult res;
try {
- res = interpret(incomplete + s, context, userSessionMap);
+ res = interpret(incomplete + s, context, sessionId);
} catch (Exception e) {
LOGGER.error("Interpreter exception", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
@@ -230,12 +230,12 @@ public class LivyHelper {
public InterpreterResult interpret(String stringLines,
final InterpreterContext context,
- final Map<String, Integer> userSessionMap)
+ int sessionId)
throws Exception {
if (stringLines.trim().equals("")) {
return new InterpreterResult(Code.SUCCESS, "");
}
- Map jsonMap = executeCommand(stringLines, context, userSessionMap);
+ Map jsonMap = executeCommand(stringLines, context, sessionId);
Integer id = ((Double) jsonMap.get("id")).intValue();
InterpreterResult res = getResultFromMap(jsonMap);
if (res != null) {
@@ -247,7 +247,7 @@ public class LivyHelper {
if (paragraphHttpMap.get(context.getParagraphId()) == null) {
return new InterpreterResult(Code.INCOMPLETE, "");
}
- jsonMap = getStatusById(context, userSessionMap, id);
+ jsonMap = getStatusById(context, sessionId, id);
InterpreterResult interpreterResult = getResultFromMap(jsonMap);
if (interpreterResult != null) {
return interpreterResult;
@@ -292,11 +292,10 @@ public class LivyHelper {
return null;
}
- private Map executeCommand(String lines, InterpreterContext context,
- Map<String, Integer> userSessionMap) throws Exception {
+ private Map executeCommand(String lines, InterpreterContext context, int sessionId)
+ throws Exception {
String json = executeHTTP(property.get("zeppelin.livy.url") + "/sessions/"
- + userSessionMap.get(context.getAuthenticationInfo().getUser())
- + "/statements",
+ + sessionId + "/statements",
"POST",
"{\"code\": \"" + StringEscapeUtils.escapeJson(lines) + "\"}",
context.getParagraphId());
@@ -316,9 +315,9 @@ public class LivyHelper {
}
private Map getStatusById(InterpreterContext context,
- Map<String, Integer> userSessionMap, Integer id) throws Exception {
+ int sessionId, Integer id) throws Exception {
String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/"
- + userSessionMap.get(context.getAuthenticationInfo().getUser())
+ + sessionId
+ "/statements/" + id,
"GET", null, context.getParagraphId());
LOGGER.debug("statement {} response: {}", id, json);
@@ -390,19 +389,18 @@ public class LivyHelper {
}
public void cancelHTTP(String paragraphId) {
+ // TODO(zjffdu), use cancel rest api of livy
paragraphHttpMap.put(paragraphId, null);
}
- public void closeSession(Map<String, Integer> userSessionMap) {
- for (Map.Entry<String, Integer> entry : userSessionMap.entrySet()) {
- try {
- executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/"
- + entry.getValue(),
- "DELETE", null, null);
- } catch (Exception e) {
- LOGGER.error(String.format("Error closing session for user with session ID: %s",
- entry.getValue()), e);
- }
+ public void closeSession(int sessionId) {
+ try {
+ executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" + sessionId,
+ "DELETE", null, null);
+ } catch (Exception e) {
+ LOGGER.error(String.format("Error closing session for user with session ID: %s",
+ sessionId), e);
}
}
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkInterpreter.java
index bd342a2..0dfce94 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkInterpreter.java
@@ -33,81 +33,14 @@ import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
-public class LivyPySparkInterpreter extends Interpreter {
-
- Logger LOGGER = LoggerFactory.getLogger(LivyPySparkInterpreter.class);
-
- protected Map<String, Integer> userSessionMap;
- protected LivyHelper livyHelper;
+public class LivyPySparkInterpreter extends BaseLivyInterprereter {
public LivyPySparkInterpreter(Properties property) {
super(property);
- userSessionMap = new HashMap<>();
- livyHelper = new LivyHelper(property);
- }
-
- @Override
- public void open() {
- }
-
- @Override
- public void close() {
- livyHelper.closeSession(userSessionMap);
- }
-
- @Override
- public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
- try {
- if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
- try {
- userSessionMap.put(
- interpreterContext.getAuthenticationInfo().getUser(),
- livyHelper.createSession(
- interpreterContext,
- "pyspark")
- );
- } catch (Exception e) {
- LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
- return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
- }
- }
-
- if (line == null || line.trim().length() == 0) {
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
- }
-
- return livyHelper.interpret(line, interpreterContext, userSessionMap);
- } catch (Exception e) {
- LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
- return new InterpreterResult(InterpreterResult.Code.ERROR,
- InterpreterUtils.getMostRelevantMessage(e));
- }
}
@Override
- public void cancel(InterpreterContext context) {
- livyHelper.cancelHTTP(context.getParagraphId());
+ public String getSessionKind() {
+ return "pyspark";
}
-
- @Override
- public FormType getFormType() {
- return FormType.SIMPLE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler(
- LivyPySparkInterpreter.class.getName() + this.hashCode());
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor) {
- return null;
- }
-
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java
index b406634..827332a 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java
@@ -28,136 +28,20 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
/**
* Livy Spark interpreter for Zeppelin.
*/
-public class LivySparkInterpreter extends Interpreter {
-
- Logger LOGGER = LoggerFactory.getLogger(LivySparkInterpreter.class);
- private LivyOutputStream out;
-
- protected static Map<String, Integer> userSessionMap;
- protected static Map<Integer, String> sessionId2AppIdMap;
- protected static Map<Integer, String> sessionId2WebUIMap;
-
- private LivyHelper livyHelper;
- private boolean displayAppInfo;
+public class LivySparkInterpreter extends BaseLivyInterprereter {
public LivySparkInterpreter(Properties property) {
super(property);
- userSessionMap = new HashMap<>();
- sessionId2AppIdMap = new HashMap<>();
- sessionId2WebUIMap = new HashMap<>();
- livyHelper = new LivyHelper(property);
- out = new LivyOutputStream();
- this.displayAppInfo = Boolean.parseBoolean(getProperty("zeppelin.livy.displayAppInfo"));
- }
-
- protected static Map<String, Integer> getUserSessionMap() {
- return userSessionMap;
- }
-
- public void setUserSessionMap(Map<String, Integer> userSessionMap) {
- this.userSessionMap = userSessionMap;
- }
-
- @Override
- public void open() {
- }
-
- @Override
- public void close() {
- livyHelper.closeSession(userSessionMap);
- }
-
- @Override
- public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
- try {
- Integer sessionId = null;
- if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
- try {
- sessionId = livyHelper.createSession(interpreterContext, "spark");
- userSessionMap.put(interpreterContext.getAuthenticationInfo().getUser(), sessionId);
- if (displayAppInfo) {
- String appId = extractStatementResult(
- livyHelper.interpret("sc.applicationId", interpreterContext, userSessionMap)
- .message().get(0).getData());
- livyHelper.interpret(
- "val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get",
- interpreterContext, userSessionMap);
- String webUI = extractStatementResult(
- livyHelper.interpret(
- "webui.getClass.getMethod(\"appUIAddress\").invoke(webui)",
- interpreterContext, userSessionMap).message().get(0).getData());
- sessionId2AppIdMap.put(sessionId, appId);
- sessionId2WebUIMap.put(sessionId, webUI);
- LOGGER.info("Create livy session with sessionId: {}, appId: {}, webUI: {}",
- sessionId, appId, webUI);
- } else {
- LOGGER.info("Create livy session with sessionId: {}", sessionId);
- }
- } catch (Exception e) {
- LOGGER.error("Exception in LivySparkInterpreter while interpret ", e);
- return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
- }
- } else {
- sessionId = userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser());
- }
- if (line == null || line.trim().length() == 0) {
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
- }
-
- return livyHelper.interpretInput(line, interpreterContext, userSessionMap, out,
- sessionId2AppIdMap.get(sessionId), sessionId2WebUIMap.get(sessionId), displayAppInfo);
- } catch (Exception e) {
- LOGGER.error("Exception in LivySparkInterpreter while interpret ", e);
- return new InterpreterResult(InterpreterResult.Code.ERROR,
- InterpreterUtils.getMostRelevantMessage(e));
- }
- }
-
- /**
- * Extract the eval result of spark shell, e.g. extract application_1473129941656_0048
- * from following:
- * res0: String = application_1473129941656_0048
- * @param result
- * @return
- */
- private static String extractStatementResult(String result) {
- int pos = -1;
- if ((pos = result.indexOf("=")) >= 0) {
- return result.substring(pos + 1).trim();
- } else {
- throw new RuntimeException("No result can be extracted from '" + result + "', " +
- "something must be wrong");
- }
- }
-
- @Override
- public void cancel(InterpreterContext context) {
- livyHelper.cancelHTTP(context.getParagraphId());
- }
-
- @Override
- public FormType getFormType() {
- return FormType.SIMPLE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler(
- LivySparkInterpreter.class.getName() + this.hashCode());
}
@Override
- public List<InterpreterCompletion> completion(String buf, int cursor) {
- return null;
+ public String getSessionKind() {
+ return "spark";
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/src/main/java/org/apache/zeppelin/livy/LivySparkRInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkRInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkRInterpreter.java
index 753b378..115b1cf 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkRInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkRInterpreter.java
@@ -33,81 +33,14 @@ import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
-public class LivySparkRInterpreter extends Interpreter {
-
- Logger LOGGER = LoggerFactory.getLogger(LivySparkRInterpreter.class);
-
- protected Map<String, Integer> userSessionMap;
- private LivyHelper livyHelper;
+public class LivySparkRInterpreter extends BaseLivyInterprereter {
public LivySparkRInterpreter(Properties property) {
super(property);
- userSessionMap = new HashMap<>();
- livyHelper = new LivyHelper(property);
- }
-
- @Override
- public void open() {
- }
-
- @Override
- public void close() {
- livyHelper.closeSession(userSessionMap);
- }
-
- @Override
- public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
- try {
- if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
- try {
- userSessionMap.put(
- interpreterContext.getAuthenticationInfo().getUser(),
- livyHelper.createSession(
- interpreterContext,
- "sparkr")
- );
- } catch (Exception e) {
- LOGGER.error("Exception in LivySparkRInterpreter while interpret ", e);
- return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
- }
- }
-
- if (line == null || line.trim().length() == 0) {
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
- }
-
- return livyHelper.interpret(line, interpreterContext, userSessionMap);
- } catch (Exception e) {
- LOGGER.error("Exception in LivySparkRInterpreter while interpret ", e);
- return new InterpreterResult(InterpreterResult.Code.ERROR,
- InterpreterUtils.getMostRelevantMessage(e));
- }
}
@Override
- public void cancel(InterpreterContext context) {
- livyHelper.cancelHTTP(context.getParagraphId());
+ public String getSessionKind() {
+ return "sparkr";
}
-
- @Override
- public FormType getFormType() {
- return FormType.SIMPLE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler(
- LivySparkRInterpreter.class.getName() + this.hashCode());
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor) {
- return null;
- }
-
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
index 6373306..089b264 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
@@ -17,6 +17,7 @@
package org.apache.zeppelin.livy;
+import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
@@ -27,66 +28,73 @@ import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
/**
- * Livy PySpark interpreter for Zeppelin.
+ * Livy SparkSQL Interpreter for Zeppelin.
*/
-public class LivySparkSQLInterpreter extends Interpreter {
+public class LivySparkSQLInterpreter extends BaseLivyInterprereter {
- Logger LOGGER = LoggerFactory.getLogger(LivySparkSQLInterpreter.class);
+ private LivySparkInterpreter sparkInterpreter;
- protected Map<String, Integer> userSessionMap;
- private LivyHelper livyHelper;
+ private boolean sqlContextCreated = false;
public LivySparkSQLInterpreter(Properties property) {
super(property);
- livyHelper = new LivyHelper(property);
- userSessionMap = LivySparkInterpreter.getUserSessionMap();
}
@Override
- public void open() {
+ public String getSessionKind() {
+ return "spark";
}
@Override
- public void close() {
- livyHelper.closeSession(userSessionMap);
+ public void open() {
+ super.open();
+ this.sparkInterpreter =
+ (LivySparkInterpreter) getInterpreterInTheSameSessionByClassName(
+ LivySparkInterpreter.class.getName());
}
@Override
- public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
+ public InterpreterResult interpret(String line, InterpreterContext context) {
try {
- if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
- try {
- userSessionMap.put(
- interpreterContext.getAuthenticationInfo().getUser(),
- livyHelper.createSession(
- interpreterContext,
- "spark")
- );
- } catch (Exception e) {
- LOGGER.error("Exception in LivySparkSQLInterpreter while interpret ", e);
- return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
- }
+ if (StringUtils.isEmpty(line)) {
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
- if (line == null || line.trim().length() == 0) {
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
+ // create sqlContext implicitly if it is not available, as in livy 0.2 sqlContext
+ // is not available.
+ synchronized (this) {
+ if (!sqlContextCreated) {
+ InterpreterResult result = sparkInterpreter.interpret("sqlContext", context);
+ if (result.code() == InterpreterResult.Code.ERROR) {
+ result = sparkInterpreter.interpret(
+ "val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n"
+ + "import sqlContext.implicits._", context);
+ if (result.code() == InterpreterResult.Code.ERROR) {
+ return new InterpreterResult(InterpreterResult.Code.ERROR,
+ "Fail to create sqlContext," + result.message());
+ }
+ }
+ sqlContextCreated = true;
+ }
}
- InterpreterResult res = livyHelper.interpret("sqlContext.sql(\"" +
- line.replaceAll("\"", "\\\\\"")
- .replaceAll("\\n", " ")
- + "\").show(" +
- property.get("zeppelin.livy.spark.sql.maxResult") + ")",
- interpreterContext, userSessionMap);
+ // delegate the work to LivySparkInterpreter in the same session.
+ // TODO(zjffdu), we may create multiple session for the same user here. This can be fixed
+ // after we move session creation to open()
+ InterpreterResult res = sparkInterpreter.interpret("sqlContext.sql(\"" +
+ line.replaceAll("\"", "\\\\\"")
+ .replaceAll("\\n", " ")
+ + "\").show(" +
+ property.get("zeppelin.livy.spark.sql.maxResult") + ")", context);
if (res.code() == InterpreterResult.Code.SUCCESS) {
StringBuilder resMsg = new StringBuilder();
resMsg.append("%table ");
- String[] rows = res.message().get(0).getData().split("\n");
-
+ String[] rows = new String(context.out.toByteArray()).split("\n");
String[] headers = rows[1].split("\\|");
for (int head = 1; head < headers.length; head++) {
resMsg.append(headers[head].trim()).append("\t");
@@ -114,7 +122,6 @@ public class LivySparkSQLInterpreter extends Interpreter {
return res;
}
-
} catch (Exception e) {
LOGGER.error("Exception in LivySparkSQLInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
@@ -127,21 +134,6 @@ public class LivySparkSQLInterpreter extends Interpreter {
}
@Override
- public void cancel(InterpreterContext context) {
- livyHelper.cancelHTTP(context.getParagraphId());
- }
-
- @Override
- public FormType getFormType() {
- return FormType.SIMPLE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
public Scheduler getScheduler() {
if (concurrentSQL()) {
int maxConcurrency = 10;
@@ -159,8 +151,7 @@ public class LivySparkSQLInterpreter extends Interpreter {
}
@Override
- public List<InterpreterCompletion> completion(String buf, int cursor) {
- return null;
+ public void close() {
+ this.sparkInterpreter.close();
}
-
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/src/test/java/org/apache/zeppelin/livy/LivyHelperTest.java
----------------------------------------------------------------------
diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyHelperTest.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyHelperTest.java
index 32d682f..e4d92e4 100644
--- a/livy/src/test/java/org/apache/zeppelin/livy/LivyHelperTest.java
+++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyHelperTest.java
@@ -56,9 +56,6 @@ public class LivyHelperTest {
@Before
public void prepareContext() throws Exception {
- interpreter.userSessionMap = new HashMap<>();
- interpreter.userSessionMap.put(null, 1);
-
Properties properties = new Properties();
properties.setProperty("zeppelin.livy.url", "http://localhost:8998");
livyHelper.property = properties;
@@ -103,10 +100,9 @@ public class LivyHelperTest {
@Test
public void checkInterpret() {
try {
- InterpreterResult result = livyHelper.interpret("print(1)", interpreterContext, interpreter.userSessionMap);
-
- collector.checkThat("check sessionId", InterpreterResult.Code.SUCCESS, CoreMatchers.equalTo(result.code()));
-
+ InterpreterResult result = livyHelper.interpret("print(1)", interpreterContext, 1);
+ collector.checkThat("check sessionId", InterpreterResult.Code.SUCCESS,
+ CoreMatchers.equalTo(result.code()));
} catch (Exception e) {
collector.addError(e);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/src/test/java/org/apache/zeppelin/livy/LivyIntegrationTest.java
----------------------------------------------------------------------
diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyIntegrationTest.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyIntegrationTest.java
deleted file mode 100644
index bc5e660..0000000
--- a/livy/src/test/java/org/apache/zeppelin/livy/LivyIntegrationTest.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.livy;
-
-
-import com.cloudera.livy.test.framework.Cluster;
-import com.cloudera.livy.test.framework.Cluster$;
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Properties;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-public class LivyIntegrationTest {
-
- private static Logger LOGGER = LoggerFactory.getLogger(LivyIntegrationTest.class);
- private static Cluster cluster;
- private static Properties properties;
-
- @BeforeClass
- public static void setUp() {
- if (!checkPreCondition()) {
- return;
- }
- cluster = Cluster$.MODULE$.get();
- LOGGER.info("Starting livy at {}", cluster.livyEndpoint());
- properties = new Properties();
- properties.setProperty("zeppelin.livy.url", cluster.livyEndpoint());
- properties.setProperty("zeppelin.livy.create.session.retries", "120");
- properties.setProperty("zeppelin.livy.spark.sql.maxResult", "100");
- }
-
- @AfterClass
- public static void tearDown() {
- if (cluster != null) {
- cluster.cleanUp();
- }
- }
-
- public static boolean checkPreCondition() {
- if (System.getenv("LIVY_HOME") == null) {
- LOGGER.warn(("livy integration is skipped because LIVY_HOME is not set"));
- return false;
- }
- if (System.getenv("SPARK_HOME") == null) {
- LOGGER.warn(("livy integration is skipped because SPARK_HOME is not set"));
- return false;
- }
- return true;
- }
-
- @Test
- public void testSparkInterpreter() {
- if (!checkPreCondition()) {
- return;
- }
-
- LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
- AuthenticationInfo authInfo = new AuthenticationInfo("user1");
- MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
- InterpreterOutput output = new InterpreterOutput(outputListener);
- InterpreterContext context = new InterpreterContext("noteId", "paragraphId", null, "title",
- "text", authInfo, null, null, null, null, null, output);
- sparkInterpreter.open();
- InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
-
- // test RDD api
- outputListener.reset();
- result = sparkInterpreter.interpret("sc.parallelize(1 to 10).sum()", context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
- assertTrue(outputListener.getOutputAppended().contains("Double = 55.0"));
-
- // test DataFrame api
- outputListener.reset();
- sparkInterpreter.interpret("val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n"
- + "import sqlContext.implicits._", context);
- result = sparkInterpreter.interpret("val df=sqlContext.createDataFrame(Seq((\"hello\",20)))\n"
- + "df.collect()" , context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
- assertTrue(outputListener.getOutputAppended()
- .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
- sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
-
- // test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter
- outputListener.reset();
- LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
- sqlInterpreter.open();
- result = sqlInterpreter.interpret("select * from df", context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
- // TODO (zjffdu), \t at the end of each line is not necessary, it is a bug of LivySparkSQLInterpreter
- assertEquals("_1\t_2\t\nhello\t20\t\n", result.message().get(0).getData());
-
- // single line comment
- outputListener.reset();
- String singleLineComment = "// my comment";
- result = sparkInterpreter.interpret(singleLineComment, context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
- assertNull(result.message().get(0).getData());
-
- // multiple line comment
- outputListener.reset();
- String multipleLineComment = "/* multiple \n" + "line \n" + "comment */";
- result = sparkInterpreter.interpret(multipleLineComment, context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
- assertNull(result.message().get(0).getData());
-
- // multi-line string
- outputListener.reset();
- String multiLineString = "val str = \"\"\"multiple\n" +
- "line\"\"\"\n" +
- "println(str)";
- result = sparkInterpreter.interpret(multiLineString, context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
- assertNull(result.message().get(0).getData());
- assertTrue(outputListener.getOutputAppended().contains("multiple\nline"));
-
- // case class
- outputListener.reset();
- String caseClassCode = "case class Person(id:Int, \n" +
- "name:String)\n" +
- "val p=Person(1, \"name_a\")";
- result = sparkInterpreter.interpret(caseClassCode, context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
- assertNull(result.message().get(0).getData());
- assertTrue(outputListener.getOutputAppended().contains("defined class Person"));
-
- // object class
- outputListener.reset();
- String objectClassCode = "object Person {}";
- result = sparkInterpreter.interpret(objectClassCode, context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
- assertNull(result.message().get(0).getData());
- assertTrue(outputListener.getOutputAppended().contains("defined module Person"));
- }
-
- @Test
- public void testPySparkInterpreter() {
- if (!checkPreCondition()) {
- return;
- }
-
- LivyPySparkInterpreter pysparkInterpreter = new LivyPySparkInterpreter(properties);
- AuthenticationInfo authInfo = new AuthenticationInfo("user1");
- MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
- InterpreterOutput output = new InterpreterOutput(outputListener);
- InterpreterContext context = new InterpreterContext("noteId", "paragraphId", null, "title",
- "text", authInfo, null, null, null, null, null, output);
- pysparkInterpreter.open();
- InterpreterResult result = pysparkInterpreter.interpret("sc.version", context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
-
- // test RDD api
- outputListener.reset();
- result = pysparkInterpreter.interpret("sc.range(1, 10).sum()", context);
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
- assertEquals("45", result.message().get(0).getData());
-
- // test DataFrame api
- outputListener.reset();
- pysparkInterpreter.interpret("from pyspark.sql import SQLContext\n"
- + "sqlContext = SQLContext(sc)", context);
- result = pysparkInterpreter.interpret("df=sqlContext.createDataFrame([(\"hello\",20)])\n"
- + "df.collect()" , context);
- assertTrue(result.message().contains("[Row(_1=u'hello', _2=20)]"));
- assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
- assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- }
-
- @Test
- public void testSparkRInterpreter() {
- if (!checkPreCondition()) {
- return;
- }
- // TODO (zjffdu), Livy's SparkRIntepreter has some issue, do it after livy-0.3 release.
- }
-
- public static class MyInterpreterOutputListener implements InterpreterOutputListener {
- private StringBuilder outputAppended = new StringBuilder();
- private StringBuilder outputUpdated = new StringBuilder();
-
- @Override
- public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
- LOGGER.info("onAppend:" + new String(line));
- outputAppended.append(new String(line));
- }
-
- @Override
- public void onUpdate(int index, InterpreterResultMessageOutput out) {
- try {
- LOGGER.info("onUpdate:" + new String(out.toByteArray()));
- outputUpdated.append(new String(out.toByteArray()));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public String getOutputAppended() {
- return outputAppended.toString();
- }
-
- public String getOutputUpdated() {
- return outputUpdated.toString();
- }
-
- public void reset() {
- outputAppended = new StringBuilder();
- outputUpdated = new StringBuilder();
- }
-
- @Override
- public void onUpdateAll(InterpreterOutput out) {
-
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
----------------------------------------------------------------------
diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
new file mode 100644
index 0000000..e996694
--- /dev/null
+++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.livy;
+
+
+import com.cloudera.livy.test.framework.Cluster;
+import com.cloudera.livy.test.framework.Cluster$;
+import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class LivyInterpreterIT {
+
+ private static Logger LOGGER = LoggerFactory.getLogger(LivyInterpreterIT.class);
+ private static Cluster cluster;
+ private static Properties properties;
+
+ @BeforeClass
+ public static void setUp() {
+ if (!checkPreCondition()) {
+ return;
+ }
+ cluster = Cluster$.MODULE$.get();
+ LOGGER.info("Starting livy at {}", cluster.livyEndpoint());
+ properties = new Properties();
+ properties.setProperty("zeppelin.livy.url", cluster.livyEndpoint());
+ properties.setProperty("zeppelin.livy.create.session.retries", "120");
+ properties.setProperty("zeppelin.livy.spark.sql.maxResult", "100");
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ if (cluster != null) {
+ cluster.cleanUp();
+ }
+ }
+
+ public static boolean checkPreCondition() {
+ if (System.getenv("LIVY_HOME") == null) {
+ LOGGER.warn(("livy integration is skipped because LIVY_HOME is not set"));
+ return false;
+ }
+ if (System.getenv("SPARK_HOME") == null) {
+ LOGGER.warn(("livy integration is skipped because SPARK_HOME is not set"));
+ return false;
+ }
+ return true;
+ }
+
+ @Test
+ public void testSparkInterpreterRDD() {
+ if (!checkPreCondition()) {
+ return;
+ }
+ InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
+ interpreterGroup.put("session_1", new ArrayList<Interpreter>());
+ LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
+ sparkInterpreter.setInterpreterGroup(interpreterGroup);
+ interpreterGroup.get("session_1").add(sparkInterpreter);
+ AuthenticationInfo authInfo = new AuthenticationInfo("user1");
+ MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
+ InterpreterOutput output = new InterpreterOutput(outputListener);
+ InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
+ "title", "text", authInfo, null, null, null, null, null, output);
+ sparkInterpreter.open();
+
+ try {
+ InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+ assertTrue(outputListener.getOutputAppended().contains("1.5.2"));
+
+ // test RDD api
+ outputListener.reset();
+ result = sparkInterpreter.interpret("sc.parallelize(1 to 10).sum()", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+ assertTrue(outputListener.getOutputAppended().contains("Double = 55.0"));
+
+ // single line comment
+ outputListener.reset();
+ String singleLineComment = "// my comment";
+ result = sparkInterpreter.interpret(singleLineComment, context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+
+ // multiple line comment
+ outputListener.reset();
+ String multipleLineComment = "/* multiple \n" + "line \n" + "comment */";
+ result = sparkInterpreter.interpret(multipleLineComment, context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+
+ // multi-line string
+ outputListener.reset();
+ String multiLineString = "val str = \"\"\"multiple\n" +
+ "line\"\"\"\n" +
+ "println(str)";
+ result = sparkInterpreter.interpret(multiLineString, context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+ assertTrue(outputListener.getOutputAppended().contains("multiple\nline"));
+
+ // case class
+ outputListener.reset();
+ String caseClassCode = "case class Person(id:Int, \n" +
+ "name:String)\n" +
+ "val p=Person(1, \"name_a\")";
+ result = sparkInterpreter.interpret(caseClassCode, context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+ assertTrue(outputListener.getOutputAppended().contains("defined class Person"));
+
+ // object class
+ outputListener.reset();
+ String objectClassCode = "object Person {}";
+ result = sparkInterpreter.interpret(objectClassCode, context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+ assertTrue(outputListener.getOutputAppended().contains("defined module Person"));
+
+ // error
+ result = sparkInterpreter.interpret("println(a)", context);
+ assertEquals(InterpreterResult.Code.ERROR, result.code());
+ assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
+ assertTrue(result.message().get(0).getData().contains("error: not found: value a"));
+ } finally {
+ sparkInterpreter.close();
+ }
+ }
+
+ @Test
+ public void testSparkInterpreterDataFrame() {
+ if (!checkPreCondition()) {
+ return;
+ }
+ InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
+ interpreterGroup.put("session_1", new ArrayList<Interpreter>());
+ LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
+ sparkInterpreter.setInterpreterGroup(interpreterGroup);
+ interpreterGroup.get("session_1").add(sparkInterpreter);
+ AuthenticationInfo authInfo = new AuthenticationInfo("user1");
+ MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
+ InterpreterOutput output = new InterpreterOutput(outputListener);
+ InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
+ "title", "text", authInfo, null, null, null, null, null, output);
+ sparkInterpreter.open();
+
+ LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
+ interpreterGroup.get("session_1").add(sqlInterpreter);
+ sqlInterpreter.setInterpreterGroup(interpreterGroup);
+ sqlInterpreter.open();
+
+ try {
+ // test DataFrame api
+ outputListener.reset();
+ sparkInterpreter.interpret("val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n"
+ + "import sqlContext.implicits._", context);
+ InterpreterResult result = sparkInterpreter.interpret("val df=sqlContext.createDataFrame(Seq((\"hello\",20)))\n"
+ + "df.collect()", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+ assertTrue(outputListener.getOutputAppended()
+ .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
+ sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
+
+ // test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter
+ outputListener.reset();
+ result = sqlInterpreter.interpret("select * from df", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
+ // TODO(zjffdu), \t at the end of each line is not necessary, it is a bug of LivySparkSQLInterpreter
+ assertEquals("_1\t_2\t\nhello\t20\t\n", result.message().get(0).getData());
+ } finally {
+ sparkInterpreter.close();
+ sqlInterpreter.close();
+ }
+ }
+
+ @Test
+ public void testSparkSQLInterpreter() {
+ if (!checkPreCondition()) {
+ return;
+ }
+ InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
+ interpreterGroup.put("session_1", new ArrayList<Interpreter>());
+ LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
+ sparkInterpreter.setInterpreterGroup(interpreterGroup);
+ interpreterGroup.get("session_1").add(sparkInterpreter);
+ LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
+ interpreterGroup.get("session_1").add(sqlInterpreter);
+ sqlInterpreter.setInterpreterGroup(interpreterGroup);
+ sqlInterpreter.open();
+
+ try {
+ AuthenticationInfo authInfo = new AuthenticationInfo("user1");
+ MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
+ InterpreterOutput output = new InterpreterOutput(outputListener);
+ InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.sql",
+ "title", "text", authInfo, null, null, null, null, null, output);
+ InterpreterResult result = sqlInterpreter.interpret("show tables", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
+ assertTrue(result.message().get(0).getData().contains("tableName"));
+ } finally {
+ sqlInterpreter.close();
+ }
+ }
+
+ @Test
+ public void testPySparkInterpreter() {
+ if (!checkPreCondition()) {
+ return;
+ }
+
+ LivyPySparkInterpreter pysparkInterpreter = new LivyPySparkInterpreter(properties);
+ AuthenticationInfo authInfo = new AuthenticationInfo("user1");
+ MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
+ InterpreterOutput output = new InterpreterOutput(outputListener);
+ InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.pyspark",
+ "title", "text", authInfo, null, null, null, null, null, output);
+ pysparkInterpreter.open();
+
+ try {
+ InterpreterResult result = pysparkInterpreter.interpret("sc.version", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+ assertTrue(outputListener.getOutputAppended().contains("1.5.2"));
+
+ // test RDD api
+ outputListener.reset();
+ result = pysparkInterpreter.interpret("sc.range(1, 10).sum()", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+ assertTrue(outputListener.getOutputAppended().contains("45"));
+
+ // test DataFrame api
+ outputListener.reset();
+ pysparkInterpreter.interpret("from pyspark.sql import SQLContext\n"
+ + "sqlContext = SQLContext(sc)", context);
+ result = pysparkInterpreter.interpret("df=sqlContext.createDataFrame([(\"hello\",20)])\n"
+ + "df.collect()", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(0, result.message().size());
+ assertTrue(outputListener.getOutputAppended().contains("[Row(_1=u'hello', _2=20)]"));
+
+ // error
+ result = pysparkInterpreter.interpret("print(a)", context);
+ assertEquals(InterpreterResult.Code.ERROR, result.code());
+ assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
+ assertTrue(result.message().get(0).getData().contains("name 'a' is not defined"));
+ } finally {
+ pysparkInterpreter.close();
+ }
+ }
+
+ @Test
+ public void testSparkRInterpreter() {
+ if (!checkPreCondition()) {
+ return;
+ }
+ // TODO(zjffdu), Livy's SparkRIntepreter has some issue, do it after livy-0.3 release.
+ }
+
+ public static class MyInterpreterOutputListener implements InterpreterOutputListener {
+ private StringBuilder outputAppended = new StringBuilder();
+ private StringBuilder outputUpdated = new StringBuilder();
+
+ @Override
+ public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
+ LOGGER.info("onAppend:" + new String(line));
+ outputAppended.append(new String(line));
+ }
+
+ @Override
+ public void onUpdate(int index, InterpreterResultMessageOutput out) {
+ try {
+ LOGGER.info("onUpdate:" + new String(out.toByteArray()));
+ outputUpdated.append(new String(out.toByteArray()));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public String getOutputAppended() {
+ return outputAppended.toString();
+ }
+
+ public String getOutputUpdated() {
+ return outputUpdated.toString();
+ }
+
+ public void reset() {
+ outputAppended = new StringBuilder();
+ outputUpdated = new StringBuilder();
+ }
+
+ @Override
+ public void onUpdateAll(InterpreterOutput out) {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterTest.java
----------------------------------------------------------------------
diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterTest.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterTest.java
deleted file mode 100644
index b21afa4..0000000
--- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.livy;
-
-
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.hamcrest.CoreMatchers;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ErrorCollector;
-import org.junit.runner.RunWith;
-import org.mockito.Answers;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.runners.MockitoJUnitRunner;
-
-import java.util.HashMap;
-import java.util.Properties;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.doReturn;
-
-@RunWith(MockitoJUnitRunner.class)
-public class LivyInterpreterTest {
-
- @Rule
- public ErrorCollector collector = new ErrorCollector();
-
- private static LivyPySparkInterpreter interpreter;
-
- @Mock(answer = Answers.RETURNS_DEEP_STUBS)
- private InterpreterContext interpreterContext;
-
- @AfterClass
- public static void tearDown() {
- interpreter.close();
- }
-
- @Before
- public void prepareContext() throws Exception {
- interpreter = new LivyPySparkInterpreter(new Properties());
- interpreter.userSessionMap = new HashMap<>();
- interpreter.userSessionMap.put(null, 0);
- interpreter.livyHelper = Mockito.mock(LivyHelper.class);
- interpreter.open();
-
- doReturn(new InterpreterResult(InterpreterResult.Code.SUCCESS)).when(interpreter.livyHelper)
- .interpret("print \"x is 1.\"", interpreterContext, interpreter.userSessionMap);
- }
-
- @Test
- public void checkInitVariables() throws Exception {
- collector.checkThat("Check that, if userSessionMap is made: ",
- interpreter.userSessionMap, CoreMatchers.notNullValue());
- }
-
- @Test
- public void checkBasicInterpreter() throws Exception {
-
- String paragraphString = "print \"x is 1.\"";
-
- final InterpreterResult actual = interpreter.interpret(paragraphString, interpreterContext);
-
- collector.checkThat("Check that, result is computed: ",
- actual.code(), CoreMatchers.equalTo(InterpreterResult.Code.SUCCESS));
- assertThat(actual).isNotNull();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/04c62e41/testing/downloadLivy.sh
----------------------------------------------------------------------
diff --git a/testing/downloadLivy.sh b/testing/downloadLivy.sh
new file mode 100755
index 0000000..8e5418d
--- /dev/null
+++ b/testing/downloadLivy.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [[ "$#" -ne 1 ]]; then
+ echo "usage) $0 [livy version]"
+ echo " eg) $0 0.2"
+ exit 0
+fi
+
+LIVY_VERSION="${1}"
+
+set -xe
+
+MAX_DOWNLOAD_TIME_SEC=590
+FWDIR="$(dirname "${BASH_SOURCE-$0}")"
+ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
+
+#######################################
+# Downloads file from the givrn URL.
+# Ties 3 times with 1s delay, 20s read and 15s connection timeouts.
+# Globals:
+# None
+# Arguments:
+# url - source URL
+# Returns:
+# None
+#######################################
+download_with_retry() {
+ local url="$1"
+ wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 3 "${url}"
+ if [[ "$?" -ne 0 ]]; then
+ echo "3 download attempts for ${url} failed"
+ fi
+}
+
+LIVY_CACHE=".livy-dist"
+LIVY_ARCHIVE="livy-assembly-${LIVY_VERSION}"
+export LIVY_HOME="${ZEPPELIN_HOME}/livy-server-$LIVY_VERSION"
+echo "LIVY_HOME is ${LIVY_HOME}"
+
+if [[ ! -d "${LIVY_HOME}" ]]; then
+ mkdir -p "${LIVY_CACHE}"
+ cd "${LIVY_CACHE}"
+ if [[ ! -f "${LIVY_ARCHIVE}.tar.gz" ]]; then
+ pwd
+ ls -la .
+ echo "${LIVY_CACHE} does not have ${LIVY_ARCHIVE} downloading ..."
+
+ # download livy from archive if not cached
+ echo "${LIVY_VERSION} being downloaded from archives"
+ STARTTIME=`date +%s`
+ download_with_retry "https://oss.sonatype.org/content/repositories/releases/com/cloudera/livy/livy-assembly/${LIVY_VERSION}/${LIVY_ARCHIVE}.zip"
+ ENDTIME=`date +%s`
+ DOWNLOADTIME="$((ENDTIME-STARTTIME))"
+ fi
+
+ # extract archive in un-cached root, clean-up on failure
+ cp "${LIVY_ARCHIVE}.zip" ..
+ cd ..
+ if ! unzip "${LIVY_ARCHIVE}.zip" ; then
+ echo "Unable to extract ${LIVY_ARCHIVE}.zip" >&2
+ rm -rf "${LIVY_ARCHIVE}"
+ rm -f "${LIVY_ARCHIVE}.zip"
+ fi
+fi
+
+set +xe