You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by js...@apache.org on 2017/11/27 08:25:58 UTC
incubator-livy git commit: [LIVY-399][TEST] Enable real test for
PySpark and SparkR interpreters
Repository: incubator-livy
Updated Branches:
refs/heads/master f893d1991 -> cd777e209
[LIVY-399][TEST] Enable real test for PySpark and SparkR interpreters
Currently because we lack pyspark and sparkr dependencies in our environment, so we neglect the pyspark and sparkr integration tests, here propose a way to enable these integration tests. The solution is mainly borrowed from Zeppelin - to download Spark binary before integration tests, so that we will have a complete Spark environment to do the test. Because of this several modules like minicluster-dependencies are not required, so here removed such modules.
Besides, we proposed to remove real test implementations, since we never used real test before and it mainly did what scripts should do with Scala code, which makes it hard to maintain. Also for now mini-cluster test is quite close to real test, so it is not so useful to still keep real test.
Author: jerryshao <ss...@hortonworks.com>
Closes #49 from jerryshao/LIVY-399.
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/cd777e20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/cd777e20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/cd777e20
Branch: refs/heads/master
Commit: cd777e209ae26d9f66814fed6565b029da3b2479
Parents: f893d19
Author: jerryshao <ss...@hortonworks.com>
Authored: Mon Nov 27 16:25:52 2017 +0800
Committer: jerryshao <ss...@hortonworks.com>
Committed: Mon Nov 27 16:25:52 2017 +0800
----------------------------------------------------------------------
.travis.yml | 15 +-
dev/travis-upload-logs.sh | 39 ---
.../minicluster-dependencies/pom.xml | 135 ---------
.../minicluster-dependencies/scala-2.10/pom.xml | 36 ---
.../minicluster-dependencies/scala-2.11/pom.xml | 36 ---
.../src/main/resources/build.marker | 0
integration-test/pom.xml | 97 ++-----
.../framework/BaseIntegrationTestSuite.scala | 18 --
.../apache/livy/test/framework/Cluster.scala | 3 -
.../livy/test/framework/MiniCluster.scala | 41 +--
.../livy/test/framework/RealCluster.scala | 277 -------------------
.../src/test/resources/test_python_api.py | 8 +-
.../scala/org/apache/livy/test/BatchIT.scala | 6 +-
.../org/apache/livy/test/InteractiveIT.scala | 7 +-
.../scala/org/apache/livy/test/JobApiIT.scala | 13 +-
pom.xml | 28 +-
.../apache/livy/repl/PythonInterpreter.scala | 2 +-
17 files changed, 76 insertions(+), 685 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 8de73e3..c31114a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -57,17 +57,17 @@ before_install:
- sudo apt-get -y install python3-pip python-dev
- sudo apt-get -y install libkrb5-dev
- sudo apt-get -y remove python-setuptools
- - pip2 install --user --upgrade pip "setuptools < 36"
- - pip3 install --user --upgrade pip "setuptools < 36"
- - pip2 install --user codecov cloudpickle
- - pip3 install --user cloudpickle
+ - sudo pip2 install --upgrade pip "setuptools < 36"
+ - sudo pip3 install --upgrade pip "setuptools < 36"
+ - sudo pip2 install codecov cloudpickle
+ - sudo pip3 install cloudpickle
install:
- mvn $MVN_FLAG install -Dskip -DskipTests -DskipITs -Dmaven.javadoc.skip=true -B -V
before_script:
- - pip2 install --user "requests >= 2.10.0" pytest flaky flake8 requests-kerberos
- - pip3 install --user "requests >= 2.10.0" pytest flaky requests-kerberos
+ - sudo pip2 install "requests >= 2.10.0" pytest flaky flake8 requests-kerberos
+ - sudo pip3 install "requests >= 2.10.0" pytest flaky requests-kerberos
script:
- mvn $MVN_FLAG verify -e
@@ -76,5 +76,4 @@ after_success:
- bash <(curl -s https://codecov.io/bash)
after_failure:
- - npm list -g azure-cli --depth=0 || npm install -g azure-cli
- - ./dev/travis-upload-logs.sh
+ - for log in `find * -name "*.log" -o -name "stderr" -o -name "stdout"`; do echo "=========$log========="; cat $log; done
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/dev/travis-upload-logs.sh
----------------------------------------------------------------------
diff --git a/dev/travis-upload-logs.sh b/dev/travis-upload-logs.sh
deleted file mode 100755
index 97333d1..0000000
--- a/dev/travis-upload-logs.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -e
-
-if [ -z "$TRAVIS_JOB_NUMBER" ]; then
- echo "TRAVIS_JOB_NUMBER isn't defined."
- exit 1
-fi
-
-if [ -z "$LOG_AZURE_STORAGE_CONNECTION_STRING" ]; then
- echo "LOG_AZURE_STORAGE_CONNECTION_STRING isn't defined."
- exit 1
-fi
-
-export AZURE_STORAGE_CONNECTION_STRING="$LOG_AZURE_STORAGE_CONNECTION_STRING"
-
-LOG_ZIP_NAME="$TRAVIS_JOB_NUMBER.zip"
-LOG_ZIP_PATH="assembly/target/$LOG_ZIP_NAME"
-
-find * -name "*.log" -o -name "stderr" -o -name "stdout" | zip -@ $LOG_ZIP_PATH
-
-azure telemetry --disable
-azure storage blob upload -q $LOG_ZIP_PATH buildlogs $LOG_ZIP_NAME && echo "===== Build log uploaded to https://livy.blob.core.windows.net/buildlogs/$LOG_ZIP_NAME ====="
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/minicluster-dependencies/pom.xml
----------------------------------------------------------------------
diff --git a/integration-test/minicluster-dependencies/pom.xml b/integration-test/minicluster-dependencies/pom.xml
deleted file mode 100644
index a1c54ee..0000000
--- a/integration-test/minicluster-dependencies/pom.xml
+++ /dev/null
@@ -1,135 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
--->
-<!-- This module is created to generate classpath for MiniCluster to run integration test with different
- Scala version -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.livy</groupId>
- <artifactId>multi-scala-project-root</artifactId>
- <relativePath>../../scala/pom.xml</relativePath>
- <version>0.5.0-incubating-SNAPSHOT</version>
- </parent>
- <artifactId>minicluster-dependencies-parent</artifactId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- <packaging>pom</packaging>
- <properties>
- <skipDeploy>true</skipDeploy>
- </properties>
- <dependencies>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>2.5</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-yarn_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
- <dependency>
- <groupId>org.objenesis</groupId>
- <artifactId>objenesis</artifactId>
- <version>1.2</version>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals><goal>build-classpath</goal></goals>
- <configuration>
- <outputFile>target/classpath</outputFile>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/minicluster-dependencies/scala-2.10/pom.xml
----------------------------------------------------------------------
diff --git a/integration-test/minicluster-dependencies/scala-2.10/pom.xml b/integration-test/minicluster-dependencies/scala-2.10/pom.xml
deleted file mode 100644
index 905c4cb..0000000
--- a/integration-test/minicluster-dependencies/scala-2.10/pom.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.livy</groupId>
- <artifactId>minicluster-dependencies_2.10</artifactId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <parent>
- <groupId>org.apache.livy</groupId>
- <artifactId>minicluster-dependencies-parent</artifactId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <properties>
- <scala.version>${scala-2.10.version}</scala.version>
- <scala.binary.version>2.10</scala.binary.version>
- </properties>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/minicluster-dependencies/scala-2.11/pom.xml
----------------------------------------------------------------------
diff --git a/integration-test/minicluster-dependencies/scala-2.11/pom.xml b/integration-test/minicluster-dependencies/scala-2.11/pom.xml
deleted file mode 100644
index 245b119..0000000
--- a/integration-test/minicluster-dependencies/scala-2.11/pom.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.livy</groupId>
- <artifactId>minicluster-dependencies_2.11</artifactId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <parent>
- <groupId>org.apache.livy</groupId>
- <artifactId>minicluster-dependencies-parent</artifactId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <properties>
- <scala.version>${scala-2.11.version}</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
- </properties>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/minicluster-dependencies/src/main/resources/build.marker
----------------------------------------------------------------------
diff --git a/integration-test/minicluster-dependencies/src/main/resources/build.marker b/integration-test/minicluster-dependencies/src/main/resources/build.marker
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index 145f215..0c08504 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -38,8 +38,6 @@
-->
<cluster.spec>default</cluster.spec>
<skipDeploy>true</skipDeploy>
- <skipITs-2.10>false</skipITs-2.10>
- <skipITs-2.11>false</skipITs-2.11>
</properties>
<dependencies>
@@ -65,44 +63,12 @@
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>minicluster-dependencies_2.10</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>minicluster-dependencies_2.11</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
<artifactId>livy-test-lib</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.decodified</groupId>
- <artifactId>scala-ssh_${scala.binary.version}</artifactId>
- <version>0.7.0</version>
- </dependency>
-
- <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
@@ -209,6 +175,26 @@
<build>
<plugins>
<plugin>
+ <groupId>com.googlecode.maven-download-plugin</groupId>
+ <artifactId>download-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>download-spark-files</id>
+ <goals>
+ <goal>wget</goal>
+ </goals>
+ <configuration>
+ <readTimeOut>60000</readTimeOut>
+ <retries>5</retries>
+ <url>${spark.bin.download.url}</url>
+ <outputDirectory>${project.build.directory}</outputDirectory>
+ <unpack>true</unpack>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
@@ -237,6 +223,7 @@
</execution>
</executions>
</plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@@ -247,6 +234,7 @@
</execution>
</executions>
</plugin>
+
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
@@ -257,37 +245,19 @@
<phase>none</phase>
</execution>
<execution>
- <id>integration-test-scala-2.10</id>
+ <id>integration-test</id>
<phase>integration-test</phase>
<goals>
<goal>test</goal>
</goals>
- <configuration>
- <environmentVariables>
- <LIVY_SPARK_SCALA_VERSION>2.10</LIVY_SPARK_SCALA_VERSION>
- </environmentVariables>
- <skipTests>${skipITs-2.10}</skipTests>
- </configuration>
- </execution>
- <execution>
- <id>integration-test-scala-2.11</id>
- <phase>integration-test</phase>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <environmentVariables>
- <LIVY_SPARK_SCALA_VERSION>2.11</LIVY_SPARK_SCALA_VERSION>
- </environmentVariables>
- <skipTests>${skipITs-2.11}</skipTests>
- </configuration>
</execution>
</executions>
<configuration>
<environmentVariables>
<LIVY_HOME>${execution.root}</LIVY_HOME>
- <LIVY_TEST>true</LIVY_TEST>
+ <LIVY_TEST>false</LIVY_TEST>
<LIVY_INTEGRATION_TEST>true</LIVY_INTEGRATION_TEST>
+ <SPARK_HOME>${project.build.directory}/${spark.bin.name}</SPARK_HOME>
</environmentVariables>
<systemProperties>
<cluster.spec>${cluster.spec}</cluster.spec>
@@ -300,18 +270,6 @@
<profiles>
<profile>
- <id>real-spark-home</id>
- <activation>
- <property>
- <name>real.spark.home</name>
- </property>
- </activation>
- <properties>
- <spark.home>${real.spark.home}</spark.home>
- </properties>
- </profile>
-
- <profile>
<id>skip-integration-test</id>
<activation>
<property>
@@ -319,11 +277,6 @@
<value>true</value>
</property>
</activation>
- <properties>
- <skipITs-2.10>true</skipITs-2.10>
- <skipITs-2.11>true</skipITs-2.11>
- </properties>
</profile>
</profiles>
-
</project>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/src/main/scala/org/apache/livy/test/framework/BaseIntegrationTestSuite.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/BaseIntegrationTestSuite.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/BaseIntegrationTestSuite.scala
index d770e7e..19a423a 100644
--- a/integration-test/src/main/scala/org/apache/livy/test/framework/BaseIntegrationTestSuite.scala
+++ b/integration-test/src/main/scala/org/apache/livy/test/framework/BaseIntegrationTestSuite.scala
@@ -69,24 +69,6 @@ abstract class BaseIntegrationTestSuite extends FunSuite with Matchers with Befo
hdfsPath.toUri().getPath()
}
- /** Wrapper around test() to be used by pyspark tests. */
- protected def pytest(desc: String)(testFn: => Unit): Unit = {
- test(desc) {
- assume(cluster.isRealSpark(), "PySpark tests require a real Spark installation.")
- testFn
- }
- }
-
- /** Wrapper around test() to be used by SparkR tests. */
- protected def rtest(desc: String)(testFn: => Unit): Unit = {
- test(desc) {
- assume(!sys.props.getOrElse("skipRTests", "false").toBoolean, "Skipping R tests.")
- assume(cluster.isRealSpark(), "SparkR tests require a real Spark installation.")
- assume(cluster.hasSparkR(), "Spark under test does not support R.")
- testFn
- }
- }
-
/** Clean up session and show info when test fails. */
protected def withSession[S <: LivyRestClient#Session, R]
(s: S)
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/src/main/scala/org/apache/livy/test/framework/Cluster.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/Cluster.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/Cluster.scala
index e1b6844..26b08a9 100644
--- a/integration-test/src/main/scala/org/apache/livy/test/framework/Cluster.scala
+++ b/integration-test/src/main/scala/org/apache/livy/test/framework/Cluster.scala
@@ -38,8 +38,6 @@ trait Cluster {
def deploy(): Unit
def cleanUp(): Unit
def configDir(): File
- def isRealSpark(): Boolean
- def hasSparkR(): Boolean
def runLivy(): Unit
def stopLivy(): Unit
@@ -101,7 +99,6 @@ object Cluster extends Logging {
var _cluster: Cluster = null
try {
_cluster = config.get(CLUSTER_TYPE) match {
- case Some("real") => new RealCluster(config)
case Some("mini") => new MiniCluster(config)
case t => throw new Exception(s"Unknown or unset cluster.type $t")
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
index e048a7d..3cfc772 100644
--- a/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
+++ b/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
@@ -53,13 +53,6 @@ private class MiniClusterConfig(val config: Map[String, String]) {
}
sealed trait MiniClusterUtils extends ClusterUtils {
- private val livySparkScalaVersionEnvVarName = "LIVY_SPARK_SCALA_VERSION"
-
- protected def getSparkScalaVersion(): String = {
- sys.env.getOrElse(livySparkScalaVersionEnvVarName, {
- throw new RuntimeException(s"Please specify env var $livySparkScalaVersionEnvVarName.")
- })
- }
protected def saveConfig(conf: Configuration, dest: File): Unit = {
val redacted = new Configuration(conf)
@@ -155,7 +148,6 @@ object MiniLivyMain extends MiniClusterBase {
var livyConf = Map(
LivyConf.LIVY_SPARK_MASTER.key -> "yarn",
LivyConf.LIVY_SPARK_DEPLOY_MODE.key -> "cluster",
- LivyConf.LIVY_SPARK_SCALA_VERSION.key -> getSparkScalaVersion(),
LivyConf.HEARTBEAT_WATCHDOG_INTERVAL.key -> "1s",
LivyConf.YARN_POLL_INTERVAL.key -> "500ms",
LivyConf.RECOVERY_MODE.key -> "recovery",
@@ -210,15 +202,6 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU
override def hdfsScratchDir(): Path = _hdfsScrathDir
- override def isRealSpark(): Boolean = {
- new File(sys.env("SPARK_HOME") + File.separator + "RELEASE").isFile()
- }
-
- override def hasSparkR(): Boolean = {
- val path = Seq(sys.env("SPARK_HOME"), "R", "lib", "sparkr.zip").mkString(File.separator)
- new File(path).isFile()
- }
-
override def doAsClusterUser[T](task: => T): T = task
// Explicitly remove the "test-lib" dependency from the classpath of child processes. We
@@ -237,29 +220,7 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU
assert(tempDir.mkdir(), "Cannot create temp test dir.")
sparkConfDir = mkdir("spark-conf")
- // When running a real Spark cluster, don't set the classpath.
- val extraCp = if (!isRealSpark()) {
- val sparkScalaVersion = getSparkScalaVersion()
- val classPathFile =
- new File(s"minicluster-dependencies/scala-$sparkScalaVersion/target/classpath")
- assert(classPathFile.isFile,
- s"Cannot read MiniCluster classpath file: ${classPathFile.getCanonicalPath}")
- val sparkClassPath =
- FileUtils.readFileToString(classPathFile, Charset.defaultCharset())
-
- val dummyJar = Files.createTempFile(Paths.get(tempDir.toURI), "dummy", "jar").toFile
- Map(
- SparkLauncher.DRIVER_EXTRA_CLASSPATH -> sparkClassPath,
- SparkLauncher.EXECUTOR_EXTRA_CLASSPATH -> sparkClassPath,
- // Used for Spark 2.0. Spark 2.0 will upload specified jars to distributed cache in yarn
- // mode, if not specified it will check jars folder. Here since jars folder is not
- // existed, so it will throw exception.
- "spark.yarn.jars" -> dummyJar.getAbsolutePath)
- } else {
- Map()
- }
-
- val sparkConf = extraCp ++ Map(
+ val sparkConf = Map(
"spark.executor.instances" -> "1",
"spark.scheduler.minRegisteredResourcesRatio" -> "0.0",
"spark.ui.enabled" -> "false",
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/src/main/scala/org/apache/livy/test/framework/RealCluster.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/RealCluster.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/RealCluster.scala
deleted file mode 100644
index 0acc580..0000000
--- a/integration-test/src/main/scala/org/apache/livy/test/framework/RealCluster.scala
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.livy.test.framework
-
-import java.io.{File, IOException}
-import java.nio.file.Files
-import java.security.PrivilegedExceptionAction
-import javax.servlet.http.HttpServletResponse._
-
-import scala.collection.JavaConverters._
-import scala.concurrent.duration._
-import scala.language.postfixOps
-import scala.sys.process._
-import scala.util.Random
-
-import com.decodified.scalassh._
-import com.ning.http.client.AsyncHttpClient
-import org.apache.hadoop.fs.{FileSystem, Path}
-import org.apache.hadoop.fs.permission.FsPermission
-import org.apache.hadoop.security.UserGroupInformation
-import org.scalatest.concurrent.Eventually._
-
-import org.apache.livy.{LivyConf, Logging}
-
-private class RealClusterConfig(config: Map[String, String]) {
- val ip = config("ip")
-
- val sshLogin = config("ssh.login")
- val sshPubKey = config("ssh.pubkey")
- val livyPort = config.getOrElse(LivyConf.SERVER_PORT.key, "8998").toInt
- val livyClasspath = config.getOrElse("livy.classpath", "")
-
- val deployLivy = config.getOrElse("deploy-livy", "true").toBoolean
- val noDeployLivyHome = config.get("livy-home")
-
- val sparkHome = config("env.spark_home")
- val sparkConf = config.getOrElse("env.spark_conf", "/etc/spark/conf")
- val hadoopConf = config.getOrElse("env.hadoop_conf", "/etc/hadoop/conf")
-
- val javaHome = config.getOrElse("env.java_home", "/usr/java/default")
-}
-
-class RealCluster(_config: Map[String, String])
- extends Cluster with ClusterUtils with Logging {
-
- private val config = new RealClusterConfig(_config)
-
- private var livyIsRunning = false
- private var livyHomePath: String = _
- private var livyEpoch = 0
-
- private var _configDir: File = _
-
- private var hdfsScratch: Path = _
-
- private var sparkConfDir: String = _
- private var tempDirPath: String = _
- private var _hasSparkR: Boolean = _
-
- override def isRealSpark(): Boolean = true
-
- override def hasSparkR(): Boolean = _hasSparkR
-
- override def configDir(): File = _configDir
-
- override def hdfsScratchDir(): Path = hdfsScratch
-
- override def doAsClusterUser[T](task: => T): T = {
- val user = UserGroupInformation.createRemoteUser(config.sshLogin)
- user.doAs(new PrivilegedExceptionAction[T] {
- override def run(): T = task
- })
- }
-
- private def sshClient[T](body: SshClient => SSH.Result[T]): T = {
- val sshLogin = PublicKeyLogin(
- config.sshLogin, None, config.sshPubKey :: Nil)
- val hostConfig = HostConfig(login = sshLogin, hostKeyVerifier = HostKeyVerifiers.DontVerify)
- SSH(config.ip, hostConfig)(body) match {
- case Left(err) => throw new IOException(err)
- case Right(result) => result
- }
- }
-
- private def exec(cmd: String): CommandResult = {
- info(s"Running command: $cmd")
- val result = sshClient(_.exec(cmd))
- result.exitCode match {
- case Some(ec) if ec > 0 =>
- throw new IOException(s"Command '$cmd' failed: $ec\n" +
- s"stdout: ${result.stdOutAsString()}\n" +
- s"stderr: ${result.stdErrAsString()}\n")
- case _ =>
- }
- result
- }
-
- private def upload(local: String, remote: String): Unit = {
- info(s"Uploading local path $local")
- sshClient(_.upload(local, remote))
- }
-
- private def download(remote: String, local: String): Unit = {
- info(s"Downloading remote path $remote")
- sshClient(_.download(remote, local))
- }
-
- override def deploy(): Unit = {
- // Make sure Livy is not running.
- stopLivy()
-
- // Check whether SparkR is supported in YARN (need the sparkr.zip archive).\
- _hasSparkR = try {
- exec(s"test -f ${config.sparkHome}/R/lib/sparkr.zip")
- true
- } catch {
- case e: IOException => false
- }
-
- // Copy the remove Hadoop configuration to a local temp dir so that tests can use it to
- // talk to HDFS and YARN.
- val localTemp = new File(sys.props("java.io.tmpdir") + File.separator + "hadoop-conf")
- download(config.hadoopConf, localTemp.getAbsolutePath())
- _configDir = localTemp
-
- // Create a temp directory where test files will be written.
- tempDirPath = s"/tmp/livy-it-${Random.alphanumeric.take(16).mkString}"
- exec(s"mkdir -p $tempDirPath")
-
- // Also create an HDFS scratch directory for tests.
- doAsClusterUser {
- hdfsScratch = fs.makeQualified(
- new Path(s"/tmp/livy-it-${Random.alphanumeric.take(16).mkString}"))
- fs.mkdirs(hdfsScratch)
- fs.setPermission(hdfsScratch, new FsPermission("777"))
- }
-
- // Create a copy of the Spark configuration, and make sure the master is "yarn-cluster".
- sparkConfDir = s"$tempDirPath/spark-conf"
- val sparkProps = s"$sparkConfDir/spark-defaults.conf"
- Seq(
- s"cp -Lr ${config.sparkConf} $sparkConfDir",
- s"touch $sparkProps",
- s"sed -i.old '/spark.master.*/d' $sparkProps",
- s"sed -i.old '/spark.submit.deployMode.*/d' $sparkProps",
- s"echo 'spark.master=yarn-cluster' >> $sparkProps"
- ).foreach(exec)
-
- if (config.deployLivy) {
- try {
- info(s"Deploying Livy to ${config.ip}...")
- val version = sys.props("project.version")
- val assemblyZip = new File(s"../assembly/target/livy-server-$version.zip")
- assert(assemblyZip.isFile,
- s"Can't find livy assembly zip at ${assemblyZip.getCanonicalPath}")
- val assemblyName = assemblyZip.getName()
-
- // SSH to the node to unzip and install Livy.
- upload(assemblyZip.getCanonicalPath, s"$tempDirPath/$assemblyName")
- exec(s"unzip -o $tempDirPath/$assemblyName -d $tempDirPath")
- livyHomePath = s"$tempDirPath/livy-server-$version"
- info(s"Deployed Livy to ${config.ip} at $livyHomePath.")
- } catch {
- case e: Exception =>
- error(s"Failed to deploy Livy to ${config.ip}.", e)
- cleanUp()
- throw e
- }
- } else {
- livyHomePath = config.noDeployLivyHome.get
- info("Skipping deployment.")
- }
-
- runLivy()
- }
-
- override def cleanUp(): Unit = {
- stopLivy()
- if (tempDirPath != null) {
- exec(s"rm -rf $tempDirPath")
- }
- if (hdfsScratch != null) {
- doAsClusterUser {
- // Cannot use the shared `fs` since this runs in a shutdown hook, and that instance
- // may have been closed already.
- val fs = FileSystem.newInstance(hadoopConf)
- try {
- fs.delete(hdfsScratch, true)
- } finally {
- fs.close()
- }
- }
- }
- }
-
- override def runLivy(): Unit = synchronized {
- assert(!livyIsRunning, "Livy is running already.")
-
- val livyConf = Map(
- "livy.server.port" -> config.livyPort.toString,
- // "livy.server.recovery.mode=local",
- "livy.environment" -> "development",
- LivyConf.LIVY_SPARK_MASTER.key -> "yarn",
- LivyConf.LIVY_SPARK_DEPLOY_MODE.key -> "cluster")
- val livyConfFile = File.createTempFile("livy.", ".properties")
- saveProperties(livyConf, livyConfFile)
- upload(livyConfFile.getAbsolutePath(), s"$livyHomePath/conf/livy.conf")
-
- val env = Map(
- "JAVA_HOME" -> config.javaHome,
- "HADOOP_CONF_DIR" -> config.hadoopConf,
- "SPARK_CONF_DIR" -> sparkConfDir,
- "SPARK_HOME" -> config.sparkHome,
- "CLASSPATH" -> config.livyClasspath,
- "LIVY_PID_DIR" -> s"$tempDirPath/pid",
- "LIVY_LOG_DIR" -> s"$tempDirPath/logs",
- "LIVY_MAX_LOG_FILES" -> "16",
- "LIVY_IDENT_STRING" -> "it"
- )
- val livyEnvFile = File.createTempFile("livy-env.", ".sh")
- saveProperties(env, livyEnvFile)
- upload(livyEnvFile.getAbsolutePath(), s"$livyHomePath/conf/livy-env.sh")
-
- info(s"Starting Livy @ port ${config.livyPort}...")
- exec(s"env -i $livyHomePath/bin/livy-server start")
- livyIsRunning = true
-
- val httpClient = new AsyncHttpClient()
- eventually(timeout(1 minute), interval(1 second)) {
- assert(httpClient.prepareGet(livyEndpoint).execute().get().getStatusCode == SC_OK)
- }
- info(s"Started Livy.")
- }
-
- override def stopLivy(): Unit = synchronized {
- info("Stopping Livy Server")
- try {
- exec(s"$livyHomePath/bin/livy-server stop")
- } catch {
- case e: Exception =>
- if (livyIsRunning) {
- throw e
- }
- }
-
- if (livyIsRunning) {
- // Wait a tiny bit so that the process finishes closing its output files.
- Thread.sleep(2)
-
- livyEpoch += 1
- val logName = "livy-it-server.out"
- val localName = s"livy-it-server-$livyEpoch.out"
- val logPath = s"$tempDirPath/logs/$logName"
- val localLog = sys.props("java.io.tmpdir") + File.separator + localName
- download(logPath, localLog)
- info(s"Log for epoch $livyEpoch available at $localLog")
- }
- }
-
- override def livyEndpoint: String = s"http://${config.ip}:${config.livyPort}"
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/src/test/resources/test_python_api.py
----------------------------------------------------------------------
diff --git a/integration-test/src/test/resources/test_python_api.py b/integration-test/src/test/resources/test_python_api.py
index 7a8b22e..3749e69 100644
--- a/integration-test/src/test/resources/test_python_api.py
+++ b/integration-test/src/test/resources/test_python_api.py
@@ -46,7 +46,7 @@ def process_job(job, expected_result, is_error_job=False):
pickled_job = cloudpickle.dumps(job)
base64_pickled_job = base64.b64encode(pickled_job).decode('utf-8')
- base64_pickled_job_json = json.dumps({'job': base64_pickled_job})
+ base64_pickled_job_json = json.dumps({'job': base64_pickled_job, 'jobType': 'pyspark'})
request_url = livy_end_point + "/sessions/" + str(session_id) + "/submit-job"
header = {'Content-Type': 'application/json', 'X-Requested-By': 'livy'}
response = requests.request('POST', request_url, headers=header, data=base64_pickled_job_json)
@@ -54,8 +54,8 @@ def process_job(job, expected_result, is_error_job=False):
assert response.status_code == httplib.CREATED
job_id = response.json()['id']
- poll_time = 0.1
- max_poll_time = 10
+ poll_time = 1
+ max_poll_time = 30
poll_response = None
while (poll_response is None or poll_response.json()['state'] == 'STARTED') and poll_time < \
max_poll_time:
@@ -81,7 +81,7 @@ def process_job(job, expected_result, is_error_job=False):
def delay_rerun(*args):
- time.sleep(8)
+ time.sleep(10)
return True
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/src/test/scala/org/apache/livy/test/BatchIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/scala/org/apache/livy/test/BatchIT.scala b/integration-test/src/test/scala/org/apache/livy/test/BatchIT.scala
index 32794e7..b02301b 100644
--- a/integration-test/src/test/scala/org/apache/livy/test/BatchIT.scala
+++ b/integration-test/src/test/scala/org/apache/livy/test/BatchIT.scala
@@ -66,7 +66,7 @@ class BatchIT extends BaseIntegrationTestSuite with BeforeAndAfterAll {
}
}
- pytest("submit a pyspark application") {
+ test("submit a pyspark application") {
val scriptPath = uploadResource("batch.py")
val output = newOutputPath()
withScript(scriptPath, List(output)) { s =>
@@ -75,9 +75,7 @@ class BatchIT extends BaseIntegrationTestSuite with BeforeAndAfterAll {
}
}
- // This is disabled since R scripts don't seem to work in yarn-cluster mode. There's a
- // TODO comment in Spark's ApplicationMaster.scala.
- ignore("submit a SparkR application") {
+ test("submit a SparkR application") {
val hdfsPath = uploadResource("rtest.R")
withScript(hdfsPath, List.empty) { s =>
s.verifySessionSuccess()
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala b/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
index 43d66cb..740fa19 100644
--- a/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
+++ b/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
@@ -66,10 +66,9 @@ class InteractiveIT extends BaseIntegrationTestSuite {
}
}
- pytest("pyspark interactive session") {
+ test("pyspark interactive session") {
withNewSession(PySpark) { s =>
s.run("1+1").verifyResult("2")
- s.run("sqlContext").verifyResult(startsWith("<pyspark.sql.context.HiveContext"))
s.run("sc.parallelize(range(100)).map(lambda x: x * 2).reduce(lambda x, y: x + y)")
.verifyResult("9900")
s.run("from pyspark.sql.types import Row").verifyResult("")
@@ -81,7 +80,7 @@ class InteractiveIT extends BaseIntegrationTestSuite {
}
}
- rtest("R interactive session") {
+ test("R interactive session") {
withNewSession(SparkR) { s =>
// R's output sometimes includes the count of statements, which makes it annoying to test
// things. This helps a bit.
@@ -89,8 +88,6 @@ class InteractiveIT extends BaseIntegrationTestSuite {
def count: Int = curr.incrementAndGet()
s.run("1+1").verifyResult(startsWith(s"[$count] 2"))
- s.run("sqlContext <- sparkRSQL.init(sc)").verifyResult(null)
- s.run("hiveContext <- sparkRHive.init(sc)").verifyResult(null)
s.run("""localDF <- data.frame(name=c("John", "Smith", "Sarah"), age=c(19, 23, 18))""")
.verifyResult(null)
s.run("df <- createDataFrame(sqlContext, localDF)").verifyResult(null)
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala b/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
index 8fe9dc8..fdad6df 100644
--- a/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
+++ b/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
@@ -188,8 +188,13 @@ class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logg
protected def scalaTest(desc: String)(testFn: => Unit): Unit = {
test(desc) {
- assume(sys.env("LIVY_SPARK_SCALA_VERSION") ==
- LivySparkUtils.formatScalaVersion(Properties.versionNumberString),
+ val livyConf = new LivyConf()
+ val (sparkVersion, scalaVersion) = LivySparkUtils.sparkSubmitVersion(livyConf)
+ val formattedSparkVersion = LivySparkUtils.formatSparkVersion(sparkVersion)
+ val versionString =
+ LivySparkUtils.sparkScalaVersion(formattedSparkVersion, scalaVersion, livyConf)
+
+ assume(versionString == LivySparkUtils.formatScalaVersion(Properties.versionNumberString),
s"Scala test can only be run with ${Properties.versionString}")
testFn
}
@@ -233,7 +238,7 @@ class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logg
sessionId = -1
}
- pytest("validate Python-API requests") {
+ test("validate Python-API requests") {
val addFileContent = "hello from addfile"
val addFilePath = createTempFilesForTest("add_file", ".txt", addFileContent, true)
val addPyFileContent = "def test_add_pyfile(): return \"hello from addpyfile\""
@@ -253,7 +258,7 @@ class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logg
env.put("UPLOAD_FILE_URL", uploadFilePath)
env.put("UPLOAD_PYFILE_URL", uploadPyFilePath)
- builder.redirectOutput(new File(sys.props("java.io.tmpdir") + "/pytest_results.txt"))
+ builder.redirectOutput(new File(sys.props("java.io.tmpdir") + "/pytest_results.log"))
builder.redirectErrorStream(true)
val process = builder.start()
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5fb2a7a..97887ae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -108,6 +108,10 @@
<test.redirectToFile>true</test.redirectToFile>
<execution.root>${user.dir}</execution.root>
<spark.home>${execution.root}/dev/spark</spark.home>
+ <spark.bin.download.url>
+ https://d3kbcqa49mib13.cloudfront.net/spark-1.6.2-bin-hadoop2.6.tgz
+ </spark.bin.download.url>
+ <spark.bin.name>spark-1.6.2-bin-hadoop2.6</spark.bin.name>
<!-- used for testing, NCSARequestLog use it for access log -->
<livy.log.dir>${basedir}/target</livy.log.dir>
@@ -228,9 +232,6 @@
<module>scala-api/scala-2.11</module>
<module>server</module>
<module>test-lib</module>
- <module>integration-test/minicluster-dependencies</module>
- <module>integration-test/minicluster-dependencies/scala-2.10</module>
- <module>integration-test/minicluster-dependencies/scala-2.11</module>
<module>integration-test</module>
</modules>
@@ -791,6 +792,12 @@
<version>1.13</version>
</plugin>
+ <plugin>
+ <groupId>com.googlecode.maven-download-plugin</groupId>
+ <artifactId>download-maven-plugin</artifactId>
+ <version>1.3.0</version>
+ </plugin>
+
</plugins>
</pluginManagement>
@@ -1097,6 +1104,11 @@
</activation>
<properties>
<spark.version>2.0.1</spark.version>
+ <py4j.version>0.10.3</py4j.version>
+ <spark.bin.download.url>
+ https://d3kbcqa49mib13.cloudfront.net/spark-2.0.1-bin-hadoop2.7.tgz
+ </spark.bin.download.url>
+ <spark.bin.name>spark-2.0.1-bin-hadoop2.7</spark.bin.name>
</properties>
</profile>
@@ -1109,6 +1121,11 @@
</activation>
<properties>
<spark.version>2.1.0</spark.version>
+ <py4j.version>0.10.4</py4j.version>
+ <spark.bin.download.url>
+ https://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz
+ </spark.bin.download.url>
+ <spark.bin.name>spark-2.1.0-bin-hadoop2.7</spark.bin.name>
</properties>
</profile>
@@ -1122,6 +1139,11 @@
<properties>
<spark.version>2.2.0</spark.version>
<java.version>1.8</java.version>
+ <py4j.version>0.10.4</py4j.version>
+ <spark.bin.download.url>
+ https://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz
+ </spark.bin.download.url>
+ <spark.bin.name>spark-2.2.0-bin-hadoop2.7</spark.bin.name>
</properties>
</profile>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/cd777e20/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala b/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
index f4f5e41..b141bef 100644
--- a/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
+++ b/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
@@ -193,7 +193,7 @@ private class PythonInterpreter(
override def kind: String = "pyspark"
- private[repl] val pysparkJobProcessor =
+ private[repl] lazy val pysparkJobProcessor =
PythonInterpreter.initiatePy4jCallbackGateway(gatewayServer)
override def close(): Unit = {