You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by js...@apache.org on 2018/09/20 03:49:27 UTC
incubator-livy git commit: [LIVY-510] Remove support for JDK7.
Repository: incubator-livy
Updated Branches:
refs/heads/master f99cc32df -> d39ab356b
[LIVY-510] Remove support for JDK7.
With this change Livy will only build and run with JDK8. While this
doesn't necessarily preclude using Java 7 for running Spark itself,
that is not encouraged due to possible issues with serialization.
There are a few small changes elsewhere that I made so tests would pass,
but aren't related to the java 8 change:
- a small tweak to the fallback code that initializes the python
interpreter, which was failing with a weird reflection error.
- expose the user's home directory in the MiniYarnCluster, to allow
the tests to see pip packages installed in the user's home.
- tweak the python API tests so that the pytest cache does not pollute
the source directory.
- changed Travis to use Oracle's JDK, since the Open JDK path was
mixing Java 8 and 9 and causing errors.
- updated jacoco to latest stable.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #111 from vanzin/LIVY-510.
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/d39ab356
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/d39ab356
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/d39ab356
Branch: refs/heads/master
Commit: d39ab356b4e36d8780bd44b9d5020bfeec90cb2f
Parents: f99cc32
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Thu Sep 20 11:49:22 2018 +0800
Committer: jerryshao <sa...@gmail.com>
Committed: Thu Sep 20 11:49:22 2018 +0800
----------------------------------------------------------------------
.travis.yml | 6 +-
README.md | 6 +-
.../livy/test/framework/MiniCluster.scala | 7 +-
.../src/test/resources/test_python_api.py | 3 +-
.../scala/org/apache/livy/test/JobApiIT.scala | 16 ++--
pom.xml | 82 ++------------------
python-api/setup.cfg | 1 +
.../apache/livy/repl/PythonInterpreter.scala | 8 +-
8 files changed, 34 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 6c5eee3..d38cf4f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,17 +33,13 @@ matrix:
# Spark 2.2+ will only be verified using JDK8
# Thriftserver requires JDK8
- env: MVN_FLAG='-Pthriftserver -Pspark-2.2-it -DskipTests'
- jdk: oraclejdk8
- env: MVN_FLAG='-Pthriftserver -Pspark-2.2 -DskipITs'
- jdk: oraclejdk8
- env: MVN_FLAG='-Pthriftserver -Pspark-2.3-it -DskipTests'
- jdk: oraclejdk8
- env: MVN_FLAG='-Pthriftserver -Pspark-2.3 -DskipITs'
- jdk: oraclejdk8
jdk:
- - openjdk7
+ - oraclejdk8
addons:
apt:
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 2dc33c2..ff6ae16 100644
--- a/README.md
+++ b/README.md
@@ -28,19 +28,19 @@ To build Livy, you will need:
Debian/Ubuntu:
* mvn (from ``maven`` package or maven3 tarball)
- * openjdk-7-jdk (or Oracle Java7 jdk)
+ * openjdk-8-jdk (or Oracle JDK 8)
* Python 2.6+
* R 3.x
Redhat/CentOS:
* mvn (from ``maven`` package or maven3 tarball)
- * java-1.7.0-openjdk (or Oracle Java7 jdk)
+ * java-1.8.0-openjdk (or Oracle JDK 8)
* Python 2.6+
* R 3.x
MacOS:
* Xcode command line tools
- * Oracle's JDK 1.7+
+ * Oracle's JDK 1.8
* Maven (Homebrew)
* Python 2.6+
* R 3.x
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
index c4d4eb5..b58abe2 100644
--- a/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
+++ b/integration-test/src/main/scala/org/apache/livy/test/framework/MiniCluster.scala
@@ -101,6 +101,10 @@ object MiniYarnMain extends MiniClusterBase {
config.localDirCount, config.logDirCount)
yarnCluster.init(baseConfig)
+ // This allows applications run by YARN during the integration tests to find PIP modules
+ // installed in the user's home directory (instead of just the global ones).
+ baseConfig.set(YarnConfiguration.NM_USER_HOME_DIR, sys.env("HOME"))
+
// Install a shutdown hook for stop the service and kill all running applications.
Runtime.getRuntime().addShutdownHook(new Thread() {
override def run(): Unit = yarnCluster.stop()
@@ -283,8 +287,7 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU
sys.props("java.home") + "/bin/java",
"-Dtest.appender=console",
"-Djava.io.tmpdir=" + procTmp.getAbsolutePath(),
- "-cp", childClasspath + File.pathSeparator + configDir.getAbsolutePath(),
- "-XX:MaxPermSize=256m") ++
+ "-cp", childClasspath + File.pathSeparator + configDir.getAbsolutePath()) ++
extraJavaArgs ++
Seq(
klass.getName().stripSuffix("$"),
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/integration-test/src/test/resources/test_python_api.py
----------------------------------------------------------------------
diff --git a/integration-test/src/test/resources/test_python_api.py b/integration-test/src/test/resources/test_python_api.py
index 3749e69..d3023c0 100644
--- a/integration-test/src/test/resources/test_python_api.py
+++ b/integration-test/src/test/resources/test_python_api.py
@@ -217,7 +217,6 @@ def test_upload_pyfile():
if __name__ == '__main__':
- test_dir_path = os.getcwd() + "/src"
- value = pytest.main(test_dir_path)
+ value = pytest.main([os.path.dirname(__file__)])
if value != 0:
raise Exception("One or more test cases have failed.")
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
----------------------------------------------------------------------
diff --git a/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala b/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
index 7c0e560..93b2524 100644
--- a/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
+++ b/integration-test/src/test/scala/org/apache/livy/test/JobApiIT.scala
@@ -249,7 +249,12 @@ class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logg
val uploadPyFilePath = createTempFilesForTest("upload_pyfile", ".py",
uploadPyFileContent, false)
- val builder = new ProcessBuilder(Seq("python", createPyTestsForPythonAPI().toString).asJava)
+ val tmpDir = new File(sys.props("java.io.tmpdir")).getAbsoluteFile()
+ val testDir = Files.createTempDirectory(tmpDir.toPath(), "python-tests-").toFile()
+ val testFile = createPyTestsForPythonAPI(testDir)
+
+ val builder = new ProcessBuilder(Seq("python", testFile.getAbsolutePath()).asJava)
+ builder.directory(testDir)
val env = builder.environment()
env.put("LIVY_END_POINT", livyEndpoint)
@@ -258,8 +263,8 @@ class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logg
env.put("UPLOAD_FILE_URL", uploadFilePath)
env.put("UPLOAD_PYFILE_URL", uploadPyFilePath)
- builder.redirectOutput(new File(sys.props("java.io.tmpdir") + "/pytest_results.log"))
builder.redirectErrorStream(true)
+ builder.redirectOutput(new File(tmpDir, "pytest_results.log"))
val process = builder.start()
@@ -268,11 +273,10 @@ class JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logg
assert(process.exitValue() === 0)
}
- private def createPyTestsForPythonAPI(): File = {
- var source: InputStream = null
+ private def createPyTestsForPythonAPI(testDir: File): File = {
+ val file = Files.createTempFile(testDir.toPath(), "test_python_api-", ".py").toFile()
+ val source = getClass().getClassLoader().getResourceAsStream("test_python_api.py")
try {
- source = getClass.getClassLoader.getResourceAsStream("test_python_api.py")
- val file = Files.createTempFile("", "").toFile
Files.copy(source, file.toPath, StandardCopyOption.REPLACE_EXISTING)
file
} finally {
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e07f39d..b49c524 100644
--- a/pom.xml
+++ b/pom.xml
@@ -90,7 +90,7 @@
<httpcore.version>4.4.4</httpcore.version>
<jackson.version>2.9.5</jackson.version>
<javax.servlet-api.version>3.1.0</javax.servlet-api.version>
- <jetty.version>9.2.16.v20160414</jetty.version>
+ <jetty.version>9.3.8.v20160314</jetty.version>
<json4s.version>3.2.10</json4s.version>
<junit.version>4.11</junit.version>
<libthrift.version>0.9.3</libthrift.version>
@@ -108,9 +108,7 @@
<scala.version>${scala-2.11.version}</scala.version>
<scalatest.version>2.2.4</scalatest.version>
<scalatra.version>2.3.0</scalatra.version>
- <java.version>1.7</java.version>
- <minJavaVersion>1.7</minJavaVersion>
- <maxJavaVersion>1.8</maxJavaVersion>
+ <java.version>1.8</java.version>
<test.redirectToFile>true</test.redirectToFile>
<execution.root>${user.dir}</execution.root>
<spark.home>${execution.root}/dev/spark</spark.home>
@@ -580,26 +578,6 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
- <version>1.4.1</version>
- <executions>
- <execution>
- <id>enforce-versions</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <requireJavaVersion>
- <version>1.7</version>
- </requireJavaVersion>
- </rules>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
</plugin>
@@ -675,7 +653,6 @@
<jvmArgs>
<jvmArg>-Xms1024m</jvmArg>
<jvmArg>-Xmx1024m</jvmArg>
- <jvmArg>-XX:MaxPermSize=${MaxPermGen}</jvmArg>
<jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg>
</jvmArgs>
</configuration>
@@ -714,7 +691,7 @@
</systemProperties>
<redirectTestOutputToFile>${test.redirectToFile}</redirectTestOutputToFile>
<useFile>${test.redirectToFile}</useFile>
- <argLine>${argLine} -Xmx2g -XX:MaxPermSize=512m</argLine>
+ <argLine>${argLine} -Xmx2g</argLine>
<failIfNoTests>false</failIfNoTests>
</configuration>
</plugin>
@@ -743,7 +720,7 @@
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>WDF TestSuite.txt</filereports>
- <argLine>${argLine} -Xmx2g -XX:MaxPermSize=512m</argLine>
+ <argLine>${argLine} -Xmx2g</argLine>
</configuration>
<executions>
<execution>
@@ -818,12 +795,12 @@
<inherited>false</inherited>
<configuration>
<rules>
+ <requireJavaVersion>
+ <version>${java.version}</version>
+ </requireJavaVersion>
<requireMavenVersion>
<version>[3.0.0,)</version>
</requireMavenVersion>
- <requireJavaVersion>
- <version>[${minJavaVersion}.0,${maxJavaVersion}.1000}]</version>
- </requireJavaVersion>
<requireOS>
<family>unix</family>
</requireOS>
@@ -831,26 +808,12 @@
</configuration>
<executions>
<execution>
- <id>clean</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <phase>pre-clean</phase>
- </execution>
- <execution>
<id>default</id>
<goals>
<goal>enforce</goal>
</goals>
<phase>validate</phase>
</execution>
- <execution>
- <id>site</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <phase>pre-site</phase>
- </execution>
</executions>
</plugin>
@@ -996,7 +959,7 @@
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
- <version>0.7.7.201606060606</version>
+ <version>0.8.2</version>
<executions>
<execution>
<goals>
@@ -1065,33 +1028,6 @@
<modules>
<module>thriftserver/server</module>
</modules>
- <properties>
- <jetty.version>9.3.8.v20160314</jetty.version>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
- <executions>
- <execution>
- <id>enforce-java</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <requireJavaVersion>
- <version>1.8</version>
- </requireJavaVersion>
- </rules>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
</profile>
<!-- Spark version profiles -->
@@ -1184,7 +1120,6 @@
<spark.scala-2.11.version>2.2.0</spark.scala-2.11.version>
<spark.scala-2.10.version>2.2.0</spark.scala-2.10.version>
<spark.version>${spark.scala-2.11.version}</spark.version>
- <java.version>1.8</java.version>
<py4j.version>0.10.4</py4j.version>
<json4s.version>3.2.11</json4s.version>
</properties>
@@ -1218,7 +1153,6 @@
<spark.version>${spark.scala-2.11.version}</spark.version>
<netty.spark-2.11.version>4.1.17.Final</netty.spark-2.11.version>
<netty.spark-2.10.version>4.0.37.Final</netty.spark-2.10.version>
- <java.version>1.8</java.version>
<py4j.version>0.10.7</py4j.version>
<json4s.version>3.2.11</json4s.version>
</properties>
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/python-api/setup.cfg
----------------------------------------------------------------------
diff --git a/python-api/setup.cfg b/python-api/setup.cfg
index e2121f9..8a8799b 100644
--- a/python-api/setup.cfg
+++ b/python-api/setup.cfg
@@ -4,3 +4,4 @@ test=pytest
[tool:pytest]
addopts = --verbose
python_files = src/test/python/*/*.py
+cache_dir = target/.pytest_cache
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d39ab356/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala b/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
index 97e746f..bab2bfa 100644
--- a/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
+++ b/repl/src/main/scala/org/apache/livy/repl/PythonInterpreter.scala
@@ -201,9 +201,11 @@ object PythonInterpreter extends Logging {
.newInstance(parts(0), gateway)
} catch {
case NonFatal(e) =>
- classOf[PythonProxyHandler].getConstructor(classOf[String],
- Class.forName("py4j.CallbackClient"), classOf[Gateway])
- .newInstance(parts(0), gateway.getCallbackClient, gateway)
+ val cbClient = gateway.getClass().getMethod("getCallbackClient").invoke(gateway)
+ val cbClass = Class.forName("py4j.CallbackClient")
+ classOf[PythonProxyHandler]
+ .getConstructor(classOf[String], cbClass, classOf[Gateway])
+ .newInstance(parts(0), cbClient, gateway)
}
Proxy.newProxyInstance(Thread.currentThread.getContextClassLoader,