You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lr...@apache.org on 2022/02/10 03:46:28 UTC
[incubator-toree] branch master updated: [TOREE-525] Add support for Java 11 and Spark 3.2.0 (#194)
This is an automated email from the ASF dual-hosted git repository.
lresende pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-toree.git
The following commit(s) were added to refs/heads/master by this push:
new ca831fb [TOREE-525] Add support for Java 11 and Spark 3.2.0 (#194)
ca831fb is described below
commit ca831fb76d89f74760a899be0545c2bb0d48037b
Author: Marek P <64...@users.noreply.github.com>
AuthorDate: Thu Feb 10 04:44:34 2022 +0100
[TOREE-525] Add support for Java 11 and Spark 3.2.0 (#194)
---
.github/workflows/build.yml | 4 +++-
.jvmopts | 1 -
build.sbt | 2 +-
client/src/main/scala/org/apache/toree/comm/ClientCommWriter.scala | 3 +++
.../apache/toree/dependencies/CoursierDependencyDownloader.scala | 4 ++--
.../scala/org/apache/toree/boot/layer/InterpreterManager.scala | 5 +++--
.../src/main/scala/org/apache/toree/plugins/PluginManager.scala | 3 ++-
.../test/scala/org/apache/toree/plugins/PluginManagerSpec.scala | 3 ++-
project/Dependencies.scala | 7 ++++++-
project/build.properties | 2 +-
.../apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala | 3 +++
11 files changed, 26 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 369a995..92c2274 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -29,6 +29,8 @@ jobs:
runs-on: ubuntu-latest
strategy:
fail-fast: false
+ matrix:
+ java: [ '8', '11' ]
env:
# define Java options for both official sbt and sbt-extras
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
@@ -46,7 +48,7 @@ jobs:
uses: actions/setup-java@v2
with:
distribution: 'zulu'
- java-version: '8'
+ java-version: ${{ matrix.java }}
- name: Add SBT launcher
run: |
mkdir -p $HOME/.sbt/launchers/1.3.12
diff --git a/.jvmopts b/.jvmopts
index 3c565ee..0c9757c 100644
--- a/.jvmopts
+++ b/.jvmopts
@@ -22,7 +22,6 @@
-XX:+UseCompressedClassPointers
-XX:ReservedCodeCacheSize=256M
-XX:+TieredCompilation
--XX:+CMSPermGenSweepingEnabled
-XX:+CMSClassUnloadingEnabled
-XX:+UseConcMarkSweepGC
-XX:+HeapDumpOnOutOfMemoryError
diff --git a/build.sbt b/build.sbt
index 44b6606..7b98b56 100644
--- a/build.sbt
+++ b/build.sbt
@@ -22,7 +22,7 @@ version in ThisBuild := Properties.envOrElse("VERSION", "0.0.0-dev") +
(if ((isSnapshot in ThisBuild).value) "-SNAPSHOT" else "")
isSnapshot in ThisBuild := Properties.envOrElse("IS_SNAPSHOT","true").toBoolean
organization in ThisBuild := "org.apache.toree.kernel"
-crossScalaVersions in ThisBuild := Seq("2.12.12")
+crossScalaVersions in ThisBuild := Seq("2.12.15") // https://github.com/scala/bug/issues/12475, for Spark 3.2.0
scalaVersion in ThisBuild := (crossScalaVersions in ThisBuild).value.head
Dependencies.sparkVersion in ThisBuild := {
val envVar = "APACHE_SPARK_VERSION"
diff --git a/client/src/main/scala/org/apache/toree/comm/ClientCommWriter.scala b/client/src/main/scala/org/apache/toree/comm/ClientCommWriter.scala
index bc8bf8e..0a755e7 100644
--- a/client/src/main/scala/org/apache/toree/comm/ClientCommWriter.scala
+++ b/client/src/main/scala/org/apache/toree/comm/ClientCommWriter.scala
@@ -57,6 +57,9 @@ class ClientCommWriter(
case _ =>
throw new Throwable("Invalid kernel message type!")
}
+ println(">>>")
+ println(messageType)
+ print(commContent)
actorLoader.load(SocketType.ShellClient) !
kmBuilder.withHeader(messageType).withContentString(commContent).build
}
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
index 4059080..c3d9d1d 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
@@ -263,8 +263,8 @@ class CoursierDependencyDownloader extends DependencyDownloader {
"===> %s %s: Downloaded %d bytes (%.2f%%)\n",
downloadId.getOrElse(url, url),
s"(${downloadFile.get(url).map(_.getName).getOrElse("")})",
- new java.lang.Long(downloaded),
- new java.lang.Double(percent)
+ java.lang.Long.valueOf(downloaded),
+ java.lang.Double.valueOf(percent)
)
}
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/InterpreterManager.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/InterpreterManager.scala
index 94b8f82..b1e5fa9 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/InterpreterManager.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/InterpreterManager.scala
@@ -91,7 +91,7 @@ object InterpreterManager {
try {
Class
.forName(className)
- .getConstructor(Class.forName("com.typesafe.config.Config"))
+ .getDeclaredConstructor(Class.forName("com.typesafe.config.Config"))
.newInstance(config).asInstanceOf[Interpreter]
}
catch {
@@ -99,7 +99,8 @@ object InterpreterManager {
logger.debug("Using default constructor for class " + className)
Class
.forName(className)
- .newInstance().asInstanceOf[Interpreter]
+ // https://docs.oracle.com/javase/9/docs/api/java/lang/Class.html#newInstance--
+ .getDeclaredConstructor().newInstance().asInstanceOf[Interpreter]
}
}
diff --git a/plugins/src/main/scala/org/apache/toree/plugins/PluginManager.scala b/plugins/src/main/scala/org/apache/toree/plugins/PluginManager.scala
index 4b611a1..25deefe 100644
--- a/plugins/src/main/scala/org/apache/toree/plugins/PluginManager.scala
+++ b/plugins/src/main/scala/org/apache/toree/plugins/PluginManager.scala
@@ -129,7 +129,8 @@ class PluginManager(
logger.debug(s"Loading $name as plugin")
// Assume that each plugin has an empty constructor
- val tryInstance = Try(klass.newInstance())
+ // https://docs.oracle.com/javase/9/docs/api/java/lang/Class.html#newInstance--
+ val tryInstance = Try(klass.getDeclaredConstructor().newInstance())
// Log failures
tryInstance.failed.foreach(ex =>
diff --git a/plugins/src/test/scala/org/apache/toree/plugins/PluginManagerSpec.scala b/plugins/src/test/scala/org/apache/toree/plugins/PluginManagerSpec.scala
index aacba46..17a7a99 100644
--- a/plugins/src/test/scala/org/apache/toree/plugins/PluginManagerSpec.scala
+++ b/plugins/src/test/scala/org/apache/toree/plugins/PluginManagerSpec.scala
@@ -131,7 +131,8 @@ class PluginManagerSpec extends FunSpec with Matchers
val p = pluginManager.loadPlugin("", classOf[SomeClass])
p.isFailure should be (true)
- p.failed.get shouldBe an [InstantiationException]
+ // NB: Java 8 throws InstantiationException, Java 11 NoSuchMethodException
+ p.failed.get shouldBe an [Exception]
}
it("should return an unknown plugin type failure if created class but not a plugin") {
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index c4dcd1a..f0a8520 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -67,7 +67,12 @@ object Dependencies {
val springCore = "org.springframework" % "spring-core" % "5.2.2.RELEASE"// Apache v2
- val guava = "com.google.guava" % "guava" % "14.0.1" // Apache v2
+ // NB: Updated from 14.0.1 as suggested in
+ // https://github.com/google/guava/issues/3249,
+ // https://github.com/google/guava/issues/3345
+ // to prevent test org.apache.toree.magic.builtin.BuiltinLoaderSpec
+ // from failing on OpenJDK 11.
+ val guava = "com.google.guava" % "guava" % "31.0.1-jre" // Apache v2
// Projects
diff --git a/project/build.properties b/project/build.properties
index dec0cf4..f7cc74b 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -15,4 +15,4 @@
# limitations under the License
#
-sbt.version=1.3.12
+sbt.version=1.5.5
diff --git a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
index 89d0482..82b7c9b 100644
--- a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
+++ b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
@@ -91,6 +91,9 @@ class ScalaInterpreter(private val config:Config = ConfigFactory.load) extends I
settings = newSettings(args)
settings = appendClassPath(settings)
+ // https://issues.apache.org/jira/browse/TOREE-132?focusedCommentId=15104495&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-15104495
+ settings.usejavacp.value = true
+
start()
// ensure bindings are defined before allowing user code to run