You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lr...@apache.org on 2018/03/30 06:08:59 UTC
incubator-toree git commit: [TOREE-469] Remove binary jars used to
test addJar functionality
Repository: incubator-toree
Updated Branches:
refs/heads/master c064a0d97 -> 5d26bdabe
[TOREE-469] Remove binary jars used to test addJar functionality
Apache source releases does not allow binaries on releases
and these test artifacts were stopping releases to be approved.
We now create a test jar as part of the test execution and use
those to test the addJar functionality.
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/5d26bdab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/5d26bdab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/5d26bdab
Branch: refs/heads/master
Commit: 5d26bdabe466f3aa855d8ec74bf268baafd0db31
Parents: c064a0d
Author: Luciano Resende <lr...@apache.org>
Authored: Wed Mar 28 08:41:32 2018 -0700
Committer: Luciano Resende <lr...@apache.org>
Committed: Thu Mar 29 23:08:42 2018 -0700
----------------------------------------------------------------------
.travis.yml | 6 +-
.../src/test/resources/ScalaTestJar.jar | Bin 1313 -> 0 bytes
.../src/test/resources/TestJar.jar | Bin 849 -> 0 bytes
.../src/test/resources/TestJar2.jar | Bin 736 -> 0 bytes
.../AddExternalJarMagicSpecForIntegration.scala | 92 ++++++++++---------
.../spark/toree/test/utils/JarUtils.scala | 73 +++++++++++++++
test_toree.py | 2 +-
7 files changed, 126 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 98039a7..c5b8180 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,7 +43,7 @@ cache:
- $HOME/.sbt/boot/
- $HOME/.coursier/cache/v1
-branches:
- only:
- - master
+#branches:
+# only:
+# - master
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/scala-interpreter/src/test/resources/ScalaTestJar.jar
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/test/resources/ScalaTestJar.jar b/scala-interpreter/src/test/resources/ScalaTestJar.jar
deleted file mode 100644
index de9bed6..0000000
Binary files a/scala-interpreter/src/test/resources/ScalaTestJar.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/scala-interpreter/src/test/resources/TestJar.jar
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/test/resources/TestJar.jar b/scala-interpreter/src/test/resources/TestJar.jar
deleted file mode 100644
index 153ea3a..0000000
Binary files a/scala-interpreter/src/test/resources/TestJar.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/scala-interpreter/src/test/resources/TestJar2.jar
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/test/resources/TestJar2.jar b/scala-interpreter/src/test/resources/TestJar2.jar
deleted file mode 100644
index 1e40e7e..0000000
Binary files a/scala-interpreter/src/test/resources/TestJar2.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala b/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala
index 871afde..b6834e1 100644
--- a/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala
+++ b/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala
@@ -17,14 +17,15 @@
package integration.interpreter.scala
-import java.io.{ByteArrayOutputStream, OutputStream}
+import java.io.{ByteArrayOutputStream, File}
+import java.net.URL
+import org.apache.spark.toree.test.utils.JarUtils
import org.apache.toree.annotations.SbtForked
import org.apache.toree.global.StreamState
import org.apache.toree.interpreter._
import org.apache.toree.kernel.api.KernelLike
import org.apache.toree.kernel.interpreter.scala.ScalaInterpreter
-import org.apache.toree.utils.{MultiOutputStream, TaskManager}
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
@@ -35,6 +36,8 @@ class AddExternalJarMagicSpecForIntegration
private val outputResult = new ByteArrayOutputStream()
private var interpreter: Interpreter = _
+ private var tempdir: File = _
+
before {
interpreter = new ScalaInterpreter {
@@ -44,6 +47,8 @@ class AddExternalJarMagicSpecForIntegration
interpreter.init(mock[KernelLike])
StreamState.setStreams(outputStream = outputResult)
+
+ tempdir = JarUtils.createTemporaryDir()
}
after {
@@ -53,108 +58,109 @@ class AddExternalJarMagicSpecForIntegration
describe("ScalaInterpreter") {
describe("#addJars") {
- it("should be able to load an external jar") {
- val testJarUrl = this.getClass.getClassLoader.getResource("TestJar.jar")
+ it("should do something") {
+ interpreter.interpret("1+1")
+ }
- //
- // NOTE: This can be done with any jar. I have tested it previously by
- // downloading jgoodies, placing it in /tmp/... and loading it.
- //
+ it("should be able to load Java jars") {
+ val testJar1Url =
+ JarUtils.createDummyJar(tempdir.toString, "test1", "TestClass")
- // Should fail since jar was not added to paths
+ // Should fail since jars were not added to paths
interpreter.interpret(
- "import com.ibm.testjar.TestClass")._1 should be (Results.Error)
+ "import test1.TestClass")._1 should be (Results.Error)
- // Add jar to paths
- interpreter.addJars(testJarUrl)
+ // Add jars to paths
+ interpreter.addJars(testJar1Url)
// Should now succeed
interpreter.interpret(
- "import com.ibm.testjar.TestClass")._1 should be (Results.Success)
+ "import test1.TestClass")._1 should be (Results.Success)
// Should now run
interpreter.interpret(
- """println(new TestClass().sayHello("Chip"))"""
+ """println(test1.TestClass.sayHello("Chip"))"""
) should be ((Results.Success, Left(Map())))
outputResult.toString should be ("Hello, Chip\n")
+ outputResult.reset()
+
+ interpreter.interpret(
+ """println(test1.TestClass.addStuff(2,1))"""
+ ) should be ((Results.Success, Left(Map())))
+ outputResult.toString should be ("3\n")
+ outputResult.reset()
}
it("should support Scala jars") {
- val testJarUrl = this.getClass.getClassLoader.getResource("ScalaTestJar.jar")
+ val locationURL = "http://repo1.maven.org/maven2/org/scala-rules/rule-engine-core_2.11/0.5.1/rule-engine-core_2.11-0.5.1.jar"
+ val testJarUrl = JarUtils.downloadJar(tempdir.toString, locationURL)
// Should fail since jar was not added to paths
interpreter.interpret(
- "import com.ibm.scalatestjar.TestClass")._1 should be (Results.Error)
+ "import org.scalarules.utils._")._1 should be (Results.Error)
// Add jar to paths
interpreter.addJars(testJarUrl)
// Should now succeed
interpreter.interpret(
- "import com.ibm.scalatestjar.TestClass")._1 should be (Results.Success)
+ "import org.scalarules.utils._")._1 should be (Results.Success)
// Should now run
+ /*
interpreter.interpret(
"""println(new TestClass().runMe())"""
) should be ((Results.Success, Left(Map())))
outputResult.toString should be ("You ran me!\n")
+ */
}
it("should be able to add multiple jars at once") {
val testJar1Url =
- this.getClass.getClassLoader.getResource("TestJar.jar")
+ JarUtils.createDummyJar(tempdir.toString, "test1", "TestClass")
val testJar2Url =
- this.getClass.getClassLoader.getResource("TestJar2.jar")
-// val interpreter = new ScalaInterpreter(List(), mock[OutputStream])
-// with StandardSparkIMainProducer
-// with StandardTaskManagerProducer
-// with StandardSettingsProducer
-// interpreter.start()
+ JarUtils.createDummyJar(tempdir.toString, "test2", "TestClass")
// Should fail since jars were not added to paths
interpreter.interpret(
- "import com.ibm.testjar.TestClass")._1 should be (Results.Error)
+ "import test1.TestClass")._1 should be (Results.Error)
interpreter.interpret(
- "import com.ibm.testjar2.TestClass")._1 should be (Results.Error)
+ "import test2.TestClass")._1 should be (Results.Error)
// Add jars to paths
interpreter.addJars(testJar1Url, testJar2Url)
// Should now succeed
interpreter.interpret(
- "import com.ibm.testjar.TestClass")._1 should be (Results.Success)
+ "import test1.TestClass")._1 should be (Results.Success)
interpreter.interpret(
- "import com.ibm.testjar2.TestClass")._1 should be (Results.Success)
+ "import test2.TestClass")._1 should be (Results.Success)
// Should now run
interpreter.interpret(
- """println(new com.ibm.testjar.TestClass().sayHello("Chip"))"""
+ """println(test1.TestClass.sayHello("Chip"))"""
) should be ((Results.Success, Left(Map())))
outputResult.toString should be ("Hello, Chip\n")
outputResult.reset()
interpreter.interpret(
- """println(new com.ibm.testjar2.TestClass().CallMe())"""
+ """println(test2.TestClass.addStuff(2,1))"""
) should be ((Results.Success, Left(Map())))
outputResult.toString should be ("3\n")
+ outputResult.reset()
}
it("should be able to add multiple jars in consecutive calls to addjar") {
val testJar1Url =
- this.getClass.getClassLoader.getResource("TestJar.jar")
+ JarUtils.createDummyJar(tempdir.toString, "test1", "TestClass")
val testJar2Url =
- this.getClass.getClassLoader.getResource("TestJar2.jar")
-// val interpreter = new ScalaInterpreter(List(), mock[OutputStream])
-// with StandardSparkIMainProducer
-// with StandardTaskManagerProducer
-// with StandardSettingsProducer
-// interpreter.start()
+ JarUtils.createDummyJar(tempdir.toString, "test2", "TestClass")
// Should fail since jars were not added to paths
interpreter.interpret(
- "import com.ibm.testjar.TestClass")._1 should be (Results.Error)
+ "import test1.TestClass")._1 should be (Results.Error)
interpreter.interpret(
- "import com.ibm.testjar2.TestClass")._1 should be (Results.Error)
+ "import test2.TestClass")._1 should be (Results.Error)
// Add jars to paths
interpreter.addJars(testJar1Url)
@@ -162,19 +168,19 @@ class AddExternalJarMagicSpecForIntegration
// Should now succeed
interpreter.interpret(
- "import com.ibm.testjar.TestClass")._1 should be (Results.Success)
+ "import test1.TestClass")._1 should be (Results.Success)
interpreter.interpret(
- "import com.ibm.testjar2.TestClass")._1 should be (Results.Success)
+ "import test2.TestClass")._1 should be (Results.Success)
// Should now run
interpreter.interpret(
- """println(new com.ibm.testjar.TestClass().sayHello("Chip"))"""
+ """println(test1.TestClass.sayHello("Chip"))"""
) should be ((Results.Success, Left(Map())))
outputResult.toString should be ("Hello, Chip\n")
outputResult.reset()
interpreter.interpret(
- """println(new com.ibm.testjar2.TestClass().CallMe())"""
+ """println(test2.TestClass.addStuff(2,1))"""
) should be ((Results.Success, Left(Map())))
outputResult.toString should be ("3\n")
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/scala-interpreter/src/test/scala/org/apache/spark/toree/test/utils/JarUtils.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/test/scala/org/apache/spark/toree/test/utils/JarUtils.scala b/scala-interpreter/src/test/scala/org/apache/spark/toree/test/utils/JarUtils.scala
new file mode 100644
index 0000000..0e1c82c
--- /dev/null
+++ b/scala-interpreter/src/test/scala/org/apache/spark/toree/test/utils/JarUtils.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package org.apache.spark.toree.test.utils
+
+import sys.process._
+import java.net.URL
+import java.io.File
+import java.util.UUID
+
+import scala.language.postfixOps
+
+import org.apache.spark.TestUtils.{JavaSourceFromString, _}
+
+object JarUtils {
+
+ def createTemporaryDir() = {
+ val tempDir = System.getProperty("java.io.tmpdir")
+ val dir = new File(tempDir, UUID.randomUUID.toString)
+ dir.mkdirs()
+ dir.deleteOnExit()
+ dir.getCanonicalFile
+ }
+
+ def createDummyJar(destDir: String, packageName: String, className: String) : URL = {
+ val srcDir = new File(destDir, packageName)
+ srcDir.mkdirs()
+ val source =
+ s"""package $packageName;
+ |
+ |public class $className implements java.io.Serializable {
+ | public static String sayHello(String arg) { return "Hello, " + arg; }
+ | public static int addStuff(int arg1, int arg2) { return arg1 + arg2; }
+ |}
+ """.stripMargin
+
+ val sourceFile =
+ new JavaSourceFromString(new File(srcDir, className).toURI.getPath, source)
+ val compiledFile = createCompiledClass(className, srcDir, sourceFile, Seq.empty)
+ val jarFile = new File(destDir,
+ s"$packageName-$className-%s.jar".format(System.currentTimeMillis()))
+ val jarURL = createJar(Seq(compiledFile), jarFile, directoryPrefix = Some(packageName))
+ jarFile.deleteOnExit()
+ jarURL
+ }
+
+ def downloadJar(destDir: String, artifactURL: String) : URL = {
+ val fileName = getFileName(artifactURL).
+ replace(".jar", s"%s.jar".format(System.currentTimeMillis()))
+ val jarFile = new File(destDir, fileName)
+ jarFile.deleteOnExit()
+ (new URL(artifactURL) #> jarFile !!)
+ jarFile.toURI.toURL
+ }
+
+ private def getFileName(artifactURL: String) = {
+ artifactURL.split("/").last
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5d26bdab/test_toree.py
----------------------------------------------------------------------
diff --git a/test_toree.py b/test_toree.py
index 0f90db4..4644126 100644
--- a/test_toree.py
+++ b/test_toree.py
@@ -72,7 +72,7 @@ class ToreeScalaKernelTests(jupyter_kernel_test.KernelTests):
]
test_statements_stdout = [
- {'code': '%AddJar http://0.0.0.0:8000/TestJar.jar'},
+ {'code': '%AddJar http://home.apache.org/~lresende/toree/TestJar.jar'},
{'code': 'import com.ibm.testjar.TestClass\nprintln(new TestClass().sayHello("Person"))', 'result': 'Hello, Person\n'}
]