You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@toree.apache.org by Luciano Resende <lu...@gmail.com> on 2018/01/30 15:10:12 UTC
Removing compiled test assets from master, was Re: incubator-toree
git commit: Removed compiled assets from codebase.
Marius,
Could you please help merging this to master, I believe this is the last
thing we need to get the 0.2.0 release out.
Thanks
On Mon, Jan 23, 2017 at 8:37 AM, <ch...@apache.org> wrote:
> Repository: incubator-toree
> Updated Branches:
> refs/heads/0.1.x 2eb26cd3c -> f0bb5d238
>
>
> Removed compiled assets from codebase.
>
> Generate them during testing instead. This is adapted from code that is
> part of Apache Spark.
>
>
> Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
> Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/
> commit/f0bb5d23
> Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/f0bb5d23
> Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/f0bb5d23
>
> Branch: refs/heads/0.1.x
> Commit: f0bb5d238dc10376db3935c15e007ea46076e548
> Parents: 2eb26cd
> Author: mariusvniekerk <ma...@gmail.com>
> Authored: Mon Jan 23 10:47:42 2017 -0500
> Committer: mariusvniekerk <ma...@gmail.com>
> Committed: Mon Jan 23 10:53:30 2017 -0500
>
> ----------------------------------------------------------------------
> .../src/test/resources/ScalaTestJar.jar | Bin 1313 -> 0 bytes
> .../src/test/resources/TestJar.jar | Bin 849 -> 0 bytes
> .../src/test/resources/TestJar2.jar | Bin 736 -> 0 bytes
> .../AddExternalJarMagicSpecForIntegration.scala | 60 +++++++++++--------
> .../apache/spark/toree/testutils/JarUtils.scala | 41 +++++++++++++
> 5 files changed, 75 insertions(+), 26 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-toree/
> blob/f0bb5d23/scala-interpreter/src/test/resources/ScalaTestJar.jar
> ----------------------------------------------------------------------
> diff --git a/scala-interpreter/src/test/resources/ScalaTestJar.jar
> b/scala-interpreter/src/test/resources/ScalaTestJar.jar
> deleted file mode 100644
> index de9bed6..0000000
> Binary files a/scala-interpreter/src/test/resources/ScalaTestJar.jar and
> /dev/null differ
>
> http://git-wip-us.apache.org/repos/asf/incubator-toree/
> blob/f0bb5d23/scala-interpreter/src/test/resources/TestJar.jar
> ----------------------------------------------------------------------
> diff --git a/scala-interpreter/src/test/resources/TestJar.jar
> b/scala-interpreter/src/test/resources/TestJar.jar
> deleted file mode 100644
> index 153ea3a..0000000
> Binary files a/scala-interpreter/src/test/resources/TestJar.jar and
> /dev/null differ
>
> http://git-wip-us.apache.org/repos/asf/incubator-toree/
> blob/f0bb5d23/scala-interpreter/src/test/resources/TestJar2.jar
> ----------------------------------------------------------------------
> diff --git a/scala-interpreter/src/test/resources/TestJar2.jar
> b/scala-interpreter/src/test/resources/TestJar2.jar
> deleted file mode 100644
> index 1e40e7e..0000000
> Binary files a/scala-interpreter/src/test/resources/TestJar2.jar and
> /dev/null differ
>
> http://git-wip-us.apache.org/repos/asf/incubator-toree/
> blob/f0bb5d23/scala-interpreter/src/test/scala/
> integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala
> ----------------------------------------------------------------------
> diff --git a/scala-interpreter/src/test/scala/integration/interpreter/
> scala/AddExternalJarMagicSpecForIntegration.scala
> b/scala-interpreter/src/test/scala/integration/interpreter/scala/
> AddExternalJarMagicSpecForIntegration.scala
> index 2863bec..8fdafe7 100644
> --- a/scala-interpreter/src/test/scala/integration/interpreter/scala/
> AddExternalJarMagicSpecForIntegration.scala
> +++ b/scala-interpreter/src/test/scala/integration/interpreter/scala/
> AddExternalJarMagicSpecForIntegration.scala
> @@ -17,13 +17,14 @@
>
> package integration.interpreter.scala
>
> -import java.io.{ByteArrayOutputStream, OutputStream}
> +import java.io.{ByteArrayOutputStream, File, OutputStream}
>
> +import org.apache.spark.toree.testutils.JarUtils
> import org.apache.toree.global.StreamState
> import org.apache.toree.interpreter._
> import org.apache.toree.kernel.api.KernelLike
> import org.apache.toree.kernel.interpreter.scala.{ScalaInterpreter,
> StandardSettingsProducer, StandardSparkIMainProducer,
> StandardTaskManagerProducer}
> -import org.apache.toree.utils.{TaskManager, MultiOutputStream}
> +import org.apache.toree.utils.{MultiOutputStream, TaskManager}
> import org.scalatest.mock.MockitoSugar
> import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
>
> @@ -33,6 +34,7 @@ class AddExternalJarMagicSpecForIntegration
>
> private val outputResult = new ByteArrayOutputStream()
> private var interpreter: Interpreter = _
> + private var tempdir: File = _
>
> before {
> interpreter = new ScalaInterpreter {
> @@ -44,6 +46,8 @@ class AddExternalJarMagicSpecForIntegration
> interpreter.init(mock[KernelLike])
>
> StreamState.setStreams(outputStream = outputResult)
> +
> + tempdir = JarUtils.createTemporaryDir()
> }
>
> after {
> @@ -53,7 +57,8 @@ class AddExternalJarMagicSpecForIntegration
> describe("ScalaInterpreter") {
> describe("#addJars") {
> it("should be able to load an external jar") {
> - val testJarUrl = this.getClass.getClassLoader.
> getResource("TestJar.jar")
> +
> + val testJarUrl = JarUtils.createDummyJar(tempdir.toString,
> "testing.external", "TestClass")
>
> //
> // NOTE: This can be done with any jar. I have tested it
> previously by
> @@ -62,14 +67,14 @@ class AddExternalJarMagicSpecForIntegration
>
> // Should fail since jar was not added to paths
> interpreter.interpret(
> - "import com.ibm.testjar.TestClass")._1 should be (Results.Error)
> + "import testing.external.TestClass")._1 should be
> (Results.Error)
>
> // Add jar to paths
> interpreter.addJars(testJarUrl)
>
> // Should now succeed
> interpreter.interpret(
> - "import com.ibm.testjar.TestClass")._1 should be
> (Results.Success)
> + "import testing.external.TestClass")._1 should be
> (Results.Success)
>
> // Should now run
> interpreter.interpret(
> @@ -78,7 +83,9 @@ class AddExternalJarMagicSpecForIntegration
> outputResult.toString should be ("Hello, Chip\n")
> }
>
> - it("should support Scala jars") {
> + // Test disabled due to removal of statically compiled Scala Jar.
> + // Should replace with something that fetches a jar using coursier
> instead.
> + ignore("should support Scala jars") {
> val testJarUrl = this.getClass.getClassLoader.
> getResource("ScalaTestJar.jar")
>
> // Should fail since jar was not added to paths
> @@ -101,9 +108,9 @@ class AddExternalJarMagicSpecForIntegration
>
> it("should be able to add multiple jars at once") {
> val testJar1Url =
> - this.getClass.getClassLoader.getResource("TestJar.jar")
> + JarUtils.createDummyJar(tempdir.toString, "testing.testjar1",
> "TestClass")
> val testJar2Url =
> - this.getClass.getClassLoader.getResource("TestJar2.jar")
> + JarUtils.createDummyJar(tempdir.toString, "testing.testjar2",
> "TestClass")
> // val interpreter = new ScalaInterpreter(List(),
> mock[OutputStream])
> // with StandardSparkIMainProducer
> // with StandardTaskManagerProducer
> @@ -112,37 +119,37 @@ class AddExternalJarMagicSpecForIntegration
>
> // Should fail since jars were not added to paths
> interpreter.interpret(
> - "import com.ibm.testjar.TestClass")._1 should be (Results.Error)
> + "import testing.testjar.TestClass")._1 should be (Results.Error)
> interpreter.interpret(
> - "import com.ibm.testjar2.TestClass")._1 should be
> (Results.Error)
> + "import testing.testjar2.TestClass")._1 should be
> (Results.Error)
>
> // Add jars to paths
> interpreter.addJars(testJar1Url, testJar2Url)
>
> // Should now succeed
> interpreter.interpret(
> - "import com.ibm.testjar.TestClass")._1 should be
> (Results.Success)
> + "import testing.testjar.TestClass")._1 should be
> (Results.Success)
> interpreter.interpret(
> - "import com.ibm.testjar2.TestClass")._1 should be
> (Results.Success)
> + "import testing.testjar2.TestClass")._1 should be
> (Results.Success)
>
> // Should now run
> interpreter.interpret(
> - """println(new com.ibm.testjar.TestClass().sayHello("Chip"))"""
> + """println(new testing.testjar.TestClass().sayHello("Chip"))"""
> ) should be ((Results.Success, Left("")))
> outputResult.toString should be ("Hello, Chip\n")
> outputResult.reset()
>
> interpreter.interpret(
> - """println(new com.ibm.testjar2.TestClass().CallMe())"""
> + """println(new testing.testjar.TestClass().addStuff(1, 2))"""
> ) should be ((Results.Success, Left("")))
> outputResult.toString should be ("3\n")
> }
>
> it("should be able to add multiple jars in consecutive calls to
> addjar") {
> val testJar1Url =
> - this.getClass.getClassLoader.getResource("TestJar.jar")
> + JarUtils.createDummyJar(tempdir.toString, "testing.testjar1",
> "TestClass")
> val testJar2Url =
> - this.getClass.getClassLoader.getResource("TestJar2.jar")
> + JarUtils.createDummyJar(tempdir.toString, "testing.testjar2",
> "TestClass")
> // val interpreter = new ScalaInterpreter(List(),
> mock[OutputStream])
> // with StandardSparkIMainProducer
> // with StandardTaskManagerProducer
> @@ -151,45 +158,46 @@ class AddExternalJarMagicSpecForIntegration
>
> // Should fail since jars were not added to paths
> interpreter.interpret(
> - "import com.ibm.testjar.TestClass")._1 should be (Results.Error)
> + "import testing.testjar1.TestClass")._1 should be
> (Results.Error)
> interpreter.interpret(
> - "import com.ibm.testjar2.TestClass")._1 should be
> (Results.Error)
> + "import testing.testjar2.TestClass")._1 should be
> (Results.Error)
>
> // Add jars to paths
> interpreter.addJars(testJar1Url)
> interpreter.addJars(testJar2Url)
>
> // Should now succeed
> + // Should now succeed
> interpreter.interpret(
> - "import com.ibm.testjar.TestClass")._1 should be
> (Results.Success)
> + "import testing.testjar1.TestClass")._1 should be
> (Results.Success)
> interpreter.interpret(
> - "import com.ibm.testjar2.TestClass")._1 should be
> (Results.Success)
> + "import testing.testjar2.TestClass")._1 should be
> (Results.Success)
>
> // Should now run
> interpreter.interpret(
> - """println(new com.ibm.testjar.TestClass().sayHello("Chip"))"""
> + """println(new testing.testjar1.TestClass().
> sayHello("Chip"))"""
> ) should be ((Results.Success, Left("")))
> outputResult.toString should be ("Hello, Chip\n")
> outputResult.reset()
>
> interpreter.interpret(
> - """println(new com.ibm.testjar2.TestClass().CallMe())"""
> + """println(new testing.testjar2.TestClass().addStuff(1, 2))"""
> ) should be ((Results.Success, Left("")))
> outputResult.toString should be ("3\n")
> }
>
> it("should not have issues with previous variables") {
> val testJar1Url =
> - this.getClass.getClassLoader.getResource("TestJar.jar")
> + JarUtils.createDummyJar(tempdir.toString, "testing.testjar1",
> "TestClass")
> val testJar2Url =
> - this.getClass.getClassLoader.getResource("TestJar2.jar")
> + JarUtils.createDummyJar(tempdir.toString, "testing.testjar2",
> "TestClass")
>
> // Add a jar, which reinitializes the symbols
> interpreter.addJars(testJar1Url)
>
> interpreter.interpret(
> """
> - |val t = new com.ibm.testjar.TestClass()
> + |val t = new testing.testjar1.TestClass()
> """.stripMargin)._1 should be (Results.Success)
>
> // Add a second jar, which reinitializes the symbols and breaks
> the
> @@ -198,7 +206,7 @@ class AddExternalJarMagicSpecForIntegration
>
> interpreter.interpret(
> """
> - |def runMe(testClass: com.ibm.testjar.TestClass) =
> + |def runMe(testClass: testing.testjar1.TestClass) =
> |testClass.sayHello("Hello")
> """.stripMargin)._1 should be (Results.Success)
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-toree/
> blob/f0bb5d23/scala-interpreter/src/test/scala/org/apache/spark/toree/
> testutils/JarUtils.scala
> ----------------------------------------------------------------------
> diff --git a/scala-interpreter/src/test/scala/org/apache/spark/toree/testutils/JarUtils.scala
> b/scala-interpreter/src/test/scala/org/apache/spark/toree/
> testutils/JarUtils.scala
> new file mode 100644
> index 0000000..572aa4a
> --- /dev/null
> +++ b/scala-interpreter/src/test/scala/org/apache/spark/toree/
> testutils/JarUtils.scala
> @@ -0,0 +1,41 @@
> +package org.apache.spark.toree.testutils
> +
> +import java.io.File
> +import java.util.UUID
> +
> +import org.apache.spark.TestUtils.{JavaSourceFromString, _}
> +
> +object JarUtils {
> +
> + def createTemporaryDir() = {
> + val tempDir = System.getProperty("java.io.tmpdir")
> + val dir = new File(tempDir, UUID.randomUUID.toString)
> + dir.mkdirs()
> + dir.deleteOnExit()
> + dir.getCanonicalFile
> + }
> +
> + def createDummyJar(destDir: String, packageName: String, className:
> String) = {
> + val srcDir = new File(destDir, packageName)
> + srcDir.mkdirs()
> + val excSource = new JavaSourceFromString(
> + new File(srcDir, className).getAbsolutePath,
> + s"""package $packageName;
> + |
> + |public class $className implements java.io.Serializable {
> + | public static String sayHello(String arg) { return "Hello, " +
> arg; }
> + | public static int addStuff(int arg1, int arg2) { return arg1 +
> arg2; }
> + |}
> + """.
> + stripMargin)
> + val excFile = createCompiledClass(className, srcDir, excSource,
> Seq.empty)
> + val jarFile = new File(destDir,
> + s"$packageName-$className-%s.jar".format(System.
> currentTimeMillis()))
> + val jarURL = createJar(Seq(excFile), jarFile, directoryPrefix =
> Some(packageName))
> + jarFile.deleteOnExit()
> + jarURL
> + }
> +
> +
> +
> +}
>
>
--
Luciano Resende
http://twitter.com/lresende1975
http://lresende.blogspot.com/