You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by ma...@apache.org on 2017/02/16 18:35:33 UTC
[1/7] incubator-toree git commit: Copy ivy file to resources
Repository: incubator-toree
Updated Branches:
refs/heads/master e5ef3a104 -> 5743f068d
Copy ivy file to resources
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/464c367d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/464c367d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/464c367d
Branch: refs/heads/master
Commit: 464c367d5f2138a6faacbbf2c96666e91adaad0b
Parents: 4ad1afd
Author: Jakob Odersky <ja...@odersky.com>
Authored: Tue Feb 14 17:19:05 2017 -0800
Committer: Jakob Odersky <ja...@odersky.com>
Committed: Wed Feb 15 15:24:09 2017 -0800
----------------------------------------------------------------------
project/CommonPlugin.scala | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/464c367d/project/CommonPlugin.scala
----------------------------------------------------------------------
diff --git a/project/CommonPlugin.scala b/project/CommonPlugin.scala
index 58358d5..093fad9 100644
--- a/project/CommonPlugin.scala
+++ b/project/CommonPlugin.scala
@@ -43,7 +43,14 @@ object CommonPlugin extends AutoPlugin {
),
unmanagedResourceDirectories in Test ++= Seq(
(baseDirectory in ThisBuild).value / "resources" / "test"
- )
+ ),
+ resourceGenerators in Compile += Def.task {
+ val inputFile = (deliverLocal in Compile).value
+ val outputFile = (resourceManaged in Compile).value / s"${name.value}-ivy.xml"
+ streams.value.log.info(s"Copying ${inputFile.getPath} to ${outputFile.getPath}")
+ IO.copyFile(inputFile, outputFile)
+ Seq(outputFile)
+ }.taskValue
)
}
[3/7] incubator-toree git commit: Move common project settings to
build-level settings
Posted by ma...@apache.org.
Move common project settings to build-level settings
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/00f4122a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/00f4122a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/00f4122a
Branch: refs/heads/master
Commit: 00f4122ae23c4c04a9706af0ea7e1ac1cb807d11
Parents: bf02de5
Author: Jakob Odersky <ja...@odersky.com>
Authored: Mon Feb 13 16:16:29 2017 -0800
Committer: Jakob Odersky <ja...@odersky.com>
Committed: Wed Feb 15 15:24:09 2017 -0800
----------------------------------------------------------------------
Makefile | 5 +-
build.sbt | 206 ++++++++++++++++++++++++------
kernel-api/build.sbt | 2 +
kernel/build.sbt | 2 +
plugins/build.sbt | 2 +
project/CommonPlugin.scala | 41 ++++++
project/Dependencies.scala | 35 +++--
project/build.properties | 3 +-
project/common.scala | 255 -------------------------------------
project/plugins.sbt | 2 +-
protocol/build.sbt | 8 +-
pyspark-interpreter/build.sbt | 1 +
scala-interpreter/build.sbt | 38 +-----
sparkr-interpreter/build.sbt | 1 +
sql-interpreter/build.sbt | 1 +
15 files changed, 250 insertions(+), 352 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/Makefile
----------------------------------------------------------------------
diff --git a/Makefile b/Makefile
index ec160e5..30bf04c 100644
--- a/Makefile
+++ b/Makefile
@@ -106,8 +106,9 @@ dev-binder: .binder-image
target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): VM_WORKDIR=/src/toree-kernel
target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): ${shell find ./*/src/main/**/*}
target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): ${shell find ./*/build.sbt}
-target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): dist/toree-legal project/build.properties build.sbt project/common.scala project/plugins.sbt
- $(call RUN,$(ENV_OPTS) sbt toree/assembly)
+target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): ${shell find ./project/*.scala} ${shell find ./project/*.sbt}
+target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): dist/toree-legal project/build.properties build.sbt
+ $(call RUN,$(ENV_OPTS) sbt root/assembly)
build: target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index a3d1f2a..e60a99e 100644
--- a/build.sbt
+++ b/build.sbt
@@ -15,25 +15,117 @@
* limitations under the License
*/
-import Common._
-import sbtunidoc.Plugin.UnidocKeys._
-import sbtunidoc.Plugin._
-import com.typesafe.sbt.SbtGit.{GitKeys => git}
-import com.typesafe.sbt.SbtSite.site
-
-lazy val root = ToreeProject("toree", ".", doFork=false, needsSpark=true).
- settings(unidocSettings:_*).
- settings(site.settings:_*).
- settings(
- test in assembly := {},
- assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false),
- scalacOptions in (ScalaUnidoc, unidoc) += "-Ymacro-no-expand",
- git.gitRemoteRepo := "git://git.apache.org/incubator-toree.git",
- aggregate in assembly := false,
- unmanagedResourceDirectories in Compile += { baseDirectory.value / "dist/toree-legal" }
- ).aggregate(
+import scala.util.Properties
+
+// Version settings
+version in ThisBuild := Properties.envOrElse("VERSION", "0.0.0-dev") +
+ (if ((isSnapshot in ThisBuild).value) "-SNAPSHOT" else "")
+isSnapshot in ThisBuild := Properties.envOrElse("IS_SNAPSHOT","true").toBoolean
+organization in ThisBuild := "org.apache.toree.kernel"
+crossScalaVersions in ThisBuild := Seq("2.11.8")
+scalaVersion in ThisBuild := (crossScalaVersions in ThisBuild).value.head
+Dependencies.sparkVersion in ThisBuild := {
+ val envVar = "APACHE_SPARK_VERSION"
+ val defaultVersion = "2.0.0"
+
+ Properties.envOrNone(envVar) match {
+ case None =>
+ sLog.value.info(s"Using default Apache Spark version $defaultVersion!")
+ defaultVersion
+ case Some(version) =>
+ sLog.value.info(s"Using Apache Spark version $version, provided from $envVar")
+ version
+ }
+}
+
+// Compiler settings
+scalacOptions in ThisBuild ++= Seq(
+ "-deprecation",
+ "-unchecked",
+ "-feature",
+ "-Xfatal-warnings",
+ "-language:reflectiveCalls",
+ "-target:jvm-1.6",
+ "-Xlint" // Scala 2.11.x only
+)
+// Java-based options for compilation (all tasks)
+// NOTE: Providing a blank flag causes failures, only uncomment with options
+//javacOptions in Compile ++= Seq(""),
+// Java-based options for just the compile task
+javacOptions in ThisBuild ++= Seq(
+ "-Xlint:all", // Enable all Java-based warnings
+ "-Xlint:-path", // Suppress path warnings since we get tons of them
+ "-Xlint:-options",
+ "-Xlint:-processing",
+ "-Werror", // Treat warnings as errors
+ "-source", "1.6",
+ "-target", "1.6"
+)
+// Options provided to forked JVMs through sbt, based on our .jvmopts file
+javaOptions in ThisBuild ++= Seq(
+ "-Xms1024M", "-Xmx4096M", "-Xss2m", "-XX:MaxPermSize=1024M",
+ "-XX:ReservedCodeCacheSize=256M", "-XX:+TieredCompilation",
+ "-XX:+CMSPermGenSweepingEnabled", "-XX:+CMSClassUnloadingEnabled",
+ "-XX:+UseConcMarkSweepGC", "-XX:+HeapDumpOnOutOfMemoryError"
+)
+// Add additional test option to show time taken per test
+testOptions in (ThisBuild, Test) += Tests.Argument("-oDF")
+
+// Build-wide dependencies
+resolvers in ThisBuild ++= Seq(
+ "Apache Snapshots" at "http://repository.apache.org/snapshots/",
+ "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/"
+)
+updateOptions in ThisBuild := updateOptions.value.withCachedResolution(true)
+libraryDependencies in ThisBuild ++= Seq(
+ Dependencies.scalaTest % "test",
+ Dependencies.mockito % "test",
+ Dependencies.jacksonDatabind % "test"
+)
+
+ScoverageSbtPlugin.ScoverageKeys.coverageHighlighting in ThisBuild := false
+
+// Publish settings
+useGpg in ThisBuild := true
+pgpPassphrase in ThisBuild := Some(Properties.envOrElse("GPG_PASSWORD","").toArray)
+publishTo in ThisBuild := {
+ if (isSnapshot.value)
+ Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/snapshots/")
+ else
+ Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/staging/")
+}
+mappings in packageBin in ThisBuild := Seq(
+ file("LICENSE") -> "LICENSE",
+ file("NOTICE") -> "NOTICE"
+)
+licenses in ThisBuild := Seq("Apache 2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt"))
+pomExtra in ThisBuild := {
+ <parent>
+ <groupId>org.apache</groupId>
+ <artifactId>apache</artifactId>
+ <version>10</version>
+ </parent>
+ <url>http://toree.incubator.apache.org/</url>
+ <scm>
+ <url>git@github.com:apache/incubator-toree.git</url>
+ <connection>scm:git:git@github.com:apache/incubator-toree.git</connection>
+ <developerConnection>
+ scm:git:https://git-wip-us.apache.org/repos/asf/incubator-toree.git
+ </developerConnection>
+ <tag>HEAD</tag>
+ </scm>
+}
+credentials in ThisBuild+= Credentials(Path.userHome / ".ivy2" / ".credentials")
+
+// Project structure
+
+/** Root Toree project. */
+lazy val root = (project in file("."))
+ .settings(name := "toree")
+ .aggregate(
macros,protocol,plugins,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,pysparkInterpreter,sparkrInterpreter,kernel
- ).dependsOn(
+ )
+ .dependsOn(
macros,protocol,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,pysparkInterpreter,sparkrInterpreter,kernel
)
@@ -41,67 +133,105 @@ lazy val root = ToreeProject("toree", ".", doFork=false, needsSpark=true).
* Project representing macros in Scala that must be compiled separately from
* any other project using them.
*/
-lazy val macros = ToreeProject("macros")
+lazy val macros = (project in file("macros"))
+ .settings(name := "toree-macros")
/**
* Project representing the IPython kernel message protocol in Scala. Used
* by the client and kernel implementations.
*/
-lazy val protocol = ToreeProject("protocol").dependsOn(macros).
- enablePlugins(BuildInfoPlugin).settings(buildInfoSettings:_*)
+lazy val protocol = (project in file("protocol"))
+ .settings(name := "toree-protocol")
+ .dependsOn(macros)
/**
* Project representing base plugin system for the Toree infrastructure.
*/
-lazy val plugins = ToreeProject("plugins", doFork=true).dependsOn(macros)
+lazy val plugins = (project in file("plugins"))
+ .settings(name := "toree-plugins")
+ .dependsOn(macros)
/**
* Project representing forms of communication used as input/output for the
* client/kernel.
*/
-lazy val communication = ToreeProject("communication").dependsOn(macros, protocol)
+lazy val communication = (project in file("communication"))
+ .settings(name := "toree-communication")
+ .dependsOn(macros, protocol)
/**
* Project representing the kernel-api code used by the Spark Kernel. Others can
* import this to implement their own magics and plugins.
*/
-lazy val kernelApi = ToreeProject("kernel-api", needsSpark=true).dependsOn(macros, plugins)
+lazy val kernelApi = (project in file("kernel-api"))
+ .settings(name := "toree-kernel-api")
+ .dependsOn(macros, plugins)
/**
* Project representing the client code for connecting to the kernel backend.
*/
-lazy val client = ToreeProject("client").dependsOn(macros, protocol, communication)
+lazy val client = (project in file("client"))
+ .settings(name := "toree-client")
+ .dependsOn(macros, protocol, communication)
/**
* Project represents the scala interpreter used by the Spark Kernel.
*/
-lazy val scalaInterpreter = ToreeProject("scala-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi)
+lazy val scalaInterpreter = (project in file("scala-interpreter"))
+ .settings(name := "toree-scala-interpreter")
+ .dependsOn(plugins, protocol, kernelApi)
/**
* Project represents the SQL interpreter used by the Spark Kernel.
*/
-lazy val sqlInterpreter = ToreeProject("sql-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi)
+lazy val sqlInterpreter = (project in file("sql-interpreter"))
+ .settings(name := "toree-sql-interpreter")
+ .dependsOn(plugins, protocol, kernelApi)
/**
* Project represents the Python interpreter used by the Spark Kernel.
*/
-lazy val pysparkInterpreter = ToreeProject("pyspark-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi)
+lazy val pysparkInterpreter = (project in file("pyspark-interpreter"))
+ .settings(name := "toree-pyspark-interpreter")
+ .dependsOn(plugins, protocol, kernelApi)
/**
* Project represents the R interpreter used by the Spark Kernel.
*/
-lazy val sparkrInterpreter = ToreeProject("sparkr-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi)
+lazy val sparkrInterpreter = (project in file("sparkr-interpreter"))
+ .settings(name := "toree-sparkr-interpreter")
+ .dependsOn(plugins, protocol, kernelApi)
/**
* Project representing the kernel code for the Spark Kernel backend.
*/
-lazy val kernel = ToreeProject("kernel", doFork=true, needsSpark=true).dependsOn(
- macros % "test->test;compile->compile",
- protocol % "test->test;compile->compile",
- communication % "test->test;compile->compile",
- kernelApi % "test->test;compile->compile",
- pysparkInterpreter % "test->test;compile->compile",
- scalaInterpreter % "test->test;compile->compile",
- sparkrInterpreter % "test->test;compile->compile",
- sqlInterpreter % "test->test;compile->compile"
+lazy val kernel = (project in file("kernel"))
+ .settings(name := "toree-kernel")
+ .dependsOn(
+ macros % "test->test;compile->compile",
+ protocol % "test->test;compile->compile",
+ communication % "test->test;compile->compile",
+ kernelApi % "test->test;compile->compile",
+ pysparkInterpreter % "test->test;compile->compile",
+ scalaInterpreter % "test->test;compile->compile",
+ sparkrInterpreter % "test->test;compile->compile",
+ sqlInterpreter % "test->test;compile->compile"
+ )
+
+// Root project settings
+enablePlugins(ScalaUnidocPlugin)
+scalacOptions in (ScalaUnidoc, unidoc) ++= Seq(
+ "-Ymacro-expand:none",
+ "-skip-packages", Seq(
+ "akka",
+ "scala"
+ ).mkString(":"),
+ "-no-link-warnings" // Suppresses problems with Scaladoc @throws links
)
+
+libraryDependencies ++= Dependencies.sparkAll.value
+unmanagedResourceDirectories in Compile += { baseDirectory.value / "dist/toree-legal" }
+
+test in assembly := {}
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
+aggregate in assembly := false
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/kernel-api/build.sbt
----------------------------------------------------------------------
diff --git a/kernel-api/build.sbt b/kernel-api/build.sbt
index e1ab494..68be752 100644
--- a/kernel-api/build.sbt
+++ b/kernel-api/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License
*/
+libraryDependencies ++= Dependencies.sparkAll.value
+
//
// SCALA INTERPRETER DEPENDENCIES
//
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/kernel/build.sbt
----------------------------------------------------------------------
diff --git a/kernel/build.sbt b/kernel/build.sbt
index 752ad2a..5428b6e 100644
--- a/kernel/build.sbt
+++ b/kernel/build.sbt
@@ -14,6 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License
*/
+fork in Test := true
+libraryDependencies ++= Dependencies.sparkAll.value
//
// TEST DEPENDENCIES
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/plugins/build.sbt
----------------------------------------------------------------------
diff --git a/plugins/build.sbt b/plugins/build.sbt
index 99f1701..f41d40b 100644
--- a/plugins/build.sbt
+++ b/plugins/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License
*/
+fork in Test := true
+
// Needed for type inspection
libraryDependencies ++= Seq(
Dependencies.scalaReflect.value,
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/CommonPlugin.scala
----------------------------------------------------------------------
diff --git a/project/CommonPlugin.scala b/project/CommonPlugin.scala
new file mode 100644
index 0000000..a75a9a7
--- /dev/null
+++ b/project/CommonPlugin.scala
@@ -0,0 +1,41 @@
+import sbt._
+import sbt.Keys._
+
+object CommonPlugin extends AutoPlugin {
+
+ override def requires = plugins.JvmPlugin
+ override def trigger = allRequirements
+
+ object autoImport {
+ lazy val UnitTest = config("unit") extend Test
+ lazy val IntegrationTest = config("integration") extend Test
+ lazy val SystemTest = config("system") extend Test
+ }
+ import autoImport._
+
+ override def projectSettings = {
+ inConfig(UnitTest)(Defaults.testSettings) ++
+ inConfig(IntegrationTest)(Defaults.testSettings) ++
+ inConfig(SystemTest)(Defaults.testSettings) ++
+ Seq(
+ testOptions in UnitTest := Seq(Tests.Filter(unitFilter)),
+ testOptions in IntegrationTest := Seq(Tests.Filter(intFilter)),
+ testOptions in SystemTest := Seq(Tests.Filter(sysFilter)),
+ // Add a global resource directory with compile/ and test/ for resources in all projects
+ unmanagedResourceDirectories in Compile ++= Seq(
+ (baseDirectory in ThisBuild).value / "resources" / "compile"
+ ),
+ unmanagedResourceDirectories in Test ++= Seq(
+ (baseDirectory in ThisBuild).value / "resources" / "test"
+ )
+ )
+ }
+
+ def sysFilter(name: String): Boolean =
+ (name endsWith "SpecForSystem") || (name startsWith "system.")
+ def intFilter(name: String): Boolean =
+ (name endsWith "SpecForIntegration") || (name startsWith "integration.")
+ def unitFilter(name: String): Boolean =
+ (name endsWith "Spec") && !intFilter(name) && !sysFilter(name)
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/Dependencies.scala
----------------------------------------------------------------------
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 2efdf35..508461f 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -4,20 +4,7 @@ import scala.util.Properties
object Dependencies {
- val sparkVersion = Def.setting{
- //sLog.value.warn("danger!")
- val envVar = "APACHE_SPARK_VERSION"
- val defaultVersion = "2.0.0"
-
- Properties.envOrNone(envVar) match {
- case None =>
- sLog.value.info(s"Using default Apache Spark version $defaultVersion!")
- defaultVersion
- case Some(version) =>
- sLog.value.info(s"Using Apache Spark version $version, provided from $envVar")
- version
- }
- }
+ // Libraries
val akkaActor = "com.typesafe.akka" %% "akka-actor" % "2.4.17"
val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % "2.4.17"
@@ -54,6 +41,7 @@ object Dependencies {
val slf4jApi = "org.slf4j" % "slf4j-api" % "1.7.21" // MIT
+ val sparkVersion = settingKey[String]("Version of Apache Spark to use in Toree") // defined in root build
val sparkCore = Def.setting{ "org.apache.spark" %% "spark-core" % sparkVersion.value }
val sparkGraphX = Def.setting{ "org.apache.spark" %% "spark-graphx" % sparkVersion.value }
val sparkMllib = Def.setting{ "org.apache.spark" %% "spark-mllib" % sparkVersion.value }
@@ -63,4 +51,23 @@ object Dependencies {
val springCore = "org.springframework" % "spring-core" % "4.1.1.RELEASE"// Apache v2
+ // Projects
+
+ val sparkAll = Def.setting{
+ Seq(
+ sparkCore.value % "provided" excludeAll(
+ // Exclude netty (org.jboss.netty is for 3.2.2.Final only)
+ ExclusionRule(
+ organization = "org.jboss.netty",
+ name = "netty"
+ )
+ ),
+ sparkGraphX.value % "provided",
+ sparkMllib.value % "provided",
+ sparkRepl.value % "provided",
+ sparkSql.value % "provided",
+ sparkStreaming.value % "provided"
+ )
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/build.properties
----------------------------------------------------------------------
diff --git a/project/build.properties b/project/build.properties
index 69fc25e..8dd784c 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -15,5 +15,4 @@
# limitations under the License
#
-sbt.version=0.13.9
-
+sbt.version=0.13.13
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/common.scala
----------------------------------------------------------------------
diff --git a/project/common.scala b/project/common.scala
deleted file mode 100644
index 2e87135..0000000
--- a/project/common.scala
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-import org.apache.commons.io.FileUtils
-import sbt._
-import sbt.Keys._
-import sbtbuildinfo._
-import sbtbuildinfo.BuildInfoKeys._
-import scoverage.ScoverageSbtPlugin
-//import coursier.Keys._
-import com.typesafe.sbt.pgp.PgpKeys._
-import scala.util.{Try, Properties}
-
-import Dependencies.sparkVersion
-
-object Common {
- // Parameters for publishing to artifact repositories
- private val versionNumber = Properties.envOrElse("VERSION", "0.0.0-dev")
- private val snapshot = Properties.envOrElse("IS_SNAPSHOT","true").toBoolean
- private val gpgLocation = Properties.envOrElse("GPG","/usr/local/bin/gpg")
- private val gpgPassword = Properties.envOrElse("GPG_PASSWORD","")
- private val buildOrganization = "org.apache.toree.kernel"
- private val buildVersion = if (snapshot) s"$versionNumber-SNAPSHOT" else versionNumber
- private val buildScalaVersion = "2.11.8"
-// private val buildScalaVersion = "2.10.6"
-
- val buildInfoSettings = Seq(
- buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, "sparkVersion" -> sparkVersion.value),
- buildInfoPackage := buildOrganization,
- buildInfoUsePackageAsPath := true,
- buildInfoOptions += BuildInfoOption.BuildTime
- )
-
- def ToreeProject(projectName: String, doFork: Boolean = false, needsSpark: Boolean = false):Project = (
- ToreeProject(s"toree-$projectName", projectName, doFork, needsSpark)
- )
-
- def ToreeProject(projectName: String, projectDir: String, doFork: Boolean, needsSpark: Boolean):Project = (
- Project(projectName, file(projectDir))
- .configs( UnitTest )
- .configs( IntegrationTest )
- .configs( SystemTest )
- .configs( ScratchTest )
- .settings(commonSettings:_*)
- .settings( inConfig(UnitTest)(Defaults.testTasks) : _*)
- .settings( inConfig(IntegrationTest)(Defaults.testTasks) : _*)
- .settings( inConfig(SystemTest)(Defaults.testTasks) : _*)
- .settings( inConfig(ScratchTest)(Defaults.testTasks) : _*)
- .settings(
- testOptions in UnitTest := Seq(Tests.Filter(unitFilter)),
- testOptions in IntegrationTest := Seq(Tests.Filter(intFilter)),
- testOptions in SystemTest := Seq(Tests.Filter(sysFilter)),
- testOptions in ScratchTest := Seq(Tests.Filter(scratchFilter))
- ).settings(
- fork in Test := doFork,
- fork in UnitTest := doFork,
- fork in IntegrationTest := doFork,
- fork in SystemTest := doFork,
- fork in ScratchTest := doFork,
- libraryDependencies ++= (
- if (needsSpark) Seq(
- Dependencies.sparkCore.value % "provided" excludeAll(
- // Exclude netty (org.jboss.netty is for 3.2.2.Final only)
- ExclusionRule(
- organization = "org.jboss.netty",
- name = "netty"
- )
- ),
- Dependencies.sparkGraphX.value % "provided",
- Dependencies.sparkMllib.value % "provided",
- Dependencies.sparkRepl.value % "provided",
- Dependencies.sparkSql.value % "provided",
- Dependencies.sparkStreaming.value % "provided"
- )
- else Nil
- )
- )
- )
-
- def scratchFilter(name: String): Boolean =
- (name endsWith "SpecForScratch") || (name startsWith "scratch.")
- def sysFilter(name: String): Boolean =
- (name endsWith "SpecForSystem") || (name startsWith "system.")
- def intFilter(name: String): Boolean =
- (name endsWith "SpecForIntegration") || (name startsWith "integration.")
- def unitFilter(name: String): Boolean =
- (name endsWith "Spec") && !intFilter(name) &&
- !sysFilter(name) && !scratchFilter(name)
-
- lazy val UnitTest = config("unit") extend Test
- lazy val IntegrationTest = config("integration") extend Test
- lazy val SystemTest = config("system") extend Test
- lazy val ScratchTest = config("scratch") extend Test
-
- val commonSettings: Seq[Def.Setting[_]] = Seq(
- organization := buildOrganization,
- useGpg := true,
- gpgCommand := gpgLocation,
- pgpPassphrase in Global := Some(gpgPassword.toArray),
- version := buildVersion,
- scalaVersion := buildScalaVersion,
-// crossScalaVersions := Seq("2.10.5", "2.11.8"),
- crossScalaVersions := Seq("2.11.8"),
- isSnapshot := snapshot,
- updateOptions := updateOptions.value.withCachedResolution(true),
- resolvers ++= Seq(
- "Apache Snapshots" at "http://repository.apache.org/snapshots/",
- "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/"
- ),
- // Test dependencies
- libraryDependencies ++= Seq(
- Dependencies.scalaTest % "test",
- Dependencies.mockito % "test",
- Dependencies.jacksonDatabind % "test"
- ),
- ScoverageSbtPlugin.ScoverageKeys.coverageHighlighting := false,
- pomExtra :=
- <parent>
- <groupId>org.apache</groupId>
- <artifactId>apache</artifactId>
- <version>10</version>
- </parent>
- <licenses>
- <license>
- <name>Apache 2</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
- <url>http://toree.incubator.apache.org/</url>
- <scm>
- <connection>scm:git:git@github.com:apache/incubator-toree.git</connection>
- <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-toree.git</developerConnection>
- <url>scm:git:git@github.com:apache/incubator-toree.git</url>
- <tag>HEAD</tag>
- </scm>,
-
- mappings in packageBin in Compile += file("LICENSE") -> "LICENSE",
- mappings in packageBin in Compile += file("NOTICE") -> "NOTICE",
-
-// coursierVerbosity := {
-// val level = Try(Integer.valueOf(Properties.envOrElse(
-// "TOREE_RESOLUTION_VERBOSITY", "1")
-// ).toInt).getOrElse(1)
-//
-// scala.Console.out.println(
-// s"[INFO] Toree Resolution Verbosity Level = $level"
-// )
-//
-// level
-// },
-
- scalacOptions in (Compile, doc) ++= Seq(
- // Ignore packages (for Scaladoc) not from our project
- "-skip-packages", Seq(
- "akka",
- "scala"
- ).mkString(":")
- ),
-
- // Scala-based options for compilation
- scalacOptions ++= Seq(
- "-deprecation",
- "-unchecked", "-feature",
- //"-Xlint", // Scala 2.11.x only
- "-Xfatal-warnings",
- //"-Ywarn-all",
- "-language:reflectiveCalls",
- "-target:jvm-1.6"
- ),
-
- // Java-based options for compilation (all tasks)
- // NOTE: Providing a blank flag causes failures, only uncomment with options
- //javacOptions in Compile ++= Seq(""),
-
- // Java-based options for just the compile task
- javacOptions in (Compile, compile) ++= Seq(
- "-Xlint:all", // Enable all Java-based warnings
- "-Xlint:-path", // Suppress path warnings since we get tons of them
- "-Xlint:-options",
- "-Xlint:-processing",
- "-Werror", // Treat warnings as errors
- "-source", "1.6",
- "-target", "1.6"
- ),
-
- scalacOptions in (Compile, doc) ++= Seq(
- "-no-link-warnings" // Suppresses problems with Scaladoc @throws links
- ),
-
- // Options provided to forked JVMs through sbt, based on our .jvmopts file
- javaOptions ++= Seq(
- "-Xms1024M", "-Xmx4096M", "-Xss2m", "-XX:MaxPermSize=1024M",
- "-XX:ReservedCodeCacheSize=256M", "-XX:+TieredCompilation",
- "-XX:+CMSPermGenSweepingEnabled", "-XX:+CMSClassUnloadingEnabled",
- "-XX:+UseConcMarkSweepGC", "-XX:+HeapDumpOnOutOfMemoryError"
- ),
-
- // Add additional test option to show time taken per test
- testOptions in Test += Tests.Argument("-oDF"),
-
- // Add a global resource directory with compile/ and test/ for resources in all projects
- unmanagedResourceDirectories in Compile += file("resources/compile"),
- unmanagedResourceDirectories in Test += file("resources/test"),
-
- // Publish Settings
- publishTo := {
- if (isSnapshot.value)
- Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/snapshots/")
- else
- Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/staging/")
- },
- credentials += Credentials(Path.userHome / ".ivy2" / ".credentials"),
-
- // Add rebuild ivy xml to the following tasks
- compile <<= (compile in Compile) dependsOn (rebuildIvyXml dependsOn deliverLocal)
- ) ++ rebuildIvyXmlSettings // Include our rebuild ivy xml settings
-
- // ==========================================================================
- // = REBUILD IVY XML SETTINGS BELOW
- // ==========================================================================
-
- lazy val rebuildIvyXml = TaskKey[Unit](
- "rebuild-ivy-xml",
- "Rebuilds the ivy xml using deliver-local and copies it to src " +
- "resource directories"
- )
-
- // TODO: Figure out how to retrieve the configuration being used to avoid
- // this duplication
- lazy val rebuildIvyXmlSettings = Seq(
- rebuildIvyXml := {
- val s: TaskStreams = streams.value
- val inputFile = (crossTarget.value / s"ivy-${version.value}.xml").getAbsoluteFile
- val outputFile =
- ((resourceDirectory in Compile).value / s"${name.value}-ivy.xml").getAbsoluteFile
- s.log.info(s"Copying ${inputFile.getPath} to ${outputFile.getPath}")
- FileUtils.copyFile(inputFile, outputFile)
- }
- )
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 61c6905..2b299da 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -27,7 +27,7 @@ resolvers += "Apache Snapshots" at "http://repository.apache.org/snapshots/"
addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0")
// Provides the ability to generate unifed documentation for multiple projects
-addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.3.1")
+addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.0")
// Provides the ability to list dependencies in a readable format using
// `sbt dependencyTree`; there are other commands provided as well
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/protocol/build.sbt
----------------------------------------------------------------------
diff --git a/protocol/build.sbt b/protocol/build.sbt
index ca70698..0ed902e 100644
--- a/protocol/build.sbt
+++ b/protocol/build.sbt
@@ -25,6 +25,8 @@ libraryDependencies ++= Seq(
Dependencies.slf4jApi
)
-//
-// TEST DEPENDENCIES
-//
+enablePlugins(BuildInfoPlugin)
+buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, "sparkVersion" -> Dependencies.sparkVersion.value)
+buildInfoPackage := organization.value
+buildInfoUsePackageAsPath := true
+buildInfoOptions += BuildInfoOption.BuildTime
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/pyspark-interpreter/build.sbt
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/build.sbt b/pyspark-interpreter/build.sbt
index 9444250..ac89909 100644
--- a/pyspark-interpreter/build.sbt
+++ b/pyspark-interpreter/build.sbt
@@ -14,3 +14,4 @@
* See the License for the specific language governing permissions and
* limitations under the License
*/
+libraryDependencies ++= Dependencies.sparkAll.value
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/scala-interpreter/build.sbt
----------------------------------------------------------------------
diff --git a/scala-interpreter/build.sbt b/scala-interpreter/build.sbt
index 5d6c95f..c07b032 100644
--- a/scala-interpreter/build.sbt
+++ b/scala-interpreter/build.sbt
@@ -16,41 +16,5 @@ import sbt.Tests.{Group, SubProcess}
* See the License for the specific language governing permissions and
* limitations under the License
*/
-// The chunk below is used to ensure that the test tagged with SbtForked ACTUALLY runs with
-// a forked jvm,
-
-def isAnnotatedWithRequiresSpark(definition: xsbti.api.Definition): Boolean = {
- definition.annotations().exists { annotation: xsbti.api.Annotation =>
- annotation.base match {
- case proj: xsbti.api.Projection if (proj.id() == "SbtForked") => true
- case _ => false
- }
- }
-}
-
-// Note the type TaskKey[Seq[String]] must be explicitly specified otherwise an error occurs
-lazy val testsAnnotatedWithRequiresSpark: TaskKey[Seq[String]] = taskKey[Seq[String]]("Returns list of FQCNs of tests annotated with SbtForked")
-testsAnnotatedWithRequiresSpark := {
- val analysis = (compile in Test).value
- analysis.apis.internal.values.flatMap({ source =>
- source.api().definitions().filter(isAnnotatedWithRequiresSpark).map(_.name())
- }).toSeq
-}
-
-
-def forkedJvmPerTest(testDefs: Seq[TestDefinition], testsToFork: Seq[String]) = {
-
- val (forkedTests, otherTests) = testDefs.partition { testDef => testsToFork.contains(testDef.name) }
-
- val otherTestsGroup = new Group(name = "Single JVM tests", tests = otherTests, runPolicy = SubProcess(javaOptions = Seq.empty[String]))
- val forkedTestGroups = forkedTests map { test =>
- new Group(
- name = test.name,
- tests = Seq(test),
- runPolicy = SubProcess(javaOptions = Seq.empty[String]))
- }
- Seq(otherTestsGroup) ++ forkedTestGroups
-}
-
-testGrouping in Test <<= (definedTests in Test, testsAnnotatedWithRequiresSpark) map forkedJvmPerTest
+libraryDependencies ++= Dependencies.sparkAll.value
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/sparkr-interpreter/build.sbt
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/build.sbt b/sparkr-interpreter/build.sbt
index 9444250..ac89909 100644
--- a/sparkr-interpreter/build.sbt
+++ b/sparkr-interpreter/build.sbt
@@ -14,3 +14,4 @@
* See the License for the specific language governing permissions and
* limitations under the License
*/
+libraryDependencies ++= Dependencies.sparkAll.value
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/sql-interpreter/build.sbt
----------------------------------------------------------------------
diff --git a/sql-interpreter/build.sbt b/sql-interpreter/build.sbt
index 9444250..ac89909 100644
--- a/sql-interpreter/build.sbt
+++ b/sql-interpreter/build.sbt
@@ -14,3 +14,4 @@
* See the License for the specific language governing permissions and
* limitations under the License
*/
+libraryDependencies ++= Dependencies.sparkAll.value
[4/7] incubator-toree git commit: Consolidate dependencies
Posted by ma...@apache.org.
Consolidate dependencies
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/bf02de54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/bf02de54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/bf02de54
Branch: refs/heads/master
Commit: bf02de5415fab3cad7a815e88624a64b8a59825c
Parents: abcc0c8
Author: Jakob Odersky <ja...@odersky.com>
Authored: Mon Feb 13 15:17:41 2017 -0800
Committer: Jakob Odersky <ja...@odersky.com>
Committed: Wed Feb 15 15:24:09 2017 -0800
----------------------------------------------------------------------
client/build.sbt | 8 ++---
communication/build.sbt | 8 ++---
kernel-api/build.sbt | 20 ++++++-------
kernel/build.sbt | 3 +-
macros/build.sbt | 8 +++--
plugins/build.sbt | 11 +++----
project/Dependencies.scala | 66 +++++++++++++++++++++++++++++++++++++++++
project/common.scala | 64 +++++++++++++--------------------------
protocol/build.sbt | 8 ++---
9 files changed, 117 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bf02de54/client/build.sbt
----------------------------------------------------------------------
diff --git a/client/build.sbt b/client/build.sbt
index 5c2ca7e..1ee62a7 100644
--- a/client/build.sbt
+++ b/client/build.sbt
@@ -19,7 +19,7 @@ scalacOptions += "-language:reflectiveCalls"
// Main library dependencies to function
libraryDependencies ++= Seq(
- "com.typesafe.akka" %% "akka-actor" % "2.4.8",
- "com.typesafe.akka" %% "akka-slf4j" % "2.4.8",
- "com.typesafe.akka" %% "akka-testkit" % "2.4.8" % "test"
-)
\ No newline at end of file
+ Dependencies.akkaActor,
+ Dependencies.akkaSlf4j,
+ Dependencies.akkaTestkit % "test"
+)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bf02de54/communication/build.sbt
----------------------------------------------------------------------
diff --git a/communication/build.sbt b/communication/build.sbt
index 8d0bee6..92db418 100644
--- a/communication/build.sbt
+++ b/communication/build.sbt
@@ -16,8 +16,8 @@
*/
libraryDependencies ++= Seq(
- "org.zeromq" % "jeromq" % "0.3.6",
- "com.typesafe.akka" %% "akka-actor" % "2.4.8",
- "com.typesafe.akka" %% "akka-slf4j" % "2.4.8",
- "com.typesafe.akka" %% "akka-testkit" % "2.4.8" % "test"
+ Dependencies.jeroMq,
+ Dependencies.akkaActor,
+ Dependencies.akkaSlf4j,
+ Dependencies.akkaTestkit % "test"
)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bf02de54/kernel-api/build.sbt
----------------------------------------------------------------------
diff --git a/kernel-api/build.sbt b/kernel-api/build.sbt
index 26e9d6f..e1ab494 100644
--- a/kernel-api/build.sbt
+++ b/kernel-api/build.sbt
@@ -28,35 +28,33 @@ ivyConfigurations += Configurations.ScalaTool
// Add the usual dependency on the library as well on the compiler in the
// 'scala-tool' configuration
libraryDependencies ++= Seq(
- "org.scala-lang" % "scala-library" % scalaVersion.value,
- "org.scala-lang" % "scala-compiler" % scalaVersion.value % "scala-tool",
- "org.scala-lang" % "scala-reflect" % scalaVersion.value
+ Dependencies.scalaLibrary.value,
+ Dependencies.scalaCompiler.value % "scala-tool",
+ Dependencies.scalaReflect.value
)
//
// EXECUTION DEPENDENCIES
//
-libraryDependencies += "org.apache.commons" % "commons-exec" % "1.3"
+libraryDependencies += Dependencies.commonsExec
//
// CLI DEPENDENCIES
//
-libraryDependencies += "net.sf.jopt-simple" % "jopt-simple" % "4.6" // MIT
+libraryDependencies += Dependencies.joptSimple
-libraryDependencies += "com.typesafe" % "config" % "1.3.0"
+libraryDependencies += Dependencies.config
//
// MAGIC DEPENDENCIES
//
libraryDependencies ++= Seq(
- // Used to find and download jars from Maven-based repositories
- "org.apache.ivy" % "ivy" % "2.4.0-rc1", // Apache v2
- "io.get-coursier" %% "coursier" % "1.0.0-M14", // Apache v2
- "io.get-coursier" %% "coursier-cache" % "1.0.0-M14" // Apache v2
+ Dependencies.coursier,
+ Dependencies.coursierCache
)
// Brought in in order to simplify the reading of each project's ivy.xml file
// from the classpath. If we really want we can write our own class and remove
// this dependency but the wheel has already been invented.
-libraryDependencies += "org.springframework" % "spring-core" % "4.1.1.RELEASE" // Apache v2
+libraryDependencies += Dependencies.springCore // Apache v2
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bf02de54/kernel/build.sbt
----------------------------------------------------------------------
diff --git a/kernel/build.sbt b/kernel/build.sbt
index 0965dce..752ad2a 100644
--- a/kernel/build.sbt
+++ b/kernel/build.sbt
@@ -18,6 +18,5 @@
//
// TEST DEPENDENCIES
//
-libraryDependencies +=
- "com.typesafe.akka" %% "akka-testkit" % "2.4.8" % "test" // MIT
+libraryDependencies += Dependencies.akkaTestkit % "test"
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bf02de54/macros/build.sbt
----------------------------------------------------------------------
diff --git a/macros/build.sbt b/macros/build.sbt
index 4083071..65eda08 100644
--- a/macros/build.sbt
+++ b/macros/build.sbt
@@ -15,9 +15,11 @@
* limitations under the License
*/
// Do not export a jar for this or publish anything (should serve just as a pre-processor)
-libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value
+libraryDependencies += Dependencies.scalaReflect.value
libraryDependencies ++= (
- if (scalaVersion.value.startsWith("2.10")) List("org.scalamacros" %% "quasiquotes" % "2.1.0")
- else Nil
+ if (scalaVersion.value.startsWith("2.10"))
+ List("org.scalamacros" %% "quasiquotes" % "2.1.0")
+ else
+ Nil
)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bf02de54/plugins/build.sbt
----------------------------------------------------------------------
diff --git a/plugins/build.sbt b/plugins/build.sbt
index 56cdc85..99f1701 100644
--- a/plugins/build.sbt
+++ b/plugins/build.sbt
@@ -17,13 +17,10 @@
// Needed for type inspection
libraryDependencies ++= Seq(
- "org.scala-lang" % "scala-reflect" % scalaVersion.value,
- // BSD 3-clause license, used for detecting plugins
- "org.clapper" %% "classutil" % "1.0.12",
- "org.slf4j" % "slf4j-api" % "1.7.21" // MIT
+ Dependencies.scalaReflect.value,
+ Dependencies.clapper,
+ Dependencies.slf4jApi
)
// Test dependencies
-libraryDependencies ++= Seq(
- "org.scala-lang" % "scala-compiler" % scalaVersion.value % "test"
-)
+libraryDependencies += Dependencies.scalaCompiler.value % "test"
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bf02de54/project/Dependencies.scala
----------------------------------------------------------------------
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
new file mode 100644
index 0000000..2efdf35
--- /dev/null
+++ b/project/Dependencies.scala
@@ -0,0 +1,66 @@
+import sbt._
+import sbt.Keys._
+import scala.util.Properties
+
+object Dependencies {
+
+ val sparkVersion = Def.setting{
+ //sLog.value.warn("danger!")
+ val envVar = "APACHE_SPARK_VERSION"
+ val defaultVersion = "2.0.0"
+
+ Properties.envOrNone(envVar) match {
+ case None =>
+ sLog.value.info(s"Using default Apache Spark version $defaultVersion!")
+ defaultVersion
+ case Some(version) =>
+ sLog.value.info(s"Using Apache Spark version $version, provided from $envVar")
+ version
+ }
+ }
+
+ val akkaActor = "com.typesafe.akka" %% "akka-actor" % "2.4.17"
+ val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % "2.4.17"
+ val akkaTestkit = "com.typesafe.akka" %% "akka-testkit" % "2.4.17"
+
+ val clapper = "org.clapper" %% "classutil" % "1.0.12" // BSD 3-clause license, used for detecting plugins
+
+ val commonsExec = "org.apache.commons" % "commons-exec" % "1.3"
+
+ val config = "com.typesafe" % "config" % "1.3.0"
+
+ // Apache v2
+ val coursier = "io.get-coursier" %% "coursier" % "1.0.0-M15-1"
+ val coursierCache = "io.get-coursier" %% "coursier-cache" % "1.0.0-M15-1"
+
+ val ivy = "org.apache.ivy" % "ivy" % "2.4.0-rc1" // Apache v2
+
+ // use the same jackson version in test than the one provided at runtime by Spark 2.0.0
+ val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.5" // Apache v2
+
+ val jeroMq = "org.zeromq" % "jeromq" % "0.3.6"
+
+ val joptSimple = "net.sf.jopt-simple" % "jopt-simple" % "4.6" // MIT
+
+ val mockito = "org.mockito" % "mockito-all" % "1.10.19" // MIT
+
+ val playJson = "com.typesafe.play" %% "play-json" % "2.3.10" // Apache v2
+
+ val scalaCompiler = Def.setting{ "org.scala-lang" % "scala-compiler" % scalaVersion.value }
+ val scalaLibrary = Def.setting{ "org.scala-lang" % "scala-library" % scalaVersion.value }
+ val scalaReflect = Def.setting{ "org.scala-lang" % "scala-reflect" % scalaVersion.value }
+
+ val scalaTest = "org.scalatest" %% "scalatest" % "2.2.6" // Apache v2
+
+ val slf4jApi = "org.slf4j" % "slf4j-api" % "1.7.21" // MIT
+
+ val sparkCore = Def.setting{ "org.apache.spark" %% "spark-core" % sparkVersion.value }
+ val sparkGraphX = Def.setting{ "org.apache.spark" %% "spark-graphx" % sparkVersion.value }
+ val sparkMllib = Def.setting{ "org.apache.spark" %% "spark-mllib" % sparkVersion.value }
+ val sparkRepl = Def.setting{ "org.apache.spark" %% "spark-repl" % sparkVersion.value }
+ val sparkSql = Def.setting{ "org.apache.spark" %% "spark-sql" % sparkVersion.value }
+ val sparkStreaming = Def.setting{ "org.apache.spark" %% "spark-streaming" % sparkVersion.value }
+
+ val springCore = "org.springframework" % "spring-core" % "4.1.1.RELEASE"// Apache v2
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bf02de54/project/common.scala
----------------------------------------------------------------------
diff --git a/project/common.scala b/project/common.scala
index 560ff4a..2e87135 100644
--- a/project/common.scala
+++ b/project/common.scala
@@ -24,6 +24,8 @@ import scoverage.ScoverageSbtPlugin
import com.typesafe.sbt.pgp.PgpKeys._
import scala.util.{Try, Properties}
+import Dependencies.sparkVersion
+
object Common {
// Parameters for publishing to artifact repositories
private val versionNumber = Properties.envOrElse("VERSION", "0.0.0-dev")
@@ -36,7 +38,7 @@ object Common {
// private val buildScalaVersion = "2.10.6"
val buildInfoSettings = Seq(
- buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, "sparkVersion" -> sparkVersion),
+ buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, "sparkVersion" -> sparkVersion.value),
buildInfoPackage := buildOrganization,
buildInfoUsePackageAsPath := true,
buildInfoOptions += BuildInfoOption.BuildTime
@@ -69,7 +71,20 @@ object Common {
fork in SystemTest := doFork,
fork in ScratchTest := doFork,
libraryDependencies ++= (
- if (needsSpark) sparkLibraries
+ if (needsSpark) Seq(
+ Dependencies.sparkCore.value % "provided" excludeAll(
+ // Exclude netty (org.jboss.netty is for 3.2.2.Final only)
+ ExclusionRule(
+ organization = "org.jboss.netty",
+ name = "netty"
+ )
+ ),
+ Dependencies.sparkGraphX.value % "provided",
+ Dependencies.sparkMllib.value % "provided",
+ Dependencies.sparkRepl.value % "provided",
+ Dependencies.sparkSql.value % "provided",
+ Dependencies.sparkStreaming.value % "provided"
+ )
else Nil
)
)
@@ -90,44 +105,6 @@ object Common {
lazy val SystemTest = config("system") extend Test
lazy val ScratchTest = config("scratch") extend Test
- private lazy val sparkVersion = {
- val sparkEnvironmentVariable = "APACHE_SPARK_VERSION"
- val defaultSparkVersion = "2.0.0"
-
- val _sparkVersion = Properties.envOrNone(sparkEnvironmentVariable)
-
- if (_sparkVersion.isEmpty) {
- scala.Console.out.println(
- s"""
- |[INFO] Using default Apache Spark $defaultSparkVersion!
- """.stripMargin.trim.replace('\n', ' '))
- defaultSparkVersion
- } else {
- val version = _sparkVersion.get
- scala.Console.out.println(
- s"""
- |[INFO] Using Apache Spark $version provided from
- |$sparkEnvironmentVariable!
- """.stripMargin.trim.replace('\n', ' '))
- version
- }
- }
-
- private val sparkLibraries = Seq(
- "org.apache.spark" %% "spark-core" % sparkVersion % "provided" excludeAll( // Apache v2
- // Exclude netty (org.jboss.netty is for 3.2.2.Final only)
- ExclusionRule(
- organization = "org.jboss.netty",
- name = "netty"
- )
- ),
- "org.apache.spark" %% "spark-streaming" % sparkVersion % "provided",
- "org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
- "org.apache.spark" %% "spark-mllib" % sparkVersion % "provided",
- "org.apache.spark" %% "spark-graphx" % sparkVersion % "provided",
- "org.apache.spark" %% "spark-repl" % sparkVersion % "provided"
- )
-
val commonSettings: Seq[Def.Setting[_]] = Seq(
organization := buildOrganization,
useGpg := true,
@@ -145,10 +122,9 @@ object Common {
),
// Test dependencies
libraryDependencies ++= Seq(
- "org.scalatest" %% "scalatest" % "2.2.6" % "test", // Apache v2
- "org.mockito" % "mockito-all" % "1.10.19" % "test", // MIT
- // use the same jackson version in test than the one provided at runtime by Spark 2.0.0
- "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.5" % "test" // Apache v2
+ Dependencies.scalaTest % "test",
+ Dependencies.mockito % "test",
+ Dependencies.jacksonDatabind % "test"
),
ScoverageSbtPlugin.ScoverageKeys.coverageHighlighting := false,
pomExtra :=
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bf02de54/protocol/build.sbt
----------------------------------------------------------------------
diff --git a/protocol/build.sbt b/protocol/build.sbt
index 097eccd..ca70698 100644
--- a/protocol/build.sbt
+++ b/protocol/build.sbt
@@ -19,10 +19,10 @@
// JSON DEPENDENCIES
//
libraryDependencies ++= Seq(
- "com.typesafe.play" %% "play-json" % "2.3.10" excludeAll( // Apache v2
- ExclusionRule(organization = "com.fasterxml.jackson.core")
- ),
- "org.slf4j" % "slf4j-api" % "1.7.21" // MIT
+ Dependencies.playJson excludeAll(
+ ExclusionRule(organization = "com.fasterxml.jackson.core")
+ ),
+ Dependencies.slf4jApi
)
//
[5/7] incubator-toree git commit: Remove unused plugins
Posted by ma...@apache.org.
Remove unused plugins
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/8bdfe01c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/8bdfe01c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/8bdfe01c
Branch: refs/heads/master
Commit: 8bdfe01c9900c509a73399ddd8d925a5c6513300
Parents: 00f4122
Author: Jakob Odersky <ja...@odersky.com>
Authored: Tue Feb 14 15:23:38 2017 -0800
Committer: Jakob Odersky <ja...@odersky.com>
Committed: Wed Feb 15 15:24:09 2017 -0800
----------------------------------------------------------------------
build.sbt | 2 --
project/plugins.sbt | 28 ----------------------------
2 files changed, 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/8bdfe01c/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index e60a99e..7943cc1 100644
--- a/build.sbt
+++ b/build.sbt
@@ -83,8 +83,6 @@ libraryDependencies in ThisBuild ++= Seq(
Dependencies.jacksonDatabind % "test"
)
-ScoverageSbtPlugin.ScoverageKeys.coverageHighlighting in ThisBuild := false
-
// Publish settings
useGpg in ThisBuild := true
pgpPassphrase in ThisBuild := Some(Properties.envOrElse("GPG_PASSWORD","").toArray)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/8bdfe01c/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 2b299da..1bfc875 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -17,42 +17,14 @@
logLevel := Level.Warn
-resolvers += Classpaths.sbtPluginReleases
-
-resolvers += "jgit-repo" at "http://download.eclipse.org/jgit/maven"
-
-resolvers += "Apache Snapshots" at "http://repository.apache.org/snapshots/"
-
-// Provides the ability to create an IntelliJ project using `sbt gen-idea`
-addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0")
-
// Provides the ability to generate unifed documentation for multiple projects
addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.0")
-// Provides the ability to list dependencies in a readable format using
-// `sbt dependencyTree`; there are other commands provided as well
-addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2")
-
// Provides abilit to create an uber-jar
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.0")
// Provides a generated build info object to sync between build and application
addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.6.1")
-// Provides code coverage support
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4")
-
-// Provides coveralls integration (for use with Travis-ci)
-addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0.BETA1")
-
-// Provides site generation functionality
-addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "0.8.1")
-
-// Provides auto-generating and publishing a gh-pages site
-addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.3")
-
-// Provides alternative resolving/downloading over sbt
-//addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-M14")
-
// Used for signing jars published via `sbt publish-signed`
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
[2/7] incubator-toree git commit: fix typo: remove space after
akka-slf4j version
Posted by ma...@apache.org.
fix typo: remove space after akka-slf4j version
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/4ad1afda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/4ad1afda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/4ad1afda
Branch: refs/heads/master
Commit: 4ad1afda0ca43167da1ffb02b6e8f5d23f45dee6
Parents: c00b4a3
Author: Jakob Odersky <ja...@odersky.com>
Authored: Tue Feb 14 17:00:55 2017 -0800
Committer: Jakob Odersky <ja...@odersky.com>
Committed: Wed Feb 15 15:24:09 2017 -0800
----------------------------------------------------------------------
project/Dependencies.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/4ad1afda/project/Dependencies.scala
----------------------------------------------------------------------
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index c827760..670a758 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -23,7 +23,7 @@ object Dependencies {
// Libraries
val akkaActor = "com.typesafe.akka" %% "akka-actor" % "2.4.17" // Apache v2
- val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % "2.4.17 "// Apache v2
+ val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % "2.4.17" // Apache v2
val akkaTestkit = "com.typesafe.akka" %% "akka-testkit" % "2.4.17" // Apache v2
val clapper = "org.clapper" %% "classutil" % "1.0.12" // BSD 3-clause license, used for detecting plugins
[6/7] incubator-toree git commit: Add misssing license headers
Posted by ma...@apache.org.
Add misssing license headers
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/c00b4a37
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/c00b4a37
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/c00b4a37
Branch: refs/heads/master
Commit: c00b4a37f8c74f7ed2bf29f4176a743ba445497d
Parents: 8bdfe01
Author: Jakob Odersky <ja...@odersky.com>
Authored: Tue Feb 14 15:24:36 2017 -0800
Committer: Jakob Odersky <ja...@odersky.com>
Committed: Wed Feb 15 15:24:09 2017 -0800
----------------------------------------------------------------------
.gitattributes | 17 ++++++++++++++
project/CommonPlugin.scala | 16 +++++++++++++
project/Dependencies.scala | 51 ++++++++++++++++++++++++++---------------
3 files changed, 66 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c00b4a37/.gitattributes
----------------------------------------------------------------------
diff --git a/.gitattributes b/.gitattributes
index a8edefd..2098a15 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
# Set the default behavior to have all files normalized to Unix-style
# line endings upon check-in.
* text=auto
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c00b4a37/project/CommonPlugin.scala
----------------------------------------------------------------------
diff --git a/project/CommonPlugin.scala b/project/CommonPlugin.scala
index a75a9a7..58358d5 100644
--- a/project/CommonPlugin.scala
+++ b/project/CommonPlugin.scala
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
import sbt._
import sbt.Keys._
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c00b4a37/project/Dependencies.scala
----------------------------------------------------------------------
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 508461f..c827760 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
import sbt._
import sbt.Keys._
import scala.util.Properties
@@ -6,26 +22,25 @@ object Dependencies {
// Libraries
- val akkaActor = "com.typesafe.akka" %% "akka-actor" % "2.4.17"
- val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % "2.4.17"
- val akkaTestkit = "com.typesafe.akka" %% "akka-testkit" % "2.4.17"
+ val akkaActor = "com.typesafe.akka" %% "akka-actor" % "2.4.17" // Apache v2
+ val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % "2.4.17 "// Apache v2
+ val akkaTestkit = "com.typesafe.akka" %% "akka-testkit" % "2.4.17" // Apache v2
val clapper = "org.clapper" %% "classutil" % "1.0.12" // BSD 3-clause license, used for detecting plugins
- val commonsExec = "org.apache.commons" % "commons-exec" % "1.3"
+ val commonsExec = "org.apache.commons" % "commons-exec" % "1.3" // Apache v2
- val config = "com.typesafe" % "config" % "1.3.0"
+ val config = "com.typesafe" % "config" % "1.3.0" // Apache v2
- // Apache v2
- val coursier = "io.get-coursier" %% "coursier" % "1.0.0-M15-1"
- val coursierCache = "io.get-coursier" %% "coursier-cache" % "1.0.0-M15-1"
+ val coursier = "io.get-coursier" %% "coursier" % "1.0.0-M15-1" // Apache v2
+ val coursierCache = "io.get-coursier" %% "coursier-cache" % "1.0.0-M15-1" // Apache v2
val ivy = "org.apache.ivy" % "ivy" % "2.4.0-rc1" // Apache v2
// use the same jackson version in test than the one provided at runtime by Spark 2.0.0
val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.5" // Apache v2
- val jeroMq = "org.zeromq" % "jeromq" % "0.3.6"
+ val jeroMq = "org.zeromq" % "jeromq" % "0.3.6" // MPL v2
val joptSimple = "net.sf.jopt-simple" % "jopt-simple" % "4.6" // MIT
@@ -33,21 +48,21 @@ object Dependencies {
val playJson = "com.typesafe.play" %% "play-json" % "2.3.10" // Apache v2
- val scalaCompiler = Def.setting{ "org.scala-lang" % "scala-compiler" % scalaVersion.value }
- val scalaLibrary = Def.setting{ "org.scala-lang" % "scala-library" % scalaVersion.value }
- val scalaReflect = Def.setting{ "org.scala-lang" % "scala-reflect" % scalaVersion.value }
+ val scalaCompiler = Def.setting{ "org.scala-lang" % "scala-compiler" % scalaVersion.value } // BSD 3-clause
+ val scalaLibrary = Def.setting{ "org.scala-lang" % "scala-library" % scalaVersion.value } // BSD 3-clause
+ val scalaReflect = Def.setting{ "org.scala-lang" % "scala-reflect" % scalaVersion.value } // BSD 3-clause
val scalaTest = "org.scalatest" %% "scalatest" % "2.2.6" // Apache v2
val slf4jApi = "org.slf4j" % "slf4j-api" % "1.7.21" // MIT
val sparkVersion = settingKey[String]("Version of Apache Spark to use in Toree") // defined in root build
- val sparkCore = Def.setting{ "org.apache.spark" %% "spark-core" % sparkVersion.value }
- val sparkGraphX = Def.setting{ "org.apache.spark" %% "spark-graphx" % sparkVersion.value }
- val sparkMllib = Def.setting{ "org.apache.spark" %% "spark-mllib" % sparkVersion.value }
- val sparkRepl = Def.setting{ "org.apache.spark" %% "spark-repl" % sparkVersion.value }
- val sparkSql = Def.setting{ "org.apache.spark" %% "spark-sql" % sparkVersion.value }
- val sparkStreaming = Def.setting{ "org.apache.spark" %% "spark-streaming" % sparkVersion.value }
+ val sparkCore = Def.setting{ "org.apache.spark" %% "spark-core" % sparkVersion.value } // Apache v2
+ val sparkGraphX = Def.setting{ "org.apache.spark" %% "spark-graphx" % sparkVersion.value } // Apache v2
+ val sparkMllib = Def.setting{ "org.apache.spark" %% "spark-mllib" % sparkVersion.value } // Apache v2
+ val sparkRepl = Def.setting{ "org.apache.spark" %% "spark-repl" % sparkVersion.value } // Apache v2
+ val sparkSql = Def.setting{ "org.apache.spark" %% "spark-sql" % sparkVersion.value } // Apache v2
+ val sparkStreaming = Def.setting{ "org.apache.spark" %% "spark-streaming" % sparkVersion.value } // Apache v2
val springCore = "org.springframework" % "spring-core" % "4.1.1.RELEASE"// Apache v2
[7/7] incubator-toree git commit: TOREE-382 Merging pr #101
Posted by ma...@apache.org.
TOREE-382 Merging pr #101
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/5743f068
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/5743f068
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/5743f068
Branch: refs/heads/master
Commit: 5743f068d9ee65132a20598a7bae9279f2a5cebf
Parents: e5ef3a1 464c367
Author: mariusvniekerk <ma...@gmail.com>
Authored: Thu Feb 16 13:35:27 2017 -0500
Committer: mariusvniekerk <ma...@gmail.com>
Committed: Thu Feb 16 13:35:27 2017 -0500
----------------------------------------------------------------------
.gitattributes | 17 +++
Makefile | 5 +-
build.sbt | 204 ++++++++++++++++++++++-----
client/build.sbt | 8 +-
communication/build.sbt | 8 +-
kernel-api/build.sbt | 22 +--
kernel/build.sbt | 5 +-
macros/build.sbt | 8 +-
plugins/build.sbt | 13 +-
project/CommonPlugin.scala | 64 +++++++++
project/Dependencies.scala | 88 ++++++++++++
project/build.properties | 3 +-
project/common.scala | 279 -------------------------------------
project/plugins.sbt | 30 +---
protocol/build.sbt | 16 ++-
pyspark-interpreter/build.sbt | 1 +
scala-interpreter/build.sbt | 38 +----
sparkr-interpreter/build.sbt | 1 +
sql-interpreter/build.sbt | 1 +
19 files changed, 386 insertions(+), 425 deletions(-)
----------------------------------------------------------------------