You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by ma...@apache.org on 2017/02/16 18:35:35 UTC
[3/7] incubator-toree git commit: Move common project settings to
build-level settings
Move common project settings to build-level settings
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/00f4122a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/00f4122a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/00f4122a
Branch: refs/heads/master
Commit: 00f4122ae23c4c04a9706af0ea7e1ac1cb807d11
Parents: bf02de5
Author: Jakob Odersky <ja...@odersky.com>
Authored: Mon Feb 13 16:16:29 2017 -0800
Committer: Jakob Odersky <ja...@odersky.com>
Committed: Wed Feb 15 15:24:09 2017 -0800
----------------------------------------------------------------------
Makefile | 5 +-
build.sbt | 206 ++++++++++++++++++++++++------
kernel-api/build.sbt | 2 +
kernel/build.sbt | 2 +
plugins/build.sbt | 2 +
project/CommonPlugin.scala | 41 ++++++
project/Dependencies.scala | 35 +++--
project/build.properties | 3 +-
project/common.scala | 255 -------------------------------------
project/plugins.sbt | 2 +-
protocol/build.sbt | 8 +-
pyspark-interpreter/build.sbt | 1 +
scala-interpreter/build.sbt | 38 +-----
sparkr-interpreter/build.sbt | 1 +
sql-interpreter/build.sbt | 1 +
15 files changed, 250 insertions(+), 352 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/Makefile
----------------------------------------------------------------------
diff --git a/Makefile b/Makefile
index ec160e5..30bf04c 100644
--- a/Makefile
+++ b/Makefile
@@ -106,8 +106,9 @@ dev-binder: .binder-image
target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): VM_WORKDIR=/src/toree-kernel
target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): ${shell find ./*/src/main/**/*}
target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): ${shell find ./*/build.sbt}
-target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): dist/toree-legal project/build.properties build.sbt project/common.scala project/plugins.sbt
- $(call RUN,$(ENV_OPTS) sbt toree/assembly)
+target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): ${shell find ./project/*.scala} ${shell find ./project/*.sbt}
+target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR): dist/toree-legal project/build.properties build.sbt
+ $(call RUN,$(ENV_OPTS) sbt root/assembly)
build: target/scala-$(SCALA_VERSION)/$(ASSEMBLY_JAR)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index a3d1f2a..e60a99e 100644
--- a/build.sbt
+++ b/build.sbt
@@ -15,25 +15,117 @@
* limitations under the License
*/
-import Common._
-import sbtunidoc.Plugin.UnidocKeys._
-import sbtunidoc.Plugin._
-import com.typesafe.sbt.SbtGit.{GitKeys => git}
-import com.typesafe.sbt.SbtSite.site
-
-lazy val root = ToreeProject("toree", ".", doFork=false, needsSpark=true).
- settings(unidocSettings:_*).
- settings(site.settings:_*).
- settings(
- test in assembly := {},
- assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false),
- scalacOptions in (ScalaUnidoc, unidoc) += "-Ymacro-no-expand",
- git.gitRemoteRepo := "git://git.apache.org/incubator-toree.git",
- aggregate in assembly := false,
- unmanagedResourceDirectories in Compile += { baseDirectory.value / "dist/toree-legal" }
- ).aggregate(
+import scala.util.Properties
+
+// Version settings
+version in ThisBuild := Properties.envOrElse("VERSION", "0.0.0-dev") +
+ (if ((isSnapshot in ThisBuild).value) "-SNAPSHOT" else "")
+isSnapshot in ThisBuild := Properties.envOrElse("IS_SNAPSHOT","true").toBoolean
+organization in ThisBuild := "org.apache.toree.kernel"
+crossScalaVersions in ThisBuild := Seq("2.11.8")
+scalaVersion in ThisBuild := (crossScalaVersions in ThisBuild).value.head
+Dependencies.sparkVersion in ThisBuild := {
+ val envVar = "APACHE_SPARK_VERSION"
+ val defaultVersion = "2.0.0"
+
+ Properties.envOrNone(envVar) match {
+ case None =>
+ sLog.value.info(s"Using default Apache Spark version $defaultVersion!")
+ defaultVersion
+ case Some(version) =>
+ sLog.value.info(s"Using Apache Spark version $version, provided from $envVar")
+ version
+ }
+}
+
+// Compiler settings
+scalacOptions in ThisBuild ++= Seq(
+ "-deprecation",
+ "-unchecked",
+ "-feature",
+ "-Xfatal-warnings",
+ "-language:reflectiveCalls",
+ "-target:jvm-1.6",
+ "-Xlint" // Scala 2.11.x only
+)
+// Java-based options for compilation (all tasks)
+// NOTE: Providing a blank flag causes failures, only uncomment with options
+//javacOptions in Compile ++= Seq(""),
+// Java-based options for just the compile task
+javacOptions in ThisBuild ++= Seq(
+ "-Xlint:all", // Enable all Java-based warnings
+ "-Xlint:-path", // Suppress path warnings since we get tons of them
+ "-Xlint:-options",
+ "-Xlint:-processing",
+ "-Werror", // Treat warnings as errors
+ "-source", "1.6",
+ "-target", "1.6"
+)
+// Options provided to forked JVMs through sbt, based on our .jvmopts file
+javaOptions in ThisBuild ++= Seq(
+ "-Xms1024M", "-Xmx4096M", "-Xss2m", "-XX:MaxPermSize=1024M",
+ "-XX:ReservedCodeCacheSize=256M", "-XX:+TieredCompilation",
+ "-XX:+CMSPermGenSweepingEnabled", "-XX:+CMSClassUnloadingEnabled",
+ "-XX:+UseConcMarkSweepGC", "-XX:+HeapDumpOnOutOfMemoryError"
+)
+// Add additional test option to show time taken per test
+testOptions in (ThisBuild, Test) += Tests.Argument("-oDF")
+
+// Build-wide dependencies
+resolvers in ThisBuild ++= Seq(
+ "Apache Snapshots" at "http://repository.apache.org/snapshots/",
+ "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/"
+)
+updateOptions in ThisBuild := updateOptions.value.withCachedResolution(true)
+libraryDependencies in ThisBuild ++= Seq(
+ Dependencies.scalaTest % "test",
+ Dependencies.mockito % "test",
+ Dependencies.jacksonDatabind % "test"
+)
+
+ScoverageSbtPlugin.ScoverageKeys.coverageHighlighting in ThisBuild := false
+
+// Publish settings
+useGpg in ThisBuild := true
+pgpPassphrase in ThisBuild := Some(Properties.envOrElse("GPG_PASSWORD","").toArray)
+publishTo in ThisBuild := {
+ if (isSnapshot.value)
+ Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/snapshots/")
+ else
+ Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/staging/")
+}
+mappings in packageBin in ThisBuild := Seq(
+ file("LICENSE") -> "LICENSE",
+ file("NOTICE") -> "NOTICE"
+)
+licenses in ThisBuild := Seq("Apache 2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt"))
+pomExtra in ThisBuild := {
+ <parent>
+ <groupId>org.apache</groupId>
+ <artifactId>apache</artifactId>
+ <version>10</version>
+ </parent>
+ <url>http://toree.incubator.apache.org/</url>
+ <scm>
+ <url>git@github.com:apache/incubator-toree.git</url>
+ <connection>scm:git:git@github.com:apache/incubator-toree.git</connection>
+ <developerConnection>
+ scm:git:https://git-wip-us.apache.org/repos/asf/incubator-toree.git
+ </developerConnection>
+ <tag>HEAD</tag>
+ </scm>
+}
+credentials in ThisBuild+= Credentials(Path.userHome / ".ivy2" / ".credentials")
+
+// Project structure
+
+/** Root Toree project. */
+lazy val root = (project in file("."))
+ .settings(name := "toree")
+ .aggregate(
macros,protocol,plugins,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,pysparkInterpreter,sparkrInterpreter,kernel
- ).dependsOn(
+ )
+ .dependsOn(
macros,protocol,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,pysparkInterpreter,sparkrInterpreter,kernel
)
@@ -41,67 +133,105 @@ lazy val root = ToreeProject("toree", ".", doFork=false, needsSpark=true).
* Project representing macros in Scala that must be compiled separately from
* any other project using them.
*/
-lazy val macros = ToreeProject("macros")
+lazy val macros = (project in file("macros"))
+ .settings(name := "toree-macros")
/**
* Project representing the IPython kernel message protocol in Scala. Used
* by the client and kernel implementations.
*/
-lazy val protocol = ToreeProject("protocol").dependsOn(macros).
- enablePlugins(BuildInfoPlugin).settings(buildInfoSettings:_*)
+lazy val protocol = (project in file("protocol"))
+ .settings(name := "toree-protocol")
+ .dependsOn(macros)
/**
* Project representing base plugin system for the Toree infrastructure.
*/
-lazy val plugins = ToreeProject("plugins", doFork=true).dependsOn(macros)
+lazy val plugins = (project in file("plugins"))
+ .settings(name := "toree-plugins")
+ .dependsOn(macros)
/**
* Project representing forms of communication used as input/output for the
* client/kernel.
*/
-lazy val communication = ToreeProject("communication").dependsOn(macros, protocol)
+lazy val communication = (project in file("communication"))
+ .settings(name := "toree-communication")
+ .dependsOn(macros, protocol)
/**
* Project representing the kernel-api code used by the Spark Kernel. Others can
* import this to implement their own magics and plugins.
*/
-lazy val kernelApi = ToreeProject("kernel-api", needsSpark=true).dependsOn(macros, plugins)
+lazy val kernelApi = (project in file("kernel-api"))
+ .settings(name := "toree-kernel-api")
+ .dependsOn(macros, plugins)
/**
* Project representing the client code for connecting to the kernel backend.
*/
-lazy val client = ToreeProject("client").dependsOn(macros, protocol, communication)
+lazy val client = (project in file("client"))
+ .settings(name := "toree-client")
+ .dependsOn(macros, protocol, communication)
/**
* Project represents the scala interpreter used by the Spark Kernel.
*/
-lazy val scalaInterpreter = ToreeProject("scala-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi)
+lazy val scalaInterpreter = (project in file("scala-interpreter"))
+ .settings(name := "toree-scala-interpreter")
+ .dependsOn(plugins, protocol, kernelApi)
/**
* Project represents the SQL interpreter used by the Spark Kernel.
*/
-lazy val sqlInterpreter = ToreeProject("sql-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi)
+lazy val sqlInterpreter = (project in file("sql-interpreter"))
+ .settings(name := "toree-sql-interpreter")
+ .dependsOn(plugins, protocol, kernelApi)
/**
* Project represents the Python interpreter used by the Spark Kernel.
*/
-lazy val pysparkInterpreter = ToreeProject("pyspark-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi)
+lazy val pysparkInterpreter = (project in file("pyspark-interpreter"))
+ .settings(name := "toree-pyspark-interpreter")
+ .dependsOn(plugins, protocol, kernelApi)
/**
* Project represents the R interpreter used by the Spark Kernel.
*/
-lazy val sparkrInterpreter = ToreeProject("sparkr-interpreter", needsSpark=true).dependsOn(plugins, protocol, kernelApi)
+lazy val sparkrInterpreter = (project in file("sparkr-interpreter"))
+ .settings(name := "toree-sparkr-interpreter")
+ .dependsOn(plugins, protocol, kernelApi)
/**
* Project representing the kernel code for the Spark Kernel backend.
*/
-lazy val kernel = ToreeProject("kernel", doFork=true, needsSpark=true).dependsOn(
- macros % "test->test;compile->compile",
- protocol % "test->test;compile->compile",
- communication % "test->test;compile->compile",
- kernelApi % "test->test;compile->compile",
- pysparkInterpreter % "test->test;compile->compile",
- scalaInterpreter % "test->test;compile->compile",
- sparkrInterpreter % "test->test;compile->compile",
- sqlInterpreter % "test->test;compile->compile"
+lazy val kernel = (project in file("kernel"))
+ .settings(name := "toree-kernel")
+ .dependsOn(
+ macros % "test->test;compile->compile",
+ protocol % "test->test;compile->compile",
+ communication % "test->test;compile->compile",
+ kernelApi % "test->test;compile->compile",
+ pysparkInterpreter % "test->test;compile->compile",
+ scalaInterpreter % "test->test;compile->compile",
+ sparkrInterpreter % "test->test;compile->compile",
+ sqlInterpreter % "test->test;compile->compile"
+ )
+
+// Root project settings
+enablePlugins(ScalaUnidocPlugin)
+scalacOptions in (ScalaUnidoc, unidoc) ++= Seq(
+ "-Ymacro-expand:none",
+ "-skip-packages", Seq(
+ "akka",
+ "scala"
+ ).mkString(":"),
+ "-no-link-warnings" // Suppresses problems with Scaladoc @throws links
)
+
+libraryDependencies ++= Dependencies.sparkAll.value
+unmanagedResourceDirectories in Compile += { baseDirectory.value / "dist/toree-legal" }
+
+test in assembly := {}
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
+aggregate in assembly := false
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/kernel-api/build.sbt
----------------------------------------------------------------------
diff --git a/kernel-api/build.sbt b/kernel-api/build.sbt
index e1ab494..68be752 100644
--- a/kernel-api/build.sbt
+++ b/kernel-api/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License
*/
+libraryDependencies ++= Dependencies.sparkAll.value
+
//
// SCALA INTERPRETER DEPENDENCIES
//
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/kernel/build.sbt
----------------------------------------------------------------------
diff --git a/kernel/build.sbt b/kernel/build.sbt
index 752ad2a..5428b6e 100644
--- a/kernel/build.sbt
+++ b/kernel/build.sbt
@@ -14,6 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License
*/
+fork in Test := true
+libraryDependencies ++= Dependencies.sparkAll.value
//
// TEST DEPENDENCIES
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/plugins/build.sbt
----------------------------------------------------------------------
diff --git a/plugins/build.sbt b/plugins/build.sbt
index 99f1701..f41d40b 100644
--- a/plugins/build.sbt
+++ b/plugins/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License
*/
+fork in Test := true
+
// Needed for type inspection
libraryDependencies ++= Seq(
Dependencies.scalaReflect.value,
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/CommonPlugin.scala
----------------------------------------------------------------------
diff --git a/project/CommonPlugin.scala b/project/CommonPlugin.scala
new file mode 100644
index 0000000..a75a9a7
--- /dev/null
+++ b/project/CommonPlugin.scala
@@ -0,0 +1,41 @@
+import sbt._
+import sbt.Keys._
+
+object CommonPlugin extends AutoPlugin {
+
+ override def requires = plugins.JvmPlugin
+ override def trigger = allRequirements
+
+ object autoImport {
+ lazy val UnitTest = config("unit") extend Test
+ lazy val IntegrationTest = config("integration") extend Test
+ lazy val SystemTest = config("system") extend Test
+ }
+ import autoImport._
+
+ override def projectSettings = {
+ inConfig(UnitTest)(Defaults.testSettings) ++
+ inConfig(IntegrationTest)(Defaults.testSettings) ++
+ inConfig(SystemTest)(Defaults.testSettings) ++
+ Seq(
+ testOptions in UnitTest := Seq(Tests.Filter(unitFilter)),
+ testOptions in IntegrationTest := Seq(Tests.Filter(intFilter)),
+ testOptions in SystemTest := Seq(Tests.Filter(sysFilter)),
+ // Add a global resource directory with compile/ and test/ for resources in all projects
+ unmanagedResourceDirectories in Compile ++= Seq(
+ (baseDirectory in ThisBuild).value / "resources" / "compile"
+ ),
+ unmanagedResourceDirectories in Test ++= Seq(
+ (baseDirectory in ThisBuild).value / "resources" / "test"
+ )
+ )
+ }
+
+ def sysFilter(name: String): Boolean =
+ (name endsWith "SpecForSystem") || (name startsWith "system.")
+ def intFilter(name: String): Boolean =
+ (name endsWith "SpecForIntegration") || (name startsWith "integration.")
+ def unitFilter(name: String): Boolean =
+ (name endsWith "Spec") && !intFilter(name) && !sysFilter(name)
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/Dependencies.scala
----------------------------------------------------------------------
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 2efdf35..508461f 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -4,20 +4,7 @@ import scala.util.Properties
object Dependencies {
- val sparkVersion = Def.setting{
- //sLog.value.warn("danger!")
- val envVar = "APACHE_SPARK_VERSION"
- val defaultVersion = "2.0.0"
-
- Properties.envOrNone(envVar) match {
- case None =>
- sLog.value.info(s"Using default Apache Spark version $defaultVersion!")
- defaultVersion
- case Some(version) =>
- sLog.value.info(s"Using Apache Spark version $version, provided from $envVar")
- version
- }
- }
+ // Libraries
val akkaActor = "com.typesafe.akka" %% "akka-actor" % "2.4.17"
val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % "2.4.17"
@@ -54,6 +41,7 @@ object Dependencies {
val slf4jApi = "org.slf4j" % "slf4j-api" % "1.7.21" // MIT
+ val sparkVersion = settingKey[String]("Version of Apache Spark to use in Toree") // defined in root build
val sparkCore = Def.setting{ "org.apache.spark" %% "spark-core" % sparkVersion.value }
val sparkGraphX = Def.setting{ "org.apache.spark" %% "spark-graphx" % sparkVersion.value }
val sparkMllib = Def.setting{ "org.apache.spark" %% "spark-mllib" % sparkVersion.value }
@@ -63,4 +51,23 @@ object Dependencies {
val springCore = "org.springframework" % "spring-core" % "4.1.1.RELEASE"// Apache v2
+ // Projects
+
+ val sparkAll = Def.setting{
+ Seq(
+ sparkCore.value % "provided" excludeAll(
+ // Exclude netty (org.jboss.netty is for 3.2.2.Final only)
+ ExclusionRule(
+ organization = "org.jboss.netty",
+ name = "netty"
+ )
+ ),
+ sparkGraphX.value % "provided",
+ sparkMllib.value % "provided",
+ sparkRepl.value % "provided",
+ sparkSql.value % "provided",
+ sparkStreaming.value % "provided"
+ )
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/build.properties
----------------------------------------------------------------------
diff --git a/project/build.properties b/project/build.properties
index 69fc25e..8dd784c 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -15,5 +15,4 @@
# limitations under the License
#
-sbt.version=0.13.9
-
+sbt.version=0.13.13
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/common.scala
----------------------------------------------------------------------
diff --git a/project/common.scala b/project/common.scala
deleted file mode 100644
index 2e87135..0000000
--- a/project/common.scala
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-import org.apache.commons.io.FileUtils
-import sbt._
-import sbt.Keys._
-import sbtbuildinfo._
-import sbtbuildinfo.BuildInfoKeys._
-import scoverage.ScoverageSbtPlugin
-//import coursier.Keys._
-import com.typesafe.sbt.pgp.PgpKeys._
-import scala.util.{Try, Properties}
-
-import Dependencies.sparkVersion
-
-object Common {
- // Parameters for publishing to artifact repositories
- private val versionNumber = Properties.envOrElse("VERSION", "0.0.0-dev")
- private val snapshot = Properties.envOrElse("IS_SNAPSHOT","true").toBoolean
- private val gpgLocation = Properties.envOrElse("GPG","/usr/local/bin/gpg")
- private val gpgPassword = Properties.envOrElse("GPG_PASSWORD","")
- private val buildOrganization = "org.apache.toree.kernel"
- private val buildVersion = if (snapshot) s"$versionNumber-SNAPSHOT" else versionNumber
- private val buildScalaVersion = "2.11.8"
-// private val buildScalaVersion = "2.10.6"
-
- val buildInfoSettings = Seq(
- buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, "sparkVersion" -> sparkVersion.value),
- buildInfoPackage := buildOrganization,
- buildInfoUsePackageAsPath := true,
- buildInfoOptions += BuildInfoOption.BuildTime
- )
-
- def ToreeProject(projectName: String, doFork: Boolean = false, needsSpark: Boolean = false):Project = (
- ToreeProject(s"toree-$projectName", projectName, doFork, needsSpark)
- )
-
- def ToreeProject(projectName: String, projectDir: String, doFork: Boolean, needsSpark: Boolean):Project = (
- Project(projectName, file(projectDir))
- .configs( UnitTest )
- .configs( IntegrationTest )
- .configs( SystemTest )
- .configs( ScratchTest )
- .settings(commonSettings:_*)
- .settings( inConfig(UnitTest)(Defaults.testTasks) : _*)
- .settings( inConfig(IntegrationTest)(Defaults.testTasks) : _*)
- .settings( inConfig(SystemTest)(Defaults.testTasks) : _*)
- .settings( inConfig(ScratchTest)(Defaults.testTasks) : _*)
- .settings(
- testOptions in UnitTest := Seq(Tests.Filter(unitFilter)),
- testOptions in IntegrationTest := Seq(Tests.Filter(intFilter)),
- testOptions in SystemTest := Seq(Tests.Filter(sysFilter)),
- testOptions in ScratchTest := Seq(Tests.Filter(scratchFilter))
- ).settings(
- fork in Test := doFork,
- fork in UnitTest := doFork,
- fork in IntegrationTest := doFork,
- fork in SystemTest := doFork,
- fork in ScratchTest := doFork,
- libraryDependencies ++= (
- if (needsSpark) Seq(
- Dependencies.sparkCore.value % "provided" excludeAll(
- // Exclude netty (org.jboss.netty is for 3.2.2.Final only)
- ExclusionRule(
- organization = "org.jboss.netty",
- name = "netty"
- )
- ),
- Dependencies.sparkGraphX.value % "provided",
- Dependencies.sparkMllib.value % "provided",
- Dependencies.sparkRepl.value % "provided",
- Dependencies.sparkSql.value % "provided",
- Dependencies.sparkStreaming.value % "provided"
- )
- else Nil
- )
- )
- )
-
- def scratchFilter(name: String): Boolean =
- (name endsWith "SpecForScratch") || (name startsWith "scratch.")
- def sysFilter(name: String): Boolean =
- (name endsWith "SpecForSystem") || (name startsWith "system.")
- def intFilter(name: String): Boolean =
- (name endsWith "SpecForIntegration") || (name startsWith "integration.")
- def unitFilter(name: String): Boolean =
- (name endsWith "Spec") && !intFilter(name) &&
- !sysFilter(name) && !scratchFilter(name)
-
- lazy val UnitTest = config("unit") extend Test
- lazy val IntegrationTest = config("integration") extend Test
- lazy val SystemTest = config("system") extend Test
- lazy val ScratchTest = config("scratch") extend Test
-
- val commonSettings: Seq[Def.Setting[_]] = Seq(
- organization := buildOrganization,
- useGpg := true,
- gpgCommand := gpgLocation,
- pgpPassphrase in Global := Some(gpgPassword.toArray),
- version := buildVersion,
- scalaVersion := buildScalaVersion,
-// crossScalaVersions := Seq("2.10.5", "2.11.8"),
- crossScalaVersions := Seq("2.11.8"),
- isSnapshot := snapshot,
- updateOptions := updateOptions.value.withCachedResolution(true),
- resolvers ++= Seq(
- "Apache Snapshots" at "http://repository.apache.org/snapshots/",
- "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/"
- ),
- // Test dependencies
- libraryDependencies ++= Seq(
- Dependencies.scalaTest % "test",
- Dependencies.mockito % "test",
- Dependencies.jacksonDatabind % "test"
- ),
- ScoverageSbtPlugin.ScoverageKeys.coverageHighlighting := false,
- pomExtra :=
- <parent>
- <groupId>org.apache</groupId>
- <artifactId>apache</artifactId>
- <version>10</version>
- </parent>
- <licenses>
- <license>
- <name>Apache 2</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
- <url>http://toree.incubator.apache.org/</url>
- <scm>
- <connection>scm:git:git@github.com:apache/incubator-toree.git</connection>
- <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-toree.git</developerConnection>
- <url>scm:git:git@github.com:apache/incubator-toree.git</url>
- <tag>HEAD</tag>
- </scm>,
-
- mappings in packageBin in Compile += file("LICENSE") -> "LICENSE",
- mappings in packageBin in Compile += file("NOTICE") -> "NOTICE",
-
-// coursierVerbosity := {
-// val level = Try(Integer.valueOf(Properties.envOrElse(
-// "TOREE_RESOLUTION_VERBOSITY", "1")
-// ).toInt).getOrElse(1)
-//
-// scala.Console.out.println(
-// s"[INFO] Toree Resolution Verbosity Level = $level"
-// )
-//
-// level
-// },
-
- scalacOptions in (Compile, doc) ++= Seq(
- // Ignore packages (for Scaladoc) not from our project
- "-skip-packages", Seq(
- "akka",
- "scala"
- ).mkString(":")
- ),
-
- // Scala-based options for compilation
- scalacOptions ++= Seq(
- "-deprecation",
- "-unchecked", "-feature",
- //"-Xlint", // Scala 2.11.x only
- "-Xfatal-warnings",
- //"-Ywarn-all",
- "-language:reflectiveCalls",
- "-target:jvm-1.6"
- ),
-
- // Java-based options for compilation (all tasks)
- // NOTE: Providing a blank flag causes failures, only uncomment with options
- //javacOptions in Compile ++= Seq(""),
-
- // Java-based options for just the compile task
- javacOptions in (Compile, compile) ++= Seq(
- "-Xlint:all", // Enable all Java-based warnings
- "-Xlint:-path", // Suppress path warnings since we get tons of them
- "-Xlint:-options",
- "-Xlint:-processing",
- "-Werror", // Treat warnings as errors
- "-source", "1.6",
- "-target", "1.6"
- ),
-
- scalacOptions in (Compile, doc) ++= Seq(
- "-no-link-warnings" // Suppresses problems with Scaladoc @throws links
- ),
-
- // Options provided to forked JVMs through sbt, based on our .jvmopts file
- javaOptions ++= Seq(
- "-Xms1024M", "-Xmx4096M", "-Xss2m", "-XX:MaxPermSize=1024M",
- "-XX:ReservedCodeCacheSize=256M", "-XX:+TieredCompilation",
- "-XX:+CMSPermGenSweepingEnabled", "-XX:+CMSClassUnloadingEnabled",
- "-XX:+UseConcMarkSweepGC", "-XX:+HeapDumpOnOutOfMemoryError"
- ),
-
- // Add additional test option to show time taken per test
- testOptions in Test += Tests.Argument("-oDF"),
-
- // Add a global resource directory with compile/ and test/ for resources in all projects
- unmanagedResourceDirectories in Compile += file("resources/compile"),
- unmanagedResourceDirectories in Test += file("resources/test"),
-
- // Publish Settings
- publishTo := {
- if (isSnapshot.value)
- Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/snapshots/")
- else
- Some("Apache Staging Repo" at "https://repository.apache.org/content/repositories/staging/")
- },
- credentials += Credentials(Path.userHome / ".ivy2" / ".credentials"),
-
- // Add rebuild ivy xml to the following tasks
- compile <<= (compile in Compile) dependsOn (rebuildIvyXml dependsOn deliverLocal)
- ) ++ rebuildIvyXmlSettings // Include our rebuild ivy xml settings
-
- // ==========================================================================
- // = REBUILD IVY XML SETTINGS BELOW
- // ==========================================================================
-
- lazy val rebuildIvyXml = TaskKey[Unit](
- "rebuild-ivy-xml",
- "Rebuilds the ivy xml using deliver-local and copies it to src " +
- "resource directories"
- )
-
- // TODO: Figure out how to retrieve the configuration being used to avoid
- // this duplication
- lazy val rebuildIvyXmlSettings = Seq(
- rebuildIvyXml := {
- val s: TaskStreams = streams.value
- val inputFile = (crossTarget.value / s"ivy-${version.value}.xml").getAbsoluteFile
- val outputFile =
- ((resourceDirectory in Compile).value / s"${name.value}-ivy.xml").getAbsoluteFile
- s.log.info(s"Copying ${inputFile.getPath} to ${outputFile.getPath}")
- FileUtils.copyFile(inputFile, outputFile)
- }
- )
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 61c6905..2b299da 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -27,7 +27,7 @@ resolvers += "Apache Snapshots" at "http://repository.apache.org/snapshots/"
addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0")
// Provides the ability to generate unifed documentation for multiple projects
-addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.3.1")
+addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.0")
// Provides the ability to list dependencies in a readable format using
// `sbt dependencyTree`; there are other commands provided as well
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/protocol/build.sbt
----------------------------------------------------------------------
diff --git a/protocol/build.sbt b/protocol/build.sbt
index ca70698..0ed902e 100644
--- a/protocol/build.sbt
+++ b/protocol/build.sbt
@@ -25,6 +25,8 @@ libraryDependencies ++= Seq(
Dependencies.slf4jApi
)
-//
-// TEST DEPENDENCIES
-//
+enablePlugins(BuildInfoPlugin)
+buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, "sparkVersion" -> Dependencies.sparkVersion.value)
+buildInfoPackage := organization.value
+buildInfoUsePackageAsPath := true
+buildInfoOptions += BuildInfoOption.BuildTime
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/pyspark-interpreter/build.sbt
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/build.sbt b/pyspark-interpreter/build.sbt
index 9444250..ac89909 100644
--- a/pyspark-interpreter/build.sbt
+++ b/pyspark-interpreter/build.sbt
@@ -14,3 +14,4 @@
* See the License for the specific language governing permissions and
* limitations under the License
*/
+libraryDependencies ++= Dependencies.sparkAll.value
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/scala-interpreter/build.sbt
----------------------------------------------------------------------
diff --git a/scala-interpreter/build.sbt b/scala-interpreter/build.sbt
index 5d6c95f..c07b032 100644
--- a/scala-interpreter/build.sbt
+++ b/scala-interpreter/build.sbt
@@ -16,41 +16,5 @@ import sbt.Tests.{Group, SubProcess}
* See the License for the specific language governing permissions and
* limitations under the License
*/
-// The chunk below is used to ensure that the test tagged with SbtForked ACTUALLY runs with
-// a forked jvm,
-
-def isAnnotatedWithRequiresSpark(definition: xsbti.api.Definition): Boolean = {
- definition.annotations().exists { annotation: xsbti.api.Annotation =>
- annotation.base match {
- case proj: xsbti.api.Projection if (proj.id() == "SbtForked") => true
- case _ => false
- }
- }
-}
-
-// Note the type TaskKey[Seq[String]] must be explicitly specified otherwise an error occurs
-lazy val testsAnnotatedWithRequiresSpark: TaskKey[Seq[String]] = taskKey[Seq[String]]("Returns list of FQCNs of tests annotated with SbtForked")
-testsAnnotatedWithRequiresSpark := {
- val analysis = (compile in Test).value
- analysis.apis.internal.values.flatMap({ source =>
- source.api().definitions().filter(isAnnotatedWithRequiresSpark).map(_.name())
- }).toSeq
-}
-
-
-def forkedJvmPerTest(testDefs: Seq[TestDefinition], testsToFork: Seq[String]) = {
-
- val (forkedTests, otherTests) = testDefs.partition { testDef => testsToFork.contains(testDef.name) }
-
- val otherTestsGroup = new Group(name = "Single JVM tests", tests = otherTests, runPolicy = SubProcess(javaOptions = Seq.empty[String]))
- val forkedTestGroups = forkedTests map { test =>
- new Group(
- name = test.name,
- tests = Seq(test),
- runPolicy = SubProcess(javaOptions = Seq.empty[String]))
- }
- Seq(otherTestsGroup) ++ forkedTestGroups
-}
-
-testGrouping in Test <<= (definedTests in Test, testsAnnotatedWithRequiresSpark) map forkedJvmPerTest
+libraryDependencies ++= Dependencies.sparkAll.value
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/sparkr-interpreter/build.sbt
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/build.sbt b/sparkr-interpreter/build.sbt
index 9444250..ac89909 100644
--- a/sparkr-interpreter/build.sbt
+++ b/sparkr-interpreter/build.sbt
@@ -14,3 +14,4 @@
* See the License for the specific language governing permissions and
* limitations under the License
*/
+libraryDependencies ++= Dependencies.sparkAll.value
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/00f4122a/sql-interpreter/build.sbt
----------------------------------------------------------------------
diff --git a/sql-interpreter/build.sbt b/sql-interpreter/build.sbt
index 9444250..ac89909 100644
--- a/sql-interpreter/build.sbt
+++ b/sql-interpreter/build.sbt
@@ -14,3 +14,4 @@
* See the License for the specific language governing permissions and
* limitations under the License
*/
+libraryDependencies ++= Dependencies.sparkAll.value