You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gearpump.apache.org by hu...@apache.org on 2016/11/11 05:29:48 UTC
[1/2] incubator-gearpump git commit: [GEARPUMP-226] refactor build
files
Repository: incubator-gearpump
Updated Branches:
refs/heads/master cbf59fb77 -> 0b56de399
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildGearpump.scala
----------------------------------------------------------------------
diff --git a/project/BuildGearpump.scala b/project/BuildGearpump.scala
new file mode 100644
index 0000000..1a52c40
--- /dev/null
+++ b/project/BuildGearpump.scala
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.typesafe.sbt.SbtPgp.autoImport._
+import BuildExamples.examples
+import BuildExperiments.experiments
+import BuildExternals.externals
+import BuildIntegrationTests.integrationTests
+import BuildDashboard.services
+import Dependencies._
+import Docs._
+import de.johoop.jacoco4sbt.JacocoPlugin.jacoco
+import sbt.Keys._
+import sbt._
+import Pack.packProject
+import sbtassembly.AssemblyPlugin.autoImport._
+
+import xerial.sbt.Sonatype._
+
+object BuildGearpump extends sbt.Build {
+
+ val apacheRepo = "https://repository.apache.org/"
+ val distDirectory = "output"
+ val projectName = "gearpump"
+
+ val commonSettings = Seq(jacoco.settings: _*) ++ sonatypeSettings ++
+ Seq(
+ resolvers ++= Seq(
+ "apache-repo" at "https://repository.apache.org/content/repositories",
+ "maven1-repo" at "http://repo1.maven.org/maven2",
+ "maven2-repo" at "http://mvnrepository.com/artifact",
+ "sonatype" at "https://oss.sonatype.org/content/repositories/releases",
+ "clojars" at "http://clojars.org/repo"
+ )
+ ) ++
+ Seq(
+ scalaVersion := scalaVersionNumber,
+ crossScalaVersions := crossScalaVersionNumbers,
+ organization := "org.apache.gearpump",
+ useGpg := false,
+ pgpSecretRing := file("./secring.asc"),
+ pgpPublicRing := file("./pubring.asc"),
+ scalacOptions ++= Seq("-Yclosure-elim", "-Yinline"),
+ publishMavenStyle := true,
+
+ pgpPassphrase := Option(System.getenv().get("PASSPHRASE")).map(_.toArray),
+ credentials += Credentials(
+ "Sonatype Nexus Repository Manager",
+ "repository.apache.org",
+ System.getenv().get("SONATYPE_USERNAME"),
+ System.getenv().get("SONATYPE_PASSWORD")),
+
+ pomIncludeRepository := { _ => false },
+
+ publishTo := {
+ if (isSnapshot.value) {
+ Some("snapshots" at apacheRepo + "content/repositories/snapshots")
+ } else {
+ Some("releases" at apacheRepo + "content/repositories/releases")
+ }
+ },
+
+ publishArtifact in Test := true,
+
+ pomExtra := {
+ <url>https://github.com/apache/incubator-gearpump</url>
+ <licenses>
+ <license>
+ <name>Apache 2</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ </license>
+ </licenses>
+ <scm>
+ <connection>scm:git://git.apache.org/incubator-gearpump.git</connection>
+ <developerConnection>scm:git:git@github.com:apache/incubator-gearpump</developerConnection>
+ <url>github.com/apache/incubator-gearpump</url>
+ </scm>
+ <developers>
+ <developer>
+ <id>gearpump</id>
+ <name>Gearpump Team</name>
+ <url>http://gearpump.incubator.apache.org/community.html#who-we-are</url>
+ </developer>
+ </developers>
+ },
+
+ pomPostProcess := {
+ (node: xml.Node) => changeShadedDeps(
+ Set(
+ "org.scoverage",
+ "org.scala-lang"
+ ),
+ List.empty[xml.Node],
+ node)
+ }
+ )
+
+ val noPublish = Seq(
+ publish := {},
+ publishLocal := {},
+ publishArtifact := false,
+ publishArtifact in Test := false
+ )
+
+ lazy val myAssemblySettings = Seq(
+ test in assembly := {},
+ assemblyOption in assembly ~= {
+ _.copy(includeScala = false)
+ },
+ assemblyJarName in assembly := {
+ s"${name.value}_${scalaBinaryVersion.value}-${version.value}.jar"
+ },
+ assemblyShadeRules in assembly := Seq(
+ ShadeRule.rename("com.romix.**" -> "org.apache.gearpump.@0").inAll,
+ ShadeRule.rename("com.esotericsoftware.**" ->
+ "org.apache.gearpump.@0").inAll,
+ ShadeRule.rename("org.objenesis.**" -> "org.apache.gearpump.@0").inAll,
+ ShadeRule.rename("com.google.common.**" -> "org.apache.gearpump.@0").inAll,
+ ShadeRule.rename("com.google.thirdparty.**" -> "org.apache.gearpump.@0").inAll,
+ ShadeRule.rename("com.codahale.metrics.**" ->
+ "org.apache.gearpump.@0").inAll,
+ ShadeRule.rename("com.gs.collections.**" ->
+ "org.apache.gearpump.gs.collections.@0").inAll
+ ),
+ target in assembly := baseDirectory.value / "target" / scalaBinaryVersion.value
+ )
+
+ lazy val aggregated: Seq[ProjectReference] = Seq[ProjectReference](
+ core,
+ streaming,
+ services,
+ gearpumpHadoop,
+ packProject
+ ) ++ examples ++ experiments ++ externals ++ integrationTests
+
+ lazy val root = Project(
+ id = "gearpump",
+ base = file("."),
+ settings = commonSettings ++ noPublish ++ gearpumpUnidocSetting,
+ aggregate = aggregated)
+ .settings(Defaults.itSettings: _*)
+ .disablePlugins(sbtassembly.AssemblyPlugin)
+
+ lazy val core = Project(
+ id = "gearpump-core",
+ base = file("core"),
+ settings = commonSettings ++ myAssemblySettings ++ javadocSettings ++ coreDependencies ++
+ addArtifact(Artifact("gearpump-core"), sbtassembly.AssemblyKeys.assembly) ++ Seq(
+
+ assemblyOption in assembly ~= {
+ _.copy(includeScala = true)
+ },
+
+ pomPostProcess := {
+ (node: xml.Node) => changeShadedDeps(
+ Set(
+ "com.github.romix.akka",
+ "com.google.guava",
+ "com.codahale.metrics",
+ "org.scoverage"
+ ), List.empty[xml.Node], node)
+ }
+ ))
+
+ lazy val streaming = Project(
+ id = "gearpump-streaming",
+ base = file("streaming"),
+ settings = commonSettings ++ myAssemblySettings ++ javadocSettings ++
+ addArtifact(Artifact("gearpump-streaming"), sbtassembly.AssemblyKeys.assembly) ++
+ Seq(
+ assemblyMergeStrategy in assembly := {
+ case "geardefault.conf" =>
+ MergeStrategy.last
+ case x =>
+ val oldStrategy = (assemblyMergeStrategy in assembly).value
+ oldStrategy(x)
+ },
+
+ libraryDependencies ++= Seq(
+ "com.goldmansachs" % "gs-collections" % gsCollectionsVersion
+ ),
+
+ pomPostProcess := {
+ (node: xml.Node) => changeShadedDeps(
+ Set(
+ "com.goldmansachs",
+ "org.scala-lang",
+ "org.scoverage"
+ ),
+ List(
+ getShadedDepXML(organization.value, s"${core.id}_${scalaBinaryVersion.value}",
+ version.value, "provided")),
+ node)
+ }
+ )
+ ).dependsOn(core % "test->test;provided")
+
+ lazy val gearpumpHadoop = Project(
+ id = "gearpump-hadoop",
+ base = file("gearpump-hadoop"),
+ settings = commonSettings ++ noPublish ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion,
+ "org.apache.hadoop" % "hadoop-common" % hadoopVersion
+ )
+ )
+ ).dependsOn(core % "compile->compile").disablePlugins(sbtassembly.AssemblyPlugin)
+
+ private def changeShadedDeps(toExclude: Set[String], toInclude: List[xml.Node],
+ node: xml.Node): xml.Node = {
+ node match {
+ case elem: xml.Elem =>
+ val child =
+ if (elem.label == "dependencies") {
+ elem.child.filterNot { dep =>
+ dep.child.find(_.label == "groupId").exists(gid => toExclude.contains(gid.text))
+ } ++ toInclude
+ } else {
+ elem.child.map(changeShadedDeps(toExclude, toInclude, _))
+ }
+ xml.Elem(elem.prefix, elem.label, elem.attributes, elem.scope, false, child: _*)
+ case _ =>
+ node
+ }
+ }
+
+ private def getShadedDepXML(groupId: String, artifactId: String,
+ version: String, scope: String): scala.xml.Node = {
+ <dependency>
+ <groupId>{groupId}</groupId>
+ <artifactId>{artifactId}</artifactId>
+ <version>{version}</version>
+ <scope>{scope}</scope>
+ </dependency>
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildIntegrationTest.scala
----------------------------------------------------------------------
diff --git a/project/BuildIntegrationTest.scala b/project/BuildIntegrationTest.scala
deleted file mode 100644
index 6eed7a2..0000000
--- a/project/BuildIntegrationTest.scala
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import sbt.Keys._
-import sbt._
-import Build._
-import sbtassembly.AssemblyPlugin.autoImport._
-
-object BuildIntegrationTest extends sbt.Build {
-
- val jsonSimpleVersion = "1.1"
- val storm09Version = "0.9.6"
-
- lazy val integration_test = Project(
- id = "gearpump-integrationtest",
- base = file("integrationtest"),
- settings = commonSettings ++ noPublish
- ).aggregate(it_core, it_storm09, it_storm010).
- disablePlugins(sbtassembly.AssemblyPlugin)
-
- val itTestFilter: String => Boolean = { name => name endsWith "Suite" }
- lazy val it_core = Project(
- id = "gearpump-integrationtest-core",
- base = file("integrationtest/core"),
- settings = commonSettings ++ noPublish ++ Seq(
- testOptions in IntegrationTest += Tests.Filter(itTestFilter),
- libraryDependencies ++= Seq(
- "com.lihaoyi" %% "upickle" % upickleVersion,
- "org.scalatest" %% "scalatest" % scalaTestVersion % "it",
- "org.pegdown" % "pegdown" % "1.4.2" % "it",
- "org.parboiled" % "parboiled-core" % "1.1.7" % "it",
- "org.parboiled" % "parboiled-java" % "1.1.7" % "it",
- "org.mortbay.jetty" % "jetty-util" % "6.1.26" % "it",
- "org.ow2.asm" % "asm-all" % "5.0.3" % "it"
- )
- )
- ).configs(IntegrationTest).settings(Defaults.itSettings: _*)
- .dependsOn(
- streaming % "test->test; provided",
- services % "test->test; provided",
- external_kafka,
- storm,
- external_serializer
- ).disablePlugins(sbtassembly.AssemblyPlugin)
-
- // Integration test for Storm 0.9.x
- lazy val it_storm09 = Project(
- id = "gearpump-integrationtest-storm09",
- base = file("integrationtest/storm09"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- libraryDependencies ++= Seq(
- "org.apache.storm" % "storm-kafka" % storm09Version,
- "org.apache.storm" % "storm-starter" % storm09Version,
- "com.googlecode.json-simple" % "json-simple" % jsonSimpleVersion,
- "org.apache.kafka" %% "kafka" % kafkaVersion
- ),
- target in assembly := baseDirectory.value.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn (storm % "provided")
-
- // Integration test for Storm 0.10.x
- lazy val it_storm010 = Project(
- id = "gearpump-integrationtest-storm010",
- base = file("integrationtest/storm010"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- libraryDependencies ++= Seq(
- "org.apache.storm" % "storm-kafka" % stormVersion,
- "org.apache.storm" % "storm-starter" % stormVersion,
- "org.apache.kafka" %% "kafka" % kafkaVersion
- ),
- target in assembly := baseDirectory.value.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn (storm % "provided")
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildIntegrationTests.scala
----------------------------------------------------------------------
diff --git a/project/BuildIntegrationTests.scala b/project/BuildIntegrationTests.scala
new file mode 100644
index 0000000..161d906
--- /dev/null
+++ b/project/BuildIntegrationTests.scala
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import sbt.Keys._
+import sbt._
+import BuildGearpump._
+import BuildExperiments.storm
+import BuildDashboard.services
+import BuildExternals.{external_kafka, external_serializer}
+import Dependencies._
+import sbtassembly.AssemblyPlugin.autoImport._
+
+object BuildIntegrationTests extends sbt.Build {
+
+ lazy val integrationTests: Seq[ProjectReference] = Seq(
+ it_core,
+ it_storm09,
+ it_storm010
+ )
+
+ lazy val it_core = Project(
+ id = "gearpump-integrationtest-core",
+ base = file("integrationtest/core"),
+ settings = commonSettings ++ noPublish ++ Seq(
+ testOptions in IntegrationTest += Tests.Filter(_.endsWith("Suite")),
+ libraryDependencies ++= Seq(
+ "com.lihaoyi" %% "upickle" % upickleVersion,
+ "org.scalatest" %% "scalatest" % scalaTestVersion % "it",
+ "org.pegdown" % "pegdown" % "1.4.2" % "it",
+ "org.parboiled" % "parboiled-core" % "1.1.7" % "it",
+ "org.parboiled" % "parboiled-java" % "1.1.7" % "it",
+ "org.mortbay.jetty" % "jetty-util" % "6.1.26" % "it",
+ "org.ow2.asm" % "asm-all" % "5.0.3" % "it"
+ )
+ )
+ ).configs(IntegrationTest).settings(Defaults.itSettings: _*)
+ .dependsOn(
+ core % "provided",
+ streaming % "test->test; provided",
+ services % "test->test; provided",
+ external_kafka,
+ storm,
+ external_serializer
+ ).disablePlugins(sbtassembly.AssemblyPlugin)
+
+ // Integration test for Storm 0.9.x
+ lazy val it_storm09 = Project(
+ id = "gearpump-integrationtest-storm09",
+ base = file("integrationtest/storm09"),
+ settings = commonSettings ++ noPublish ++ myAssemblySettings ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.apache.storm" % "storm-kafka" % storm09Version,
+ "org.apache.storm" % "storm-starter" % storm09Version,
+ "com.googlecode.json-simple" % "json-simple" % jsonSimpleVersion,
+ "org.apache.kafka" %% "kafka" % kafkaVersion
+ ),
+ target in assembly := baseDirectory.value.getParentFile / "target" /
+ CrossVersion.binaryScalaVersion(scalaVersion.value)
+ )
+ ).dependsOn(core % "provided", storm % "provided")
+
+ // Integration test for Storm 0.10.x
+ lazy val it_storm010 = Project(
+ id = "gearpump-integrationtest-storm010",
+ base = file("integrationtest/storm010"),
+ settings = commonSettings ++ noPublish ++ myAssemblySettings ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.apache.storm" % "storm-kafka" % stormVersion,
+ "org.apache.storm" % "storm-starter" % stormVersion,
+ "org.apache.kafka" %% "kafka" % kafkaVersion
+ ),
+ target in assembly := baseDirectory.value.getParentFile / "target" /
+ CrossVersion.binaryScalaVersion(scalaVersion.value)
+ )
+ ).dependsOn(core % "provided", storm % "provided")
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildShaded.scala
----------------------------------------------------------------------
diff --git a/project/BuildShaded.scala b/project/BuildShaded.scala
deleted file mode 100644
index a43587c..0000000
--- a/project/BuildShaded.scala
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import sbt.Keys._
-import sbt._
-import sbtassembly.AssemblyPlugin.autoImport._
-
-object BuildShaded extends sbt.Build {
-
- val guavaVersion = "16.0.1"
- val codahaleVersion = "3.0.2"
- val kryoVersion = "0.4.1"
- val gsCollectionsVersion = "6.2.0"
- private val scalaVersionMajor = "2.11"
-
- val shadeAssemblySettings = Build.commonSettings ++ Seq(
- scalaVersion := Build.scalaVersionNumber,
- test in assembly := {},
- assemblyOption in assembly ~= {
- _.copy(includeScala = false)
- },
- assemblyJarName in assembly := {
- s"${name.value}_$scalaVersionMajor-${version.value}.jar"
- },
- target in assembly := baseDirectory.value.getParentFile / "target" / scalaVersionMajor
- )
-
- val shaded = Project(
- id = "gearpump-shaded",
- base = file("shaded")
- ).aggregate(shaded_akka_kryo, shaded_gs_collections, shaded_guava, shaded_metrics_graphite)
- .disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val shaded_akka_kryo = Project(
- id = "gearpump-shaded-akka-kryo",
- base = file("shaded/akka-kryo"),
- settings = shadeAssemblySettings ++ addArtifact(Artifact("gearpump-shaded-akka-kryo"),
- sbtassembly.AssemblyKeys.assembly) ++
- Seq(
- assemblyShadeRules in assembly := Seq(
- ShadeRule.zap("com.google.protobuf.**").inAll,
- ShadeRule.zap("com.typesafe.config.**").inAll,
- ShadeRule.zap("akka.**").inAll,
- ShadeRule.zap("org.jboss.netty.**").inAll,
- ShadeRule.zap("net.jpountz.lz4.**").inAll,
- ShadeRule.zap("org.uncommons.maths.**").inAll,
- ShadeRule.rename("com.romix.**" -> "org.apache.gearpump.romix.@1").inAll,
- ShadeRule.rename("com.esotericsoftware.**" ->
- "org.apache.gearpump.esotericsoftware.@1").inAll,
- ShadeRule.rename("org.objenesis.**" -> "org.apache.gearpump.objenesis.@1").inAll
- )
- ) ++
- Seq(
- libraryDependencies ++= Seq(
- "com.github.romix.akka" %% "akka-kryo-serialization" % kryoVersion
- )
- )
- )
-
- lazy val shaded_gs_collections = Project(
- id = "gearpump-shaded-gs-collections",
- base = file("shaded/gs-collections"),
- settings = shadeAssemblySettings ++ addArtifact(Artifact("gearpump-shaded-gs-collections"),
- sbtassembly.AssemblyKeys.assembly) ++
- Seq(
- assemblyShadeRules in assembly := Seq(
- ShadeRule.rename("com.gs.collections.**" ->
- "org.apache.gearpump.gs.collections.@1").inAll
- )
- ) ++
- Seq(
- libraryDependencies ++= Seq(
- "com.goldmansachs" % "gs-collections" % gsCollectionsVersion
- )
- )
- )
-
- lazy val shaded_guava = Project(
- id = "gearpump-shaded-guava",
- base = file("shaded/guava"),
- settings = shadeAssemblySettings ++ addArtifact(Artifact("gearpump-shaded-guava"),
- sbtassembly.AssemblyKeys.assembly) ++
- Seq(
- assemblyShadeRules in assembly := Seq(
- ShadeRule.rename("com.google.**" -> "org.apache.gearpump.google.@1").inAll
- )
- ) ++
- Seq(
- libraryDependencies ++= Seq(
- "com.google.guava" % "guava" % guavaVersion
- )
- )
- )
-
- lazy val shaded_metrics_graphite = Project(
- id = "gearpump-shaded-metrics-graphite",
- base = file("shaded/metrics-graphite"),
- settings = shadeAssemblySettings ++ addArtifact(Artifact("gearpump-shaded-metrics-graphite"),
- sbtassembly.AssemblyKeys.assembly) ++
- Seq(
- assemblyShadeRules in assembly := Seq(
- ShadeRule.rename("com.codahale.metrics.**" ->
- "org.apache.gearpump.codahale.metrics.@1").inAll
- )
- ) ++
- Seq(
- libraryDependencies ++= Seq(
- "com.codahale.metrics" % "metrics-graphite" % codahaleVersion,
- "com.codahale.metrics" % "metrics-jvm" % codahaleVersion
- )
- )
- )
-
- def getShadedJarFile(name: String, gearpumpVersion: String): File = {
- shaded.base / "target" / scalaVersionMajor /
- s"${name}_$scalaVersionMajor-$gearpumpVersion.jar"
- }
-
- def getShadedDepXML(groupId: String, artifactId: String, version: String): scala.xml.Node = {
- <dependency>
- <groupId>{groupId}</groupId>
- <artifactId>{artifactId}</artifactId>
- <version>{version}</version>
- </dependency>
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/Dependencies.scala
----------------------------------------------------------------------
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
new file mode 100644
index 0000000..6949497
--- /dev/null
+++ b/project/Dependencies.scala
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import sbt._
+import Keys._
+
+object Dependencies {
+
+ val crossScalaVersionNumbers = Seq("2.11.8")
+ val scalaVersionNumber = crossScalaVersionNumbers.last
+ val akkaVersion = "2.4.3"
+ val hadoopVersion = "2.6.0"
+ val hbaseVersion = "1.0.0"
+ val commonsHttpVersion = "3.1"
+ val commonsLoggingVersion = "1.1.3"
+ val commonsLangVersion = "2.6"
+ val commonsIOVersion = "2.4"
+ val dataReplicationVersion = "0.7"
+ val upickleVersion = "0.3.4"
+ val junitVersion = "4.12"
+ val kafkaVersion = "0.8.2.1"
+ val jsonSimpleVersion = "1.1"
+ val storm09Version = "0.9.6"
+ val stormVersion = "0.10.0"
+ val slf4jVersion = "1.7.16"
+ val guavaVersion = "16.0.1"
+ val codahaleVersion = "3.0.2"
+ val kryoVersion = "0.4.1"
+ val gsCollectionsVersion = "6.2.0"
+ val sprayVersion = "1.3.2"
+ val sprayJsonVersion = "1.3.1"
+ val scalaTestVersion = "2.2.0"
+ val scalaCheckVersion = "1.11.3"
+ val mockitoVersion = "1.10.17"
+ val bijectionVersion = "0.8.0"
+ val scalazVersion = "7.1.1"
+ val algebirdVersion = "0.9.0"
+ val chillVersion = "0.6.0"
+ val jedisVersion = "2.9.0"
+
+ val coreDependencies = Seq(
+ libraryDependencies ++= Seq(
+ "org.slf4j" % "slf4j-api" % slf4jVersion,
+ "org.slf4j" % "slf4j-log4j12" % slf4jVersion,
+ "commons-lang" % "commons-lang" % commonsLangVersion,
+
+ /**
+ * Overrides Netty version 3.10.3.Final used by Akka 2.4.2 to work-around netty hang issue
+ * (https://github.com/gearpump/gearpump/issues/2020)
+ *
+ * Akka 2.4.2 by default use Netty 3.10.3.Final, which has a serious issue which can hang
+ * the network. The same issue also happens in version range (3.10.0.Final, 3.10.5.Final)
+ * Netty 3.10.6.Final have this issue fixed, however, we find there is a 20% performance
+ * drop. So we decided to downgrade netty to 3.8.0.Final (Same version used in akka 2.3.12).
+ *
+ * @see https://github.com/gearpump/gearpump/pull/2017 for more discussions.
+ */
+ "io.netty" % "netty" % "3.8.0.Final",
+ "com.typesafe.akka" %% "akka-remote" % akkaVersion
+ exclude("io.netty", "netty"),
+
+ "com.typesafe.akka" %% "akka-cluster" % akkaVersion,
+ "com.typesafe.akka" %% "akka-cluster-tools" % akkaVersion,
+ "commons-logging" % "commons-logging" % commonsLoggingVersion,
+ "com.typesafe.akka" %% "akka-distributed-data-experimental" % akkaVersion,
+ "com.typesafe.akka" %% "akka-actor" % akkaVersion,
+ "com.typesafe.akka" %% "akka-agent" % akkaVersion,
+ "com.typesafe.akka" %% "akka-slf4j" % akkaVersion,
+ "com.typesafe.akka" %% "akka-kernel" % akkaVersion,
+ "com.typesafe.akka" %% "akka-http-experimental" % akkaVersion,
+ "com.typesafe.akka" %% "akka-http-spray-json-experimental" % akkaVersion,
+ "org.scala-lang" % "scala-reflect" % scalaVersionNumber,
+ "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4",
+ "com.github.romix.akka" %% "akka-kryo-serialization" % kryoVersion,
+ "com.google.guava" % "guava" % guavaVersion,
+ "com.codahale.metrics" % "metrics-graphite" % codahaleVersion
+ exclude("org.slf4j", "slf4j-api"),
+ "com.codahale.metrics" % "metrics-jvm" % codahaleVersion
+ exclude("org.slf4j", "slf4j-api"),
+ "com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test",
+ "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
+ "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "test",
+ "org.mockito" % "mockito-core" % mockitoVersion % "test",
+ "junit" % "junit" % junitVersion % "test"
+ )
+ )
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/Docs.scala
----------------------------------------------------------------------
diff --git a/project/Docs.scala b/project/Docs.scala
new file mode 100644
index 0000000..f8d433e
--- /dev/null
+++ b/project/Docs.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import BuildGearpump.{core, streaming}
+import BuildExternals.externals
+import sbt.Keys._
+import sbt._
+import sbtunidoc.Plugin.UnidocKeys._
+import sbtunidoc.Plugin._
+
+object Docs extends sbt.Build {
+ lazy val javadocSettings = Seq(
+ addCompilerPlugin(
+ "com.typesafe.genjavadoc" %% "genjavadoc-plugin" % "0.9" cross CrossVersion.full),
+ scalacOptions += s"-P:genjavadoc:out=${target.value}/java"
+ )
+
+ lazy val gearpumpUnidocSetting = scalaJavaUnidocSettings ++ Seq(
+ unidocProjectFilter in(ScalaUnidoc, unidoc) := projectsWithDoc,
+ unidocProjectFilter in(JavaUnidoc, unidoc) := projectsWithDoc,
+
+ unidocAllSources in(ScalaUnidoc, unidoc) := {
+ ignoreUndocumentedPackages((unidocAllSources in(ScalaUnidoc, unidoc)).value)
+ },
+
+ // Skip class names containing $ and some internal packages in Javadocs
+ unidocAllSources in(JavaUnidoc, unidoc) := {
+ ignoreUndocumentedPackages((unidocAllSources in(JavaUnidoc, unidoc)).value)
+ }
+ )
+
+ private lazy val projectsWithDoc = {
+ val projects: Seq[ProjectReference] = Seq[ProjectReference](
+ core,
+ streaming
+ ) ++ externals
+
+ inProjects(projects: _*)
+ }
+
+ private def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = {
+ packages
+ .map(_.filterNot(_.getName.contains("$")))
+ .map(_.filterNot(_.getCanonicalPath.contains("akka")))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/Pack.scala
----------------------------------------------------------------------
diff --git a/project/Pack.scala b/project/Pack.scala
index 54b1d43..2eb0ca7 100644
--- a/project/Pack.scala
+++ b/project/Pack.scala
@@ -18,7 +18,9 @@
import sbt.Keys._
import sbt._
-import Build._
+import BuildGearpump._
+import BuildDashboard.services
+import BuildExperiments.{cgroup, storm, yarn}
import xerial.sbt.Pack._
object Pack extends sbt.Build {
@@ -69,9 +71,14 @@ object Pack extends sbt.Build {
"storm" -> "org.apache.gearpump.experiments.storm.StormRunner"
),
packJvmOpts := Map(
- "gear" -> Seq("-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}"),
+ "gear" -> Seq(
+ "-noverify",
+ "-Djava.net.preferIPv4Stack=true",
+ "-Dgearpump.home=${PROG_HOME}"),
+
"local" -> Seq(
"-server",
+ "-noverify",
"-Djava.net.preferIPv4Stack=true",
"-DlogFilename=local",
"-Dgearpump.home=${PROG_HOME}",
@@ -79,6 +86,7 @@ object Pack extends sbt.Build {
"master" -> Seq(
"-server",
+ "-noverify",
"-Djava.net.preferIPv4Stack=true",
"-DlogFilename=master",
"-Dgearpump.home=${PROG_HOME}",
@@ -86,6 +94,7 @@ object Pack extends sbt.Build {
"worker" -> Seq(
"-server",
+ "-noverify",
"-Djava.net.preferIPv4Stack=true",
"-DlogFilename=worker",
"-Dgearpump.home=${PROG_HOME}",
@@ -93,23 +102,25 @@ object Pack extends sbt.Build {
"services" -> Seq(
"-server",
+ "-noverify",
"-Djava.net.preferIPv4Stack=true",
"-Dgearpump.home=${PROG_HOME}",
"-Djava.rmi.server.hostname=localhost"),
"yarnclient" -> Seq(
"-server",
+ "-noverify",
"-Djava.net.preferIPv4Stack=true",
"-Dgearpump.home=${PROG_HOME}",
"-Djava.rmi.server.hostname=localhost"),
"storm" -> Seq(
"-server",
+ "-noverify",
"-Djava.net.preferIPv4Stack=true",
"-Dgearpump.home=${PROG_HOME}")
),
packLibDir := Map(
- "lib" -> new ProjectsToPack(core.id, cgroup.id, streaming.id),
"lib/yarn" -> new ProjectsToPack(gearpumpHadoop.id, yarn.id).
exclude(services.id, core.id),
"lib/services" -> new ProjectsToPack(services.id).exclude(core.id),
@@ -120,7 +131,9 @@ object Pack extends sbt.Build {
packResourceDir += (baseDirectory.value / ".." / "bin" -> "bin"),
packResourceDir += (baseDirectory.value / ".." / "conf" -> "conf"),
packResourceDir += (baseDirectory.value / ".." / "yarnconf" -> "conf/yarnconf"),
- packResourceDir += (baseDirectory.value / ".." / "shaded" / "target" /
+ packResourceDir += (baseDirectory.value / ".." / "core" / "target" /
+ CrossVersion.binaryScalaVersion(scalaVersion.value) -> "lib"),
+ packResourceDir += (baseDirectory.value / ".." / "streaming" / "target" /
CrossVersion.binaryScalaVersion(scalaVersion.value) -> "lib"),
packResourceDir += (baseDirectory.value / ".." / "services" / "dashboard" -> "dashboard"),
packResourceDir += (baseDirectory.value / ".." / "examples" / "target" /
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/build.properties
----------------------------------------------------------------------
diff --git a/project/build.properties b/project/build.properties
index c4df008..8df00d1 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -16,4 +16,4 @@
# limitations under the License.
#
-sbt.version=0.13.11
+sbt.version=0.13.13
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 5a9740e..5f88b5e 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -25,7 +25,7 @@ addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.8")
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
-addSbtPlugin("io.gearpump.sbt" % "sbt-pack" % "0.7.6")
+addSbtPlugin("io.gearpump.sbt" % "sbt-pack" % "0.7.7")
addSbtPlugin("de.johoop" % "jacoco4sbt" % "2.1.6")
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/appmaster/ClockService.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/appmaster/ClockService.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/appmaster/ClockService.scala
index 68db354..2085953 100644
--- a/streaming/src/main/scala/org/apache/gearpump/streaming/appmaster/ClockService.scala
+++ b/streaming/src/main/scala/org/apache/gearpump/streaming/appmaster/ClockService.scala
@@ -23,7 +23,7 @@ import java.util.Date
import java.util.concurrent.TimeUnit
import akka.actor.{Actor, Cancellable, Stash}
-import org.apache.gearpump.google.common.primitives.Longs
+import com.google.common.primitives.Longs
import org.apache.gearpump.TimeStamp
import org.apache.gearpump.cluster.ClientToMaster.GetStallingTasks
import org.apache.gearpump.streaming.AppMasterToMaster.StallingTasks
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/dsl/window/impl/WindowRunner.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/dsl/window/impl/WindowRunner.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/dsl/window/impl/WindowRunner.scala
index 9af5e61..640d090 100644
--- a/streaming/src/main/scala/org/apache/gearpump/streaming/dsl/window/impl/WindowRunner.scala
+++ b/streaming/src/main/scala/org/apache/gearpump/streaming/dsl/window/impl/WindowRunner.scala
@@ -22,10 +22,10 @@ import java.time.Instant
import akka.actor.ActorSystem
import org.apache.gearpump.Message
import org.apache.gearpump.cluster.UserConfig
-import org.apache.gearpump.gs.collections.api.block.procedure.Procedure
-import org.apache.gearpump.gs.collections.impl.list.mutable.FastList
-import org.apache.gearpump.gs.collections.impl.map.mutable.UnifiedMap
-import org.apache.gearpump.gs.collections.impl.map.sorted.mutable.TreeSortedMap
+import com.gs.collections.api.block.procedure.Procedure
+import com.gs.collections.impl.list.mutable.FastList
+import com.gs.collections.impl.map.mutable.UnifiedMap
+import com.gs.collections.impl.map.sorted.mutable.TreeSortedMap
import org.apache.gearpump.streaming.Constants._
import org.apache.gearpump.streaming.dsl.plan.functions.{EmitFunction, SingleInputFunction}
import org.apache.gearpump.streaming.dsl.window.api.Discarding
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/metrics/ProcessorAggregator.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/metrics/ProcessorAggregator.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/metrics/ProcessorAggregator.scala
index d045def..8f8b7ab 100644
--- a/streaming/src/main/scala/org/apache/gearpump/streaming/metrics/ProcessorAggregator.scala
+++ b/streaming/src/main/scala/org/apache/gearpump/streaming/metrics/ProcessorAggregator.scala
@@ -20,12 +20,11 @@ package org.apache.gearpump.streaming.metrics
import java.util
+import com.google.common.collect.Iterators
import com.typesafe.config.Config
-
import org.apache.gearpump.TimeStamp
import org.apache.gearpump.cluster.ClientToMaster.ReadOption
import org.apache.gearpump.cluster.MasterToClient.HistoryMetricsItem
-import org.apache.gearpump.google.common.collect.Iterators
import org.apache.gearpump.metrics.Metrics.{Histogram, Meter}
import org.apache.gearpump.metrics.MetricsAggregator
import org.apache.gearpump.streaming.metrics.ProcessorAggregator._
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/task/SerializerResolver.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/task/SerializerResolver.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/task/SerializerResolver.scala
index 902c663..3b32163 100644
--- a/streaming/src/main/scala/org/apache/gearpump/streaming/task/SerializerResolver.scala
+++ b/streaming/src/main/scala/org/apache/gearpump/streaming/task/SerializerResolver.scala
@@ -18,7 +18,7 @@
package org.apache.gearpump.streaming.task
-import org.apache.gearpump.esotericsoftware.kryo.util.{IntMap, ObjectMap}
+import com.esotericsoftware.kryo.util.{IntMap, ObjectMap}
import org.apache.gearpump.streaming.task.SerializerResolver.Registration
private[task] class SerializerResolver {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/task/Subscription.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/task/Subscription.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/task/Subscription.scala
index 16f9e93..4193fbf 100644
--- a/streaming/src/main/scala/org/apache/gearpump/streaming/task/Subscription.scala
+++ b/streaming/src/main/scala/org/apache/gearpump/streaming/task/Subscription.scala
@@ -20,7 +20,7 @@ package org.apache.gearpump.streaming.task
import org.slf4j.Logger
-import org.apache.gearpump.google.common.primitives.Shorts
+import com.google.common.primitives.Shorts
import org.apache.gearpump.streaming.partitioner.{MulticastPartitioner, Partitioner, UnicastPartitioner}
import org.apache.gearpump.streaming.AppMasterToExecutor.MsgLostException
import org.apache.gearpump.streaming.LifeTime
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/streaming/src/main/scala/org/apache/gearpump/streaming/task/TaskActor.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/gearpump/streaming/task/TaskActor.scala b/streaming/src/main/scala/org/apache/gearpump/streaming/task/TaskActor.scala
index 9a52cc6..92f6672 100644
--- a/streaming/src/main/scala/org/apache/gearpump/streaming/task/TaskActor.scala
+++ b/streaming/src/main/scala/org/apache/gearpump/streaming/task/TaskActor.scala
@@ -23,10 +23,10 @@ import java.util
import java.util.concurrent.TimeUnit
import akka.actor._
+import com.gs.collections.impl.map.mutable.primitive.IntShortHashMap
import org.apache.gearpump.streaming.source.Watermark
import org.slf4j.Logger
import org.apache.gearpump.cluster.UserConfig
-import org.apache.gearpump.gs.collections.impl.map.mutable.primitive.IntShortHashMap
import org.apache.gearpump.metrics.Metrics
import org.apache.gearpump.serializer.SerializationFramework
import org.apache.gearpump.streaming.AppMasterToExecutor._
[2/2] incubator-gearpump git commit: [GEARPUMP-226] refactor build
files
Posted by hu...@apache.org.
[GEARPUMP-226] refactor build files
Refactored build files as follows
1. break `Build.scala` into parts for different module categories, e.g. `BuildExternals`, `BuildDashboard`.
2. libraries to shade are now added as managed dependencies, shaded with their depending projects and finally assembled to a uber jar. For example, `gearpump-core-$version.jar` will now contain both the gearpump sources and shaded libraries, whose package names are both `org.apache.gearpump`. The original dependencies are excluded from `pack`.
3. package names in the codebase are changed as needed by 2
4. `gearpump-core` is now `provided` for `gearpump-streaming` and won't be transitive dependencies for projects depending on`gearpump-streaming`. `gearpump-core` needs to be explicitly included as `provided` dependency so I add it to the published pom files for users.
Author: manuzhang <ow...@gmail.com>
Closes #108 from manuzhang/fix_build.
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/0b56de39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/0b56de39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/0b56de39
Branch: refs/heads/master
Commit: 0b56de39919452787b2533ac3d10ac1dd46b2f9d
Parents: cbf59fb
Author: manuzhang <ow...@gmail.com>
Authored: Fri Nov 11 13:10:51 2016 +0800
Committer: huafengw <fv...@gmail.com>
Committed: Fri Nov 11 13:10:51 2016 +0800
----------------------------------------------------------------------
.travis.yml | 6 +-
.../gearpump/transport/netty/MessageBatch.java | 2 +-
.../apache/gearpump/cluster/UserConfig.scala | 2 +-
.../apache/gearpump/metrics/AkkaReporter.scala | 2 +-
.../org/apache/gearpump/metrics/Counter.scala | 4 +-
.../org/apache/gearpump/metrics/Histogram.scala | 4 +-
.../apache/gearpump/metrics/JvmMetricsSet.scala | 4 +-
.../org/apache/gearpump/metrics/Meter.scala | 2 +-
.../org/apache/gearpump/metrics/Metrics.scala | 2 +-
.../metrics/MetricsReporterService.scala | 7 +-
.../serializer/FastKryoSerializer.scala | 11 +-
.../serializer/GearpumpSerialization.scala | 3 +-
.../org/apache/gearpump/util/FileUtils.scala | 2 +-
.../scala/org/apache/gearpump/util/Util.scala | 10 +-
.../gearpump/jarstore/FileServerSpec.scala | 7 +-
.../apache/gearpump/metrics/MetricsSpec.scala | 3 +-
.../gearpump/serializer/SerializerSpec.scala | 21 +-
.../apache/gearpump/util/FileUtilsSpec.scala | 4 +-
.../gearpump/cluster/utils/SystemOperation.java | 2 +-
.../cluster/worker/CGroupProcessLauncher.scala | 2 +-
.../apache/gearpump/redis/RedisMessage.scala | 10 +-
project/Build.scala | 559 -------------------
project/BuildDashboard.scala | 119 ++++
project/BuildExample.scala | 236 --------
project/BuildExamples.scala | 181 ++++++
project/BuildExperiments.scala | 118 ++++
project/BuildExternals.scala | 124 ++++
project/BuildGearpump.scala | 252 +++++++++
project/BuildIntegrationTest.scala | 93 ---
project/BuildIntegrationTests.scala | 93 +++
project/BuildShaded.scala | 142 -----
project/Dependencies.scala | 102 ++++
project/Docs.scala | 61 ++
project/Pack.scala | 21 +-
project/build.properties | 2 +-
project/plugins.sbt | 2 +-
.../streaming/appmaster/ClockService.scala | 2 +-
.../dsl/window/impl/WindowRunner.scala | 8 +-
.../streaming/metrics/ProcessorAggregator.scala | 3 +-
.../streaming/task/SerializerResolver.scala | 2 +-
.../gearpump/streaming/task/Subscription.scala | 2 +-
.../gearpump/streaming/task/TaskActor.scala | 2 +-
42 files changed, 1135 insertions(+), 1099 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index e7e8abd..7151f88 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,10 +3,8 @@ language:
- scala
sudo: false
before_script:
-- rm -rf $HOME/.m2
-- rm -rf $HOME/.ivy2
-- mkdir -p $HOME/.sbt/launchers/0.13.11/
-- curl -L -o $HOME/.sbt/launchers/0.13.11/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.11/sbt-launch.jar
+- mkdir -p $HOME/.sbt/launchers/0.13.13/
+- curl -L -o $HOME/.sbt/launchers/0.13.13/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.13/sbt-launch.jar
script:
- echo "TRAVIS_PULL_REQUEST" $TRAVIS_PULL_REQUEST
- echo "TRAVIS_BRANCH" $TRAVIS_BRANCH
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java b/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java
index d9de871..13ce9fa 100644
--- a/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java
+++ b/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java
@@ -18,7 +18,7 @@
package org.apache.gearpump.transport.netty;
-import org.apache.gearpump.google.common.io.Closeables;
+import com.google.common.io.Closeables;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBufferOutputStream;
import org.jboss.netty.buffer.ChannelBuffers;
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala b/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala
index 3f42808..1946cfb 100644
--- a/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala
+++ b/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala
@@ -20,8 +20,8 @@ package org.apache.gearpump.cluster
import akka.actor.{ActorSystem, ExtendedActorSystem}
import akka.serialization.JavaSerializer
+import com.google.common.io.BaseEncoding
-import org.apache.gearpump.google.common.io.BaseEncoding
/**
* Immutable configuration
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/AkkaReporter.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/metrics/AkkaReporter.scala b/core/src/main/scala/org/apache/gearpump/metrics/AkkaReporter.scala
index acd9493..9092beb 100644
--- a/core/src/main/scala/org/apache/gearpump/metrics/AkkaReporter.scala
+++ b/core/src/main/scala/org/apache/gearpump/metrics/AkkaReporter.scala
@@ -22,7 +22,7 @@ import scala.collection.JavaConverters._
import akka.actor.{ActorRef, ActorSystem}
-import org.apache.gearpump.codahale.metrics.{Gauge => CodaGauge, MetricRegistry}
+import com.codahale.metrics.{Gauge => CodaGauge, MetricRegistry}
import org.apache.gearpump.metrics.Metrics.{Counter => CounterData, Gauge => GaugeData, Histogram => HistogramData, Meter => MeterData}
import org.apache.gearpump.metrics.MetricsReporterService.ReportTo
import org.apache.gearpump.util.LogUtil
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/Counter.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/metrics/Counter.scala b/core/src/main/scala/org/apache/gearpump/metrics/Counter.scala
index 2a4a195..e778e3d 100644
--- a/core/src/main/scala/org/apache/gearpump/metrics/Counter.scala
+++ b/core/src/main/scala/org/apache/gearpump/metrics/Counter.scala
@@ -18,10 +18,10 @@
package org.apache.gearpump.metrics
-import org.apache.gearpump.codahale.metrics.{Counter => CodaHaleCounter}
+import com.codahale.metrics.{Counter => CodaHaleCounter}
/**
- * @see org.apache.gearpump.codahale.metrics.Counter
+ * @see com.codahale.metrics.Counter
*/
class Counter(val name: String, counter: CodaHaleCounter, sampleRate: Int = 1) {
private var sampleCount = 0L
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/Histogram.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/metrics/Histogram.scala b/core/src/main/scala/org/apache/gearpump/metrics/Histogram.scala
index 9d1e500..3483c1e 100644
--- a/core/src/main/scala/org/apache/gearpump/metrics/Histogram.scala
+++ b/core/src/main/scala/org/apache/gearpump/metrics/Histogram.scala
@@ -18,10 +18,10 @@
package org.apache.gearpump.metrics
-import org.apache.gearpump.codahale.metrics.{Histogram => CodaHaleHistogram}
+import com.codahale.metrics.{Histogram => CodaHaleHistogram}
/**
- * @see org.apache.gearpump.codahale.metrics.Histogram
+ * @see com.codahale.metrics.Histogram
*/
class Histogram(val name: String, histogram: CodaHaleHistogram, sampleRate: Int = 1) {
private var sampleCount = 0L
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/JvmMetricsSet.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/metrics/JvmMetricsSet.scala b/core/src/main/scala/org/apache/gearpump/metrics/JvmMetricsSet.scala
index 436d14f..56b4743 100644
--- a/core/src/main/scala/org/apache/gearpump/metrics/JvmMetricsSet.scala
+++ b/core/src/main/scala/org/apache/gearpump/metrics/JvmMetricsSet.scala
@@ -21,8 +21,8 @@ package org.apache.gearpump.metrics
import java.util
import scala.collection.JavaConverters._
-import org.apache.gearpump.codahale.metrics.jvm.{MemoryUsageGaugeSet, ThreadStatesGaugeSet}
-import org.apache.gearpump.codahale.metrics.{Metric, MetricSet}
+import com.codahale.metrics.jvm.{MemoryUsageGaugeSet, ThreadStatesGaugeSet}
+import com.codahale.metrics.{Metric, MetricSet}
class JvmMetricsSet(name: String) extends MetricSet {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/Meter.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/metrics/Meter.scala b/core/src/main/scala/org/apache/gearpump/metrics/Meter.scala
index d4b7871..55aa73f 100644
--- a/core/src/main/scala/org/apache/gearpump/metrics/Meter.scala
+++ b/core/src/main/scala/org/apache/gearpump/metrics/Meter.scala
@@ -18,7 +18,7 @@
package org.apache.gearpump.metrics
-import org.apache.gearpump.codahale.metrics.{Meter => CodaHaleMeter}
+import com.codahale.metrics.{Meter => CodaHaleMeter}
/** See org.apache.gearpump.codahale.metrics.Meter */
class Meter(val name: String, meter: CodaHaleMeter, sampleRate: Int = 1) {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/Metrics.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/metrics/Metrics.scala b/core/src/main/scala/org/apache/gearpump/metrics/Metrics.scala
index 1ee3798..3737361 100644
--- a/core/src/main/scala/org/apache/gearpump/metrics/Metrics.scala
+++ b/core/src/main/scala/org/apache/gearpump/metrics/Metrics.scala
@@ -23,7 +23,7 @@ import scala.collection.JavaConverters._
import akka.actor._
import org.slf4j.Logger
-import org.apache.gearpump.codahale.metrics._
+import com.codahale.metrics._
import org.apache.gearpump.metrics
import org.apache.gearpump.util.LogUtil
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/metrics/MetricsReporterService.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/metrics/MetricsReporterService.scala b/core/src/main/scala/org/apache/gearpump/metrics/MetricsReporterService.scala
index 94aa114..620dc61 100644
--- a/core/src/main/scala/org/apache/gearpump/metrics/MetricsReporterService.scala
+++ b/core/src/main/scala/org/apache/gearpump/metrics/MetricsReporterService.scala
@@ -20,12 +20,11 @@ package org.apache.gearpump.metrics
import java.net.InetSocketAddress
import java.util.concurrent.TimeUnit
-import scala.concurrent.duration._
+import scala.concurrent.duration._
import akka.actor.{Actor, ActorRef}
-
-import org.apache.gearpump.codahale.metrics.graphite.{Graphite, GraphiteReporter}
-import org.apache.gearpump.codahale.metrics.{MetricFilter, Slf4jReporter}
+import com.codahale.metrics.{MetricFilter, Slf4jReporter}
+import com.codahale.metrics.graphite.{Graphite, GraphiteReporter}
import org.apache.gearpump.metrics.Metrics.{DemandMoreMetrics, ReportMetrics}
import org.apache.gearpump.metrics.MetricsReporterService.ReportTo
import org.apache.gearpump.util.Constants._
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/serializer/FastKryoSerializer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/serializer/FastKryoSerializer.scala b/core/src/main/scala/org/apache/gearpump/serializer/FastKryoSerializer.scala
index f258c0f..cb3d2fd 100644
--- a/core/src/main/scala/org/apache/gearpump/serializer/FastKryoSerializer.scala
+++ b/core/src/main/scala/org/apache/gearpump/serializer/FastKryoSerializer.scala
@@ -19,19 +19,18 @@
package org.apache.gearpump.serializer
import akka.actor.ExtendedActorSystem
-
-import org.apache.gearpump.esotericsoftware.kryo.Kryo.DefaultInstantiatorStrategy
-import org.apache.gearpump.objenesis.strategy.StdInstantiatorStrategy
-import org.apache.gearpump.romix.serialization.kryo.KryoSerializerWrapper
+import com.esotericsoftware.kryo.Kryo.DefaultInstantiatorStrategy
+import com.romix.akka.serialization.kryo.{KryoBasedSerializer, KryoSerializer}
import org.apache.gearpump.serializer.FastKryoSerializer.KryoSerializationException
import org.apache.gearpump.util.LogUtil
+import org.objenesis.strategy.StdInstantiatorStrategy
class FastKryoSerializer(system: ExtendedActorSystem) extends Serializer {
private val LOG = LogUtil.getLogger(getClass)
private val config = system.settings.config
- private val kryoSerializer = new KryoSerializerWrapper(system)
+ private val kryoSerializer: KryoBasedSerializer = new KryoSerializer(system).serializer
private val kryo = kryoSerializer.kryo
val strategy = new DefaultInstantiatorStrategy
strategy.setFallbackInstantiatorStrategy(new StdInstantiatorStrategy)
@@ -40,7 +39,7 @@ class FastKryoSerializer(system: ExtendedActorSystem) extends Serializer {
override def serialize(message: Any): Array[Byte] = {
try {
- kryoSerializer.toBinary(message)
+ kryoSerializer.toBinary(message.asInstanceOf[AnyRef])
} catch {
case ex: java.lang.IllegalArgumentException =>
val clazz = message.getClass
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/serializer/GearpumpSerialization.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/serializer/GearpumpSerialization.scala b/core/src/main/scala/org/apache/gearpump/serializer/GearpumpSerialization.scala
index 524089d..45a5481 100644
--- a/core/src/main/scala/org/apache/gearpump/serializer/GearpumpSerialization.scala
+++ b/core/src/main/scala/org/apache/gearpump/serializer/GearpumpSerialization.scala
@@ -18,10 +18,9 @@
package org.apache.gearpump.serializer
+import com.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer}
import com.typesafe.config.Config
import org.slf4j.Logger
-
-import org.apache.gearpump.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer}
import org.apache.gearpump.util.{Constants, LogUtil}
class GearpumpSerialization(config: Config) {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/util/FileUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/util/FileUtils.scala b/core/src/main/scala/org/apache/gearpump/util/FileUtils.scala
index e3df37b..283a64a 100644
--- a/core/src/main/scala/org/apache/gearpump/util/FileUtils.scala
+++ b/core/src/main/scala/org/apache/gearpump/util/FileUtils.scala
@@ -21,7 +21,7 @@ package org.apache.gearpump.util
import java.io.{File, IOException}
import java.nio.charset.Charset
-import org.apache.gearpump.google.common.io.Files
+import com.google.common.io.Files
object FileUtils {
private val UTF8 = Charset.forName("UTF-8")
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/main/scala/org/apache/gearpump/util/Util.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/gearpump/util/Util.scala b/core/src/main/scala/org/apache/gearpump/util/Util.scala
index 0faa46a..8ee0e26 100644
--- a/core/src/main/scala/org/apache/gearpump/util/Util.scala
+++ b/core/src/main/scala/org/apache/gearpump/util/Util.scala
@@ -66,8 +66,14 @@ object Util {
def startProcess(options: Array[String], classPath: Array[String], mainClass: String,
arguments: Array[String]): RichProcess = {
val java = System.getProperty("java.home") + "/bin/java"
- val command = List(java) ++ options ++
- List("-cp", classPath.mkString(File.pathSeparator), mainClass) ++ arguments
+
+ val command = List(java) ++
+ // java.lang.VerifyError will be caused without "-noverify"
+ // TODO: investigate the cause and remove this
+ Array("-noverify") ++
+ options ++
+ List("-cp", classPath.mkString(File.pathSeparator), mainClass) ++
+ arguments
LOG.info(s"Starting executor process java $mainClass ${arguments.mkString(" ")} " +
s"\n ${options.mkString(" ")}")
val logger = new ProcessLogRedirector()
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/test/scala/org/apache/gearpump/jarstore/FileServerSpec.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/gearpump/jarstore/FileServerSpec.scala b/core/src/test/scala/org/apache/gearpump/jarstore/FileServerSpec.scala
index c99a031..39b6261 100644
--- a/core/src/test/scala/org/apache/gearpump/jarstore/FileServerSpec.scala
+++ b/core/src/test/scala/org/apache/gearpump/jarstore/FileServerSpec.scala
@@ -22,9 +22,9 @@ import java.io.File
import java.util.concurrent.TimeUnit
import akka.actor.ActorSystem
-import com.typesafe.config.{ConfigValueFactory, ConfigValue}
+import com.google.common.io.Files
+import com.typesafe.config.ConfigValueFactory
import org.apache.gearpump.cluster.TestUtil
-import org.apache.gearpump.google.common.io.Files
import org.apache.gearpump.jarstore.local.LocalJarStore
import org.apache.gearpump.util.{FileUtils, LogUtil}
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
@@ -39,7 +39,7 @@ class FileServerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll {
val host = "localhost"
private val LOG = LogUtil.getLogger(getClass)
- var system: ActorSystem = null
+ var system: ActorSystem = _
override def afterAll {
if (null != system) {
@@ -75,7 +75,6 @@ class FileServerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll {
"The file server" should {
"serve the data previously stored" in {
-
val rootDir = Files.createTempDir()
val localJarStore: JarStore = new LocalJarStore
val conf = TestUtil.DEFAULT_CONFIG.withValue("gearpump.jarstore.rootpath",
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/test/scala/org/apache/gearpump/metrics/MetricsSpec.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/gearpump/metrics/MetricsSpec.scala b/core/src/test/scala/org/apache/gearpump/metrics/MetricsSpec.scala
index 5881640..0855553 100644
--- a/core/src/test/scala/org/apache/gearpump/metrics/MetricsSpec.scala
+++ b/core/src/test/scala/org/apache/gearpump/metrics/MetricsSpec.scala
@@ -18,12 +18,13 @@
package org.apache.gearpump.metrics
+import com.codahale.metrics.{Counter => CodaHaleCounter, Histogram => CodaHaleHistogram, Meter => CodaHaleMeter}
+
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.mock.MockitoSugar
import org.scalatest.{FlatSpec, Matchers}
-import org.apache.gearpump.codahale.metrics.{Counter => CodaHaleCounter, Histogram => CodaHaleHistogram, Meter => CodaHaleMeter}
class MetricsSpec extends FlatSpec with Matchers with MockitoSugar {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/test/scala/org/apache/gearpump/serializer/SerializerSpec.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/gearpump/serializer/SerializerSpec.scala b/core/src/test/scala/org/apache/gearpump/serializer/SerializerSpec.scala
index f4bd114..0772a5e 100644
--- a/core/src/test/scala/org/apache/gearpump/serializer/SerializerSpec.scala
+++ b/core/src/test/scala/org/apache/gearpump/serializer/SerializerSpec.scala
@@ -18,20 +18,23 @@
package org.apache.gearpump.serializer
-import scala.collection.JavaConverters._
-import scala.concurrent.Await
-import scala.concurrent.duration.Duration
-
import akka.actor.{ActorSystem, ExtendedActorSystem}
+
+import com.esotericsoftware.kryo.io.{Input, Output}
+import com.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer}
import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
-import org.scalatest.mock.MockitoSugar
-import org.scalatest.{FlatSpec, Matchers}
import org.apache.gearpump.cluster.TestUtil
-import org.apache.gearpump.esotericsoftware.kryo.io.{Input, Output}
-import org.apache.gearpump.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer}
import org.apache.gearpump.serializer.SerializerSpec._
+import org.scalatest.mock.MockitoSugar
+import org.scalatest.{FlatSpec, Matchers}
+
+import scala.collection.JavaConverters._
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
+
class SerializerSpec extends FlatSpec with Matchers with MockitoSugar {
val config = ConfigFactory.empty.withValue("gearpump.serializers",
ConfigValueFactory.fromAnyRef(Map(classOf[ClassA].getName -> classOf[ClassASerializer].getName,
@@ -70,7 +73,7 @@ object SerializerSpec {
class ClassASerializer extends KryoSerializer[ClassA] {
override def write(kryo: Kryo, output: Output, `object`: ClassA): Unit = {
- output.writeString(classOf[ClassA].getName.toString)
+ output.writeString(classOf[ClassA].getName)
}
override def read(kryo: Kryo, input: Input, `type`: Class[ClassA]): ClassA = {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/core/src/test/scala/org/apache/gearpump/util/FileUtilsSpec.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/gearpump/util/FileUtilsSpec.scala b/core/src/test/scala/org/apache/gearpump/util/FileUtilsSpec.scala
index 66abc36..97b35ad 100644
--- a/core/src/test/scala/org/apache/gearpump/util/FileUtilsSpec.scala
+++ b/core/src/test/scala/org/apache/gearpump/util/FileUtilsSpec.scala
@@ -18,13 +18,13 @@
package org.apache.gearpump.util
+import com.google.common.io.Files
+
import java.io.File
import java.util
import org.scalatest.FlatSpec
-import org.apache.gearpump.google.common.io.Files
-
class FileUtilsSpec extends FlatSpec {
val TXT =
"""
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java
----------------------------------------------------------------------
diff --git a/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java b/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java
index 4983dde..f52afc7 100644
--- a/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java
+++ b/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java
@@ -17,7 +17,7 @@
*/
package org.apache.gearpump.cluster.utils;
-import org.apache.gearpump.google.common.io.CharStreams;
+import com.google.common.io.CharStreams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/experiments/cgroup/src/main/scala/org/apache/gearpump/cluster/worker/CGroupProcessLauncher.scala
----------------------------------------------------------------------
diff --git a/experiments/cgroup/src/main/scala/org/apache/gearpump/cluster/worker/CGroupProcessLauncher.scala b/experiments/cgroup/src/main/scala/org/apache/gearpump/cluster/worker/CGroupProcessLauncher.scala
index 74a4047..bf291cf 100644
--- a/experiments/cgroup/src/main/scala/org/apache/gearpump/cluster/worker/CGroupProcessLauncher.scala
+++ b/experiments/cgroup/src/main/scala/org/apache/gearpump/cluster/worker/CGroupProcessLauncher.scala
@@ -43,7 +43,7 @@ class CGroupProcessLauncher(val config: Config) extends ExecutorProcessLauncher
}
override def createProcess(
- appId: Int, executorId: Int, resource: Resource, appConfig: Config, options: Array[String],
+ appId: Int, executorId: Int, resource: Resource, appConfig: Config, options: Array[String],
classPath: Array[String], mainClass: String, arguments: Array[String]): RichProcess = {
val cgroupCommand = if (executorId != APP_MASTER) {
cgroupManager.map(_.startNewExecutor(appConfig, resource.slots, appId,
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/experiments/redis/src/main/scala/org/apache/gearpump/redis/RedisMessage.scala
----------------------------------------------------------------------
diff --git a/experiments/redis/src/main/scala/org/apache/gearpump/redis/RedisMessage.scala b/experiments/redis/src/main/scala/org/apache/gearpump/redis/RedisMessage.scala
index 2988f5b..ea738d6 100644
--- a/experiments/redis/src/main/scala/org/apache/gearpump/redis/RedisMessage.scala
+++ b/experiments/redis/src/main/scala/org/apache/gearpump/redis/RedisMessage.scala
@@ -147,7 +147,7 @@ object RedisMessage {
*/
case class LPUSH(key: Array[Byte], value: Array[Byte]) {
def this(key: String, value: String) = {
- this(key, toBytes(value))
+ this(toBytes(key), toBytes(value))
}
}
@@ -184,7 +184,7 @@ object RedisMessage {
*/
case class RPUSH(key: Array[Byte], value: Array[Byte]) {
def this(key: String, value: String) = {
- this(key, toBytes(value))
+ this(toBytes(key), toBytes(value))
}
}
@@ -338,7 +338,7 @@ object RedisMessage {
*/
case class SADD(key: Array[Byte], members: Array[Byte]) {
def this(key: String, members: String) = {
- this(key, toBytes(members))
+ this(toBytes(key), toBytes(members))
}
}
@@ -365,7 +365,7 @@ object RedisMessage {
*/
case class SREM(key: Array[Byte], member: Array[Byte]) {
def this(key: String, member: String) = {
- this(key, toBytes(member))
+ this(toBytes(key), toBytes(member))
}
}
@@ -438,7 +438,7 @@ object RedisMessage {
* @param increment
*/
case class INCRBYFLOAT(key: Array[Byte], increment: Double) {
- def this(key: String, increment: Number) = {
+ def this(key: String, increment: Double) = {
this(toBytes(key), increment)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/Build.scala
----------------------------------------------------------------------
diff --git a/project/Build.scala b/project/Build.scala
deleted file mode 100644
index 40b5743..0000000
--- a/project/Build.scala
+++ /dev/null
@@ -1,559 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import com.typesafe.sbt.SbtPgp.autoImport._
-import BuildExample.examples
-import BuildIntegrationTest.integration_test
-import BuildShaded._
-import de.johoop.jacoco4sbt.JacocoPlugin.jacoco
-import org.scalajs.sbtplugin.ScalaJSPlugin.autoImport._
-import sbt.Keys._
-import sbt._
-import Pack.packProject
-import org.scalajs.sbtplugin.cross.CrossProject
-import sbtassembly.AssemblyPlugin.autoImport._
-import sbtunidoc.Plugin.UnidocKeys._
-import sbtunidoc.Plugin._
-import xerial.sbt.Sonatype._
-
-object Build extends sbt.Build {
-
- val copySharedSourceFiles = TaskKey[Unit]("copied shared services source code")
-
- val akkaVersion = "2.4.3"
- val apacheRepo = "https://repository.apache.org/"
- val hadoopVersion = "2.6.0"
- val hbaseVersion = "1.0.0"
- val commonsHttpVersion = "3.1"
- val commonsLoggingVersion = "1.1.3"
- val commonsLangVersion = "2.6"
- val commonsIOVersion = "2.4"
- val dataReplicationVersion = "0.7"
- val upickleVersion = "0.3.4"
- val junitVersion = "4.12"
- val kafkaVersion = "0.8.2.1"
- val stormVersion = "0.10.0"
- val slf4jVersion = "1.7.7"
-
- val crossScalaVersionNumbers = Seq("2.11.8")
- val scalaVersionNumber = crossScalaVersionNumbers.last
- val sprayVersion = "1.3.2"
- val sprayJsonVersion = "1.3.1"
- val scalaTestVersion = "2.2.0"
- val scalaCheckVersion = "1.11.3"
- val mockitoVersion = "1.10.17"
- val bijectionVersion = "0.8.0"
- val scalazVersion = "7.1.1"
- val algebirdVersion = "0.9.0"
- val chillVersion = "0.6.0"
- val distDirectory = "output"
- val projectName = "gearpump"
-
- override def projects: Seq[Project] = (super.projects.toList ++ BuildExample.projects.toList
- ++ Pack.projects.toList).toSeq
-
- val commonSettings = Seq(jacoco.settings: _*) ++ sonatypeSettings ++
- Seq(
- resolvers ++= Seq(
- "patriknw at bintray" at "http://dl.bintray.com/patriknw/maven",
- "apache-repo" at "https://repository.apache.org/content/repositories",
- "maven1-repo" at "http://repo1.maven.org/maven2",
- "maven2-repo" at "http://mvnrepository.com/artifact",
- "sonatype" at "https://oss.sonatype.org/content/repositories/releases",
- "bintray/non" at "http://dl.bintray.com/non/maven",
- "clockfly" at "http://dl.bintray.com/clockfly/maven",
- "clojars" at "http://clojars.org/repo"
- )
- // ,addCompilerPlugin("org.scalamacros" % "paradise" % "2.1.0-M5" cross CrossVersion.full)
- ) ++
- Seq(
- scalaVersion := scalaVersionNumber,
- crossScalaVersions := crossScalaVersionNumbers,
- organization := "org.apache.gearpump",
- useGpg := false,
- pgpSecretRing := file("./secring.asc"),
- pgpPublicRing := file("./pubring.asc"),
- scalacOptions ++= Seq("-Yclosure-elim", "-Yinline"),
- publishMavenStyle := true,
-
- pgpPassphrase := Option(System.getenv().get("PASSPHRASE")).map(_.toArray),
- credentials += Credentials(
- "Sonatype Nexus Repository Manager",
- "repository.apache.org",
- System.getenv().get("SONATYPE_USERNAME"),
- System.getenv().get("SONATYPE_PASSWORD")),
-
- pomIncludeRepository := { _ => false },
-
- publishTo := {
- if (isSnapshot.value) {
- Some("snapshots" at apacheRepo + "content/repositories/snapshots")
- } else {
- Some("releases" at apacheRepo + "content/repositories/releases")
- }
- },
-
- publishArtifact in Test := true,
-
- pomExtra := {
- <url>https://github.com/apache/incubator-gearpump</url>
- <licenses>
- <license>
- <name>Apache 2</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- </license>
- </licenses>
- <scm>
- <connection>scm:git://git.apache.org/incubator-gearpump.git</connection>
- <developerConnection>scm:git:git@github.com:apache/incubator-gearpump</developerConnection>
- <url>github.com/apache/incubator-gearpump</url>
- </scm>
- <developers>
- <developer>
- <id>gearpump</id>
- <name>Gearpump Team</name>
- <url>http://gearpump.incubator.apache.org/community.html#who-we-are</url>
- </developer>
- </developers>
- }
- )
-
- val noPublish = Seq(
- publish := {},
- publishLocal := {},
- publishArtifact := false,
- publishArtifact in Test := false
- )
-
- val coreDependencies = Seq(
- libraryDependencies ++= Seq(
- "org.slf4j" % "slf4j-api" % slf4jVersion,
- "org.slf4j" % "slf4j-log4j12" % slf4jVersion,
- "commons-lang" % "commons-lang" % commonsLangVersion,
- "com.google.code.findbugs" % "jsr305" % "1.3.9" % "compile",
-
- /**
- * Overrides Netty version 3.10.3.Final used by Akka 2.4.2 to work-around netty hang issue
- * (https://github.com/gearpump/gearpump/issues/2020)
- *
- * Akka 2.4.2 by default use Netty 3.10.3.Final, which has a serious issue which can hang
- * the network. The same issue also happens in version range (3.10.0.Final, 3.10.5.Final)
- * Netty 3.10.6.Final have this issue fixed, however, we find there is a 20% performance
- * drop. So we decided to downgrade netty to 3.8.0.Final (Same version used in akka 2.3.12).
- *
- * @see https://github.com/gearpump/gearpump/pull/2017 for more discussions.
- */
- "io.netty" % "netty" % "3.8.0.Final",
- "com.typesafe.akka" %% "akka-remote" % akkaVersion
- exclude("io.netty", "netty"),
-
- "com.typesafe.akka" %% "akka-cluster" % akkaVersion,
- "com.typesafe.akka" %% "akka-cluster-tools" % akkaVersion,
- "commons-logging" % "commons-logging" % commonsLoggingVersion,
- "com.typesafe.akka" %% "akka-distributed-data-experimental" % akkaVersion,
- "com.typesafe.akka" %% "akka-actor" % akkaVersion,
- "com.typesafe.akka" %% "akka-agent" % akkaVersion,
- "com.typesafe.akka" %% "akka-slf4j" % akkaVersion,
- "com.typesafe.akka" %% "akka-kernel" % akkaVersion,
- "com.typesafe.akka" %% "akka-http-experimental" % akkaVersion,
- "com.typesafe.akka" %% "akka-http-spray-json-experimental" % akkaVersion,
- "org.scala-lang" % "scala-reflect" % scalaVersionNumber,
- "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4",
- "com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test",
- "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
- "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "test",
- "org.mockito" % "mockito-core" % mockitoVersion % "test",
- "junit" % "junit" % junitVersion % "test"
- ),
-
- unmanagedJars in Compile ++= Seq(
- getShadedJarFile(shaded_metrics_graphite.id, version.value),
- getShadedJarFile(shaded_guava.id, version.value),
- getShadedJarFile(shaded_akka_kryo.id, version.value)
- )
- )
-
- lazy val javadocSettings = Seq(
- addCompilerPlugin("com.typesafe.genjavadoc" %% "genjavadoc-plugin" %
- "0.9" cross CrossVersion.full),
- scalacOptions += s"-P:genjavadoc:out=${target.value}/java"
- )
-
- val myAssemblySettings = Seq(
- test in assembly := {},
- assemblyOption in assembly ~= {
- _.copy(includeScala = false)
- },
- assemblyJarName in assembly := {
- s"${name.value.split("-").last}-${scalaBinaryVersion.value}-${version.value}-assembly.jar"
- }
- )
-
- val projectsWithDoc = inProjects(
- core,
- streaming,
- external_kafka,
- external_monoid,
- external_serializer,
- external_hbase,
- external_hadoopfs)
-
- lazy val gearpumpUnidocSetting = scalaJavaUnidocSettings ++ Seq(
- unidocProjectFilter in(ScalaUnidoc, unidoc) := projectsWithDoc,
- unidocProjectFilter in(JavaUnidoc, unidoc) := projectsWithDoc,
-
- unidocAllSources in(ScalaUnidoc, unidoc) := {
- ignoreUndocumentedPackages((unidocAllSources in(ScalaUnidoc, unidoc)).value)
- },
-
- // Skip class names containing $ and some internal packages in Javadocs
- unidocAllSources in(JavaUnidoc, unidoc) := {
- ignoreUndocumentedPackages((unidocAllSources in(JavaUnidoc, unidoc)).value)
- }
- )
-
- private def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = {
- packages
- .map(_.filterNot(_.getName.contains("$")))
- .map(_.filterNot(_.getCanonicalPath.contains("akka")))
- }
-
- private def addShadedDeps(deps: Seq[xml.Node], node: xml.Node): xml.Node = {
- node match {
- case elem: xml.Elem =>
- val child = if (elem.label == "dependencies") {
- elem.child ++ deps
- } else {
- elem.child.map(addShadedDeps(deps, _))
- }
- xml.Elem(elem.prefix, elem.label, elem.attributes, elem.scope, false, child: _*)
- case _ =>
- node
- }
- }
-
- lazy val root = Project(
- id = "gearpump",
- base = file("."),
- settings = commonSettings ++ noPublish ++ gearpumpUnidocSetting)
- .aggregate(shaded, core, streaming, services, external_kafka, external_monoid,
- external_serializer, examples, storm, yarn, external_hbase, gearpumpHadoop, packProject,
- external_hadoopfs, integration_test).settings(Defaults.itSettings: _*)
- .disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val core = Project(
- id = "gearpump-core",
- base = file("core"),
- settings = commonSettings ++ javadocSettings ++ coreDependencies ++ Seq(
- pomPostProcess := {
- (node: xml.Node) => addShadedDeps(List(
- getShadedDepXML(organization.value, shaded_akka_kryo.id, version.value),
- getShadedDepXML(organization.value, shaded_guava.id, version.value),
- getShadedDepXML(organization.value, shaded_metrics_graphite.id, version.value)), node)
- }
- )).disablePlugins(sbtassembly.AssemblyPlugin)
-
-
- lazy val cgroup = Project(
- id = "gearpump-experimental-cgroup",
- base = file("experiments/cgroup"),
- settings = commonSettings ++ noPublish)
- .dependsOn (core % "test->test; compile->compile")
- .disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val streaming = Project(
- id = "gearpump-streaming",
- base = file("streaming"),
- settings = commonSettings ++ javadocSettings ++ Seq(
- unmanagedJars in Compile ++= Seq(
- getShadedJarFile(shaded_gs_collections.id, version.value)
- ),
-
- pomPostProcess := {
- (node: xml.Node) => addShadedDeps(List(
- getShadedDepXML(organization.value, shaded_gs_collections.id, version.value)), node)
- }
- ))
- .dependsOn(core % "test->test; compile->compile", shaded_gs_collections)
- .disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val external_kafka = Project(
- id = "gearpump-external-kafka",
- base = file("external/kafka"),
- settings = commonSettings ++ javadocSettings ++
- Seq(
- libraryDependencies ++= Seq(
- "org.apache.kafka" %% "kafka" % kafkaVersion,
- "com.twitter" %% "bijection-core" % bijectionVersion,
- ("org.apache.kafka" %% "kafka" % kafkaVersion classifier ("test")) % "test"
- )
- ))
- .dependsOn (streaming % "test->test; provided")
- .disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val services_full = CrossProject("gearpump-services", file("services"), CrossType.Full).
- settings(
- publish := {},
- publishLocal := {}
- ).disablePlugins(sbtassembly.AssemblyPlugin)
-
- val distDashboardDirectory = s"${distDirectory}/target/pack/dashboard/views/scalajs"
-
- // ScalaJs project need to be build seperately.
- // sbt "project gearpump-servicesJS" compile
- lazy val serviceJS: Project = services_full.js.settings(serviceJSSettings: _*)
-
- lazy val services: Project = services_full.jvm.
- settings(serviceJvmSettings: _*)
- .settings(compile in Compile <<= (compile in Compile))
- .dependsOn(streaming % "test->test;compile->compile")
-
- lazy val serviceJvmSettings = commonSettings ++ noPublish ++ Seq(
- libraryDependencies ++= Seq(
- "com.typesafe.akka" %% "akka-http-testkit" % akkaVersion % "test",
- "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
- "com.lihaoyi" %% "upickle" % upickleVersion,
- "com.softwaremill.akka-http-session" %% "core" % "0.2.5",
- "com.typesafe.akka" %% "akka-http-spray-json-experimental" % akkaVersion,
- "com.github.scribejava" % "scribejava-apis" % "2.4.0",
- "com.ning" % "async-http-client" % "1.9.33",
- "org.webjars" % "angularjs" % "1.4.9",
-
- // angular 1.5 breaks ui-select, but we need ng-touch 1.5
- "org.webjars.npm" % "angular-touch" % "1.5.0",
- "org.webjars" % "angular-ui-router" % "0.2.15",
- "org.webjars" % "bootstrap" % "3.3.6",
- "org.webjars" % "d3js" % "3.5.6",
- "org.webjars" % "momentjs" % "2.10.6",
- "org.webjars" % "lodash" % "3.10.1",
- "org.webjars" % "font-awesome" % "4.5.0",
- "org.webjars" % "jquery" % "2.2.0",
- "org.webjars" % "jquery-cookie" % "1.4.1",
- "org.webjars.bower" % "angular-loading-bar" % "0.8.0",
- "org.webjars.bower" % "angular-smart-table" % "2.1.6",
- "org.webjars.bower" % "angular-motion" % "0.4.3",
- "org.webjars.bower" % "bootstrap-additions" % "0.3.1",
- "org.webjars.bower" % "angular-strap" % "2.3.5",
- "org.webjars.npm" % "ui-select" % "0.14.2",
- "org.webjars.bower" % "ng-file-upload" % "5.0.9",
- "org.webjars.bower" % "vis" % "4.7.0",
- "org.webjars.bower" % "clipboard.js" % "0.1.1",
- "org.webjars.npm" % "dashing-deps" % "0.1.2",
- "org.webjars.npm" % "dashing" % "0.4.8"
- ).map(_.exclude("org.scalamacros", "quasiquotes_2.10"))
- .map(_.exclude("org.scalamacros", "quasiquotes_2.10.3")))
-
- lazy val serviceJSSettings = Seq(
- scalaVersion := scalaVersionNumber,
- crossScalaVersions := crossScalaVersionNumbers,
- checksums := Seq(""),
- requiresDOM := true,
- libraryDependencies ++= Seq(
- "com.lihaoyi" %%% "upickle" % upickleVersion,
- "com.lihaoyi" %%% "utest" % "0.3.1"
- ),
- scalaJSStage in Global := FastOptStage,
- testFrameworks += new TestFramework("utest.runner.Framework"),
- requiresDOM := true,
- persistLauncher in Compile := false,
- persistLauncher in Test := false,
- skip in packageJSDependencies := false,
- scoverage.ScoverageSbtPlugin.ScoverageKeys.coverageExcludedPackages :=
- ".*gearpump\\.dashboard.*",
-
- copySharedSourceFiles := {
- // scalastyle:off println
- println(s"Copy shared source code to project services...")
- // scalastyle:on println
- },
-
- artifactPath in fastOptJS in Compile :=
- new java.io.File(distDashboardDirectory, moduleName.value + "-fastopt.js"),
-
- fastOptJS in Compile <<= (fastOptJS in Compile).dependsOn(copySharedSourceFiles),
-
- relativeSourceMaps := true,
- jsEnv in Test := new PhantomJS2Env(scalaJSPhantomJSClassLoader.value))
-
- lazy val akkastream = Project(
- id = "gearpump-experiments-akkastream",
- base = file("experiments/akkastream"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- libraryDependencies ++= Seq(
- "org.json4s" %% "json4s-jackson" % "3.2.11"
- ),
- mainClass in(Compile, packageBin) := Some("akka.stream.gearpump.example.Test")
- ))
- .dependsOn(streaming % "test->test; provided")
-
- lazy val redis = Project(
- id = "gearpump-experiments-redis",
- base = file("experiments/redis"),
- settings = commonSettings ++ noPublish ++
- Seq(
- libraryDependencies ++= Seq(
- "redis.clients" % "jedis" % "2.9.0"
- )
- )
- ).dependsOn(streaming % "test->test; provided")
-
- lazy val storm = Project(
- id = "gearpump-experiments-storm",
- base = file("experiments/storm"),
- settings = commonSettings ++ noPublish ++
- Seq(
- libraryDependencies ++= Seq(
- "commons-io" % "commons-io" % commonsIOVersion,
- "org.apache.storm" % "storm-core" % stormVersion
- exclude("clj-stacktrace", "clj-stacktrace")
- exclude("ch.qos.logback", "logback-classic")
- exclude("ch.qos.logback", "logback-core")
- exclude("clj-time", "clj-time")
- exclude("clout", "clout")
- exclude("compojure", "compojure")
- exclude("hiccup", "hiccup")
- exclude("jline", "jline")
- exclude("joda-time", "joda-time")
- exclude("org.clojure", "core.incubator")
- exclude("org.clojure", "math.numeric-tower")
- exclude("org.clojure", "tools.logging")
- exclude("org.clojure", "tools.cli")
- exclude("org.clojure", "tools.macro")
- exclude("org.mortbay.jetty", "jetty-util")
- exclude("org.mortbay.jetty", "jetty")
- exclude("org.ow2.asm", "asm")
- exclude("org.slf4j", "log4j-over-slf4j")
- exclude("org.apache.logging.log4j", "log4j-slf4j-impl")
- exclude("ring", "ring-core")
- exclude("ring", "ring-devel")
- exclude("ring", "ring-jetty-adapter")
- exclude("ring", "ring-servlet")
- )
- ))
- .dependsOn (streaming % "test->test; compile->compile")
- .disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val gearpumpHadoop = Project(
- id = "gearpump-hadoop",
- base = file("gearpump-hadoop"),
- settings = commonSettings ++ noPublish ++
- Seq(
- libraryDependencies ++= Seq(
- "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion,
- "org.apache.hadoop" % "hadoop-common" % hadoopVersion
- )
- )
- ).dependsOn(core % "compile->compile").disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val yarn = Project(
- id = "gearpump-experiments-yarn",
- base = file("experiments/yarn"),
- settings = commonSettings ++ noPublish ++
- Seq(
- libraryDependencies ++= Seq(
- "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion,
- "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion,
- "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion,
- "commons-httpclient" % "commons-httpclient" % commonsHttpVersion,
- "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVersion,
- "org.apache.hadoop" % "hadoop-yarn-server-resourcemanager" % hadoopVersion % "provided",
- "org.apache.hadoop" % "hadoop-yarn-server-nodemanager" % hadoopVersion % "provided"
- )
- ))
- .dependsOn(services % "test->test;compile->compile",
- core % "provided", gearpumpHadoop).disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val external_hbase = Project(
- id = "gearpump-external-hbase",
- base = file("external/hbase"),
- settings = commonSettings ++ javadocSettings ++
- Seq(
- libraryDependencies ++= Seq(
- "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided",
- "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion % "provided",
- "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVersion % "provided",
- "org.codehaus.jackson" % "jackson-core-asl" % "1.9.13" % "provided",
- "org.codehaus.jackson" % "jackson-mapper-asl" % "1.9.13" % "provided",
- "org.apache.hbase" % "hbase-client" % hbaseVersion
- exclude("com.github.stephenc.findbugs", "findbugs-annotations")
- exclude("com.google.guava", "guava")
- exclude("commons-codec", "commons-codec")
- exclude("commons-io", "commons-io")
- exclude("commons-lang", "commons-lang")
- exclude("commons-logging", "commons-logging")
- exclude("io.netty", "netty")
- exclude("junit", "junit")
- exclude("log4j", "log4j")
- exclude("org.apache.zookeeper", "zookeeper")
- exclude("org.codehaus.jackson", "jackson-mapper-asl"),
- "org.apache.hbase" % "hbase-client" % hbaseVersion,
- "org.apache.hbase" % "hbase-common" % hbaseVersion
- exclude("com.github.stephenc.findbugs", "findbugs-annotations")
- exclude("com.google.guava", "guava")
- exclude("commons-codec", "commons-codec")
- exclude("commons-collections", "commons-collections")
- exclude("commons-io", "commons-io")
- exclude("commons-lang", "commons-lang")
- exclude("commons-logging", "commons-logging")
- exclude("junit", "junit")
- exclude("log4j", "log4j")
- )
- ))
- .dependsOn (streaming % "test->test; provided")
- .disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val external_monoid = Project(
- id = "gearpump-external-monoid",
- base = file("external/monoid"),
- settings = commonSettings ++ javadocSettings ++
- Seq(
- libraryDependencies ++= Seq(
- "com.twitter" %% "algebird-core" % algebirdVersion
- )
- ))
- .dependsOn (streaming % "provided")
- .disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val external_serializer = Project(
- id = "gearpump-external-serializer",
- base = file("external/serializer"),
- settings = commonSettings ++ javadocSettings ++
- Seq(
- libraryDependencies ++= Seq(
- "com.twitter" %% "chill-bijection" % chillVersion
- exclude("com.esotericsoftware.kryo", "kyro")
- exclude("com.esotericsoftware.minlog", "minlog")
- )
- ))
- .dependsOn (streaming % "provided")
- .disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val external_hadoopfs = Project(
- id = "gearpump-external-hadoopfs",
- base = file("external/hadoopfs"),
- settings = commonSettings ++ javadocSettings ++
- Seq(
- libraryDependencies ++= Seq(
- "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided",
- "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion % "provided"
- )
- ))
- .dependsOn (streaming % "test->test; provided")
- .disablePlugins(sbtassembly.AssemblyPlugin)
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildDashboard.scala
----------------------------------------------------------------------
diff --git a/project/BuildDashboard.scala b/project/BuildDashboard.scala
new file mode 100644
index 0000000..c14b9d6
--- /dev/null
+++ b/project/BuildDashboard.scala
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import BuildGearpump._
+import Dependencies._
+import org.scalajs.sbtplugin.cross.{CrossProject, CrossType}
+import org.scalajs.sbtplugin.ScalaJSPlugin.autoImport._
+import sbt._
+import sbt.Keys._
+
+object BuildDashboard extends sbt.Build {
+
+ lazy val services: Project = services_full.jvm
+ .settings(serviceJvmSettings: _*)
+ .dependsOn(core % "provided", streaming % "test->test; provided")
+
+ // ScalaJs project need to be build separately.
+ // sbt "project gearpump-servicesJS" compile
+ private lazy val serviceJS: Project = services_full.js.settings(serviceJSSettings: _*)
+
+ private lazy val services_full = CrossProject("gearpump-services", file("services"),
+ CrossType.Full)
+ .settings(
+ publish := {},
+ publishLocal := {}
+ ).disablePlugins(sbtassembly.AssemblyPlugin)
+
+ private val copySharedSourceFiles = TaskKey[Unit]("copied shared services source code")
+
+ private val distDashboardDirectory = s"${distDirectory}/target/pack/dashboard/views/scalajs"
+
+ private lazy val serviceJvmSettings = commonSettings ++ noPublish ++ Seq(
+ libraryDependencies ++= Seq(
+ "com.typesafe.akka" %% "akka-http-testkit" % akkaVersion % "test",
+ "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
+ "com.lihaoyi" %% "upickle" % upickleVersion,
+ "com.softwaremill.akka-http-session" %% "core" % "0.2.5",
+ "com.typesafe.akka" %% "akka-http-spray-json-experimental" % akkaVersion,
+ "com.github.scribejava" % "scribejava-apis" % "2.4.0",
+ "com.ning" % "async-http-client" % "1.9.33",
+ "org.webjars" % "angularjs" % "1.4.9",
+
+ // angular 1.5 breaks ui-select, but we need ng-touch 1.5
+ "org.webjars.npm" % "angular-touch" % "1.5.0",
+ "org.webjars" % "angular-ui-router" % "0.2.15",
+ "org.webjars" % "bootstrap" % "3.3.6",
+ "org.webjars" % "d3js" % "3.5.6",
+ "org.webjars" % "momentjs" % "2.10.6",
+ "org.webjars" % "lodash" % "3.10.1",
+ "org.webjars" % "font-awesome" % "4.5.0",
+ "org.webjars" % "jquery" % "2.2.0",
+ "org.webjars" % "jquery-cookie" % "1.4.1",
+ "org.webjars.bower" % "angular-loading-bar" % "0.8.0"
+ exclude("org.webjars.bower", "angular"),
+ "org.webjars.bower" % "angular-smart-table" % "2.1.6"
+ exclude("org.webjars.bower", "angular"),
+ "org.webjars.bower" % "angular-motion" % "0.4.3",
+ "org.webjars.bower" % "bootstrap-additions" % "0.3.1",
+ "org.webjars.bower" % "angular-strap" % "2.3.5"
+ exclude("org.webjars.bower", "angular"),
+ "org.webjars.npm" % "ui-select" % "0.14.2",
+ "org.webjars.bower" % "ng-file-upload" % "5.0.9",
+ "org.webjars.bower" % "vis" % "4.7.0",
+ "org.webjars.bower" % "clipboard.js" % "0.1.1",
+ "org.webjars.npm" % "dashing-deps" % "0.1.2",
+ "org.webjars.npm" % "dashing" % "0.4.8"
+ ).map(_.exclude("org.scalamacros", "quasiquotes_2.10"))
+ .map(_.exclude("org.scalamacros", "quasiquotes_2.10.3")))
+
+ private lazy val serviceJSSettings = Seq(
+ scalaVersion := scalaVersionNumber,
+ crossScalaVersions := crossScalaVersionNumbers,
+ checksums := Seq(""),
+ requiresDOM := true,
+ libraryDependencies ++= Seq(
+ "com.lihaoyi" %%% "upickle" % upickleVersion,
+ "com.lihaoyi" %%% "utest" % "0.3.1"
+ ),
+ scalaJSStage in Global := FastOptStage,
+ testFrameworks += new TestFramework("utest.runner.Framework"),
+ requiresDOM := true,
+ persistLauncher in Compile := false,
+ persistLauncher in Test := false,
+ skip in packageJSDependencies := false,
+ scoverage.ScoverageSbtPlugin.ScoverageKeys.coverageExcludedPackages :=
+ ".*gearpump\\.dashboard.*",
+
+ copySharedSourceFiles := {
+ // scalastyle:off println
+ println(s"Copy shared source code to project services...")
+ // scalastyle:on println
+ },
+
+ artifactPath in fastOptJS in Compile :=
+ new java.io.File(distDashboardDirectory, moduleName.value + "-fastopt.js"),
+
+ fastOptJS in Compile := {
+ (fastOptJS in Compile).dependsOn(copySharedSourceFiles).value
+ },
+
+ relativeSourceMaps := true,
+ jsEnv in Test := new PhantomJS2Env(scalaJSPhantomJSClassLoader.value)
+ )
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildExample.scala
----------------------------------------------------------------------
diff --git a/project/BuildExample.scala b/project/BuildExample.scala
deleted file mode 100644
index fadc1ec..0000000
--- a/project/BuildExample.scala
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import sbt.Keys._
-import sbt._
-import Build._
-import sbtassembly.AssemblyPlugin.autoImport._
-
-object BuildExample extends sbt.Build {
-
- lazy val examples = Project(
- id = "gearpump-examples",
- base = file("examples"),
- settings = commonSettings ++ noPublish
- ).aggregate(wordcount, wordcountJava, complexdag, sol, fsio, examples_kafka,
- distributedshell, stockcrawler, transport, examples_state, pagerank, distributeservice).
- disablePlugins(sbtassembly.AssemblyPlugin)
-
- lazy val wordcountJava = Project(
- id = "gearpump-examples-wordcountjava",
- base = file("examples/streaming/wordcount-java"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.streaming.examples.wordcountjava.WordCount"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn(streaming % "test->test; provided")
-
- lazy val wordcount = Project(
- id = "gearpump-examples-wordcount",
- base = file("examples/streaming/wordcount"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.streaming.examples.wordcount.WordCount"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn(streaming % "test->test; provided")
-
- lazy val sol = Project(
- id = "gearpump-examples-sol",
- base = file("examples/streaming/sol"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- mainClass in(Compile, packageBin) := Some("org.apache.gearpump.streaming.examples.sol.SOL"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
-
- )
- ) dependsOn (streaming % "test->test; provided")
-
- lazy val complexdag = Project(
- id = "gearpump-examples-complexdag",
- base = file("examples/streaming/complexdag"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.streaming.examples.complexdag.Dag"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn (streaming % "test->test; provided")
-
- lazy val transport = Project(
- id = "gearpump-examples-transport",
- base = file("examples/streaming/transport"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- libraryDependencies ++= Seq(
- "io.spray" %% "spray-can" % sprayVersion,
- "io.spray" %% "spray-routing-shapeless2" % sprayVersion,
- "io.spray" %% "spray-json" % sprayJsonVersion,
- "com.lihaoyi" %% "upickle" % upickleVersion
- ),
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.streaming.examples.transport.Transport"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn (streaming % "test->test; provided")
-
- lazy val distributedshell = Project(
- id = "gearpump-examples-distributedshell",
- base = file("examples/distributedshell"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.examples.distributedshell.DistributedShell"),
-
- target in assembly := baseDirectory.value.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn (core % "test->test; provided")
-
- lazy val distributeservice = Project(
- id = "gearpump-examples-distributeservice",
- base = file("examples/distributeservice"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- libraryDependencies ++= Seq(
- "commons-httpclient" % "commons-httpclient" % commonsHttpVersion,
- "commons-lang" % "commons-lang" % commonsLangVersion,
- "commons-io" % "commons-io" % commonsIOVersion,
- "io.spray" %% "spray-can" % sprayVersion,
- "io.spray" %% "spray-routing-shapeless2" % sprayVersion
- ),
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.experiments.distributeservice.DistributeService"),
-
- target in assembly := baseDirectory.value.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn (core % "test->test; provided")
-
- lazy val fsio = Project(
- id = "gearpump-examples-fsio",
- base = file("examples/streaming/fsio"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- libraryDependencies ++= Seq(
- "org.apache.hadoop" % "hadoop-common" % hadoopVersion
- exclude("org.mortbay.jetty", "jetty-util")
- exclude("org.mortbay.jetty", "jetty")
- exclude("org.fusesource.leveldbjni", "leveldbjni-all")
- exclude("tomcat", "jasper-runtime")
- exclude("commons-beanutils", "commons-beanutils-core")
- exclude("commons-beanutils", "commons-beanutils")
- exclude("asm", "asm")
- exclude("org.ow2.asm", "asm")
- ),
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.streaming.examples.fsio.SequenceFileIO"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn (streaming % "test->test; provided")
-
- lazy val examples_kafka = Project(
- id = "gearpump-examples-kafka",
- base = file("examples/streaming/kafka"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.streaming.examples.kafka.wordcount.KafkaWordCount"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn(streaming % "test->test; provided", external_kafka)
-
- lazy val stockcrawler = Project(
- id = "gearpump-examples-stockcrawler",
- base = file("examples/streaming/stockcrawler"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- libraryDependencies ++= Seq(
- "io.spray" %% "spray-can" % sprayVersion,
- "io.spray" %% "spray-routing-shapeless2" % sprayVersion,
- "com.lihaoyi" %% "upickle" % upickleVersion,
- "commons-httpclient" % "commons-httpclient" % commonsHttpVersion,
- "net.sourceforge.htmlcleaner" % "htmlcleaner" % "2.2",
- "joda-time" % "joda-time" % "2.7",
- "org.joda" % "joda-convert" % "1.2",
- "io.spray" %% "spray-json" % sprayJsonVersion
- ),
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.streaming.examples.stock.main.Stock"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn(streaming % "test->test; provided", external_kafka % "test->test")
-
- lazy val examples_state = Project(
- id = "gearpump-examples-state",
- base = file("examples/streaming/state"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- libraryDependencies ++= Seq(
- "org.apache.hadoop" % "hadoop-common" % hadoopVersion
- exclude("org.mortbay.jetty", "jetty-util")
- exclude("org.mortbay.jetty", "jetty")
- exclude("org.fusesource.leveldbjni", "leveldbjni-all")
- exclude("tomcat", "jasper-runtime")
- exclude("commons-beanutils", "commons-beanutils-core")
- exclude("commons-beanutils", "commons-beanutils")
- exclude("asm", "asm")
- exclude("org.ow2.asm", "asm"),
- "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion
- ),
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.streaming.examples.state.MessageCountApp"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ).dependsOn(streaming % "test->test; provided",
- external_hadoopfs, external_monoid, external_serializer, external_kafka)
-
- lazy val pagerank = Project(
- id = "gearpump-examples-pagerank",
- base = file("examples/pagerank"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
- Seq(
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.experiments.pagerank.example.PageRankExample"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn (streaming % "test->test; provided")
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildExamples.scala
----------------------------------------------------------------------
diff --git a/project/BuildExamples.scala b/project/BuildExamples.scala
new file mode 100644
index 0000000..bac84a8
--- /dev/null
+++ b/project/BuildExamples.scala
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import sbt.Keys._
+import sbt._
+import BuildGearpump._
+import BuildExternals._
+import Dependencies._
+import sbtassembly.AssemblyPlugin.autoImport._
+
+object BuildExamples extends sbt.Build {
+
+ lazy val examples: Seq[ProjectReference] = Seq(
+ complexdag,
+ distributedshell,
+ distributeservice,
+ examples_kafka,
+ examples_state,
+ fsio,
+ pagerank,
+ sol,
+ stockcrawler,
+ transport,
+ wordcount,
+ wordcountJava
+ )
+
+ lazy val wordcountJava = Project(
+ id = "gearpump-examples-wordcountjava",
+ base = file("examples/streaming/wordcount-java"),
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcountjava.WordCount")
+ ).dependsOn(core % "provided", streaming % "test->test; provided")
+
+ lazy val wordcount = Project(
+ id = "gearpump-examples-wordcount",
+ base = file("examples/streaming/wordcount"),
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcount.WordCount")
+ ).dependsOn(core % "provided", streaming % "test->test; provided")
+
+ lazy val sol = Project(
+ id = "gearpump-examples-sol",
+ base = file("examples/streaming/sol"),
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.sol.SOL")
+ ).dependsOn(core % "provided", streaming % "test->test; provided")
+
+ lazy val complexdag = Project(
+ id = "gearpump-examples-complexdag",
+ base = file("examples/streaming/complexdag"),
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.complexdag.Dag")
+ ).dependsOn(core % "provided", streaming % "test->test; provided")
+
+ lazy val transport = Project(
+ id = "gearpump-examples-transport",
+ base = file("examples/streaming/transport"),
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.transport.Transport") ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "io.spray" %% "spray-can" % sprayVersion,
+ "io.spray" %% "spray-routing-shapeless2" % sprayVersion,
+ "io.spray" %% "spray-json" % sprayJsonVersion,
+ "com.lihaoyi" %% "upickle" % upickleVersion
+ )
+ )
+ ).dependsOn(core % "provided", streaming % "test->test; provided")
+
+ lazy val distributedshell = Project(
+ id = "gearpump-examples-distributedshell",
+ base = file("examples/distributedshell"),
+ settings = exampleSettings("org.apache.gearpump.examples.distributedshell.DistributedShell")
+ ).dependsOn(core % "test->test; provided")
+
+ lazy val distributeservice = Project(
+ id = "gearpump-examples-distributeservice",
+ base = file("examples/distributeservice"),
+ settings =
+ exampleSettings("org.apache.gearpump.experiments.distributeservice.DistributeService") ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "commons-httpclient" % "commons-httpclient" % commonsHttpVersion,
+ "commons-lang" % "commons-lang" % commonsLangVersion,
+ "commons-io" % "commons-io" % commonsIOVersion,
+ "io.spray" %% "spray-can" % sprayVersion,
+ "io.spray" %% "spray-routing-shapeless2" % sprayVersion
+ )
+ )
+ ).dependsOn(core % "test->test; provided")
+
+ lazy val fsio = Project(
+ id = "gearpump-examples-fsio",
+ base = file("examples/streaming/fsio"),
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.fsio.SequenceFileIO") ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.apache.hadoop" % "hadoop-common" % hadoopVersion
+ exclude("org.mortbay.jetty", "jetty-util")
+ exclude("org.mortbay.jetty", "jetty")
+ exclude("org.fusesource.leveldbjni", "leveldbjni-all")
+ exclude("tomcat", "jasper-runtime")
+ exclude("commons-beanutils", "commons-beanutils-core")
+ exclude("commons-beanutils", "commons-beanutils")
+ exclude("asm", "asm")
+ exclude("org.ow2.asm", "asm")
+ )
+ )
+ ).dependsOn(core % "provided", streaming % "test->test; provided")
+
+ lazy val examples_kafka = Project(
+ id = "gearpump-examples-kafka",
+ base = file("examples/streaming/kafka"),
+ settings =
+ exampleSettings("org.apache.gearpump.streaming.examples.kafka.wordcount.KafkaWordCount")
+ ).dependsOn(core % "provided", streaming % "test->test; provided", external_kafka)
+
+ lazy val stockcrawler = Project(
+ id = "gearpump-examples-stockcrawler",
+ base = file("examples/streaming/stockcrawler"),
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.stock.main.Stock") ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "io.spray" %% "spray-can" % sprayVersion,
+ "io.spray" %% "spray-routing-shapeless2" % sprayVersion,
+ "com.lihaoyi" %% "upickle" % upickleVersion,
+ "commons-httpclient" % "commons-httpclient" % commonsHttpVersion,
+ "net.sourceforge.htmlcleaner" % "htmlcleaner" % "2.2",
+ "joda-time" % "joda-time" % "2.7",
+ "org.joda" % "joda-convert" % "1.2",
+ "io.spray" %% "spray-json" % sprayJsonVersion
+ )
+ )
+ ).dependsOn(core % "provided", streaming % "test->test; provided", external_kafka % "test->test")
+
+ lazy val examples_state = Project(
+ id = "gearpump-examples-state",
+ base = file("examples/streaming/state"),
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.state.MessageCountApp") ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.apache.hadoop" % "hadoop-common" % hadoopVersion
+ exclude("org.mortbay.jetty", "jetty-util")
+ exclude("org.mortbay.jetty", "jetty")
+ exclude("org.fusesource.leveldbjni", "leveldbjni-all")
+ exclude("tomcat", "jasper-runtime")
+ exclude("commons-beanutils", "commons-beanutils-core")
+ exclude("commons-beanutils", "commons-beanutils")
+ exclude("asm", "asm")
+ exclude("org.ow2.asm", "asm"),
+ "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion
+ )
+ )
+ ).dependsOn(core % "provided", streaming % "test->test; provided",
+ external_hadoopfs, external_monoid, external_serializer, external_kafka)
+
+ lazy val pagerank = Project(
+ id = "gearpump-examples-pagerank",
+ base = file("examples/pagerank"),
+ settings = exampleSettings("org.apache.gearpump.experiments.pagerank.example.PageRankExample")
+ ).dependsOn(core % "provided", streaming % "test->test; provided")
+
+ private def exampleSettings(className: String): Seq[Def.Setting[_]] =
+ commonSettings ++ noPublish ++ myAssemblySettings ++ Seq(
+ mainClass in(Compile, packageBin) :=
+ Some(className),
+ target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
+ CrossVersion.binaryScalaVersion(scalaVersion.value)
+ )
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildExperiments.scala
----------------------------------------------------------------------
diff --git a/project/BuildExperiments.scala b/project/BuildExperiments.scala
new file mode 100644
index 0000000..92a26c6
--- /dev/null
+++ b/project/BuildExperiments.scala
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import BuildGearpump._
+import BuildDashboard.services
+import Dependencies._
+import sbt._
+import sbt.Keys._
+
+object BuildExperiments extends sbt.Build {
+
+ lazy val experiments: Seq[ProjectReference] = Seq(
+ // akkastream,
+ cgroup,
+ redis,
+ storm,
+ yarn
+ )
+
+ lazy val yarn = Project(
+ id = "gearpump-experiments-yarn",
+ base = file("experiments/yarn"),
+ settings = commonSettings ++ noPublish ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion,
+ "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion,
+ "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion,
+ "commons-httpclient" % "commons-httpclient" % commonsHttpVersion,
+ "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVersion,
+ "org.apache.hadoop" % "hadoop-yarn-server-resourcemanager" % hadoopVersion % "provided",
+ "org.apache.hadoop" % "hadoop-yarn-server-nodemanager" % hadoopVersion % "provided"
+ )
+ ))
+ .dependsOn(services % "test->test;compile->compile",
+ core % "provided", gearpumpHadoop).disablePlugins(sbtassembly.AssemblyPlugin)
+
+ lazy val akkastream = Project(
+ id = "gearpump-experiments-akkastream",
+ base = file("experiments/akkastream"),
+ settings = commonSettings ++ noPublish ++ myAssemblySettings ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.json4s" %% "json4s-jackson" % "3.2.11"
+ ),
+ mainClass in(Compile, packageBin) := Some("akka.stream.gearpump.example.Test")
+ ))
+ .dependsOn(core % "provided", streaming % "test->test; provided")
+
+ lazy val redis = Project(
+ id = "gearpump-experiments-redis",
+ base = file("experiments/redis"),
+ settings = commonSettings ++ noPublish ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "redis.clients" % "jedis" % jedisVersion
+ )
+ )
+ ).dependsOn(core % "provided", streaming % "test->test; provided")
+
+ lazy val storm = Project(
+ id = "gearpump-experiments-storm",
+ base = file("experiments/storm"),
+ settings = commonSettings ++ noPublish ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "commons-io" % "commons-io" % commonsIOVersion,
+ "org.apache.storm" % "storm-core" % stormVersion
+ exclude("clj-stacktrace", "clj-stacktrace")
+ exclude("ch.qos.logback", "logback-classic")
+ exclude("ch.qos.logback", "logback-core")
+ exclude("clj-time", "clj-time")
+ exclude("clout", "clout")
+ exclude("compojure", "compojure")
+ exclude("hiccup", "hiccup")
+ exclude("jline", "jline")
+ exclude("joda-time", "joda-time")
+ exclude("org.clojure", "core.incubator")
+ exclude("org.clojure", "math.numeric-tower")
+ exclude("org.clojure", "tools.logging")
+ exclude("org.clojure", "tools.cli")
+ exclude("org.clojure", "tools.macro")
+ exclude("org.mortbay.jetty", "jetty-util")
+ exclude("org.mortbay.jetty", "jetty")
+ exclude("org.ow2.asm", "asm")
+ exclude("org.slf4j", "log4j-over-slf4j")
+ exclude("org.apache.logging.log4j", "log4j-slf4j-impl")
+ exclude("ring", "ring-core")
+ exclude("ring", "ring-devel")
+ exclude("ring", "ring-jetty-adapter")
+ exclude("ring", "ring-servlet")
+ )
+ ))
+ .dependsOn (core % "provided", streaming % "test->test; provided")
+ .disablePlugins(sbtassembly.AssemblyPlugin)
+
+ lazy val cgroup = Project(
+ id = "gearpump-experimental-cgroup",
+ base = file("experiments/cgroup"),
+ settings = commonSettings ++ noPublish)
+ .dependsOn (core % "test->test; compile->compile")
+ .disablePlugins(sbtassembly.AssemblyPlugin)
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0b56de39/project/BuildExternals.scala
----------------------------------------------------------------------
diff --git a/project/BuildExternals.scala b/project/BuildExternals.scala
new file mode 100644
index 0000000..48715cb
--- /dev/null
+++ b/project/BuildExternals.scala
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import BuildGearpump._
+import Dependencies._
+import Docs._
+import sbt._
+import sbt.Keys._
+
+object BuildExternals extends sbt.Build {
+
+ lazy val externals: Seq[ProjectReference] = Seq(
+ external_hbase,
+ external_kafka,
+ external_monoid,
+ external_hadoopfs
+ )
+
+ lazy val external_kafka = Project(
+ id = "gearpump-external-kafka",
+ base = file("external/kafka"),
+ settings = commonSettings ++ javadocSettings ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.apache.kafka" %% "kafka" % kafkaVersion,
+ "com.twitter" %% "bijection-core" % bijectionVersion,
+ ("org.apache.kafka" %% "kafka" % kafkaVersion classifier ("test")) % "test"
+ )
+ ))
+ .dependsOn(core % "provided", streaming % "test->test; provided")
+ .disablePlugins(sbtassembly.AssemblyPlugin)
+
+ lazy val external_hbase = Project(
+ id = "gearpump-external-hbase",
+ base = file("external/hbase"),
+ settings = commonSettings ++ javadocSettings ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided",
+ "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion % "provided",
+ "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVersion % "provided",
+ "org.codehaus.jackson" % "jackson-core-asl" % "1.9.13" % "provided",
+ "org.codehaus.jackson" % "jackson-mapper-asl" % "1.9.13" % "provided",
+ "org.apache.hbase" % "hbase-client" % hbaseVersion
+ exclude("com.github.stephenc.findbugs", "findbugs-annotations")
+ exclude("com.google.guava", "guava")
+ exclude("commons-codec", "commons-codec")
+ exclude("commons-io", "commons-io")
+ exclude("commons-lang", "commons-lang")
+ exclude("commons-logging", "commons-logging")
+ exclude("io.netty", "netty")
+ exclude("junit", "junit")
+ exclude("log4j", "log4j")
+ exclude("org.apache.zookeeper", "zookeeper")
+ exclude("org.codehaus.jackson", "jackson-mapper-asl"),
+ "org.apache.hbase" % "hbase-common" % hbaseVersion
+ exclude("com.github.stephenc.findbugs", "findbugs-annotations")
+ exclude("com.google.guava", "guava")
+ exclude("commons-codec", "commons-codec")
+ exclude("commons-collections", "commons-collections")
+ exclude("commons-io", "commons-io")
+ exclude("commons-lang", "commons-lang")
+ exclude("commons-logging", "commons-logging")
+ exclude("junit", "junit")
+ exclude("log4j", "log4j")
+ )
+ ))
+ .dependsOn (core % "provided", streaming % "test->test; provided")
+ .disablePlugins(sbtassembly.AssemblyPlugin)
+
+ lazy val external_monoid = Project(
+ id = "gearpump-external-monoid",
+ base = file("external/monoid"),
+ settings = commonSettings ++ javadocSettings ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "com.twitter" %% "algebird-core" % algebirdVersion
+ )
+ ))
+ .dependsOn (core % "provided", streaming % "provided")
+ .disablePlugins(sbtassembly.AssemblyPlugin)
+
+ lazy val external_serializer = Project(
+ id = "gearpump-external-serializer",
+ base = file("external/serializer"),
+ settings = commonSettings ++ javadocSettings ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "com.twitter" %% "chill-bijection" % chillVersion
+ exclude("com.esotericsoftware.kryo", "kyro")
+ exclude("com.esotericsoftware.minlog", "minlog")
+ )
+ ))
+ .dependsOn (core % "provided", streaming % "provided")
+ .disablePlugins(sbtassembly.AssemblyPlugin)
+
+ lazy val external_hadoopfs = Project(
+ id = "gearpump-external-hadoopfs",
+ base = file("external/hadoopfs"),
+ settings = commonSettings ++ javadocSettings ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided",
+ "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion % "provided"
+ )
+ ))
+ .dependsOn(core % "provided", streaming % "test->test; provided")
+ .disablePlugins(sbtassembly.AssemblyPlugin)
+}
\ No newline at end of file