You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gearpump.apache.org by ap...@apache.org on 2016/04/11 20:26:03 UTC
[01/50] incubator-gearpump git commit: Merge pull request #1940 from
clockfly/issue_1932
Repository: incubator-gearpump
Updated Branches:
refs/heads/master [created] d5343681e
Merge pull request #1940 from clockfly/issue_1932
fix #1932, client config overwrites critical system config
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/20c62e5a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/20c62e5a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/20c62e5a
Branch: refs/heads/master
Commit: 20c62e5a331b436be4f5cb91663bc1c0057ae2fb
Parents: 2566c0e 847fdf8
Author: Sean Zhong <xi...@intel.com>
Authored: Fri Jan 29 16:57:43 2016 +0800
Committer: Sean Zhong <xi...@intel.com>
Committed: Fri Jan 29 16:57:43 2016 +0800
----------------------------------------------------------------------
.../io/gearpump/cluster/ClusterConfig.scala | 2 +-
.../io/gearpump/cluster/worker/Worker.scala | 57 +++++++++++---------
2 files changed, 34 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
[42/50] incubator-gearpump git commit: Merge pull request #1973 from
manuzhang/kafka_consumer
Posted by ap...@apache.org.
Merge pull request #1973 from manuzhang/kafka_consumer
fix #1972, backoff retry kafka consuming on exception
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/c7ba3d51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/c7ba3d51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/c7ba3d51
Branch: refs/heads/master
Commit: c7ba3d5128ab84d26759fe71b96993d55a870b80
Parents: c3df2ed 9b1085c
Author: Sean Zhong <xi...@intel.com>
Authored: Sat Feb 27 13:54:38 2016 +0800
Committer: Sean Zhong <xi...@intel.com>
Committed: Sat Feb 27 13:54:38 2016 +0800
----------------------------------------------------------------------
.../consumer/ExponentialBackoffSleeper.scala | 58 ++++++++++
.../kafka/lib/consumer/FetchThread.scala | 56 ++++++++--
.../kafka/lib/consumer/KafkaConsumer.scala | 2 +
.../streaming/kafka/lib/FetchThreadSpec.scala | 108 -------------------
.../streaming/kafka/lib/KafkaConsumerSpec.scala | 89 ---------------
.../ExponentialBackoffSleeperSpec.scala | 72 +++++++++++++
.../kafka/lib/consumer/FetchThreadSpec.scala | 108 +++++++++++++++++++
.../kafka/lib/consumer/KafkaConsumerSpec.scala | 88 +++++++++++++++
8 files changed, 373 insertions(+), 208 deletions(-)
----------------------------------------------------------------------
[34/50] incubator-gearpump git commit: Merge pull request #1965 from
stanleyxu2005/issue_1892
Posted by ap...@apache.org.
Merge pull request #1965 from stanleyxu2005/issue_1892
fix #1892: added instruction text to operate network graph
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/1db8606b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/1db8606b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/1db8606b
Branch: refs/heads/master
Commit: 1db8606b031577dafc248f0335067aa00463398a
Parents: 9feb159 3bf290e
Author: Sean Zhong <xi...@intel.com>
Authored: Mon Feb 15 10:29:02 2016 +0800
Committer: Sean Zhong <xi...@intel.com>
Committed: Mon Feb 15 10:29:02 2016 +0800
----------------------------------------------------------------------
services/dashboard/index.html | 4 +-
services/dashboard/views/apps/apps.js | 4 +-
.../dashboard/views/apps/compose/compose.js | 6 +-
.../dashboard/views/apps/streamingapp/dag.html | 8 ++-
.../dashboard/views/apps/streamingapp/dag.js | 48 +++++++++++---
.../views/apps/streamingapp/popups/dag_edit.js | 2 +-
.../streamingapp/popups/processor_menu.html | 15 -----
.../views/apps/streamingapp/vis/network.js | 69 +++++++++++---------
.../views/widgets/metrics_period_switcher.html | 3 -
.../views/widgets/metrics_period_switcher.js | 32 ---------
.../dashboard/views/widgets/radio_group.html | 12 ----
services/dashboard/views/widgets/radio_group.js | 29 --------
.../widgets/metrics_period_switcher.html | 3 +
.../widgets/metrics_period_switcher.js | 32 +++++++++
services/dashboard/widgets/radio_group.html | 12 ++++
services/dashboard/widgets/radio_group.js | 29 ++++++++
16 files changed, 164 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
[13/50] incubator-gearpump git commit: change
topologicalOrderIterator to topologicalOrderWithCirclesIterator
Posted by ap...@apache.org.
change topologicalOrderIterator to topologicalOrderWithCirclesIterator
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/dcd2ca46
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/dcd2ca46
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/dcd2ca46
Branch: refs/heads/master
Commit: dcd2ca46335be5682cfd00e64df6a04f99ff8f82
Parents: 55135fb
Author: pangolulu <gy...@gmail.com>
Authored: Tue Feb 2 20:18:31 2016 +0800
Committer: pangolulu <gy...@gmail.com>
Committed: Tue Feb 2 20:18:31 2016 +0800
----------------------------------------------------------------------
.../src/main/scala/io/gearpump/streaming/StreamApplication.scala | 2 +-
.../main/scala/io/gearpump/streaming/appmaster/ClockService.scala | 2 +-
.../src/main/scala/io/gearpump/streaming/dsl/plan/Planner.scala | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/dcd2ca46/streaming/src/main/scala/io/gearpump/streaming/StreamApplication.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/io/gearpump/streaming/StreamApplication.scala b/streaming/src/main/scala/io/gearpump/streaming/StreamApplication.scala
index cba19a4..b68054a 100644
--- a/streaming/src/main/scala/io/gearpump/streaming/StreamApplication.scala
+++ b/streaming/src/main/scala/io/gearpump/streaming/StreamApplication.scala
@@ -144,7 +144,7 @@ object StreamApplication {
LOG.warn(s"Detected cycles in DAG of application $name!")
}
- val indices = dag.topologicalOrderIterator.toList.zipWithIndex.toMap
+ val indices = dag.topologicalOrderWithCirclesIterator.toList.zipWithIndex.toMap
val graph = dag.mapVertex {processor =>
val updatedProcessor = ProcessorToProcessorDescription(indices(processor), processor)
updatedProcessor
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/dcd2ca46/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala
index 038620a..578a15d 100644
--- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala
+++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala
@@ -380,7 +380,7 @@ object ClockService {
}
if (isClockStalling) {
- val processorId = dag.graph.topologicalOrderIterator.toList.find { processorId =>
+ val processorId = dag.graph.topologicalOrderWithCirclesIterator.toList.find { processorId =>
val clock = processorClocks.get(processorId)
if (clock.isDefined) {
clock.get.min == minClock.appClock
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/dcd2ca46/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/Planner.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/Planner.scala b/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/Planner.scala
index b3714f2..ee0818d 100644
--- a/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/Planner.scala
+++ b/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/Planner.scala
@@ -55,7 +55,7 @@ class Planner {
private def optimize(dag: Graph[Op, OpEdge]): Graph[OpChain, OpEdge] = {
val newGraph = dag.mapVertex(op => OpChain(List(op)))
- val nodes = newGraph.topologicalOrderIterator.toList.reverse
+ val nodes = newGraph.topologicalOrderWithCirclesIterator.toList.reverse
for (node <- nodes) {
val outGoingEdges = newGraph.outgoingEdgesOf(node)
for (edge <- outGoingEdges) {
[14/50] incubator-gearpump git commit: Merge pull request #1949 from
whjiang/timeout-1910
Posted by ap...@apache.org.
Merge pull request #1949 from whjiang/timeout-1910
fix #1910 enlarge timeout in MasterClient
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/89292aca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/89292aca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/89292aca
Branch: refs/heads/master
Commit: 89292aca9707efdd00a91e3ab464afbb09186346
Parents: e8d224e f7347a8
Author: Sean Zhong <xi...@intel.com>
Authored: Wed Feb 3 09:12:53 2016 +0800
Committer: Sean Zhong <xi...@intel.com>
Committed: Wed Feb 3 09:12:53 2016 +0800
----------------------------------------------------------------------
core/src/main/resources/geardefault.conf | 3 +++
.../io/gearpump/cluster/client/ClientContext.scala | 14 ++++++++++----
.../io/gearpump/cluster/client/MasterClient.scala | 5 +++--
core/src/main/scala/io/gearpump/util/Constants.scala | 1 +
4 files changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[25/50] incubator-gearpump git commit: Merge pull request #1959 from
stanleyxu2005/issue_1958
Posted by ap...@apache.org.
Merge pull request #1959 from stanleyxu2005/issue_1958
fix #1958: better test report organizing
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/1445d12d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/1445d12d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/1445d12d
Branch: refs/heads/master
Commit: 1445d12dc96def113195b9f00aea551e109005e1
Parents: 058bbf5 f048d6f
Author: Qian Xu <st...@users.noreply.github.com>
Authored: Sun Feb 7 14:23:57 2016 +0800
Committer: Qian Xu <st...@users.noreply.github.com>
Committed: Sun Feb 7 14:23:57 2016 +0800
----------------------------------------------------------------------
integrationtest/autorun/autorun.sh | 17 ++++-----
.../minicluster/BaseContainer.scala | 14 ++++++--
.../minicluster/RestClient.scala | 38 ++++++++++++--------
3 files changed, 43 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
[04/50] incubator-gearpump git commit: fix #1935,
add topological sort algorithm for graph with circles
Posted by ap...@apache.org.
fix #1935, add topological sort algorithm for graph with circles
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/55135fbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/55135fbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/55135fbb
Branch: refs/heads/master
Commit: 55135fbb459935064dd0f6fa0843601698dd71e3
Parents: 20c62e5
Author: pangolulu <gy...@gmail.com>
Authored: Thu Jan 28 23:32:06 2016 +0800
Committer: pangolulu <gy...@gmail.com>
Committed: Mon Feb 1 19:12:42 2016 +0800
----------------------------------------------------------------------
.../src/main/scala/io/gearpump/util/Graph.scala | 88 ++++++++++++++++++++
.../test/scala/io/gearpump/util/GraphSpec.scala | 17 ++++
2 files changed, 105 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/55135fbb/core/src/main/scala/io/gearpump/util/Graph.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/util/Graph.scala b/core/src/main/scala/io/gearpump/util/Graph.scala
index c3035d0..6bff9da 100644
--- a/core/src/main/scala/io/gearpump/util/Graph.scala
+++ b/core/src/main/scala/io/gearpump/util/Graph.scala
@@ -254,6 +254,94 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial
}
/**
+ * Return all circles in graph.
+ * The reference of this algorithm is:
+ * https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
+ */
+ private def findCircles: mutable.MutableList[mutable.MutableList[N]] = {
+ val inStack = mutable.Map.empty[N, Boolean]
+ val stack = mutable.Stack[N]()
+ val indexMap = mutable.Map.empty[N, Int]
+ val lowLink = mutable.Map.empty[N, Int]
+ var index = 0
+
+ val circles = mutable.MutableList.empty[mutable.MutableList[N]]
+
+ def tarjan(node: N): Unit = {
+ indexMap(node) = index
+ lowLink(node) = index
+ index += 1
+ inStack(node) = true
+ stack.push(node)
+
+ outgoingEdgesOf(node).foreach {
+ edge => {
+ if (!indexMap.contains(edge._3)) {
+ tarjan(edge._3)
+ if (lowLink.get(edge._3).get < lowLink.get(node).get)
+ lowLink(node) = lowLink(edge._3)
+ } else {
+ if (inStack.get(edge._3).get && (indexMap.get(edge._3).get < lowLink.get(node).get))
+ lowLink(node) = indexMap(edge._3)
+ }
+ }
+ }
+
+ if (indexMap.get(node).get == lowLink.get(node).get) {
+ val circle = mutable.MutableList.empty[N]
+ var n = node
+ do {
+ n = stack.pop()
+ inStack(n) = false
+ circle += n
+ } while (n != node)
+ circles += circle
+ }
+ }
+
+ vertices.foreach {
+ node => {
+ if (!indexMap.contains(node)) tarjan(node)
+ }
+ }
+
+ circles
+ }
+
+ /**
+ * Return an iterator of vertex in topological order of graph with circles
+ * The node returned by Iterator is stable sorted.
+ * The reference of this algorithm is:
+ * http://www.drdobbs.com/database/topological-sorting/184410262
+ */
+ def topologicalOrderWithCirclesIterator: Iterator[N] = {
+ val circles = findCircles
+ val newGraph = Graph.empty[mutable.MutableList[N], E]
+ circles.foreach {
+ circle => {
+ newGraph.addVertex(circle)
+ }
+ }
+
+ for (circle1 <- circles; circle2 <- circles; if circle1 != circle2) yield {
+ for (node1 <- circle1; node2 <- circle2) yield {
+ var edges = outgoingEdgesOf(node1)
+ for (edge <- edges; if edge._3 == node2) yield {
+ newGraph.addEdge(circle1, edge._2, circle2)
+ }
+
+ edges = outgoingEdgesOf(node2)
+ for (edge <- edges; if edge._3 == node1) yield {
+ newGraph.addEdge(circle2, edge._2, circle1)
+ }
+ }
+ }
+
+ val topo = newGraph.topologicalOrderIterator
+ topo.flatMap(_.sortBy(_indexs(_)).iterator)
+ }
+
+ /**
* check whether there is a loop
*/
def hasCycle(): Boolean = {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/55135fbb/core/src/test/scala/io/gearpump/util/GraphSpec.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/io/gearpump/util/GraphSpec.scala b/core/src/test/scala/io/gearpump/util/GraphSpec.scala
index 892350c..0028446 100644
--- a/core/src/test/scala/io/gearpump/util/GraphSpec.scala
+++ b/core/src/test/scala/io/gearpump/util/GraphSpec.scala
@@ -178,6 +178,23 @@ class GraphSpec extends PropSpec with PropertyChecks with Matchers {
assert(graph.hasCycle())
}
+ property("topologicalOrderIterator " +
+ "and topologicalOrderWithCirclesIterator method should return equal order of graph with no circle") {
+ val graph = Graph(1 ~> 2 ~> 3, 4 ~> 2, 2 ~> 5)
+ val topoNoCircles = graph.topologicalOrderIterator
+ val topoWithCircles = graph.topologicalOrderWithCirclesIterator
+
+ assert(topoNoCircles.zip(topoWithCircles).forall(x => x._1 == x._2))
+ }
+
+ property("Topological sort of graph with circles should work properly") {
+ val graph = Graph(0 ~> 1 ~> 3 ~> 4 ~> 6 ~> 5 ~> 7, 4 ~> 1, 1 ~> 2 ~> 4, 7 ~> 6, 8 ~> 2, 6 ~> 9, 4 ~> 10)
+ val topoWithCircles = graph.topologicalOrderWithCirclesIterator
+ val trueTopoWithCircles = Iterator[Int](0, 8, 1, 3, 4, 2, 6 ,5, 7, 10, 9)
+
+ assert(trueTopoWithCircles.zip(topoWithCircles).forall(x => x._1 == x._2))
+ }
+
property("Duplicated edges detecting should work properly") {
val graph = Graph.empty[String, String]
val defaultEdge = "edge"
[12/50] incubator-gearpump git commit: fix #1910 enlarge timeout in
MasterClient
Posted by ap...@apache.org.
fix #1910 enlarge timeout in MasterClient
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/f7347a81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/f7347a81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/f7347a81
Branch: refs/heads/master
Commit: f7347a812a78459ef1eedbf98d073d60edc19347
Parents: e8d224e
Author: Weihua Jiang <we...@intel.com>
Authored: Tue Feb 2 15:52:10 2016 +0800
Committer: Weihua Jiang <we...@intel.com>
Committed: Tue Feb 2 15:52:10 2016 +0800
----------------------------------------------------------------------
core/src/main/resources/geardefault.conf | 3 +++
.../io/gearpump/cluster/client/ClientContext.scala | 14 ++++++++++----
.../io/gearpump/cluster/client/MasterClient.scala | 5 +++--
core/src/main/scala/io/gearpump/util/Constants.scala | 1 +
4 files changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7347a81/core/src/main/resources/geardefault.conf
----------------------------------------------------------------------
diff --git a/core/src/main/resources/geardefault.conf b/core/src/main/resources/geardefault.conf
index e5dc554..d775696 100644
--- a/core/src/main/resources/geardefault.conf
+++ b/core/src/main/resources/geardefault.conf
@@ -206,6 +206,9 @@ gearpump {
### the appmaster will shutdown itself.
resource-allocation-timeout-seconds = 10
+ ### client's timeout (in second) to connect to master and wait for the response
+ masterclient.timeout = 90
+
### Define where the submitted jar file will be stored at
### This path follows the hadoop path schema
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7347a81/core/src/main/scala/io/gearpump/cluster/client/ClientContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/cluster/client/ClientContext.scala b/core/src/main/scala/io/gearpump/cluster/client/ClientContext.scala
index 5b6d0fc..536d8c1 100644
--- a/core/src/main/scala/io/gearpump/cluster/client/ClientContext.scala
+++ b/core/src/main/scala/io/gearpump/cluster/client/ClientContext.scala
@@ -36,6 +36,7 @@ import scala.collection.JavaConversions._
import scala.concurrent.Await
import scala.concurrent.duration.Duration
import scala.concurrent.Future
+import scala.util.Try
/**
@@ -92,7 +93,7 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) {
}
private def submit(app : AppDescription, jarPath: String) : Int = {
- val client = new MasterClient(master)
+ val client = getMasterClient
val appName = checkAndAddNamePrefix(app.name, System.getProperty(GEARPUMP_APP_NAME_PREFIX))
val updatedApp = AppDescription(appName, app.appMaster, app.userConfig, app.clusterConfig)
if (jarPath == null) {
@@ -115,17 +116,17 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) {
}
def listApps: AppMastersData = {
- val client = new MasterClient(master)
+ val client = getMasterClient
client.listApplications
}
def shutdown(appId : Int) : Unit = {
- val client = new MasterClient(master)
+ val client = getMasterClient
client.shutdownApplication(appId)
}
def resolveAppID(appId: Int) : ActorRef = {
- val client = new MasterClient(master)
+ val client = getMasterClient
client.resolveAppId(appId)
}
@@ -156,6 +157,11 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) {
}
fullName
}
+
+ private def getMasterClient: MasterClient = {
+ val timeout = Try(config.getInt(Constants.GEARPUMP_MASTERCLIENT_TIMEOUT)).getOrElse(90)
+ new MasterClient(master, akka.util.Timeout(timeout, TimeUnit.SECONDS))
+ }
}
object ClientContext {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7347a81/core/src/main/scala/io/gearpump/cluster/client/MasterClient.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/cluster/client/MasterClient.scala b/core/src/main/scala/io/gearpump/cluster/client/MasterClient.scala
index 4fa586b..5800e8d 100644
--- a/core/src/main/scala/io/gearpump/cluster/client/MasterClient.scala
+++ b/core/src/main/scala/io/gearpump/cluster/client/MasterClient.scala
@@ -20,6 +20,7 @@ package io.gearpump.cluster.client
import akka.actor.ActorRef
import akka.pattern.ask
+import akka.util.Timeout
import io.gearpump.cluster.ClientToMaster._
import io.gearpump.cluster.MasterToAppMaster.{AppMastersData, AppMastersDataRequest, ReplayFromTimestampWindowTrailingEdge}
import io.gearpump.cluster.MasterToClient.{ReplayApplicationResult, ResolveAppIdResult, ShutdownApplicationResult, SubmitApplicationResult}
@@ -34,8 +35,8 @@ import scala.util.{Failure, Success}
* Client to Master node.
* Stateless, thread safe
*/
-class MasterClient(master : ActorRef) {
- implicit val timeout = Constants.FUTURE_TIMEOUT
+class MasterClient(master : ActorRef, timeout: Timeout) {
+ implicit val masterClientTimeout = timeout
def submitApplication(app : AppDescription, appJar: Option[AppJar]) : Int = {
val result = Await.result( (master ? SubmitApplication(app, appJar)).asInstanceOf[Future[SubmitApplicationResult]], Duration.Inf)
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f7347a81/core/src/main/scala/io/gearpump/util/Constants.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/util/Constants.scala b/core/src/main/scala/io/gearpump/util/Constants.scala
index aa43b00..5e340e3 100644
--- a/core/src/main/scala/io/gearpump/util/Constants.scala
+++ b/core/src/main/scala/io/gearpump/util/Constants.scala
@@ -43,6 +43,7 @@ object Constants {
val GEARPUMP_SERIALIZERS = "gearpump.serializers"
val GEARPUMP_TASK_DISPATCHER = "gearpump.task-dispatcher"
val GEARPUMP_CLUSTER_MASTERS = "gearpump.cluster.masters"
+ val GEARPUMP_MASTERCLIENT_TIMEOUT = "gearpump.masterclient.timeout"
val GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS = "gearpump.worker.executor-share-same-jvm-as-worker"
val GEARPUMP_HOME = "gearpump.home"
[35/50] incubator-gearpump git commit: Merge pull request #1941 from
pangolulu/fix_1935
Posted by ap...@apache.org.
Merge pull request #1941 from pangolulu/fix_1935
fix #1935, add topological sort algorithm for graph with circles.
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/aff1fe3a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/aff1fe3a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/aff1fe3a
Branch: refs/heads/master
Commit: aff1fe3a546638712aa1b7bbb669a53be7c948e8
Parents: 1db8606 dcd2ca4
Author: Sean Zhong <xi...@intel.com>
Authored: Wed Feb 17 14:24:54 2016 +0800
Committer: Sean Zhong <xi...@intel.com>
Committed: Wed Feb 17 14:24:54 2016 +0800
----------------------------------------------------------------------
.../src/main/scala/io/gearpump/util/Graph.scala | 88 ++++++++++++++++++++
.../test/scala/io/gearpump/util/GraphSpec.scala | 17 ++++
.../gearpump/streaming/StreamApplication.scala | 2 +-
.../streaming/appmaster/ClockService.scala | 2 +-
.../gearpump/streaming/dsl/plan/Planner.scala | 2 +-
5 files changed, 108 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[27/50] incubator-gearpump git commit: Merge pull request #1960 from
rvvincelli/doctypos_II
Posted by ap...@apache.org.
Merge pull request #1960 from rvvincelli/doctypos_II
A few documentation points
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/d1a8c0ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/d1a8c0ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/d1a8c0ae
Branch: refs/heads/master
Commit: d1a8c0aef3bba29d8908feb2080e16ccb91ccd8f
Parents: 1445d12 f64af57
Author: Kam Kasravi <ka...@yahoo.com>
Authored: Wed Feb 10 09:57:06 2016 -0800
Committer: Kam Kasravi <ka...@yahoo.com>
Committed: Wed Feb 10 09:57:06 2016 -0800
----------------------------------------------------------------------
docs/deployment-security.md | 2 +-
docs/dev-connectors.md | 17 ++++++++++++++++-
docs/dev-custom-serializer.md | 13 +++++++++++++
3 files changed, 30 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[11/50] incubator-gearpump git commit: Merge pull request #1948 from
manuzhang/standalone_nimbus
Posted by ap...@apache.org.
Merge pull request #1948 from manuzhang/standalone_nimbus
fix #1895, make GearpumpNimbus a standalone service
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/e8d224e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/e8d224e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/e8d224e0
Branch: refs/heads/master
Commit: e8d224e060b6ec74cd812d939a53e1458ae2d0a5
Parents: ca018b7 400f627
Author: ManuZhang <Ow...@gmail.com>
Authored: Tue Feb 2 14:32:42 2016 +0800
Committer: ManuZhang <Ow...@gmail.com>
Committed: Tue Feb 2 14:32:42 2016 +0800
----------------------------------------------------------------------
docs/dev-storm.md | 40 ++-
.../gearpump/experiments/storm/Commands.scala | 36 ---
.../storm/GearpumpThriftServer.scala | 143 ----------
.../experiments/storm/StormRunner.scala | 136 ++--------
.../experiments/storm/main/GearpumpNimbus.scala | 267 +++++++++++++++++++
.../storm/main/GearpumpStormClient.scala | 70 +++++
.../storm/topology/GearpumpStormTopology.scala | 52 +---
.../experiments/storm/GearpumpNimbusSpec.scala | 68 -----
.../storm/GearpumpThriftServerSpec.scala | 48 ----
.../experiments/storm/StormRunnerSpec.scala | 58 ----
.../topology/GearpumpStormTopologySpec.scala | 20 +-
.../suites/StandaloneModeSuite.scala | 4 +-
.../integrationtest/storm/StormClient.scala | 14 +-
project/Pack.scala | 2 +-
.../io/gearpump/services/MasterService.scala | 2 +-
15 files changed, 421 insertions(+), 539 deletions(-)
----------------------------------------------------------------------
[02/50] incubator-gearpump git commit: fix #1895,
make GearpumpNimbus a standalone service
Posted by ap...@apache.org.
fix #1895, make GearpumpNimbus a standalone service
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/502dbae9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/502dbae9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/502dbae9
Branch: refs/heads/master
Commit: 502dbae996440cfde7b2b9473a44af9cd1714cc8
Parents: 20c62e5
Author: manuzhang <ow...@gmail.com>
Authored: Fri Jan 29 16:55:53 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Feb 1 16:14:27 2016 +0800
----------------------------------------------------------------------
docs/dev-storm.md | 40 ++-
.../gearpump/experiments/storm/Commands.scala | 36 ---
.../storm/GearpumpThriftServer.scala | 143 -----------
.../experiments/storm/StormRunner.scala | 137 ++---------
.../experiments/storm/main/GearpumpNimbus.scala | 245 +++++++++++++++++++
.../storm/main/GearpumpStormClient.scala | 69 ++++++
.../storm/topology/GearpumpStormTopology.scala | 52 +---
.../experiments/storm/GearpumpNimbusSpec.scala | 68 -----
.../storm/GearpumpThriftServerSpec.scala | 48 ----
.../experiments/storm/StormRunnerSpec.scala | 58 -----
.../topology/GearpumpStormTopologySpec.scala | 20 +-
.../integrationtest/storm/StormClient.scala | 14 +-
project/Pack.scala | 2 +-
.../io/gearpump/services/MasterService.scala | 2 +-
14 files changed, 397 insertions(+), 537 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/docs/dev-storm.md
----------------------------------------------------------------------
diff --git a/docs/dev-storm.md b/docs/dev-storm.md
index a05c4ed..a22ed6c 100644
--- a/docs/dev-storm.md
+++ b/docs/dev-storm.md
@@ -77,23 +77,43 @@ This section shows how to run an existing Storm jar in a local Gearpump cluster.
1. launch a local cluster
```
- ./target/pack/bin/local
+ bin/local
```
-2. submit a topology from storm-starter.
+2. start a local Gearpump Nimbus server
+
+ Users need server's thrift port to submit topologies later. The thrift port is written to a yaml config file set with `-output` option.
+ Users can provide an existing config file where only `nimbus.thrift.port` is overwritten. If not provided, a new file `app.yaml` is created with the config.
+
```
- bin/storm -verbose -config storm.yaml -jar storm-starter-${STORM_VERSION}.jar storm.starter.ExclamationTopology exclamation
+ bin/storm nimbus -output [conf <custom yaml config>]
```
+
+3. submit Storm applications
+
+ Users can either submit Storm applications through command line or UI.
+
+ a. submit Storm applications through command line
+
+
+ ```
+ bin/storm app -verbose -config app.yaml -jar storm-starter-${STORM_VERSION}.jar storm.starter.ExclamationTopology exclamation
+ ```
- Users are able to configure their applications through following options
+ Users are able to configure their applications through following options
- * `jar` - set the path of a storm application jar
- * `config` - submit a customized storm configuration file
+ * `jar` - set the path of a Storm application jar
+ * `config` - submit the custom configuration file generated when launching Nimbus
- That's it. Check the dashboard and you should see data flowing through your topology.
+ b. submit Storm application through UI
- *Note that submission from UI is not supported yet*.
+ 1. Click on the "Create" button on the applications page on UI.
+ 2. Click on the "Submit Storm Application" item in the pull down menu.
+ 3. In the popup console, upload the Storm application jar and the configuration file generated when launching Nimbus,
+ and fill in `storm.starter.ExclamationTopology exclamation` as arguments.
+ 4. Click on the "Submit" button
+ Either way, check the dashboard and you should see data flowing through your topology.
## How is it different from running on Storm
@@ -145,11 +165,11 @@ Gearpump has flow control between tasks such that [sender cannot flood receiver]
All Storm configurations are respected with the following priority order
```
-defaults.yaml < storm.yaml < application config < component config < custom user config
+defaults.yaml < custom file config < application config < component config
```
where
* application config is submit from Storm application along with the topology
* component config is set in spout / bolt with `getComponentConfiguration`
-* custom user config is specified with the `-config` option when submitting Storm application from command line
\ No newline at end of file
+* custom file config is specified with the `-config` option when submitting Storm application from command line or uploaded from UI
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala
deleted file mode 100644
index 1d5b903..0000000
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.experiments.storm
-
-import backtype.storm.generated.{KillOptions, StormTopology, SubmitOptions}
-
-object Commands {
-
- case class Submit(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology, options: SubmitOptions)
-
- case class AppSubmitted(name: String, appId: Int)
-
- case class Kill(name: String, option: KillOptions)
-
- case class AppKilled(name: String, appId: Int)
-
- case object GetClusterInfo
-
- case class GetTopology(id: String)
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala
deleted file mode 100644
index 640c20c..0000000
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.experiments.storm
-
-import java.nio.ByteBuffer
-import java.util.concurrent.TimeUnit
-import java.util.{HashMap => JHashMap, Map => JMap}
-
-import akka.actor.ActorRef
-import backtype.storm.Config
-import backtype.storm.generated._
-import backtype.storm.security.auth.{ThriftConnectionType, ThriftServer}
-import backtype.storm.utils.Utils
-import io.gearpump.experiments.storm.Commands.{GetClusterInfo, _}
-import io.gearpump.experiments.storm.util.StormUtil
-import io.gearpump.util.ActorUtil.askActor
-
-import scala.concurrent.Await
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-object GearpumpThriftServer {
- val THRIFT_PORT = StormUtil.getThriftPort
- implicit val timeOut = akka.util.Timeout(3, TimeUnit.SECONDS)
-
- private def createServer(handler: ActorRef): ThriftServer = {
- val processor = new Nimbus.Processor[GearpumpNimbus](new GearpumpNimbus(handler))
- val connectionType = ThriftConnectionType.NIMBUS
- val config = Utils.readDefaultConfig().asInstanceOf[JMap[AnyRef, AnyRef]]
- config.put(Config.NIMBUS_THRIFT_PORT, s"$THRIFT_PORT")
- new ThriftServer(config, processor, connectionType)
- }
-
- def apply(handler: ActorRef): GearpumpThriftServer = {
- new GearpumpThriftServer(createServer(handler))
- }
-
- class GearpumpNimbus(handler: ActorRef) extends Nimbus.Iface {
-
- override def submitTopology(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology): Unit = {
- val ask = askActor(handler, Submit(name, uploadedJarLocation, jsonConf, topology, new SubmitOptions(TopologyInitialStatus.ACTIVE)))
- Await.result(ask, 30 seconds)
- }
-
- override def killTopologyWithOpts(name: String, options: KillOptions): Unit = {
- Await.result(askActor(handler,Kill(name, options)), 10 seconds)
- }
-
- override def submitTopologyWithOpts(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology, options: SubmitOptions): Unit = {
- Await.result(askActor(handler,Submit(name, uploadedJarLocation, jsonConf, topology, options)), 10 seconds)
- }
-
- override def uploadChunk(location: String, chunk: ByteBuffer): Unit = {
- }
-
- override def getNimbusConf: String = {
- throw new UnsupportedOperationException
- }
-
- override def getTopology(id: String): StormTopology = {
- Await.result(askActor[StormTopology](handler, GetTopology(id)), 10 seconds)
- }
-
- override def getTopologyConf(id: String): String = {
- throw new UnsupportedOperationException
- }
-
- override def beginFileDownload(file: String): String = {
- throw new UnsupportedOperationException
- }
-
- override def getUserTopology(id: String): StormTopology = getTopology(id)
-
- override def activate(name: String): Unit = {
- throw new UnsupportedOperationException
- }
-
- override def rebalance(name: String, options: RebalanceOptions): Unit = {
- throw new UnsupportedOperationException
- }
-
- override def deactivate(name: String): Unit = {
- throw new UnsupportedOperationException
- }
-
- override def getTopologyInfo(id: String): TopologyInfo = {
- throw new UnsupportedOperationException
- }
-
- override def getTopologyInfoWithOpts(s: String, getInfoOptions: GetInfoOptions): TopologyInfo = {
- throw new UnsupportedOperationException
- }
-
- override def killTopology(name: String): Unit = killTopologyWithOpts(name, new KillOptions())
-
- override def downloadChunk(id: String): ByteBuffer = {
- throw new UnsupportedOperationException
- }
-
- override def beginFileUpload(): String = {
- "local thrift server"
- }
-
- override def getClusterInfo: ClusterSummary = {
- Await.result(askActor[ClusterSummary](handler, GetClusterInfo), 10 seconds)
- }
-
- override def finishFileUpload(location: String): Unit = {
- }
-
- override def uploadNewCredentials(s: String, credentials: Credentials): Unit = {
- throw new UnsupportedOperationException
- }
- }
-}
-
-class GearpumpThriftServer(server: ThriftServer) extends Thread {
-
- override def run(): Unit = {
- server.serve()
- }
-
- def close(): Unit = {
- server.stop()
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
index 84d9460..dd8d782 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
@@ -18,131 +18,36 @@
package io.gearpump.experiments.storm
-import java.io.File
-import java.util.{Map => JMap}
+import io.gearpump.experiments.storm.main.{GearpumpNimbus, GearpumpStormClient}
+import io.gearpump.util.LogUtil
+import org.slf4j.Logger
-import akka.actor.{Actor, ActorSystem, Props}
-import backtype.storm.Config
-import backtype.storm.generated.{ClusterSummary, StormTopology, SupervisorSummary, TopologySummary}
-import com.typesafe.config.ConfigValueFactory
-import io.gearpump.cluster.UserConfig
-import io.gearpump.cluster.client.ClientContext
-import io.gearpump.cluster.main.{ArgumentsParser, CLIOption}
-import io.gearpump.experiments.storm.Commands.{GetClusterInfo, _}
-import io.gearpump.experiments.storm.topology.GearpumpStormTopology
-import io.gearpump.experiments.storm.util.{GraphBuilder, StormConstants}
-import io.gearpump.streaming.StreamApplication
-import io.gearpump.util.Constants._
-import io.gearpump.util.{AkkaApp, Constants, LogUtil, Util}
+object StormRunner {
+ private val LOG: Logger = LogUtil.getLogger(getClass)
-import scala.collection.JavaConverters._
+ private val commands = Map("nimbus" -> GearpumpNimbus, "app" -> GearpumpStormClient)
-object StormRunner extends AkkaApp with ArgumentsParser {
- override val options: Array[(String, CLIOption[Any])] = Array(
- "jar" -> CLIOption[String]("<storm jar>", required = true),
- "config" -> CLIOption[String]("<storm config path>", required = false),
- "verbose" -> CLIOption("<print verbose log on console>", required = false, defaultValue = Some(false)))
-
- override val remainArgs = Array("topology_name")
-
- override def main(inputAkkaConf: Config, args: Array[String]): Unit = {
-
- val akkaConf = updateConfig(inputAkkaConf)
- val config = parse(args)
-
- val verbose = config.getBoolean("verbose")
- if (verbose) {
- LogUtil.verboseLogToConsole
- }
-
- val jar = config.getString("jar")
- val stormConfig = config.getString("config")
- val topology = config.remainArgs(0)
- val stormArgs = config.remainArgs.drop(1)
-
- val system = ActorSystem("storm", akkaConf)
- val clientContext = new ClientContext(akkaConf, system, null)
-
- val gearpumpNimbus = system.actorOf(Props(new Handler(clientContext, jar, stormConfig)))
- val thriftServer = GearpumpThriftServer(gearpumpNimbus)
- thriftServer.start()
-
- val stormOptions = Array("-Dstorm.options=" +
- s"${Config.NIMBUS_HOST}=127.0.0.1,${Config.NIMBUS_THRIFT_PORT}=${GearpumpThriftServer.THRIFT_PORT}",
- "-Dstorm.jar=" + jar,
- s"-D${PREFER_IPV4}=true"
- )
-
- val classPath = Array(System.getProperty("java.class.path"), jar)
- val process = Util.startProcess(stormOptions, classPath, topology, stormArgs)
-
- // wait till the process exit
- val exit = process.exitValue()
-
- thriftServer.close()
- clientContext.close()
- system.shutdown()
- system.awaitTermination()
-
- if (exit != 0) {
- throw new Exception(s"failed to submit jar, exit code $exit, error summary: ${process.logger.error}")
- }
+ private def usage: Unit = {
+ println(commands)
+ val keys = commands.keys.toList.sorted
+ Console.err.println("Usage: " + "<" + keys.mkString("|") + ">")
}
- import Constants._
- private def updateConfig(config: Config): Config = {
- val storm = s"<${GEARPUMP_HOME}>/lib/storm/*"
- val appClassPath = s"$storm${File.pathSeparator}" + config.getString(GEARPUMP_APPMASTER_EXTRA_CLASSPATH)
- val executorClassPath = s"$storm${File.pathSeparator}" + config.getString(Constants.GEARPUMP_EXECUTOR_EXTRA_CLASSPATH)
-
- val updated = config.withValue(GEARPUMP_APPMASTER_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(appClassPath))
- .withValue(GEARPUMP_EXECUTOR_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(executorClassPath))
-
- if (config.hasPath(StormConstants.STORM_SERIALIZATION_FRAMEWORK)) {
- val serializerConfig = ConfigValueFactory.fromAnyRef(config.getString(StormConstants.STORM_SERIALIZATION_FRAMEWORK))
- updated.withValue(GEARPUMP_SERIALIZER_POOL, serializerConfig)
+ private def executeCommand(command : String, commandArgs : Array[String]): Unit = {
+ if (!commands.contains(command)) {
+ usage
} else {
- updated
+ commands(command).main(commandArgs)
}
}
-
- class Handler(clientContext: ClientContext, jar: String, fileConfig: String) extends Actor {
- private var applications = Map.empty[String, Int]
- private var topologies = Map.empty[String, StormTopology]
- private val LOG = LogUtil.getLogger(classOf[Handler])
-
- implicit val system = context.system
-
- def receive: Receive = {
- case Kill(name, option) =>
- topologies -= name
- clientContext.shutdown(applications.getOrElse(name, throw new RuntimeException(s"topology $name not found")))
- val appId = applications(name)
- applications -= name
- LOG.info(s"Killed topology $name")
- sender ! AppKilled(name, appId)
- case Submit(name, uploadedJarLocation, appConfig, topology, option) =>
- topologies += name -> topology
-
- val gearpumpStormTopology = GearpumpStormTopology(topology, appConfig, fileConfig)
- val stormConfig = gearpumpStormTopology.getStormConfig
- val processorGraph = GraphBuilder.build(gearpumpStormTopology)
- val config = UserConfig.empty
- .withValue[StormTopology](StormConstants.STORM_TOPOLOGY, topology)
- .withValue[JMap[AnyRef, AnyRef]](StormConstants.STORM_CONFIG, stormConfig)
- val app = StreamApplication(name, processorGraph, config)
- val appId = clientContext.submit(app, jar)
- applications += name -> appId
- LOG.info(s"Storm Application $appId submitted")
- sender ! AppSubmitted(name, appId)
- case GetClusterInfo =>
- val topologySummaryList = topologies.map { case (name, _) =>
- new TopologySummary(name, name, 0, 0, 0, 0, "")
- }.toSeq
- sender ! new ClusterSummary(List[SupervisorSummary]().asJava, 0, topologySummaryList.asJava)
- case GetTopology(id) =>
- sender ! topologies(id)
+ def main(args: Array[String]) = {
+ if (args.length == 0) {
+ usage
+ } else {
+ val command = args(0)
+ val commandArgs = args.drop(1)
+ executeCommand(command, commandArgs)
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
new file mode 100644
index 0000000..4204580
--- /dev/null
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.experiments.storm.main
+
+import java.io.{File, FileOutputStream, FileWriter}
+import java.nio.ByteBuffer
+import java.nio.channels.{Channels, WritableByteChannel}
+import java.util.{Map => JMap, UUID}
+
+import akka.actor.ActorSystem
+import backtype.storm.Config
+import backtype.storm.generated._
+import backtype.storm.security.auth.{ThriftConnectionType, ThriftServer}
+import backtype.storm.utils.TimeCacheMap.ExpiredCallback
+import backtype.storm.utils.{TimeCacheMap, Utils}
+import com.typesafe.config.ConfigValueFactory
+import io.gearpump.cluster.{MasterToAppMaster, UserConfig}
+import io.gearpump.cluster.client.ClientContext
+import io.gearpump.cluster.main.{ArgumentsParser, CLIOption}
+import io.gearpump.experiments.storm.topology.GearpumpStormTopology
+import io.gearpump.experiments.storm.util.{GraphBuilder, StormConstants, StormUtil}
+import io.gearpump.streaming.StreamApplication
+import io.gearpump.util.{FileUtils, AkkaApp, Constants, LogUtil}
+import org.apache.storm.shade.org.json.simple.JSONValue
+import org.apache.storm.shade.org.yaml.snakeyaml.Yaml
+import org.apache.storm.shade.org.yaml.snakeyaml.constructor.SafeConstructor
+import org.slf4j.Logger
+
+import scala.collection.JavaConverters._
+
+object GearpumpNimbus extends AkkaApp with ArgumentsParser {
+ private val THRIFT_PORT = StormUtil.getThriftPort
+ private val OUTPUT = "output"
+
+ override val options: Array[(String, CLIOption[Any])] = Array(
+ OUTPUT -> CLIOption[String]("<output path for configuration file>", required = false, defaultValue = Some("app.yaml"))
+ )
+
+ override def main(inputAkkaConf: Config, args: Array[String]): Unit = {
+ val parsed = parse(args)
+ val output = parsed.getString(OUTPUT)
+ val akkaConf = updateClientConfig(inputAkkaConf)
+ val system = ActorSystem("storm", akkaConf)
+ val clientContext = new ClientContext(akkaConf, system, null)
+ val stormConf = Utils.readStormConfig().asInstanceOf[JMap[AnyRef, AnyRef]]
+ val thriftConf: JMap[String, String] = Map(Config.NIMBUS_THRIFT_PORT -> s"$THRIFT_PORT").asJava
+ updateOutputStormConfig(thriftConf, output)
+ stormConf.putAll(thriftConf)
+ val thriftServer = createServer(clientContext, stormConf)
+ thriftServer.serve()
+ system.awaitTermination()
+ }
+
+ private def createServer(clientContext: ClientContext, stormConf: JMap[AnyRef, AnyRef]): ThriftServer = {
+ val processor = new Nimbus.Processor[GearpumpNimbus](new GearpumpNimbus(clientContext, stormConf))
+ val connectionType = ThriftConnectionType.NIMBUS
+ new ThriftServer(stormConf, processor, connectionType)
+ }
+
+ private def updateOutputStormConfig(conf: JMap[String, String], output: String): Unit = {
+ // read existing config
+ val outputConfig = Utils.findAndReadConfigFile(output, false).asInstanceOf[JMap[AnyRef, AnyRef]]
+ outputConfig.putAll(conf)
+ val yaml = new Yaml(new SafeConstructor)
+ val writer = new FileWriter(new File(output))
+ yaml.dump(outputConfig, writer)
+ }
+
+ import Constants._
+ private def updateClientConfig(config: Config): Config = {
+ val storm = s"<${GEARPUMP_HOME}>/lib/storm/*"
+ val appClassPath = s"$storm${File.pathSeparator}" + config.getString(GEARPUMP_APPMASTER_EXTRA_CLASSPATH)
+ val executorClassPath = s"$storm${File.pathSeparator}" + config.getString(Constants.GEARPUMP_EXECUTOR_EXTRA_CLASSPATH)
+
+ val updated = config.withValue(GEARPUMP_APPMASTER_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(appClassPath))
+ .withValue(GEARPUMP_EXECUTOR_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(executorClassPath))
+
+ if (config.hasPath(StormConstants.STORM_SERIALIZATION_FRAMEWORK)) {
+ val serializerConfig = ConfigValueFactory.fromAnyRef(config.getString(StormConstants.STORM_SERIALIZATION_FRAMEWORK))
+ updated.withValue(GEARPUMP_SERIALIZER_POOL, serializerConfig)
+ } else {
+ updated
+ }
+ }
+
+}
+
+class GearpumpNimbus(clientContext: ClientContext, stormConf: JMap[AnyRef, AnyRef]) extends Nimbus.Iface {
+ private val LOG: Logger = LogUtil.getLogger(classOf[GearpumpNimbus])
+ private var applications = Map.empty[String, Int]
+ private var topologies = Map.empty[String, (StormTopology, JMap[AnyRef, AnyRef])]
+ private val expireSeconds = StormUtil.getInt(stormConf, Config.NIMBUS_FILE_COPY_EXPIRATION_SECS).get
+ private val expiredCallback = new ExpiredCallback[String, WritableByteChannel] {
+ override def expire(k: String, v: WritableByteChannel): Unit = {
+ v.close()
+ }
+ }
+ private val fileCacheMap = new TimeCacheMap[String, WritableByteChannel](expireSeconds, expiredCallback)
+
+ override def submitTopology(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology): Unit = {
+ submitTopologyWithOpts(name, uploadedJarLocation, jsonConf, topology, new SubmitOptions(TopologyInitialStatus.ACTIVE))
+ }
+
+ override def submitTopologyWithOpts(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology, options: SubmitOptions): Unit = {
+
+ implicit val system = clientContext.system
+ val gearpumpStormTopology = GearpumpStormTopology(name, topology, jsonConf)
+ val stormConfig = gearpumpStormTopology.getStormConfig
+ val processorGraph = GraphBuilder.build(gearpumpStormTopology)
+ val config = UserConfig.empty
+ .withValue[StormTopology](StormConstants.STORM_TOPOLOGY, topology)
+ .withValue[JMap[AnyRef, AnyRef]](StormConstants.STORM_CONFIG, stormConfig)
+ val app = StreamApplication(name, processorGraph, config)
+ LOG.info(s"jar file uploaded to $uploadedJarLocation")
+ val appId = clientContext.submit(app, uploadedJarLocation)
+ applications += name -> appId
+ topologies += name -> (topology, stormConfig)
+ LOG.info(s"Storm Application $appId submitted")
+ }
+
+ override def killTopologyWithOpts(name: String, options: KillOptions): Unit = {
+ clientContext.shutdown(applications.getOrElse(name, throw new RuntimeException(s"topology $name not found")))
+ applications -= name
+ topologies -= name
+ LOG.info(s"Killed topology $name")
+ }
+
+ override def getNimbusConf: String = {
+ JSONValue.toJSONString(stormConf)
+ }
+
+ override def getTopology(name: String): StormTopology = {
+ updateApps
+ topologies.getOrElse(name,
+ throw new RuntimeException(s"topology $name not found"))._1
+ }
+
+ override def getTopologyConf(name: String): String = {
+ updateApps
+ JSONValue.toJSONString(topologies.getOrElse(name,
+ throw new RuntimeException(s"topology $name not found"))._2)
+ }
+
+ override def getUserTopology(id: String): StormTopology = getTopology(id)
+
+ override def beginFileUpload(): String = {
+ val location = s"stormjar-${UUID.randomUUID()}.jar"
+ val channel = Channels.newChannel(new FileOutputStream(location))
+ fileCacheMap.put(location, channel)
+ LOG.info(s"Uploading file from client to $location")
+ location
+ }
+
+ override def uploadChunk(location: String, chunk: ByteBuffer): Unit = {
+ if (!fileCacheMap.containsKey(location)) {
+ throw new RuntimeException(s"File for $location does not exist (or timed out)")
+ } else {
+ val channel = fileCacheMap.get(location)
+ channel.write(chunk)
+ fileCacheMap.put(location, channel)
+ }
+ }
+
+ override def finishFileUpload(location: String): Unit = {
+ if (!fileCacheMap.containsKey(location)) {
+ throw new RuntimeException(s"File for $location does not exist (or timed out)")
+ } else {
+ val channel = fileCacheMap.get(location)
+ channel.close()
+ fileCacheMap.remove(location)
+ new File(location).delete()
+ }
+ }
+
+ override def getClusterInfo: ClusterSummary = {
+ updateApps
+ val topologySummaryList = topologies.map { case (name, _) =>
+ new TopologySummary(name, name, 0, 0, 0, 0, "")
+ }.toSeq
+ new ClusterSummary(List[SupervisorSummary]().asJava, 0, topologySummaryList.asJava)
+ }
+
+ override def beginFileDownload(file: String): String = {
+ throw new UnsupportedOperationException
+ }
+
+ override def uploadNewCredentials(s: String, credentials: Credentials): Unit = {
+ throw new UnsupportedOperationException
+ }
+ override def activate(name: String): Unit = {
+ throw new UnsupportedOperationException
+ }
+
+ override def rebalance(name: String, options: RebalanceOptions): Unit = {
+ throw new UnsupportedOperationException
+ }
+
+ override def deactivate(name: String): Unit = {
+ throw new UnsupportedOperationException
+ }
+
+ override def getTopologyInfo(name: String): TopologyInfo = {
+ throw new UnsupportedOperationException
+ }
+
+ override def getTopologyInfoWithOpts(s: String, getInfoOptions: GetInfoOptions): TopologyInfo = {
+ throw new UnsupportedOperationException
+ }
+
+ override def killTopology(name: String): Unit = killTopologyWithOpts(name, new KillOptions())
+
+ override def downloadChunk(name: String): ByteBuffer = {
+ throw new UnsupportedOperationException
+ }
+
+ private def updateApps: Unit = {
+ clientContext.listApps.appMasters.foreach { app =>
+ val name = app.appName
+ if (applications.contains(app.appName)) {
+ if (app.status != MasterToAppMaster.AppMasterActive) {
+ applications -= name
+ topologies -= name
+ }
+ }
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
new file mode 100644
index 0000000..70efbf3
--- /dev/null
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.experiments.storm.main
+
+import backtype.storm.Config
+import backtype.storm.utils.Utils
+import io.gearpump.cluster.main.{ArgumentsParser, CLIOption}
+import io.gearpump.util.Constants._
+import io.gearpump.util.{AkkaApp, LogUtil, Util}
+
+object GearpumpStormClient extends AkkaApp with ArgumentsParser {
+
+ override val options: Array[(String, CLIOption[Any])] = Array(
+ "jar" -> CLIOption[String]("<storm jar>", required = true),
+ "config" -> CLIOption[Int]("<storm config file>", required = true),
+ "verbose" -> CLIOption("<print verbose log on console>", required = false, defaultValue = Some(false)))
+
+ override def main(inputAkkaConf: Config, args: Array[String]): Unit = {
+ val config = parse(args)
+
+ val verbose = config.getBoolean("verbose")
+ if (verbose) {
+ LogUtil.verboseLogToConsole
+ }
+
+ val jar = config.getString("jar")
+ val stormConfig = config.getString("config")
+ val topology = config.remainArgs(0)
+ val stormArgs = config.remainArgs.drop(1)
+ val stormOptions = Array(
+ s"-Dstorm.options=${getThriftOptions(stormConfig)}",
+ s"-Dstorm.jar=$jar",
+ s"-Dstorm.config.file=$stormConfig",
+ s"-D${PREFER_IPV4}=true"
+ )
+
+ val classPath = Array(s"${System.getProperty(GEARPUMP_HOME)}/lib/storm/*", jar)
+ val process = Util.startProcess(stormOptions, classPath, topology, stormArgs)
+
+ // wait till the process exit
+ val exit = process.exitValue()
+
+ if (exit != 0) {
+ throw new Exception(s"failed to submit jar, exit code $exit, error summary: ${process.logger.error}")
+ }
+ }
+
+ private def getThriftOptions(stormConfig: String): String = {
+ val config = Utils.findAndReadConfigFile(stormConfig, true)
+ val thriftPort = config.get(Config.NIMBUS_THRIFT_PORT)
+ s"${Config.NIMBUS_HOST}=127.0.0.1,${Config.NIMBUS_THRIFT_PORT}=$thriftPort"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
index 996498b..b88a200 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
@@ -18,7 +18,6 @@
package io.gearpump.experiments.storm.topology
-import java.io._
import java.lang.{Iterable => JIterable}
import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap}
@@ -35,8 +34,6 @@ import io.gearpump.experiments.storm.util.StormUtil._
import io.gearpump.streaming.Processor
import io.gearpump.streaming.task.Task
import io.gearpump.util.LogUtil
-import org.apache.storm.shade.org.yaml.snakeyaml.Yaml
-import org.apache.storm.shade.org.yaml.snakeyaml.constructor.SafeConstructor
import org.slf4j.Logger
import scala.collection.JavaConversions._
@@ -45,43 +42,18 @@ object GearpumpStormTopology {
private val LOG: Logger = LogUtil.getLogger(classOf[GearpumpStormTopology])
def apply(
+ name: String,
topology: StormTopology,
- appConfigInJson: String,
- fileConfig: String)(implicit system: ActorSystem): GearpumpStormTopology = {
+ appConfigInJson: String)(implicit system: ActorSystem): GearpumpStormTopology = {
new GearpumpStormTopology(
+ name,
topology,
Utils.readStormConfig().asInstanceOf[JMap[AnyRef, AnyRef]],
- parseJsonStringToMap(appConfigInJson),
- readStormConfig(fileConfig)
+ parseJsonStringToMap(appConfigInJson)
)
}
- /**
- * @param configFile user provided local config file
- * @return a config Map loaded from config file
- */
- private def readStormConfig(configFile: String): JMap[AnyRef, AnyRef] = {
- var ret: JMap[AnyRef, AnyRef] = new JHashMap[AnyRef, AnyRef]
- try {
- val yaml = new Yaml(new SafeConstructor)
- val input: InputStream = new FileInputStream(configFile)
- try {
- ret = yaml.load(new InputStreamReader(input)).asInstanceOf[JMap[AnyRef, AnyRef]]
- } catch {
- case e: IOException =>
- LOG.warn(s"failed to load config file $configFile")
- } finally {
- input.close()
- }
- } catch {
- case e: FileNotFoundException =>
- LOG.warn(s"failed to find config file $configFile")
- case t: Throwable =>
- LOG.error(t.getMessage)
- }
- ret
- }
}
/**
@@ -91,21 +63,20 @@ object GearpumpStormTopology {
* 3. provides interface for Gearpump applications to use Storm topology
*
* an implicit ActorSystem is required to create Gearpump processors
+ * @param name topology name
* @param topology Storm topology
- * @param sysConfig configs from "defaults.yaml" and "storm.yaml"
+ * @param sysConfig configs from "defaults.yaml" and custom config file
* @param appConfig config submitted from user application
- * @param fileConfig custom file config set by user in command line
*/
private[storm] class GearpumpStormTopology(
+ name: String,
topology: StormTopology,
sysConfig: JMap[AnyRef, AnyRef],
- appConfig: JMap[AnyRef, AnyRef],
- fileConfig: JMap[AnyRef, AnyRef])(implicit system: ActorSystem) {
- import io.gearpump.experiments.storm.topology.GearpumpStormTopology._
+ appConfig: JMap[AnyRef, AnyRef])(implicit system: ActorSystem) {
private val spouts = topology.get_spouts()
private val bolts = topology.get_bolts()
- private val stormConfig = mergeConfigs(sysConfig, appConfig, fileConfig, getComponentConfigs(spouts, bolts))
+ private val stormConfig = mergeConfigs(sysConfig, appConfig, getComponentConfigs(spouts, bolts))
private val spoutProcessors = spouts.map { case (id, spout) =>
id -> spoutToProcessor(id, spout, stormConfig.toMap) }.toMap
private val boltProcessors = bolts.map { case (id, bolt) =>
@@ -114,7 +85,7 @@ private[storm] class GearpumpStormTopology(
/**
* @return merged Storm config with priority
- * defaults.yaml < storm.yaml < application config < component config < custom file config
+ * defaults.yaml < custom file config < application config < component config
*/
def getStormConfig: JMap[AnyRef, AnyRef] = stormConfig
@@ -139,13 +110,12 @@ private[storm] class GearpumpStormTopology(
private def mergeConfigs(
sysConfig: JMap[AnyRef, AnyRef],
appConfig: JMap[AnyRef, AnyRef],
- fileConfig: JMap[AnyRef, AnyRef],
componentConfigs: Iterable[JMap[AnyRef, AnyRef]]): JMap[AnyRef, AnyRef] = {
val allConfig = new JHashMap[AnyRef, AnyRef]
allConfig.putAll(sysConfig)
allConfig.putAll(appConfig)
allConfig.putAll(getMergedComponentConfig(componentConfigs, allConfig.toMap))
- allConfig.putAll(fileConfig)
+ allConfig.put(Config.TOPOLOGY_NAME, name)
allConfig
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala
deleted file mode 100644
index 20b63b3..0000000
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.experiments.storm
-
-import akka.actor.ActorSystem
-import akka.testkit.TestProbe
-import backtype.storm.generated.StormTopology
-import io.gearpump.experiments.storm.util.TopologyUtil
-import io.gearpump.cluster.TestUtil
-import Commands.{GetTopology, Kill, Submit}
-import GearpumpThriftServer.GearpumpNimbus
-import org.scalatest.mock.MockitoSugar
-import org.scalatest.{Matchers, WordSpec}
-
-import scala.concurrent.Future
-
-class GearpumpNimbusSpec extends WordSpec with Matchers with MockitoSugar {
-
- "GearpumpNimbus" should {
- "submit and kill topology through ClientContext" in {
-
- implicit val system = ActorSystem("storm-test", TestUtil.DEFAULT_CONFIG)
- implicit val dispatcher = system.dispatcher
-
- val handler = TestProbe()
- val gearpumpNimbus = new GearpumpNimbus(handler.ref)
-
- val appId = 0
- val name = "test"
- val uploadedJarLocation = "local"
- val jsonConf = "storm_json_conf"
- val topology = TopologyUtil.getTestTopology
-
- Future(gearpumpNimbus.submitTopology(name, uploadedJarLocation, jsonConf, topology))
- handler.expectMsgType[Submit]
-
- Future(gearpumpNimbus.getTopology(name))
- handler.expectMsgType[GetTopology]
- handler.reply(new StormTopology)
-
- Future(gearpumpNimbus.getUserTopology(name))
- handler.expectMsgType[GetTopology]
- handler.reply(new StormTopology)
-
- Future(gearpumpNimbus.killTopology(name))
- handler.expectMsgType[Kill]
-
- system.shutdown()
- system.awaitTermination()
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala
deleted file mode 100644
index b1736f0..0000000
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.experiments.storm
-
-import backtype.storm.security.auth.ThriftServer
-import org.mockito.Mockito._
-import org.scalatest.WordSpec
-import org.scalatest.mock.MockitoSugar
-
-class GearpumpThriftServerSpec extends WordSpec with MockitoSugar {
-
- "GearpumpThriftServer" should {
- "run ThriftServer.serve" in {
- val tServer = mock[ThriftServer]
- val thriftServer = new GearpumpThriftServer(tServer)
-
- thriftServer.run()
-
- verify(tServer).serve()
- }
-
- "stop ThriftServer" in {
- val tServer = mock[ThriftServer]
- val thriftServer = new GearpumpThriftServer(tServer)
-
- thriftServer.close()
-
- verify(tServer).stop()
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala
deleted file mode 100644
index 95cf279..0000000
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package io.gearpump.experiments.storm
-
-import akka.actor.{ActorSystem, Props}
-import akka.testkit.TestProbe
-import io.gearpump.cluster.TestUtil
-import io.gearpump.cluster.client.ClientContext
-import io.gearpump.experiments.storm.Commands.{AppKilled, AppSubmitted, Kill, Submit}
-import io.gearpump.experiments.storm.StormRunner.Handler
-import io.gearpump.experiments.storm.util.TopologyUtil
-import org.mockito.Matchers._
-import org.mockito.Mockito
-import org.scalatest.{FlatSpec, Matchers}
-
-class StormRunnerSpec extends FlatSpec with Matchers {
- it should "handle submit/kill correctly" in {
- val conf = TestUtil.DEFAULT_CONFIG
- implicit val system = ActorSystem("storm-test", conf)
-
- val uploadedJarLocation = "local"
- val jsonConf = "storm_json_conf"
- val topology = TopologyUtil.getTestTopology
-
- implicit val dispatcher = system.dispatcher
- val clientContext = Mockito.mock(classOf[ClientContext])
- Mockito.when(clientContext.submit(any(), any())).thenReturn(0)
- val handler = system.actorOf(Props(new Handler(clientContext, "jar", "user_config")))
- val client = TestProbe()
-
- client.send(handler, Submit("app", uploadedJarLocation, jsonConf, topology, null))
- client.expectMsg(AppSubmitted("app", 0))
-
-
- client.send(handler, Kill("app", null))
- client.expectMsg(AppKilled("app", 0))
-
- system.shutdown()
- system.awaitTermination()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
index d1c330b..8f10886 100644
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
+++ b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
@@ -20,6 +20,7 @@ package io.gearpump.experiments.storm.topology
import java.util.{HashMap => JHashMap, Map => JMap}
+import backtype.storm.Config
import io.gearpump.experiments.storm.processor.StormProcessor
import io.gearpump.experiments.storm.producer.StormProducer
import io.gearpump.experiments.storm.util.TopologyUtil
@@ -40,28 +41,27 @@ class GearpumpStormTopologySpec extends WordSpec with Matchers with MockitoSugar
val sysConfig = newJavaConfig(name, sysVal)
val appVal = "app"
val appConfig = newJavaConfig(name, appVal)
- val fileVal = "file"
- val fileConfig = newJavaConfig(name, fileVal)
implicit val system = MockUtil.system
- val topology1 = new GearpumpStormTopology(stormTopology, newEmptyConfig, newEmptyConfig, newEmptyConfig)
- topology1.getStormConfig shouldBe empty
+ val topology1 = new GearpumpStormTopology("topology1", stormTopology, newEmptyConfig, newEmptyConfig)
+ topology1.getStormConfig(Config.TOPOLOGY_NAME) shouldBe "topology1"
+ topology1.getStormConfig should not contain name
- val topology2 = new GearpumpStormTopology(stormTopology, sysConfig, newEmptyConfig, newEmptyConfig)
+ val topology2 = new GearpumpStormTopology("topology2", stormTopology, sysConfig, newEmptyConfig)
+ topology2.getStormConfig(Config.TOPOLOGY_NAME) shouldBe "topology2"
topology2.getStormConfig.get(name) shouldBe sysVal
- val topology3 = new GearpumpStormTopology(stormTopology, sysConfig, appConfig, newEmptyConfig)
+ val topology3 = new GearpumpStormTopology("topology3", stormTopology, sysConfig, appConfig)
+ topology3.getStormConfig(Config.TOPOLOGY_NAME) shouldBe "topology3"
topology3.getStormConfig.get(name) shouldBe appVal
- val topology4 = new GearpumpStormTopology(stormTopology, sysConfig, appConfig, fileConfig)
- topology4.getStormConfig.get(name) shouldBe fileVal
}
"create Gearpump processors from Storm topology" in {
val stormTopology = TopologyUtil.getTestTopology
implicit val system = MockUtil.system
val gearpumpStormTopology =
- GearpumpStormTopology(stormTopology, null, "")
+ GearpumpStormTopology("app", stormTopology, null)
val processors = gearpumpStormTopology.getProcessors
stormTopology.get_spouts().foreach { case (spoutId, _) =>
val processor = processors(spoutId)
@@ -80,7 +80,7 @@ class GearpumpStormTopologySpec extends WordSpec with Matchers with MockitoSugar
implicit val system = MockUtil.system
val sysConfig = new JHashMap[AnyRef, AnyRef]
val gearpumpStormTopology =
- GearpumpStormTopology(stormTopology, null, "")
+ GearpumpStormTopology("app", stormTopology, null)
val targets0 = gearpumpStormTopology.getTargets("1")
targets0 should contain key "3"
targets0 should contain key "4"
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
index ed792f4..15acf4e 100644
--- a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
+++ b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
@@ -20,7 +20,6 @@ package io.gearpump.integrationtest.storm
import backtype.storm.utils.{Utils, DRPCClient}
-import io.gearpump.experiments.storm.GearpumpThriftServer
import io.gearpump.integrationtest.Docker
import io.gearpump.integrationtest.minicluster.BaseContainer
import org.apache.log4j.Logger
@@ -30,23 +29,28 @@ class StormClient(masterAddrs: Seq[(String, Int)]) {
private val LOG = Logger.getLogger(getClass)
private val STORM_HOST = "storm0"
- private val STORM_CMD = "/opt/start storm"
+ private val STORM_NIMBUS = "/opt/start storm nimbus"
+ private val STORM_APP = "/opt/start storm app"
private val STORM_DRPC = "storm-drpc"
private val CONFIG_FILE = "storm.yaml"
- private val NIMBUS_THRIFT_PORT = GearpumpThriftServer.THRIFT_PORT
private val DRPC_PORT = 3772
private val DRPC_INVOCATIONS_PORT = 3773
private val container = new BaseContainer(STORM_HOST, STORM_DRPC, masterAddrs,
- tunnelPorts = Set(NIMBUS_THRIFT_PORT, DRPC_PORT, DRPC_INVOCATIONS_PORT))
+ tunnelPorts = Set(DRPC_PORT, DRPC_INVOCATIONS_PORT))
def start(): Unit = {
container.createAndStart()
+ startNimbus
+ }
+
+ private def startNimbus: String = {
+ Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_NIMBUS -output $CONFIG_FILE")
}
def submitStormApp(jar: String, mainClass: String, args: String = ""): Int = {
try {
- Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_CMD -config $CONFIG_FILE " +
+ Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_APP -config $CONFIG_FILE " +
s"-jar $jar $mainClass $args").split("\n")
.filter(_.contains("The application id is ")).head.split(" ").last.toInt
} catch {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/project/Pack.scala
----------------------------------------------------------------------
diff --git a/project/Pack.scala b/project/Pack.scala
index b242b36..a3852c4 100644
--- a/project/Pack.scala
+++ b/project/Pack.scala
@@ -59,7 +59,7 @@ object Pack extends sbt.Build {
"worker" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-DlogFilename=worker", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"),
"services" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"),
"yarnclient" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"),
- "storm" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost")
+ "storm" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}")
),
packLibDir := Map(
"lib" -> new ProjectsToPack(core.id, streaming.id),
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/502dbae9/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
----------------------------------------------------------------------
diff --git a/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala b/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
index e2b2c47..aa90726 100644
--- a/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
+++ b/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
@@ -221,7 +221,7 @@ object MasterService {
*/
def submitStormApp(jar: Option[File], stormConf: Option[File], args: String, systemConfig: Config): Boolean = {
submitAndDeleteTempFiles(
- "io.gearpump.experiments.storm.StormRunner",
+ "io.gearpump.experiments.storm.main.GearpumpStormClient",
argsArray = spaceSeparatedArgumentsToArray(args),
fileMap = Map("jar" -> jar, "config" -> stormConf).filter(_._2.isDefined).mapValues(_.get),
classPath = getStormApplicationClassPath,
[40/50] incubator-gearpump git commit: fix #1975,
fix storm integration test
Posted by ap...@apache.org.
fix #1975, fix storm integration test
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/920ad262
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/920ad262
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/920ad262
Branch: refs/heads/master
Commit: 920ad262f9f9e88611d520df9b40d1ff8363f81c
Parents: 42806a9
Author: manuzhang <ow...@gmail.com>
Authored: Tue Feb 2 16:58:00 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Wed Feb 24 16:37:01 2016 +0800
----------------------------------------------------------------------
.../experiments/storm/main/GearpumpNimbus.scala | 27 ++++++----
.../storm/main/GearpumpStormClient.scala | 2 +-
.../checklist/StormCompatibilitySpec.scala | 51 +++++++++++--------
.../suites/StandaloneModeSuite.scala | 4 +-
.../integrationtest/storm/StormClient.scala | 52 +++++++++-----------
project/Build.scala | 3 +-
.../streaming/appmaster/ClockService.scala | 2 -
7 files changed, 74 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/920ad262/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
index 6d5b0aa..e9973e5 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
@@ -39,14 +39,15 @@ import io.gearpump.streaming.StreamApplication
import io.gearpump.util.{AkkaApp, Constants, LogUtil}
import org.apache.storm.shade.org.json.simple.JSONValue
import org.apache.storm.shade.org.yaml.snakeyaml.Yaml
-import org.apache.storm.shade.org.yaml.snakeyaml.constructor.SafeConstructor
import org.slf4j.Logger
import scala.collection.JavaConverters._
+import scala.concurrent.Future
object GearpumpNimbus extends AkkaApp with ArgumentsParser {
private val THRIFT_PORT = StormUtil.getThriftPort
private val OUTPUT = "output"
+ private val LOG: Logger = LogUtil.getLogger(classOf[GearpumpNimbus])
override val options: Array[(String, CLIOption[Any])] = Array(
OUTPUT -> CLIOption[String]("<output path for configuration file>", required = false, defaultValue = Some("app.yaml"))
@@ -57,15 +58,20 @@ object GearpumpNimbus extends AkkaApp with ArgumentsParser {
val output = parsed.getString(OUTPUT)
val akkaConf = updateClientConfig(inputAkkaConf)
val system = ActorSystem("storm", akkaConf)
+
val clientContext = new ClientContext(akkaConf, system, null)
val stormConf = Utils.readStormConfig().asInstanceOf[JMap[AnyRef, AnyRef]]
- val thriftConf: JMap[String, String] = Map(
- Config.NIMBUS_HOST -> ("" + akkaConf.getString(Constants.GEARPUMP_HOSTNAME)),
- Config.NIMBUS_THRIFT_PORT -> s"$THRIFT_PORT").asJava
+ val thriftConf: JMap[AnyRef, AnyRef] = Map(
+ Config.NIMBUS_HOST -> akkaConf.getString(Constants.GEARPUMP_HOSTNAME),
+ Config.NIMBUS_THRIFT_PORT -> s"$THRIFT_PORT").asJava.asInstanceOf[JMap[AnyRef, AnyRef]]
updateStormConfig(thriftConf, output)
stormConf.putAll(thriftConf)
- val thriftServer = createServer(clientContext, stormConf)
- thriftServer.serve()
+
+ import scala.concurrent.ExecutionContext.Implicits.global
+ Future {
+ val thriftServer = createServer(clientContext, stormConf)
+ thriftServer.serve()
+ }
system.awaitTermination()
}
@@ -75,12 +81,12 @@ object GearpumpNimbus extends AkkaApp with ArgumentsParser {
new ThriftServer(stormConf, processor, connectionType)
}
- private def updateStormConfig(thriftConfig: JMap[String, String], output: String): Unit = {
+ private def updateStormConfig(thriftConfig: JMap[AnyRef, AnyRef], output: String): Unit = {
val updatedConfig: JMap[AnyRef, AnyRef] = new JHashMap[AnyRef, AnyRef]
val outputConfig = Utils.findAndReadConfigFile(output, false).asInstanceOf[JMap[AnyRef, AnyRef]]
updatedConfig.putAll(outputConfig)
updatedConfig.putAll(thriftConfig)
- val yaml = new Yaml(new SafeConstructor)
+ val yaml = new Yaml
val serialized = yaml.dumpAsMap(updatedConfig)
val writer = new FileWriter(new File(output))
try {
@@ -112,7 +118,8 @@ object GearpumpNimbus extends AkkaApp with ArgumentsParser {
}
class GearpumpNimbus(clientContext: ClientContext, stormConf: JMap[AnyRef, AnyRef]) extends Nimbus.Iface {
- private val LOG: Logger = LogUtil.getLogger(classOf[GearpumpNimbus])
+ import io.gearpump.experiments.storm.main.GearpumpNimbus._
+
private var applications = Map.empty[String, Int]
private var topologies = Map.empty[String, TopologyData]
private val expireSeconds = StormUtil.getInt(stormConf, Config.NIMBUS_FILE_COPY_EXPIRATION_SECS).get
@@ -128,7 +135,7 @@ class GearpumpNimbus(clientContext: ClientContext, stormConf: JMap[AnyRef, AnyRe
}
override def submitTopologyWithOpts(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology, options: SubmitOptions): Unit = {
-
+ LOG.info(s"Submitted topology $name")
implicit val system = clientContext.system
val gearpumpStormTopology = GearpumpStormTopology(name, topology, jsonConf)
val stormConfig = gearpumpStormTopology.getStormConfig
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/920ad262/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
index 643e926..d4c0e8a 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
@@ -46,7 +46,7 @@ object GearpumpStormClient extends AkkaApp with ArgumentsParser {
val stormOptions = Array(
s"-Dstorm.options=${getThriftOptions(stormConfig)}",
s"-Dstorm.jar=$jar",
- s"-Dstorm.config.file=$stormConfig",
+ s"-Dstorm.conf.file=$stormConfig",
s"-D${PREFER_IPV4}=true"
)
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/920ad262/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/StormCompatibilitySpec.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/StormCompatibilitySpec.scala b/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/StormCompatibilitySpec.scala
index 30c6d57..3baabf6 100644
--- a/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/StormCompatibilitySpec.scala
+++ b/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/StormCompatibilitySpec.scala
@@ -26,7 +26,7 @@ import io.gearpump.integrationtest.{TestSpecBase, Util}
*/
class StormCompatibilitySpec extends TestSpecBase {
- private lazy val stormClient = new StormClient(cluster.getMastersAddresses)
+ private lazy val stormClient = new StormClient(cluster.getMastersAddresses, restClient)
override def beforeAll(): Unit = {
super.beforeAll()
@@ -74,27 +74,33 @@ class StormCompatibilitySpec extends TestSpecBase {
"Storm over Gearpump" should withStorm {
stormVersion =>
- s"support basic topologies ($stormVersion)" in {
+ s"support basic topologies ($stormVersion)" in {
val stormJar = getStormJar(stormVersion)
val topologyName = getTopologyName("exclamation", stormVersion)
+
// exercise
- val appId = stormClient.submitStormApp(stormJar,
- mainClass = "storm.starter.ExclamationTopology", args = topologyName)
+ val appId = stormClient.submitStormApp(
+ jar = stormJar,
+ mainClass = "storm.starter.ExclamationTopology",
+ args = topologyName,
+ appName = topologyName)
// verify
- val actual = expectAppIsRunning(appId, topologyName)
- Util.retryUntil(restClient.queryStreamingAppDetail(actual.appId).clock > 0)
+ Util.retryUntil(restClient.queryStreamingAppDetail(appId).clock > 0)
}
s"support to run a python version of wordcount ($stormVersion)" in {
val stormJar = getStormJar(stormVersion)
val topologyName = getTopologyName("wordcount", stormVersion)
+
// exercise
- val appId = stormClient.submitStormApp(stormJar,
- mainClass = "storm.starter.WordCountTopology", args = topologyName)
+ val appId = stormClient.submitStormApp(
+ jar = stormJar,
+ mainClass = "storm.starter.WordCountTopology",
+ args = topologyName,
+ appName = topologyName)
// verify
- expectAppIsRunning(appId, topologyName)
Util.retryUntil(restClient.queryStreamingAppDetail(appId).clock > 0)
}
@@ -104,8 +110,11 @@ class StormCompatibilitySpec extends TestSpecBase {
// input (user and follower) data are already prepared in memory
val stormJar = getStormJar(stormVersion)
val topologyName = getTopologyName("reach", stormVersion)
- stormClient.submitStormApp(stormJar,
- mainClass = "storm.starter.ReachTopology", args = topologyName)
+ stormClient.submitStormApp(
+ jar = stormJar,
+ mainClass = "storm.starter.ReachTopology",
+ args = topologyName,
+ appName = topologyName)
val drpcClient = stormClient.getDRPCClient(cluster.getNetworkGateway)
// verify
@@ -119,13 +128,16 @@ class StormCompatibilitySpec extends TestSpecBase {
s"support tick tuple ($stormVersion)" in {
val stormJar = getStormJar(stormVersion)
val topologyName = getTopologyName("slidingWindowCounts", stormVersion)
+
// exercise
- val appId = stormClient.submitStormApp(stormJar,
- mainClass = "storm.starter.RollingTopWords", args = s"$topologyName remote")
+ val appId = stormClient.submitStormApp(
+ jar = stormJar,
+ mainClass = "storm.starter.RollingTopWords",
+ args = s"$topologyName remote",
+ appName = topologyName)
// verify
- val actual = expectAppIsRunning(appId, topologyName)
- Util.retryUntil(restClient.queryStreamingAppDetail(actual.appId).clock > 0)
+ Util.retryUntil(restClient.queryStreamingAppDetail(appId).clock > 0)
}
s"support at-least-once semantics with Storm's Kafka connector ($stormVersion)" in {
@@ -142,8 +154,6 @@ class StormCompatibilitySpec extends TestSpecBase {
val brokerList = kafkaCluster.getBrokerListConnectString
val sourceTopic = "topic1"
val sinkTopic = "topic2"
- val appsCount = restClient.listApps().length
- val appId = appsCount + 1
val args = Array("-topologyName", topologyName, "-sourceTopic", sourceTopic,
"-sinkTopic", sinkTopic, "-zookeeperConnect", zookeeper, "-brokerList", brokerList,
@@ -155,11 +165,12 @@ class StormCompatibilitySpec extends TestSpecBase {
// generate number sequence (1, 2, 3, ...) to the topic
withDataProducer(sourceTopic, brokerList) { producer =>
- stormClient.submitStormApp(stormJar,
+ val appId = stormClient.submitStormApp(
+ jar = stormJar,
mainClass = stormKafkaTopology,
- args = args.mkString(" "))
+ args = args.mkString(" "),
+ appName = topologyName)
- expectAppIsRunning(appId, topologyName)
Util.retryUntil(restClient.queryStreamingAppDetail(appId).clock > 0)
// kill executor and verify at-least-once is guaranteed on application restart
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/920ad262/integrationtest/core/src/it/scala/io/gearpump/integrationtest/suites/StandaloneModeSuite.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/it/scala/io/gearpump/integrationtest/suites/StandaloneModeSuite.scala b/integrationtest/core/src/it/scala/io/gearpump/integrationtest/suites/StandaloneModeSuite.scala
index 944f542..c7a2c3e 100644
--- a/integrationtest/core/src/it/scala/io/gearpump/integrationtest/suites/StandaloneModeSuite.scala
+++ b/integrationtest/core/src/it/scala/io/gearpump/integrationtest/suites/StandaloneModeSuite.scala
@@ -32,8 +32,8 @@ class StandaloneModeSuite extends Suites(
new RestServiceSpec,
new ExampleSpec,
new DynamicDagSpec,
- new StabilitySpec/*,
- new StormCompatibilitySpec*/
+ new StabilitySpec,
+ new StormCompatibilitySpec
) with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/920ad262/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
index 15acf4e..a018280 100644
--- a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
+++ b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
@@ -18,46 +18,37 @@
package io.gearpump.integrationtest.storm
-
import backtype.storm.utils.{Utils, DRPCClient}
-import io.gearpump.integrationtest.Docker
-import io.gearpump.integrationtest.minicluster.BaseContainer
-import org.apache.log4j.Logger
-
+import io.gearpump.integrationtest.{Util, Docker}
+import io.gearpump.integrationtest.minicluster.{RestClient, BaseContainer}
-class StormClient(masterAddrs: Seq[(String, Int)]) {
+class StormClient(masterAddrs: Seq[(String, Int)], restClient: RestClient) {
- private val LOG = Logger.getLogger(getClass)
- private val STORM_HOST = "storm0"
- private val STORM_NIMBUS = "/opt/start storm nimbus"
- private val STORM_APP = "/opt/start storm app"
- private val STORM_DRPC = "storm-drpc"
- private val CONFIG_FILE = "storm.yaml"
+ private val CONFIG_FILE = "/opt/gearpump/storm.yaml"
+ private val DRPC_HOST = "storm0"
private val DRPC_PORT = 3772
private val DRPC_INVOCATIONS_PORT = 3773
+ private val STORM_DRPC = "storm-drpc"
+ private val NIMBUS_HOST = "storm1"
+ private val STORM_NIMBUS = "storm nimbus"
+ private val STORM_APP = "/opt/start storm app"
- private val container = new BaseContainer(STORM_HOST, STORM_DRPC, masterAddrs,
+ private val drpcContainer = new BaseContainer(DRPC_HOST, STORM_DRPC, masterAddrs,
tunnelPorts = Set(DRPC_PORT, DRPC_INVOCATIONS_PORT))
+ private val nimbusContainer = new BaseContainer(NIMBUS_HOST, s"$STORM_NIMBUS -output $CONFIG_FILE", masterAddrs)
def start(): Unit = {
- container.createAndStart()
- startNimbus
- }
-
- private def startNimbus: String = {
- Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_NIMBUS -output $CONFIG_FILE")
+ drpcContainer.createAndStart()
+ nimbusContainer.createAndStart()
}
- def submitStormApp(jar: String, mainClass: String, args: String = ""): Int = {
- try {
- Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_APP -config $CONFIG_FILE " +
- s"-jar $jar $mainClass $args").split("\n")
- .filter(_.contains("The application id is ")).head.split(" ").last.toInt
- } catch {
- case ex: Throwable =>
- LOG.warn(s"swallowed an exception: $ex")
- -1
- }
+ def submitStormApp(jar: String, mainClass: String, args: String, appName: String): Int = {
+ Util.retryUntil({
+ Docker.exec(NIMBUS_HOST, s"$STORM_APP -config $CONFIG_FILE " +
+ s"-jar $jar $mainClass $args")
+ restClient.listRunningApps().exists(_.appName == appName)
+ })
+ restClient.listRunningApps().filter(_.appName == appName).head.appId
}
def getDRPCClient(drpcServerIp: String): DRPCClient = {
@@ -66,7 +57,8 @@ class StormClient(masterAddrs: Seq[(String, Int)]) {
}
def shutDown(): Unit = {
- container.killAndRemove()
+ drpcContainer.killAndRemove()
+ nimbusContainer.killAndRemove()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/920ad262/project/Build.scala
----------------------------------------------------------------------
diff --git a/project/Build.scala b/project/Build.scala
index 830e7c2..3f91f1e 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -385,8 +385,7 @@ object Build extends sbt.Build {
"commons-httpclient" % "commons-httpclient" % commonsHttpVersion,
"org.apache.hadoop" % "hadoop-mapreduce-client-core" % clouderaVersion,
"org.apache.hadoop" % "hadoop-yarn-server-resourcemanager" % clouderaVersion % "provided",
- "org.apache.hadoop" % "hadoop-yarn-server-nodemanager" % clouderaVersion % "provided",
- "org.specs2" %% "specs2-mock" % "3.6.4" % "test"
+ "org.apache.hadoop" % "hadoop-yarn-server-nodemanager" % clouderaVersion % "provided"
)
)
) dependsOn(services % "test->test;compile->compile", core % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/920ad262/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala
index 578a15d..67fd592 100644
--- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala
+++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala
@@ -348,8 +348,6 @@ object ClockService {
def updateMinClock(taskIndex: Int, clock: TimeStamp): Unit = {
taskClocks(taskIndex) = clock
_min = Longs.min(taskClocks: _*)
-
- Console.print(s"min clock of $processorId is ${_min}")
}
}
[05/50] incubator-gearpump git commit: Merge pull request #1942 from
manuzhang/standalone_nimbus
Posted by ap...@apache.org.
Merge pull request #1942 from manuzhang/standalone_nimbus
fix #1895, make GearpumpNimbus a standalone service
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/7f48c122
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/7f48c122
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/7f48c122
Branch: refs/heads/master
Commit: 7f48c122c68826c8d41e4ccaa18b32ab003798d2
Parents: 20c62e5 f83be15
Author: Kam Kasravi <ka...@yahoo.com>
Authored: Mon Feb 1 12:54:09 2016 -0800
Committer: Kam Kasravi <ka...@yahoo.com>
Committed: Mon Feb 1 12:54:09 2016 -0800
----------------------------------------------------------------------
docs/dev-storm.md | 40 ++-
.../gearpump/experiments/storm/Commands.scala | 36 ---
.../storm/GearpumpThriftServer.scala | 143 -----------
.../experiments/storm/StormRunner.scala | 137 ++--------
.../experiments/storm/main/GearpumpNimbus.scala | 247 +++++++++++++++++++
.../storm/main/GearpumpStormClient.scala | 70 ++++++
.../storm/topology/GearpumpStormTopology.scala | 52 +---
.../experiments/storm/GearpumpNimbusSpec.scala | 68 -----
.../storm/GearpumpThriftServerSpec.scala | 48 ----
.../experiments/storm/StormRunnerSpec.scala | 58 -----
.../topology/GearpumpStormTopologySpec.scala | 20 +-
.../integrationtest/storm/StormClient.scala | 14 +-
project/Pack.scala | 2 +-
.../io/gearpump/services/MasterService.scala | 2 +-
14 files changed, 400 insertions(+), 537 deletions(-)
----------------------------------------------------------------------
[09/50] incubator-gearpump git commit: Merge pull request #1946 from
huafengw/fix#1945
Posted by ap...@apache.org.
Merge pull request #1946 from huafengw/fix#1945
fix 945 fix stability integration test
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/ca018b7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/ca018b7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/ca018b7b
Branch: refs/heads/master
Commit: ca018b7b1e6d9a2c973455788abad4de2f3fa1e9
Parents: f136b3e 7f3c3c5
Author: Qian Xu <st...@users.noreply.github.com>
Authored: Tue Feb 2 10:10:04 2016 +0800
Committer: Qian Xu <st...@users.noreply.github.com>
Committed: Tue Feb 2 10:10:04 2016 +0800
----------------------------------------------------------------------
.../io/gearpump/integrationtest/checklist/StabilitySpec.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[43/50] incubator-gearpump git commit: fix #1983,
fix KafkaUtilSpec failure
Posted by ap...@apache.org.
fix #1983, fix KafkaUtilSpec failure
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/ec59bbd1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/ec59bbd1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/ec59bbd1
Branch: refs/heads/master
Commit: ec59bbd1144abf4879d53950fc4d839c3c2cf622
Parents: c7ba3d5
Author: manuzhang <ow...@gmail.com>
Authored: Tue Mar 1 09:06:32 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Tue Mar 1 11:23:49 2016 +0800
----------------------------------------------------------------------
.../streaming/kafka/util/KafkaServerHarness.scala | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ec59bbd1/external/kafka/src/test/scala/io/gearpump/streaming/kafka/util/KafkaServerHarness.scala
----------------------------------------------------------------------
diff --git a/external/kafka/src/test/scala/io/gearpump/streaming/kafka/util/KafkaServerHarness.scala b/external/kafka/src/test/scala/io/gearpump/streaming/kafka/util/KafkaServerHarness.scala
index ddce4a4..2ee7260 100644
--- a/external/kafka/src/test/scala/io/gearpump/streaming/kafka/util/KafkaServerHarness.scala
+++ b/external/kafka/src/test/scala/io/gearpump/streaming/kafka/util/KafkaServerHarness.scala
@@ -18,6 +18,9 @@
package io.gearpump.streaming.kafka.util
+import java.util.Properties
+
+import kafka.admin.AdminUtils
import kafka.common.KafkaException
import kafka.server.{KafkaConfig => KafkaServerConfig, KafkaServer}
import kafka.utils.{Utils, TestUtils}
@@ -44,10 +47,16 @@ trait KafkaServerHarness extends ZookeeperHarness {
super.tearDown
}
- def createTopicUntilLeaderIsElected(topic: String, partitions: Int, replicas: Int) = {
+ def createTopicUntilLeaderIsElected(topic: String, partitions: Int, replicas: Int, timeout: Long = 10000) = {
val zkClient = connectZk()
try {
- TestUtils.createTopic(zkClient, topic, partitions, replicas, servers)
+ // create topic
+ AdminUtils.createTopic(zkClient, topic, partitions, replicas, new Properties)
+ // wait until the update metadata request for new topic reaches all servers
+ (0 until partitions).map { case i =>
+ TestUtils.waitUntilMetadataIsPropagated(servers, topic, i, timeout)
+ i -> TestUtils.waitUntilLeaderIsElectedOrChanged(zkClient, topic, i, timeout)
+ }.toMap
} catch {
case e: Exception => throw e
} finally {
[28/50] incubator-gearpump git commit: Fix submit command not updated.
Posted by ap...@apache.org.
Fix submit command not updated.
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/bfd510bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/bfd510bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/bfd510bb
Branch: refs/heads/master
Commit: bfd510bbfdd6dccd642c37edeb0709103917dd17
Parents: 058bbf5
Author: Kimura Sotaro <rf...@gmail.com>
Authored: Thu Feb 11 19:10:56 2016 +0900
Committer: Kimura Sotaro <rf...@gmail.com>
Committed: Thu Feb 11 19:10:56 2016 +0900
----------------------------------------------------------------------
docs/submit-your-1st-application.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/bfd510bb/docs/submit-your-1st-application.md
----------------------------------------------------------------------
diff --git a/docs/submit-your-1st-application.md b/docs/submit-your-1st-application.md
index 73e53de..b72329d 100644
--- a/docs/submit-your-1st-application.md
+++ b/docs/submit-your-1st-application.md
@@ -26,7 +26,7 @@ Open another shell,
```bash
### To run WordCount example
-bin/gear app -jar examples/gearpump-examples-assembly-{{site.SCALA_BINARY_VERSION}}-{{site.GEARPUMP_VERSION}}.jar io.gearpump.streaming.examples.wordcount.WordCount
+bin/gear app -jar examples/wordcount-{{site.SCALA_BINARY_VERSION}}-{{site.GEARPUMP_VERSION}}-assembly.jar io.gearpump.streaming.examples.wordcount.WordCount
```
### Step 2: Congratulations, you've submitted your first application.
[22/50] incubator-gearpump git commit: fix #1906: some visual
glichtes found in mobile view
Posted by ap...@apache.org.
fix #1906: some visual glichtes found in mobile view
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/25f82edc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/25f82edc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/25f82edc
Branch: refs/heads/master
Commit: 25f82edc345b2234069915467cedb0fa17b06d36
Parents: 34c62fc
Author: Qian Xu <qi...@intel.com>
Authored: Fri Feb 5 12:58:37 2016 +0800
Committer: Qian Xu <qi...@intel.com>
Committed: Fri Feb 5 12:58:37 2016 +0800
----------------------------------------------------------------------
project/Build.scala | 7 ++++---
services/dashboard/index.html | 18 +++++++++---------
.../dashboard/views/apps/streamingapp/executor.js | 2 +-
.../views/apps/streamingapp/metric_selector.js | 2 +-
services/dashboard/views/cluster/master/master.js | 2 +-
.../views/cluster/workers/worker/worker.js | 2 +-
.../views/cluster/workers/workers_listview.js | 4 +---
7 files changed, 18 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25f82edc/project/Build.scala
----------------------------------------------------------------------
diff --git a/project/Build.scala b/project/Build.scala
index 19dde53..830e7c2 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -278,7 +278,8 @@ object Build extends sbt.Build {
"org.scalatest" %% "scalatest" % scalaTestVersion % "test",
"com.lihaoyi" %% "upickle" % upickleVersion,
"com.softwaremill" %% "akka-http-session" % "0.1.4",
- "org.webjars" % "angularjs" % "1.4.8",
+ "org.webjars" % "angularjs" % "1.4.9",
+ "org.webjars.npm" % "angular-touch" % "1.5.0-rc.2", // angular 1.5 breaks ui-select, but we need ng-touch 1.5
"org.webjars" % "angular-ui-router" % "0.2.15",
"org.webjars" % "bootstrap" % "3.3.6",
"org.webjars" % "d3js" % "3.5.6",
@@ -292,12 +293,12 @@ object Build extends sbt.Build {
"org.webjars.bower" % "angular-motion" % "0.4.3",
"org.webjars.bower" % "bootstrap-additions" % "0.3.1",
"org.webjars.bower" % "angular-strap" % "2.3.5",
- "org.webjars.bower" % "angular-ui-select" % "0.13.2",
+ "org.webjars.npm" % "ui-select" % "0.14.2",
"org.webjars.bower" % "ng-file-upload" % "5.0.9",
"org.webjars.bower" % "vis" % "4.7.0",
"org.webjars.bower" % "clipboard.js" % "0.1.1",
"org.webjars.npm" % "dashing-deps" % "0.1.2",
- "org.webjars.npm" % "dashing" % "0.4.4"
+ "org.webjars.npm" % "dashing" % "0.4.5"
).map(_.exclude("org.scalamacros", "quasiquotes_2.10")).map(_.exclude("org.scalamacros", "quasiquotes_2.10.3")))
lazy val serviceJSSettings = Seq(
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25f82edc/services/dashboard/index.html
----------------------------------------------------------------------
diff --git a/services/dashboard/index.html b/services/dashboard/index.html
index 96b5742..47007cc 100644
--- a/services/dashboard/index.html
+++ b/services/dashboard/index.html
@@ -14,10 +14,10 @@
<!-- angular-strap requires the following 2 stylesheets -->
<link rel="stylesheet" href="webjars/bootstrap-additions/0.3.1/dist/bootstrap-additions.min.css"/>
<link rel="stylesheet" href="webjars/angular-motion/0.4.3/dist/angular-motion.min.css"/>
- <link rel="stylesheet" href="webjars/angular-ui-select/0.13.2/dist/select.min.css"/>
+ <link rel="stylesheet" href="webjars/ui-select/0.14.2/dist/select.min.css"/>
<link rel="stylesheet" href="webjars/angular-loading-bar/0.8.0/build/loading-bar.min.css"/>
<link rel="stylesheet" href="webjars/vis/4.7.0/dist/vis.min.css"/>
- <link rel="stylesheet" href="webjars/dashing/0.4.4/dist/dashing.min.css"/>
+ <link rel="stylesheet" href="webjars/dashing/0.4.5/dist/dashing.min.css"/>
<!-- Site styles -->
<link rel="stylesheet" href="webjars/dashing-deps/0.1.2/roboto/roboto.min.css"/>
@@ -44,16 +44,16 @@
<!-- Static Resources -->
<script src="webjars/jquery/2.2.0/jquery.min.js"></script>
<script src="webjars/bootstrap/3.3.6/js/bootstrap.min.js"></script>
-<script src="webjars/angularjs/1.4.8/angular.min.js"></script>
-<script src="webjars/angularjs/1.4.8/angular-cookies.min.js"></script>
-<script src="webjars/angularjs/1.4.8/angular-touch.min.js"></script>
-<script src="webjars/angularjs/1.4.8/angular-animate.min.js"></script>
-<script src="webjars/angularjs/1.4.8/angular-sanitize.min.js"></script>
+<script src="webjars/angularjs/1.4.9/angular.min.js"></script>
+<script src="webjars/angularjs/1.4.9/angular-cookies.min.js"></script>
+<script src="webjars/angular-touch/1.5.0-rc.2/angular-touch.min.js"></script>
+<script src="webjars/angularjs/1.4.9/angular-animate.min.js"></script>
+<script src="webjars/angularjs/1.4.9/angular-sanitize.min.js"></script>
<script src="webjars/angular-loading-bar/0.8.0/build/loading-bar.min.js"></script>
<script src="webjars/angular-ui-router/0.2.15/angular-ui-router.min.js"></script>
<script src="webjars/angular-strap/2.3.5/dist/angular-strap.min.js"></script>
<script src="webjars/angular-strap/2.3.5/dist/angular-strap.tpl.min.js"></script>
-<script src="webjars/angular-ui-select/0.13.2/dist/select.min.js"></script>
+<script src="webjars/ui-select/0.14.2/dist/select.min.js"></script>
<script src="webjars/angular-smart-table/2.1.6/dist/smart-table.min.js"></script>
<script src="webjars/d3js/3.5.6/d3.min.js"></script>
<script src="webjars/momentjs/2.10.6/min/moment.min.js"></script>
@@ -62,7 +62,7 @@
<script src="webjars/ng-file-upload/5.0.9/ng-file-upload-all.min.js"></script>
<script src="webjars/clipboard.js/0.1.1/clipboard.js"></script>
<script src="webjars/dashing-deps/0.1.2/echarts/2.2.7-compact/echarts-all.min.js"></script>
-<script src="webjars/dashing/0.4.4/dist/dashing.min.js"></script>
+<script src="webjars/dashing/0.4.5/dist/dashing.min.js"></script>
<!-- Application -->
<script src="dashboard.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25f82edc/services/dashboard/views/apps/streamingapp/executor.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/streamingapp/executor.js b/services/dashboard/views/apps/streamingapp/executor.js
index 60df92f..31cbd5f 100644
--- a/services/dashboard/views/apps/streamingapp/executor.js
+++ b/services/dashboard/views/apps/streamingapp/executor.js
@@ -30,7 +30,7 @@ angular.module('dashboard')
$scope.executorName = executor0.id === -1 ?
'AppMaster' : ('Executor ' + executor0.id);
$scope.overviewTable = [
- $ptb.text('JVM Info').help('Format: PID@hostname').done(),
+ $ptb.text('JVM Info').done(),
$ptb.text('Actor Path').done(),
$ptb.link('Worker').done(),
$ptb.number('Task Count').done(),
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25f82edc/services/dashboard/views/apps/streamingapp/metric_selector.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/streamingapp/metric_selector.js b/services/dashboard/views/apps/streamingapp/metric_selector.js
index f2025ef..88fc507 100644
--- a/services/dashboard/views/apps/streamingapp/metric_selector.js
+++ b/services/dashboard/views/apps/streamingapp/metric_selector.js
@@ -14,7 +14,7 @@ angular.module('dashboard')
scope: {
metricName: '=',
metricType: '=',
- metricDescription: '='
+ metricDescription: '=?'
},
link: function(scope) {
'use strict';
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25f82edc/services/dashboard/views/cluster/master/master.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/cluster/master/master.js b/services/dashboard/views/cluster/master/master.js
index 31cba33..44a38f7 100644
--- a/services/dashboard/views/cluster/master/master.js
+++ b/services/dashboard/views/cluster/master/master.js
@@ -28,7 +28,7 @@ angular.module('dashboard')
$scope.whatIsMaster = i18n.terminology.master;
$scope.masterInfoTable = [
- $ptb.text('JVM Info').help('Format: PID@hostname').done(),
+ $ptb.text('JVM Info').done(),
$ptb.text('Leader').done(),
$ptb.text('Master Members').done(),
$ptb.tag('Status').done(),
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25f82edc/services/dashboard/views/cluster/workers/worker/worker.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/cluster/workers/worker/worker.js b/services/dashboard/views/cluster/workers/worker/worker.js
index 599f1d6..ad92a74 100644
--- a/services/dashboard/views/cluster/workers/worker/worker.js
+++ b/services/dashboard/views/cluster/workers/worker/worker.js
@@ -36,7 +36,7 @@ angular.module('dashboard')
$scope.whatIsWorker = i18n.terminology.worker;
$scope.overviewTable = [
- $ptb.text('JVM Info').help('Format: PID@hostname').done(),
+ $ptb.text('JVM Info').done(),
$ptb.text('Actor Path').done(),
$ptb.number('Slots Capacity').done(),
$ptb.button('Quick Links').values([
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25f82edc/services/dashboard/views/cluster/workers/workers_listview.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/cluster/workers/workers_listview.js b/services/dashboard/views/cluster/workers/workers_listview.js
index 9c27327..3cdf14e 100644
--- a/services/dashboard/views/cluster/workers/workers_listview.js
+++ b/services/dashboard/views/cluster/workers/workers_listview.js
@@ -33,9 +33,7 @@ angular.module('dashboard')
$stb.indicator().key('state').canSort('state.condition+"_"+aliveFor').styleClass('td-no-padding').done(),
$stb.link('ID').key('id').canSort().sortDefaultDescent().styleClass('col-md-1').done(),
$stb.text('Address').key('akkaAddr').canSort().styleClass('col-md-1').done(),
- $stb.text('JVM Info').key('jvm')
- .help('Format: PID@hostname')
- .styleClass('col-md-2 hidden-xs').done(),
+ $stb.text('JVM Info').key('jvm').styleClass('col-md-2 hidden-xs').done(),
// group 2/3 (5-col)
$stb.number('Executors').key('executors').canSort().styleClass('col-md-1 hidden-xs').done(),
$stb.progressbar('Slots Usage').key('slots').sortBy('slots.usage').styleClass('col-md-1').done(),
[21/50] incubator-gearpump git commit: fix storm doc
Posted by ap...@apache.org.
fix storm doc
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/34c62fc7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/34c62fc7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/34c62fc7
Branch: refs/heads/master
Commit: 34c62fc7fbe465c70c94f5b873ba28c980665180
Parents: 59e0f5a
Author: manuzhang <ow...@gmail.com>
Authored: Thu Feb 4 22:06:26 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Thu Feb 4 22:06:26 2016 +0800
----------------------------------------------------------------------
docs/dev-storm.md | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/34c62fc7/docs/dev-storm.md
----------------------------------------------------------------------
diff --git a/docs/dev-storm.md b/docs/dev-storm.md
index 0ff1127..914aabe 100644
--- a/docs/dev-storm.md
+++ b/docs/dev-storm.md
@@ -185,7 +185,8 @@ Since StreamCQL already supports Storm, it's straightforward to run StreamCQL ov
3. Go to the installed stream-cql-binary, and change following settings in `conf/streaming-site.xml` with the output Nimbus configs in Step 2.
- ```xml
+
+ ```xml
<property>
<name>streaming.storm.nimbus.host</name>
<value>${nimbus.host}</value>
@@ -194,7 +195,7 @@ Since StreamCQL already supports Storm, it's straightforward to run StreamCQL ov
<name>streaming.storm.nimbus.port</name>
<value>${nimbus.thrift.port}</value>
</property>
- ```
+ ```
4. Open CQL client shell and execute a simple cql example
@@ -202,7 +203,7 @@ Since StreamCQL already supports Storm, it's straightforward to run StreamCQL ov
bin/cql
```
- ```
+ ```sql
Streaming> CREATE INPUT STREAM s
(id INT, name STRING, type INT)
SOURCE randomgen
[49/50] incubator-gearpump git commit: fix #1996,
EmbeddedCluster requires master configuration like
ClusterActorRefProvider.
Posted by ap...@apache.org.
fix #1996, EmbeddedCluster requires master configuration like ClusterActorRefProvider.
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/ecf5b468
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/ecf5b468
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/ecf5b468
Branch: refs/heads/master
Commit: ecf5b468e89c6b19b994760c4b784d11ce8eeee6
Parents: a7bbb4c
Author: Sean Zhong <cl...@gmail.com>
Authored: Wed Mar 9 15:24:31 2016 +0800
Committer: Sean Zhong <cl...@gmail.com>
Committed: Thu Mar 10 10:11:21 2016 +0800
----------------------------------------------------------------------
conf/gear.conf | 54 +++-
core/src/main/resources/geardefault.conf | 279 ++++++++++++-------
.../io/gearpump/cluster/ClusterConfig.scala | 10 +-
.../main/scala/io/gearpump/util/ActorUtil.scala | 4 -
.../main/scala/io/gearpump/util/Constants.scala | 2 +-
core/src/test/resources/test.conf | 10 +-
.../cluster/embedded/EmbeddedCluster.scala | 89 ++++++
.../gearpump/cluster/local/LocalCluster.scala | 88 ------
.../scala/io/gearpump/cluster/main/Local.scala | 1 -
.../examples/wordcountjava/WordCount.java | 20 +-
.../examples/wordcount/WordCount.scala | 20 +-
.../stream/gearpump/graph/RemoteGraph.scala | 4 +-
.../checklist/CommandLineSpec.scala | 10 +-
.../io/gearpump/streaming/task/TaskUtil.scala | 3 +-
14 files changed, 373 insertions(+), 221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/conf/gear.conf
----------------------------------------------------------------------
diff --git a/conf/gear.conf b/conf/gear.conf
index 1673cb7..e5898e9 100644
--- a/conf/gear.conf
+++ b/conf/gear.conf
@@ -3,6 +3,7 @@
### To use the application, you at least need to change gearpump.cluster to point to right master
#########################################
+### Configurations that are shared by all services and all user applications...
gearpump {
##############################
@@ -53,7 +54,9 @@ gearpump {
cpu-core-limit-per-executor = 1
}
- ## Whether we allow remote debug
+ ## Whether to enable remote-debug mode.
+ ## In remote debug mode, every executor process will bind to a free port, and
+ ## listen for remote jvm debug.
remote-debug-executor-jvm = false
### When the resource cannot be allocated in the timeout, then
@@ -104,6 +107,40 @@ gearpump {
logfile {
}
+
+ ## Coarse-grain history metrics, which have a larger timespan but sparse data points
+ retainHistoryData {
+ # max hours of history data to retain
+ # Note: due to implementation limitation(we store all history in memory),
+ # please don't set this to too big which may exhaust memory.
+ hours = 72
+
+ # time interval between two data points for history data (unit: ms)
+ # Usually this value is set to a big value so that we only store
+ # coarse-grain data
+ intervalMs = 3600000
+ }
+
+ ## fine-grain recent metrics which just happened, which have a smaller timespan but dense data points
+ retainRecentData {
+
+ # max seconds of recent data to retain,
+ # THis is for the fine-grain data
+ seconds = 300
+
+ # time interval between two data points for recent data (unit: ms)
+ intervalMs = 15000
+ }
+ akka {
+ ### For this reporter, at most we will return max-limit-on-query metric item.
+ max-limit-on-query = 1000
+
+ ### Whitelist for Metrics Aggregator class.
+ ### See class [[MetricsAggregator]] for more information.
+ metrics-aggregator-class {
+ ## Format io.gearpump.KeyFullClassName = ""
+ }
+ }
}
#######################################
@@ -124,6 +161,10 @@ gearpump {
}
+
+ ### client's timeout (in second) to connect to master and wait for the response
+ masterclient.timeout = 90
+
### Gearpump has built-in serialization framework using Kryo.
### User are allowed to use a different serialization framework, like Protobuf
### See [io.gearpump.serializer.FastKryoSerializationFramework] to find how
@@ -159,6 +200,10 @@ gearpump {
services {
host = "0.0.0.0"
http = 8090
+ supervisor-actor-path = ""
+
+ ## To get a detail config string with origin and comments, change this to false.
+ config-render-option-concise = true
}
## Time out setting to start a new executor system
@@ -366,10 +411,9 @@ gearpump-ui {
}
}
-
-## Configurations only visible on Windows operation system..
-gearpump-windows {
+## Configurations only visible on Linux or Mac.
+gearpump-linux {
### On windows, the value must be larger than 10ms, check
### https://github.com/akka/akka/blob/master/akka-actor/src/main/scala/akka/actor/Scheduler.scala#L204
- akka.scheduler.tick-duration = 10
+ akka.scheduler.tick-duration = 1
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/core/src/main/resources/geardefault.conf
----------------------------------------------------------------------
diff --git a/core/src/main/resources/geardefault.conf b/core/src/main/resources/geardefault.conf
index d775696..065e5e0 100644
--- a/core/src/main/resources/geardefault.conf
+++ b/core/src/main/resources/geardefault.conf
@@ -6,6 +6,79 @@
### Configurations that are shared by all services and all user applications...
gearpump {
+ ##############################
+ ### Required to change!!
+ ### You need to set the master cluster address here
+ ###
+ ###
+ ### For example, you may start three master
+ ### on node1: bin/master -ip node1 -port 3000
+ ### on node2: bin/master -ip node2 -port 3000
+ ### on node3: bin/master -ip node3 -port 3000
+ ###
+ ### Then you need to set the cluster.masters = ["node1:3000","node2:3000","node3:3000"]
+ cluster {
+ masters = []
+ }
+
+ ##############################
+ ### Required to change!!
+ ### You need to set the actual host name here
+ ###
+ hostname = "127.0.0.1"
+
+
+ ## The installation folder of gearpump
+ home = ""
+
+
+
+
+
+ serializer.pool = "io.gearpump.serializer.FastKryoSerializerPool"
+
+
+
+ ## How many slots each worker contains
+ worker.slots = 1000
+
+ ## The class responsable for launching the executor process.
+ ## User can switch to "io.gearpump.cluster.worker.CGroupProcessLauncher" to enable CGroup support.
+ worker.executor-process-launcher = "io.gearpump.cluster.worker.DefaultExecutorProcessLauncher"
+
+ ## To enable worker use cgroup to make resource isolation,
+ ## set gearpump.worker.executor-process-launcher = "io.gearpump.cluster.worker.CGroupProcessLauncher"
+ ##
+ ## Before enable it, you should also make sure:
+ ## 1. Linux version (>= 2.6.18)
+ ## 2. Have installed cgroup (check the file's existence:/proc/cgroups)
+ ## You can get more information from http://gearpump.io
+ ##
+ ## For cgroup root, it represents the root node in CGroup's hierarchythe.
+ ## It's full path in local file system is "${cpu_mount_point} + root".
+ ## The cpu_mount_point is the cpu subsystem's mount poing in CGroup.
+ ## The root dir should be consistent with the part configured in /etc/cgconfig.conf
+ cgroup {
+ root = "gearpump"
+
+ ## This config only works when cgroup is enabled.
+ ## The value means the number of CPU cores per executor can use, -1 means no limitation.
+ cpu-core-limit-per-executor = 1
+ }
+
+ ## Whether to enable remote-debug mode.
+ ## In remote debug mode, every executor process will bind to a free port, and
+ ## listen for remote jvm debug.
+ remote-debug-executor-jvm = false
+
+ ### When the resource cannot be allocated in the timeout, then
+ ### the appmaster will shutdown itself.
+ resource-allocation-timeout-seconds = 120
+
+ ##
+ ## Executor share same process of worker
+ worker.executor-share-same-jvm-as-worker = false
+
###########################
### Change the dispather for tasks
### If you don't know what this is about, don't change it
@@ -13,12 +86,13 @@ gearpump {
task-dispatcher = "gearpump.shared-thread-pool-dispatcher"
- ## The installation folder of gearpump
- home = ""
-
- ## Time out setting to start a new executor system
- ## It takes a bit longer time than expected as a new JVM is created
- start-executor-system-timeout-ms = 30000
+ ### verbose gc
+ ### turn on JVM verbose GC
+ ### We will use -verbose:gc -Xloggc: -XX:+PrintGCDetails
+ ### -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution
+ ### -XX:+PrintGCApplicationConcurrentTime
+ ### -XX:+PrintGCApplicationStoppedTime
+ verbose-gc = false
###########################
### Metrics setting,
@@ -34,10 +108,10 @@ gearpump {
### This config will slightly impact the application's performance, larger rate often brings higher throughput.
sample-rate = 1
- report-interval-ms = 15000
+ report-interval-ms = 3000
+ # reporter = "logfile"
# reporter = "graphite"
- # reporter = "akka"
reporter = "akka"
graphite {
@@ -85,19 +159,6 @@ gearpump {
}
}
- ## Whether to enable remote-debug mode.
- ## In remote debug mode, every executor process will bind to a free port, and
- ## listen for remote jvm debug.
- remote-debug-executor-jvm = false
-
- ### verbose gc
- ### turn on JVM verbose GC
- ### We will use -verbose:gc -Xloggc: -XX:+PrintGCDetails
- ### -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution
- ### -XX:+PrintGCApplicationConcurrentTime
- ### -XX:+PrintGCApplicationStoppedTime
- verbose-gc = false
-
#######################################
### Logging settings
#######################################
@@ -107,8 +168,6 @@ gearpump {
# The log dir for applications
log.application.dir = "logs"
- serializer.pool = "io.gearpump.serializer.FastKryoSerializerPool"
-
serializers {
## Use default serializer for these types
"scala.collection.immutable.List" = ""
@@ -133,87 +192,22 @@ gearpump {
"akka.actor.LocalActorRef" = ""
}
- ## How many slots each worker contains
- worker.slots = 100
- ## The class responsable for launching the executor process.
- ## User can switch to "io.gearpump.cluster.worker.CGroupProcessLauncher" to enable CGroup support.
- worker.executor-process-launcher = "io.gearpump.cluster.worker.DefaultExecutorProcessLauncher"
-
- ## To enable worker use cgroup to make resource isolation,
- ## set gearpump.worker.executor-process-launcher = "io.gearpump.cluster.worker.CGroupProcessLauncher"
- ##
- ## Before enable it, you should also make sure:
- ## 1. Linux version (>= 2.6.18)
- ## 2. Have installed cgroup (check the file's existence:/proc/cgroups)
- ## You can get more information from http://gearpump.io
- ##
- ## For cgroup root, it represents the root node in CGroup's hierarchythe.
- ## It's full path in local file system is "${cpu_mount_point} + root".
- ## The cpu_mount_point is the cpu subsystem's mount poing in CGroup.
- ## The root dir should be consistent with the part configured in /etc/cgconfig.conf
- cgroup {
- root = "gearpump"
-
- ## This config only works when cgroup is enabled.
- ## The value means the number of CPU cores per executor can use, -1 means no limitation.
- cpu-core-limit-per-executor = 1
- }
-
- ##
- ## Executor share same process of worker
- worker.executor-share-same-jvm-as-worker = false
-
- ###################
- ### Appmaster JVM argument configuration
- ###################
- appmaster {
- vmargs = ""
- extraClasspath = ""
- }
-
- ###################
- ### Executor argument configuration
- ### Executor JVM can contains multiple tasks
- ###################
- executor {
- vmargs = ""
- extraClasspath = ""
- }
-
- ##############################
- ### Required to change!!
- ### You need to set the master cluster address here
- ###
- ###
- ### For example, you may start three master
- ### on node1: bin/master -ip node1 -port 3000
- ### on node2: bin/master -ip node2 -port 3000
- ### on node3: bin/master -ip node3 -port 3000
- ###
- ### Then you need to set the cluster.masters = ["node1:3000","node2:3000","node3:3000"]
- cluster {
- masters = []
- }
-
- ##############################
- ### Required to change!!
- ### You need to set the actual host name here
- ###
- hostname = "127.0.0.1"
-
- ### When the resource cannot be allocated in the timeout, then
- ### the appmaster will shutdown itself.
- resource-allocation-timeout-seconds = 10
### client's timeout (in second) to connect to master and wait for the response
masterclient.timeout = 90
+ ### Gearpump has built-in serialization framework using Kryo.
+ ### User are allowed to use a different serialization framework, like Protobuf
+ ### See [io.gearpump.serializer.FastKryoSerializationFramework] to find how
+ ### a custom serialization framework can be defined.
+ serialization-framework = "io.gearpump.serializer.FastKryoSerializationFramework"
+
### Define where the submitted jar file will be stored at
### This path follows the hadoop path schema
### For HDFS, use hdfs://host:port/path/
- ### If you want to store on master nodes, then use local directory,
+ ### For shared NFS folder, use file:///your_nfs_mapping_directory
### jarstore.rootpath = "jarstore/" will points to relative directory where master is started.
### jarstore.rootpath = "/jarstore/" will points to absolute directory on master server
jarstore.rootpath = "jarstore/"
@@ -244,6 +238,10 @@ gearpump {
config-render-option-concise = true
}
+ ## Time out setting to start a new executor system
+ ## It takes a bit longer time than expected as a new JVM is created
+ start-executor-system-timeout-ms = 120000
+
#############################################
## Default Configuration for Gearpump Netty transport layer
## If you don't know what is this about, don't change it
@@ -258,6 +256,37 @@ gearpump {
dispatcher = "gearpump.shared-thread-pool-dispatcher"
}
+ ###################
+ ### Appmaster JVM argument configuration
+ ###################
+ appmaster {
+ vmargs = ""
+ extraClasspath = ""
+ }
+
+ ###################
+ ### Executor argument configuration
+ ### Executor JVM can contains multiple tasks
+ ###################
+ executor {
+ vmargs = ""
+ extraClasspath = ""
+ }
+
+ ### Streaming related configuration
+ streaming {
+ ## We will timeout the task if it cannot register itself to AppMaster in time.
+ register-task-timeout-ms = 120000
+
+ ## ack once after sending ack-once-every-message-count messages.
+ ack-once-every-message-count = 100
+
+ ## max pending message per task to task connection. If pending message size is
+ ## over this, then the flow control will not allow further sending.
+ ## This value should be bigger than ack-once-every-message-count
+ max-pending-message-count-per-connection = 1000
+ }
+
##################################
### Akka Dispatcher configurations
### If you don't know what is this about, don't change it
@@ -275,8 +304,45 @@ gearpump {
single-thread-dispatcher {
type = PinnedDispatcher
}
+
+ ###########################
+ ### Configuration for yarn module
+ ###########################
+ yarn {
+ client {
+ package-path = "/usr/lib/gearpump/gearpump.zip"
+ }
+
+ applicationmaster {
+ ## Memory of YarnAppMaster
+ command = "$JAVA_HOME/bin/java -Xmx512m"
+ memory = "512"
+ vcores = "1"
+ queue = "default"
+ }
+
+ master {
+ ## Memory of master daemon
+ command = "$JAVA_HOME/bin/java -Xmx512m"
+ memory = "512"
+ vcores = "1"
+ }
+
+ worker {
+ ## memory of worker daemon
+ command = "$JAVA_HOME/bin/java -Xmx512m"
+ containers = "1"
+ ## This also contains all memory for child executors.
+ memory = "4096"
+ vcores = "1"
+ }
+ services {
+ enabled = true
+ }
+ }
}
+
### Configuration only visible to master nodes..
gearpump-master {
## NOTE: Please add akka-related settings in gear.conf to avoid confusion in
@@ -289,11 +355,14 @@ gearpump-worker {
## config overriding.
}
-## Configurations only visible on Windows operation system..
-gearpump-windows {
-}
-### configurations only visible to UI service...
+#########################################
+### For log level of Akka class, you need to change both log4j.properties and this entry
+#########################################
+#akka.loglevel = "INFO"
+
+
+### configurations only visible to UI server.
gearpump-ui {
## Security related settings
gearpump.ui-security {
@@ -344,7 +413,9 @@ gearpump-ui {
## ticket: https://github.com/akka/akka/issues/16295
akka.io.tcp.windows-connection-abort-workaround-enabled=off
-akka.scheduler.tick-duration = 1
+### On windows, the value must be larger than 10ms, check
+### https://github.com/akka/akka/blob/master/akka-actor/src/main/scala/akka/actor/Scheduler.scala#L204
+akka.scheduler.tick-duration = 10
akka {
http {
@@ -453,3 +524,11 @@ akka {
}
}
}
+
+
+## Configurations only visible on Linux or Mac.
+gearpump-linux {
+ ### On windows, the value must be larger than 10ms, check
+ ### https://github.com/akka/akka/blob/master/akka-actor/src/main/scala/akka/actor/Scheduler.scala#L204
+ akka.scheduler.tick-duration = 1
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/core/src/main/scala/io/gearpump/cluster/ClusterConfig.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/cluster/ClusterConfig.scala b/core/src/main/scala/io/gearpump/cluster/ClusterConfig.scala
index e2fa155..64d5c42 100644
--- a/core/src/main/scala/io/gearpump/cluster/ClusterConfig.scala
+++ b/core/src/main/scala/io/gearpump/cluster/ClusterConfig.scala
@@ -133,13 +133,15 @@ object ClusterConfig {
val all = systemProperties.withFallback(user).withFallback(gear).withFallback(gearDefault)
- val windows = all.getConfig(WINDOWS_CONFIG)
+ val linux = all.getConfig(LINUX_CONFIG)
var basic = all.withoutPath(MASTER_CONFIG).withoutPath(WORKER_CONFIG).
- withoutPath(UI_CONFIG).withoutPath(WINDOWS_CONFIG)
+ withoutPath(UI_CONFIG).withoutPath(LINUX_CONFIG)
- if (akka.util.Helpers.isWindows) {
- basic = windows.withFallback(basic)
+ if (!akka.util.Helpers.isWindows) {
+
+ // Change the akka.scheduler.tick-duration to 10ms for Linux or Mac
+ basic = linux.withFallback(basic)
}
val master = replaceHost(all.getConfig(MASTER_CONFIG).withFallback(basic))
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/core/src/main/scala/io/gearpump/util/ActorUtil.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/util/ActorUtil.scala b/core/src/main/scala/io/gearpump/util/ActorUtil.scala
index 830d45e..b63733d 100644
--- a/core/src/main/scala/io/gearpump/util/ActorUtil.scala
+++ b/core/src/main/scala/io/gearpump/util/ActorUtil.scala
@@ -71,10 +71,6 @@ object ActorUtil {
}
}
- def loadClass(className: String): Class[_<:Actor] = {
- Class.forName(className).asSubclass(classOf[Actor])
- }
-
def actorNameForExecutor(appId : Int, executorId : Int) = "app" + appId + "-executor" + executorId
/**
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/core/src/main/scala/io/gearpump/util/Constants.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/util/Constants.scala b/core/src/main/scala/io/gearpump/util/Constants.scala
index 5e340e3..7d066e9 100644
--- a/core/src/main/scala/io/gearpump/util/Constants.scala
+++ b/core/src/main/scala/io/gearpump/util/Constants.scala
@@ -29,7 +29,7 @@ object Constants {
val MASTER_CONFIG = "gearpump-master"
val WORKER_CONFIG = "gearpump-worker"
val UI_CONFIG = "gearpump-ui"
- val WINDOWS_CONFIG = "gearpump-windows"
+ val LINUX_CONFIG = "gearpump-linux" // linux or Mac
val MASTER = "master"
val WORKER = "worker"
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/core/src/test/resources/test.conf
----------------------------------------------------------------------
diff --git a/core/src/test/resources/test.conf b/core/src/test/resources/test.conf
index edc01d9..3deb28b 100644
--- a/core/src/test/resources/test.conf
+++ b/core/src/test/resources/test.conf
@@ -8,6 +8,10 @@ gearpump {
worker.slots = 100
+ ### When the resource cannot be allocated in the timeout, then
+ ### the appmaster will shutdown itself.
+ resource-allocation-timeout-seconds = 10
+
worker.executor-process-launcher = "io.gearpump.cluster.worker.DefaultExecutorProcessLauncher"
cluster {
@@ -21,11 +25,11 @@ gearpump {
serialization-framework = "io.gearpump.serializer.FastKryoSerializationFramework"
}
-## Configurations only visible on Windows operation system..
-gearpump-windows {
+## Configurations only visible on Linux or Mac..
+gearpump-linux {
### On windows, the value must be larger than 10ms, check
### https://github.com/akka/akka/blob/master/akka-actor/src/main/scala/akka/actor/Scheduler.scala#L204
- akka.scheduler.tick-duration = 10
+ akka.scheduler.tick-duration = 1
}
### Configuration only visible to worker nodes...
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/daemon/src/main/scala/io/gearpump/cluster/embedded/EmbeddedCluster.scala
----------------------------------------------------------------------
diff --git a/daemon/src/main/scala/io/gearpump/cluster/embedded/EmbeddedCluster.scala b/daemon/src/main/scala/io/gearpump/cluster/embedded/EmbeddedCluster.scala
new file mode 100644
index 0000000..6be52b2
--- /dev/null
+++ b/daemon/src/main/scala/io/gearpump/cluster/embedded/EmbeddedCluster.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.cluster.embedded
+
+import akka.actor.{ActorRef, ActorSystem, Props}
+import com.typesafe.config.{ConfigValueFactory, Config}
+import io.gearpump.cluster.ClusterConfig
+import io.gearpump.cluster.client.ClientContext
+import io.gearpump.cluster.master.{Master => MasterActor}
+import io.gearpump.cluster.worker.{Worker => WorkerActor}
+import io.gearpump.util.{LogUtil, Constants, Util, ActorUtil}
+import io.gearpump.util.Constants.{GEARPUMP_METRIC_ENABLED, GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS, MASTER, GEARPUMP_CLUSTER_MASTERS}
+import scala.collection.JavaConverters._
+
+
+/**
+ * Create a in-process cluster with single worker
+ */
+class EmbeddedCluster(inputConfig: Config) {
+
+ private val workerCount: Int = 1
+ private var _master: ActorRef = null
+ private var _system: ActorSystem = null
+ private var _config: Config = null
+
+ private val LOG = LogUtil.getLogger(getClass)
+
+ def start: Unit = {
+ val port = Util.findFreePort.get
+ val akkaConf = getConfig(inputConfig, port)
+ _config = akkaConf
+ val system = ActorSystem(MASTER, akkaConf)
+
+ val master = system.actorOf(Props[MasterActor], MASTER)
+
+ 0.until(workerCount).foreach { id =>
+ system.actorOf(Props(classOf[WorkerActor], master), classOf[WorkerActor].getSimpleName + id)
+ }
+ this._master = master
+ this._system = system
+
+ LOG.info("=================================")
+ LOG.info("Local Cluster is started at: ")
+ LOG.info(s" 127.0.0.1:$port")
+ LOG.info(s"To see UI, run command: services -master 127.0.0.1:$port")
+ }
+
+ private def getConfig(inputConfig: Config, port: Int): Config = {
+ val config = inputConfig.
+ withValue("akka.remote.netty.tcp.port", ConfigValueFactory.fromAnyRef(port)).
+ withValue(GEARPUMP_CLUSTER_MASTERS, ConfigValueFactory.fromIterable(List(s"127.0.0.1:$port").asJava)).
+ withValue(GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS, ConfigValueFactory.fromAnyRef(true)).
+ withValue(GEARPUMP_METRIC_ENABLED, ConfigValueFactory.fromAnyRef(true)).
+ withValue("akka.actor.provider", ConfigValueFactory.fromAnyRef("akka.cluster.ClusterActorRefProvider"))
+ config
+ }
+
+ def newClientContext: ClientContext = {
+ ClientContext(_config, _system, _master)
+ }
+
+ def stop: Unit = {
+ _system.stop(_master)
+ _system.shutdown()
+ _system.awaitTermination()
+ }
+}
+
+object EmbeddedCluster{
+ def apply(): EmbeddedCluster = {
+ new EmbeddedCluster(ClusterConfig.master())
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/daemon/src/main/scala/io/gearpump/cluster/local/LocalCluster.scala
----------------------------------------------------------------------
diff --git a/daemon/src/main/scala/io/gearpump/cluster/local/LocalCluster.scala b/daemon/src/main/scala/io/gearpump/cluster/local/LocalCluster.scala
deleted file mode 100644
index 0276f72..0000000
--- a/daemon/src/main/scala/io/gearpump/cluster/local/LocalCluster.scala
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.cluster.local
-
-import akka.actor.{ActorRef, ActorSystem, Props}
-import com.typesafe.config.{ConfigValueFactory, Config}
-import io.gearpump.cluster.ClusterConfig
-import io.gearpump.cluster.client.ClientContext
-import io.gearpump.cluster.master.{Master => MasterActor}
-import io.gearpump.cluster.worker.{Worker => WorkerActor}
-import io.gearpump.util.{LogUtil, Constants, Util, ActorUtil}
-import io.gearpump.util.Constants.{GEARPUMP_METRIC_ENABLED, GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS, MASTER, GEARPUMP_CLUSTER_MASTERS}
-import scala.collection.JavaConverters._
-
-
-/**
- * Create a in-process cluster with single worker
- */
-class LocalCluster(inputConfig: Config) {
-
- private val workerCount: Int = 1
- private var _master: ActorRef = null
- private var _system: ActorSystem = null
- private var _config: Config = null
-
- private val LOG = LogUtil.getLogger(getClass)
-
- def start: Unit = {
- val port = Util.findFreePort.get
- val akkaConf = getConfig(inputConfig, port)
- _config = akkaConf
- val system = ActorSystem(MASTER, akkaConf)
-
- val master = system.actorOf(Props[MasterActor], MASTER)
-
- 0.until(workerCount).foreach { id =>
- system.actorOf(Props(classOf[WorkerActor], master), classOf[WorkerActor].getSimpleName + id)
- }
- this._master = master
- this._system = system
-
- LOG.info("=================================")
- LOG.info("Local Cluster is started at: ")
- LOG.info(s" 127.0.0.1:$port")
- LOG.info(s"To see UI, run command: services -master 127.0.0.1:$port")
- }
-
- private def getConfig(inputConfig: Config, port: Int): Config = {
- val config = inputConfig.
- withValue("akka.remote.netty.tcp.port", ConfigValueFactory.fromAnyRef(port)).
- withValue(GEARPUMP_CLUSTER_MASTERS, ConfigValueFactory.fromIterable(List(s"127.0.0.1:$port").asJava)).
- withValue(GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS, ConfigValueFactory.fromAnyRef(true)).
- withValue(GEARPUMP_METRIC_ENABLED, ConfigValueFactory.fromAnyRef(true))
- config
- }
-
- def newClientContext: ClientContext = {
- ClientContext(_config, _system, _master)
- }
-
- def stop: Unit = {
- _system.stop(_master)
- _system.shutdown()
- _system.awaitTermination()
- }
-}
-
-object LocalCluster{
- def apply(): LocalCluster = {
- new LocalCluster(ClusterConfig.master())
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/daemon/src/main/scala/io/gearpump/cluster/main/Local.scala
----------------------------------------------------------------------
diff --git a/daemon/src/main/scala/io/gearpump/cluster/main/Local.scala b/daemon/src/main/scala/io/gearpump/cluster/main/Local.scala
index 3f26538..d6b2479 100644
--- a/daemon/src/main/scala/io/gearpump/cluster/main/Local.scala
+++ b/daemon/src/main/scala/io/gearpump/cluster/main/Local.scala
@@ -21,7 +21,6 @@ package io.gearpump.cluster.main
import akka.actor.{ActorSystem, Props}
import com.typesafe.config.ConfigValueFactory
import io.gearpump.cluster.ClusterConfig
-import io.gearpump.cluster.local.LocalCluster
import io.gearpump.cluster.master.{Master => MasterActor}
import io.gearpump.cluster.worker.{Worker => WorkerActor}
import io.gearpump.util.Constants._
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/examples/streaming/wordcount-java/src/main/java/io/gearpump/streaming/examples/wordcountjava/WordCount.java
----------------------------------------------------------------------
diff --git a/examples/streaming/wordcount-java/src/main/java/io/gearpump/streaming/examples/wordcountjava/WordCount.java b/examples/streaming/wordcount-java/src/main/java/io/gearpump/streaming/examples/wordcountjava/WordCount.java
index 05d4e27..9e1e7d5 100644
--- a/examples/streaming/wordcount-java/src/main/java/io/gearpump/streaming/examples/wordcountjava/WordCount.java
+++ b/examples/streaming/wordcount-java/src/main/java/io/gearpump/streaming/examples/wordcountjava/WordCount.java
@@ -22,7 +22,7 @@ import com.typesafe.config.Config;
import io.gearpump.cluster.ClusterConfig;
import io.gearpump.cluster.UserConfig;
import io.gearpump.cluster.client.ClientContext;
-import io.gearpump.cluster.local.LocalCluster;
+import io.gearpump.cluster.embedded.EmbeddedCluster;
import io.gearpump.partitioner.HashPartitioner;
import io.gearpump.partitioner.Partitioner;
import io.gearpump.streaming.javaapi.Graph;
@@ -31,11 +31,11 @@ import io.gearpump.streaming.javaapi.StreamApplication;
public class WordCount {
- public static void main(String[] args) {
+ public static void main(String[] args) throws InterruptedException {
main(ClusterConfig.defaultConfig(), args);
}
- public static void main(Config akkaConf, String[] args) {
+ public static void main(Config akkaConf, String[] args) throws InterruptedException {
// For split task, we config to create two tasks
int splitTaskNumber = 2;
@@ -58,9 +58,12 @@ public class WordCount {
StreamApplication app = new StreamApplication("wordcountJava", conf, graph);
- LocalCluster localCluster = null;
- if (System.getProperty("DEBUG") != null) {
- localCluster = new LocalCluster(akkaConf);
+ EmbeddedCluster localCluster = null;
+
+ Boolean debugMode = System.getProperty("DEBUG") != null;
+
+ if (debugMode) {
+ localCluster = new EmbeddedCluster(akkaConf);
localCluster.start();
}
@@ -75,6 +78,11 @@ public class WordCount {
}
masterClient.submit(app);
+
+ if (debugMode) {
+ Thread.sleep(30 * 1000); // sleep for 30 seconds.
+ }
+
masterClient.close();
if (localCluster != null) {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/examples/streaming/wordcount/src/main/scala/io/gearpump/streaming/examples/wordcount/WordCount.scala
----------------------------------------------------------------------
diff --git a/examples/streaming/wordcount/src/main/scala/io/gearpump/streaming/examples/wordcount/WordCount.scala b/examples/streaming/wordcount/src/main/scala/io/gearpump/streaming/examples/wordcount/WordCount.scala
index 10faefe..22f8ac6 100644
--- a/examples/streaming/wordcount/src/main/scala/io/gearpump/streaming/examples/wordcount/WordCount.scala
+++ b/examples/streaming/wordcount/src/main/scala/io/gearpump/streaming/examples/wordcount/WordCount.scala
@@ -18,7 +18,7 @@
package io.gearpump.streaming.examples.wordcount
-import io.gearpump.cluster.local.LocalCluster
+import io.gearpump.cluster.embedded.{EmbeddedCluster}
import io.gearpump.streaming.{StreamApplication, Processor}
import io.gearpump.cluster.UserConfig
import io.gearpump.cluster.client.ClientContext
@@ -34,8 +34,10 @@ object WordCount extends AkkaApp with ArgumentsParser {
override val options: Array[(String, CLIOption[Any])] = Array(
"split" -> CLIOption[Int]("<how many split tasks>", required = false, defaultValue = Some(1)),
- "sum" -> CLIOption[Int]("<how many sum tasks>", required = false, defaultValue = Some(1))
- )
+ "sum" -> CLIOption[Int]("<how many sum tasks>", required = false, defaultValue = Some(1)),
+ "debug" -> CLIOption[Boolean]("<true|false>", required = false, defaultValue = Some(false)),
+ "sleep" -> CLIOption[Int]("how many seconds to sleep for debug mode", required = false, defaultValue = Some(30))
+ )
def application(config: ParseResult) : StreamApplication = {
val splitNum = config.getInt("split")
@@ -51,8 +53,11 @@ object WordCount extends AkkaApp with ArgumentsParser {
override def main(akkaConf: Config, args: Array[String]): Unit = {
val config = parse(args)
- val localCluster = if (System.getProperty("DEBUG") != null) {
- val cluster = new LocalCluster(akkaConf: Config)
+ val debugMode = config.getBoolean("debug")
+ val sleepSeconds = config.getInt("sleep")
+
+ val localCluster = if (debugMode) {
+ val cluster = new EmbeddedCluster(akkaConf: Config)
cluster.start
Some(cluster)
} else {
@@ -66,6 +71,11 @@ object WordCount extends AkkaApp with ArgumentsParser {
val app = application(config)
context.submit(app)
+
+ if (debugMode) {
+ Thread.sleep(sleepSeconds * 1000) // sleep for 30 seconds for debugging.
+ }
+
context.close()
localCluster.map(_.stop)
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/experiments/akkastream/src/main/scala/akka/stream/gearpump/graph/RemoteGraph.scala
----------------------------------------------------------------------
diff --git a/experiments/akkastream/src/main/scala/akka/stream/gearpump/graph/RemoteGraph.scala b/experiments/akkastream/src/main/scala/akka/stream/gearpump/graph/RemoteGraph.scala
index 96f1bf3..250c354 100644
--- a/experiments/akkastream/src/main/scala/akka/stream/gearpump/graph/RemoteGraph.scala
+++ b/experiments/akkastream/src/main/scala/akka/stream/gearpump/graph/RemoteGraph.scala
@@ -27,7 +27,7 @@ import akka.stream.gearpump.task.SourceBridgeTask.SourceBridgeTaskClient
import akka.stream.impl.StreamLayout.Module
import io.gearpump.cluster.ClusterConfig
import io.gearpump.cluster.client.ClientContext
-import io.gearpump.cluster.local.LocalCluster
+import io.gearpump.cluster.embedded.{EmbeddedCluster}
import io.gearpump.streaming.{StreamApplication, ProcessorId}
import io.gearpump.util.Graph
@@ -49,7 +49,7 @@ object RemoteGraph {
*/
class RemoteGraphMaterializer(useInProcessCluster: Boolean, system: ActorSystem) extends SubGraphMaterializer {
private val local = if (useInProcessCluster) {
- val cluster = LocalCluster()
+ val cluster = EmbeddedCluster()
cluster.start
Some(cluster)
} else {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/CommandLineSpec.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/CommandLineSpec.scala b/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/CommandLineSpec.scala
index 5831656..1c9fdfd 100644
--- a/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/CommandLineSpec.scala
+++ b/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/CommandLineSpec.scala
@@ -86,6 +86,14 @@ class CommandLineSpec extends TestSpecBase {
success shouldBe false
}
+ "the EmbededCluster can be used as embedded cluster in process" in {
+ // setup
+ val args = "-debug true -sleep 10"
+ val appId = expectSubmitAppSuccess(wordCountJar, args)
+ var success = commandLineClient.killApp(appId)
+ success shouldBe true
+ }
+
"should fail when attempting to kill a non-exist application" in {
// setup
val freeAppId = getNextAvailableAppId
@@ -110,7 +118,7 @@ class CommandLineSpec extends TestSpecBase {
commandLineClient.listApps().length + 1
}
- private def expectSubmitAppSuccess(jar: String): Int = {
+ private def expectSubmitAppSuccess(jar: String, args: String = ""): Int = {
val appId = commandLineClient.submitApp(jar)
appId should not equal -1
appId
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ecf5b468/streaming/src/main/scala/io/gearpump/streaming/task/TaskUtil.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/TaskUtil.scala b/streaming/src/main/scala/io/gearpump/streaming/task/TaskUtil.scala
index 79cf442..c6564ff 100644
--- a/streaming/src/main/scala/io/gearpump/streaming/task/TaskUtil.scala
+++ b/streaming/src/main/scala/io/gearpump/streaming/task/TaskUtil.scala
@@ -26,6 +26,7 @@ object TaskUtil {
* @return resolved class
*/
def loadClass(className: String): Class[_<:Task] = {
- Class.forName(className).asSubclass(classOf[Task])
+ val loader = Thread.currentThread().getContextClassLoader()
+ loader.loadClass(className).asSubclass(classOf[Task])
}
}
[41/50] incubator-gearpump git commit: Merge pull request #1976 from
manuzhang/fix_it
Posted by ap...@apache.org.
Merge pull request #1976 from manuzhang/fix_it
fix #1975, fix storm integration test
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/c3df2ed1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/c3df2ed1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/c3df2ed1
Branch: refs/heads/master
Commit: c3df2ed189d1435991e578208c83b13acd9a588f
Parents: 42806a9 920ad26
Author: ManuZhang <Ow...@gmail.com>
Authored: Wed Feb 24 19:49:35 2016 +0800
Committer: ManuZhang <Ow...@gmail.com>
Committed: Wed Feb 24 19:49:35 2016 +0800
----------------------------------------------------------------------
.../experiments/storm/main/GearpumpNimbus.scala | 27 ++++++----
.../storm/main/GearpumpStormClient.scala | 2 +-
.../checklist/StormCompatibilitySpec.scala | 51 +++++++++++--------
.../suites/StandaloneModeSuite.scala | 4 +-
.../integrationtest/storm/StormClient.scala | 52 +++++++++-----------
project/Build.scala | 3 +-
.../streaming/appmaster/ClockService.scala | 2 -
7 files changed, 74 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
[07/50] incubator-gearpump git commit: Merge pull request #1944 from
gearpump/revert-1942-standalone_nimbus
Posted by ap...@apache.org.
Merge pull request #1944 from gearpump/revert-1942-standalone_nimbus
Revert "fix #1895, make GearpumpNimbus a standalone service"
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/f136b3e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/f136b3e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/f136b3e7
Branch: refs/heads/master
Commit: f136b3e7f0cd7740bcec6ed3a3421e4cbb1e279f
Parents: 7f48c12 b8f8bb1
Author: ManuZhang <Ow...@gmail.com>
Authored: Tue Feb 2 07:57:07 2016 +0800
Committer: ManuZhang <Ow...@gmail.com>
Committed: Tue Feb 2 07:57:07 2016 +0800
----------------------------------------------------------------------
docs/dev-storm.md | 40 +--
.../gearpump/experiments/storm/Commands.scala | 36 +++
.../storm/GearpumpThriftServer.scala | 143 +++++++++++
.../experiments/storm/StormRunner.scala | 137 ++++++++--
.../experiments/storm/main/GearpumpNimbus.scala | 247 -------------------
.../storm/main/GearpumpStormClient.scala | 70 ------
.../storm/topology/GearpumpStormTopology.scala | 52 +++-
.../experiments/storm/GearpumpNimbusSpec.scala | 68 +++++
.../storm/GearpumpThriftServerSpec.scala | 48 ++++
.../experiments/storm/StormRunnerSpec.scala | 58 +++++
.../topology/GearpumpStormTopologySpec.scala | 20 +-
.../integrationtest/storm/StormClient.scala | 14 +-
project/Pack.scala | 2 +-
.../io/gearpump/services/MasterService.scala | 2 +-
14 files changed, 537 insertions(+), 400 deletions(-)
----------------------------------------------------------------------
[50/50] incubator-gearpump git commit: Merge pull request #2000 from
clockfly/issue_1996
Posted by ap...@apache.org.
Merge pull request #2000 from clockfly/issue_1996
fix #1996, EmbeddedCluster requires master configuration like ClusterActorRefProvider
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/d5343681
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/d5343681
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/d5343681
Branch: refs/heads/master
Commit: d5343681edde427022bf4c225ce6022a6904ae88
Parents: a7bbb4c ecf5b46
Author: Sean Zhong <xi...@intel.com>
Authored: Thu Mar 10 10:43:22 2016 +0800
Committer: Sean Zhong <xi...@intel.com>
Committed: Thu Mar 10 10:43:22 2016 +0800
----------------------------------------------------------------------
conf/gear.conf | 54 +++-
core/src/main/resources/geardefault.conf | 279 ++++++++++++-------
.../io/gearpump/cluster/ClusterConfig.scala | 10 +-
.../main/scala/io/gearpump/util/ActorUtil.scala | 4 -
.../main/scala/io/gearpump/util/Constants.scala | 2 +-
core/src/test/resources/test.conf | 10 +-
.../cluster/embedded/EmbeddedCluster.scala | 89 ++++++
.../gearpump/cluster/local/LocalCluster.scala | 88 ------
.../scala/io/gearpump/cluster/main/Local.scala | 1 -
.../examples/wordcountjava/WordCount.java | 20 +-
.../examples/wordcount/WordCount.scala | 20 +-
.../stream/gearpump/graph/RemoteGraph.scala | 4 +-
.../checklist/CommandLineSpec.scala | 10 +-
.../io/gearpump/streaming/task/TaskUtil.scala | 3 +-
14 files changed, 373 insertions(+), 221 deletions(-)
----------------------------------------------------------------------
[08/50] incubator-gearpump git commit: fix #1945 fix stability
integration test
Posted by ap...@apache.org.
fix #1945 fix stability integration test
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/7f3c3c57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/7f3c3c57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/7f3c3c57
Branch: refs/heads/master
Commit: 7f3c3c5787f3fa3370bb53e128c908817fd8b005
Parents: f136b3e
Author: huafengw <fv...@gmail.com>
Authored: Tue Feb 2 09:32:57 2016 +0800
Committer: huafengw <fv...@gmail.com>
Committed: Tue Feb 2 09:51:56 2016 +0800
----------------------------------------------------------------------
.../io/gearpump/integrationtest/checklist/StabilitySpec.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/7f3c3c57/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/StabilitySpec.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/StabilitySpec.scala b/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/StabilitySpec.scala
index 10500d2..0de08b6 100644
--- a/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/StabilitySpec.scala
+++ b/integrationtest/core/src/it/scala/io/gearpump/integrationtest/checklist/StabilitySpec.scala
@@ -95,7 +95,7 @@ class StabilitySpec extends TestSpecBase {
restartClusterRequired = true
val masters = cluster.getMasterHosts
val config = restClient.queryMasterConfig()
- val shutDownTimeout = Duration(config.getString("gearpump-master.akka.cluster.auto-down-unreachable-after"))
+ val shutDownTimeout = Duration(config.getString("akka.cluster.auto-down-unreachable-after"))
// exercise
masters.foreach(cluster.removeMasterNode)
[20/50] incubator-gearpump git commit: add 0.7.6 release notes
Posted by ap...@apache.org.
add 0.7.6 release notes
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/59e0f5a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/59e0f5a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/59e0f5a5
Branch: refs/heads/master
Commit: 59e0f5a546a61780d5e3c48f50b47965dce54b4e
Parents: 0d5e117
Author: manuzhang <ow...@gmail.com>
Authored: Thu Feb 4 16:37:38 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Thu Feb 4 16:37:38 2016 +0800
----------------------------------------------------------------------
notes/0.7.6.markdown | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/59e0f5a5/notes/0.7.6.markdown
----------------------------------------------------------------------
diff --git a/notes/0.7.6.markdown b/notes/0.7.6.markdown
new file mode 100644
index 0000000..1930160
--- /dev/null
+++ b/notes/0.7.6.markdown
@@ -0,0 +1,17 @@
+## Highlights:
+
+1. #1648 Support submit storm jar via dashboard
+2. Support StreamCQL over Gearpump
+
+## Change log:
+
+- #1924, fix storm test spec failure
+- #1926, fix client side configuration does not work in CGroup
+- #1648, support submit storm jar via rest interface
+- #1933, Fixes some of the documentation typos, adds SBT configuration
+- #1936, allow user to config kafka source start offset time
+- #1777, Dynamic DAG failed after changing parallelism for many times
+- #1932, Fix client config overwrites critical system config
+- #1895, make GearpumpNimbus a standalone service
+- #1910, enlarge timeout in MasterClient
+- #1952, document how to use StreamCQL over Gearpump
\ No newline at end of file
[39/50] incubator-gearpump git commit: fix #1972,
backoff retry kafka consuming on exception
Posted by ap...@apache.org.
fix #1972, backoff retry kafka consuming on exception
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/9b1085cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/9b1085cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/9b1085cf
Branch: refs/heads/master
Commit: 9b1085cfe04c7d161372b60c499a00e89aaede51
Parents: 1db8606
Author: manuzhang <ow...@gmail.com>
Authored: Fri Feb 19 12:14:36 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Fri Feb 19 21:20:42 2016 +0800
----------------------------------------------------------------------
.../consumer/ExponentialBackoffSleeper.scala | 58 ++++++++++
.../kafka/lib/consumer/FetchThread.scala | 56 ++++++++--
.../kafka/lib/consumer/KafkaConsumer.scala | 2 +
.../streaming/kafka/lib/FetchThreadSpec.scala | 108 -------------------
.../streaming/kafka/lib/KafkaConsumerSpec.scala | 89 ---------------
.../ExponentialBackoffSleeperSpec.scala | 72 +++++++++++++
.../kafka/lib/consumer/FetchThreadSpec.scala | 108 +++++++++++++++++++
.../kafka/lib/consumer/KafkaConsumerSpec.scala | 88 +++++++++++++++
8 files changed, 373 insertions(+), 208 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/9b1085cf/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/ExponentialBackoffSleeper.scala
----------------------------------------------------------------------
diff --git a/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/ExponentialBackoffSleeper.scala b/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/ExponentialBackoffSleeper.scala
new file mode 100644
index 0000000..b821f15
--- /dev/null
+++ b/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/ExponentialBackoffSleeper.scala
@@ -0,0 +1,58 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package io.gearpump.streaming.kafka.lib.consumer
+
+/**
+ * someone sleeps for exponentially increasing duration each time
+ * until the cap
+ *
+ * @param backOffMultiplier The factor by which the duration increases.
+ * @param initialDurationMs Time in milliseconds for initial sleep.
+ * @param maximumDurationMs Cap up to which we will increase the duration.
+ */
+private[consumer] class ExponentialBackoffSleeper(
+ backOffMultiplier: Double = 2.0,
+ initialDurationMs: Long = 100,
+ maximumDurationMs: Long = 10000) {
+
+ require(backOffMultiplier > 1.0, "backOffMultiplier must be greater than 1")
+ require(initialDurationMs > 0, "initialDurationMs must be positive")
+ require(maximumDurationMs >= initialDurationMs, "maximumDurationMs must be >= initialDurationMs")
+
+ private var sleepDuration = initialDurationMs
+
+ def reset(): Unit = {
+ sleepDuration = initialDurationMs
+ }
+
+ def sleep(): Unit = {
+ Thread.sleep(sleepDuration)
+ setNextSleepDuration()
+ }
+
+ def getSleepDuration: Long = sleepDuration
+
+ def setNextSleepDuration(): Unit = {
+ val next = (sleepDuration * backOffMultiplier).asInstanceOf[Long]
+ sleepDuration = math.min(math.max(initialDurationMs, next), maximumDurationMs)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/9b1085cf/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/FetchThread.scala
----------------------------------------------------------------------
diff --git a/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/FetchThread.scala b/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/FetchThread.scala
index 932939c..ee53151 100644
--- a/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/FetchThread.scala
+++ b/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/FetchThread.scala
@@ -34,29 +34,32 @@ object FetchThread {
fetchSleepMS: Long,
startOffsetTime: Long,
consumerConfig: ConsumerConfig): FetchThread = {
- val consumers: Map[TopicAndPartition, KafkaConsumer] = topicAndPartitions.map {
- tp =>
- tp -> KafkaConsumer(tp.topic, tp.partition, startOffsetTime, consumerConfig)
- }.toMap
+ val createConsumer = (tp: TopicAndPartition) =>
+ KafkaConsumer(tp.topic, tp.partition, startOffsetTime, consumerConfig)
+
val incomingQueue = new LinkedBlockingQueue[KafkaMessage]()
- new FetchThread(consumers, incomingQueue, fetchThreshold, fetchSleepMS)
+ new FetchThread(topicAndPartitions, createConsumer, incomingQueue, fetchThreshold, fetchSleepMS)
}
}
/**
* A thread to fetch messages from multiple kafka [[TopicAndPartition]]s and puts them
* onto a queue, which is asynchronously polled by a consumer
- * @param consumers [[KafkaConsumer]]s by kafka [[TopicAndPartition]]s
+ *
+ * @param createConsumer given a [[TopicAndPartition]], create a [[KafkaConsumer]] to connect to it
* @param incomingQueue a queue to buffer incoming messages
* @param fetchThreshold above which thread should stop fetching messages
* @param fetchSleepMS interval to sleep when no more messages or hitting fetchThreshold
*/
-private[kafka] class FetchThread(consumers: Map[TopicAndPartition, KafkaConsumer],
+private[kafka] class FetchThread(topicAndPartitions: Array[TopicAndPartition],
+ createConsumer: TopicAndPartition => KafkaConsumer,
incomingQueue: LinkedBlockingQueue[KafkaMessage],
fetchThreshold: Int,
fetchSleepMS: Long) extends Thread {
import FetchThread._
+ private var consumers: Map[TopicAndPartition, KafkaConsumer] = createAllConsumers
+
def setStartOffset(tp: TopicAndPartition, startOffset: Long): Unit = {
consumers(tp).setStartOffset(startOffset)
}
@@ -67,10 +70,29 @@ private[kafka] class FetchThread(consumers: Map[TopicAndPartition, KafkaConsumer
override def run(): Unit = {
try {
- while (!Thread.currentThread.isInterrupted) {
- val hasMoreMessages = fetchMessage
- if (!hasMoreMessages || incomingQueue.size >= fetchThreshold) {
- Thread.sleep(fetchSleepMS)
+ var nextOffsets = Map.empty[TopicAndPartition, Long]
+ var reset = false
+ val sleeper = new ExponentialBackoffSleeper(
+ backOffMultiplier = 2.0,
+ initialDurationMs = 100L,
+ maximumDurationMs = 10000L)
+ while (!Thread.currentThread().isInterrupted) {
+ try {
+ if (reset) {
+ nextOffsets = consumers.mapValues(_.getNextOffset)
+ resetConsumers(nextOffsets)
+ reset = false
+ }
+ val hasMoreMessages = fetchMessage
+ sleeper.reset()
+ if (!hasMoreMessages || incomingQueue.size >= fetchThreshold) {
+ Thread.sleep(fetchSleepMS)
+ }
+ } catch {
+ case exception: Exception =>
+ LOG.warn(s"resetting consumers due to $exception")
+ reset = true
+ sleeper.sleep()
}
}
} catch {
@@ -99,4 +121,16 @@ private[kafka] class FetchThread(consumers: Map[TopicAndPartition, KafkaConsumer
}
}
}
+
+ private def createAllConsumers: Map[TopicAndPartition, KafkaConsumer] = {
+ topicAndPartitions.map(tp => tp -> createConsumer(tp)).toMap
+ }
+
+ private def resetConsumers(nextOffsets: Map[TopicAndPartition, Long]): Unit = {
+ consumers.values.foreach(_.close())
+ consumers = createAllConsumers
+ consumers.foreach { case (tp, consumer) =>
+ consumer.setStartOffset(nextOffsets(tp))
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/9b1085cf/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/KafkaConsumer.scala
----------------------------------------------------------------------
diff --git a/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/KafkaConsumer.scala b/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/KafkaConsumer.scala
index 04b04ef..208a99d 100644
--- a/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/KafkaConsumer.scala
+++ b/external/kafka/src/main/scala/io/gearpump/streaming/kafka/lib/consumer/KafkaConsumer.scala
@@ -92,6 +92,8 @@ class KafkaConsumer(consumer: SimpleConsumer,
hasNextHelper(iterator, newIterator = false)
}
+ def getNextOffset: Long = nextOffset
+
def close(): Unit = {
consumer.close()
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/9b1085cf/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/FetchThreadSpec.scala
----------------------------------------------------------------------
diff --git a/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/FetchThreadSpec.scala b/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/FetchThreadSpec.scala
deleted file mode 100644
index af59296..0000000
--- a/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/FetchThreadSpec.scala
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.streaming.kafka.lib
-
-import java.util.concurrent.LinkedBlockingQueue
-
-import io.gearpump.streaming.kafka.lib.consumer.{FetchThread, KafkaMessage, KafkaConsumer}
-import kafka.common.TopicAndPartition
-import io.gearpump.streaming.kafka.lib.consumer.KafkaConsumer
-import org.mockito.Mockito._
-import org.scalacheck.Gen
-import org.scalatest.mock.MockitoSugar
-import org.scalatest.prop.PropertyChecks
-import org.scalatest.{Matchers, PropSpec}
-
-class FetchThreadSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
-
- val nonNegativeGen = Gen.choose[Int](0, 1000)
- val positiveGen = Gen.choose[Int](1, 1000)
- val startOffsetGen = Gen.choose[Long](0L, 1000L)
- property("FetchThread should set startOffset to iterators") {
- forAll(nonNegativeGen, nonNegativeGen, startOffsetGen) {
- (fetchThreshold: Int, fetchSleepMS: Int, startOffset: Long) =>
- val topicAndPartition = mock[TopicAndPartition]
- val consumer = mock[KafkaConsumer]
- val incomingQueue = new LinkedBlockingQueue[KafkaMessage]()
- val fetchThread = new FetchThread(Map(topicAndPartition -> consumer),
- incomingQueue, fetchThreshold, fetchSleepMS)
- fetchThread.setStartOffset(topicAndPartition, startOffset)
- verify(consumer).setStartOffset(startOffset)
- }
- }
-
- val topicAndPartitionGen = for {
- topic <- Gen.alphaStr
- partition <- Gen.choose[Int](0, Int.MaxValue)
- } yield TopicAndPartition(topic, partition)
- property("FetchThread should only fetchMessage when the number of messages in queue is below the threshold") {
- forAll(positiveGen, nonNegativeGen, nonNegativeGen, startOffsetGen, topicAndPartitionGen) {
- (messageNum: Int, fetchThreshold: Int, fetchSleepMS: Int,
- startOffset: Long, topicAndPartition: TopicAndPartition) =>
- val message = mock[KafkaMessage]
- val consumer = mock[KafkaConsumer]
- when(consumer.hasNext).thenReturn(true)
- when(consumer.next).thenReturn(message)
- val incomingQueue = new LinkedBlockingQueue[KafkaMessage]()
- val fetchThread = new FetchThread(
- Map(topicAndPartition -> consumer),
- incomingQueue, fetchThreshold, fetchSleepMS)
-
- 0.until(messageNum) foreach { _ =>
- fetchThread.fetchMessage
- }
-
- incomingQueue.size() shouldBe Math.min(messageNum, fetchThreshold)
- }
- }
-
- property("FetchThread poll should try to retrieve and remove the head of incoming queue") {
- val topicAndPartition = mock[TopicAndPartition]
- val consumer = mock[KafkaConsumer]
- val kafkaMsg = mock[KafkaMessage]
- val incomingQueue = new LinkedBlockingQueue[KafkaMessage]()
- incomingQueue.put(kafkaMsg)
- val fetchThread = new FetchThread(Map(topicAndPartition -> consumer), incomingQueue, 0, 0)
- fetchThread.poll shouldBe Some(kafkaMsg)
- fetchThread.poll shouldBe None
- }
-
- val tpAndHasNextGen = for {
- tp <- topicAndPartitionGen
- hasNext <- Gen.oneOf(true, false)
- } yield (tp, hasNext)
- val tpAndHasNextListGen = Gen.listOf[(TopicAndPartition, Boolean)](tpAndHasNextGen) suchThat (_.size > 0)
- property("FetchThread fetchMessage should return false when there are no more messages from any TopicAndPartition") {
- forAll(tpAndHasNextListGen, nonNegativeGen) {
- (tps: List[(TopicAndPartition, Boolean)], fetchSleepMS: Int) =>
- val tpAndIterators = tps.map { case (tp, hasNext) =>
- val consumer = mock[KafkaConsumer]
- val kafkaMsg = mock[KafkaMessage]
- when(consumer.hasNext).thenReturn(hasNext)
- when(consumer.next).thenReturn(kafkaMsg)
- tp -> consumer
- }.toMap
-
- val incomingQueue = new LinkedBlockingQueue[KafkaMessage]()
- val fetchThread = new FetchThread(
- tpAndIterators, incomingQueue, tpAndIterators.size + 1, fetchSleepMS)
- fetchThread.fetchMessage shouldBe tps.map(_._2).reduce(_ || _)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/9b1085cf/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/KafkaConsumerSpec.scala
----------------------------------------------------------------------
diff --git a/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/KafkaConsumerSpec.scala b/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/KafkaConsumerSpec.scala
deleted file mode 100644
index aae545c..0000000
--- a/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/KafkaConsumerSpec.scala
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.streaming.kafka.lib
-
-import com.twitter.bijection.Injection
-import io.gearpump.streaming.kafka.lib.consumer.KafkaConsumer
-import kafka.api.OffsetRequest
-import kafka.common.TopicAndPartition
-import kafka.consumer.SimpleConsumer
-import kafka.message.{Message, MessageAndOffset}
-import org.mockito.Mockito._
-import org.scalacheck.Gen
-import org.scalatest.mock.MockitoSugar
-import org.scalatest.prop.PropertyChecks
-import org.scalatest.{Matchers, PropSpec}
-
-class KafkaConsumerSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
- val messageGen = Gen.alphaStr map (msg => new Message(Injection[String, Array[Byte]](msg)))
- val messageNumGen = Gen.choose[Int](0, 1000)
- val topicAndPartitionGen = for {
- topic <- Gen.alphaStr
- partition <- Gen.choose[Int](0, Int.MaxValue)
- } yield (topic, partition)
-
- property("KafkaConsumer should iterate MessageAndOffset calling hasNext and next") {
- forAll(messageGen, messageNumGen, topicAndPartitionGen) {
- (message: Message, num: Int, topicAndPartition: (String, Int)) =>
- val (topic, partition) = topicAndPartition
- val consumer = mock[SimpleConsumer]
- when(consumer.earliestOrLatestOffset(TopicAndPartition(topic, partition),
- OffsetRequest.EarliestTime, -1)).thenReturn(0)
- val iterator = 0.until(num).map(index => MessageAndOffset(message, index.toLong)).iterator
- val getIterator = (offset: Long) => iterator
- val kafkaConsumer = new KafkaConsumer(consumer, topic, partition, getIterator)
- 0.until(num).foreach { i =>
- kafkaConsumer.hasNext shouldBe true
- val kafkaMessage = kafkaConsumer.next
- kafkaMessage.offset shouldBe i.toLong
- kafkaMessage.key shouldBe None
- }
- kafkaConsumer.hasNext shouldBe false
- }
- }
-
- val startOffsetGen = Gen.choose[Long](1L, 1000L)
- property("KafkaConsumer setStartOffset should reset internal iterator") {
- forAll(topicAndPartitionGen, startOffsetGen) {
- (topicAndPartition: (String, Int), startOffset: Long) =>
- val (topic, partition) = topicAndPartition
- val consumer = mock[SimpleConsumer]
- val getIterator = mock[Long => Iterator[MessageAndOffset]]
- when(consumer.earliestOrLatestOffset(TopicAndPartition(topic, partition),
- OffsetRequest.EarliestTime, -1)).thenReturn(0)
- val kafkaConsumer = new KafkaConsumer(consumer, topic, partition, getIterator)
- kafkaConsumer.setStartOffset(startOffset)
- verify(getIterator).apply(startOffset)
- }
- }
-
- property("KafkaConsumer close should close SimpleConsumer") {
- forAll(topicAndPartitionGen) {
- (topicAndPartition: (String, Int)) =>
- val (topic, partition) = topicAndPartition
- val consumer = mock[SimpleConsumer]
- when(consumer.earliestOrLatestOffset(TopicAndPartition(topic, partition),
- OffsetRequest.EarliestTime, -1)).thenReturn(0)
- val getIterator = mock[Long => Iterator[MessageAndOffset]]
- val kafkaConsumer = new KafkaConsumer(consumer, topic, partition, getIterator)
- kafkaConsumer.close()
- verify(consumer).close()
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/9b1085cf/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/ExponentialBackoffSleeperSpec.scala
----------------------------------------------------------------------
diff --git a/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/ExponentialBackoffSleeperSpec.scala b/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/ExponentialBackoffSleeperSpec.scala
new file mode 100644
index 0000000..a20e575
--- /dev/null
+++ b/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/ExponentialBackoffSleeperSpec.scala
@@ -0,0 +1,72 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package io.gearpump.streaming.kafka.lib.consumer
+
+import org.scalatest.{WordSpec, Matchers}
+
+class ExponentialBackoffSleeperSpec extends WordSpec with Matchers {
+
+ "ExponentialBackOffSleeper" should {
+ "sleep for increasing duration" in {
+ val sleeper = new ExponentialBackoffSleeper(
+ backOffMultiplier = 2.0,
+ initialDurationMs = 100,
+ maximumDurationMs = 10000
+ )
+ sleeper.getSleepDuration shouldBe 100
+ sleeper.setNextSleepDuration()
+ sleeper.getSleepDuration shouldBe 200
+ sleeper.setNextSleepDuration()
+ sleeper.getSleepDuration shouldBe 400
+ sleeper.setNextSleepDuration()
+ sleeper.getSleepDuration shouldBe 800
+ }
+
+ "sleep for no more than maximum duration" in {
+ val sleeper = new ExponentialBackoffSleeper(
+ backOffMultiplier = 2.0,
+ initialDurationMs = 6400,
+ maximumDurationMs = 10000
+ )
+ sleeper.getSleepDuration shouldBe 6400
+ sleeper.setNextSleepDuration()
+ sleeper.getSleepDuration shouldBe 10000
+ sleeper.setNextSleepDuration()
+ sleeper.getSleepDuration shouldBe 10000
+ }
+
+ "sleep for initial duration after reset" in {
+ val sleeper = new ExponentialBackoffSleeper(
+ backOffMultiplier = 2.0,
+ initialDurationMs = 100,
+ maximumDurationMs = 10000
+ )
+ sleeper.getSleepDuration shouldBe 100
+ sleeper.setNextSleepDuration()
+ sleeper.getSleepDuration shouldBe 200
+ sleeper.reset()
+ sleeper.getSleepDuration shouldBe 100
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/9b1085cf/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/FetchThreadSpec.scala
----------------------------------------------------------------------
diff --git a/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/FetchThreadSpec.scala b/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/FetchThreadSpec.scala
new file mode 100644
index 0000000..92d3c31
--- /dev/null
+++ b/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/FetchThreadSpec.scala
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.streaming.kafka.lib.consumer
+
+import java.util.concurrent.LinkedBlockingQueue
+
+import kafka.common.TopicAndPartition
+import org.mockito.Mockito._
+import org.scalacheck.Gen
+import org.scalatest.mock.MockitoSugar
+import org.scalatest.prop.PropertyChecks
+import org.scalatest.{Matchers, PropSpec}
+
+class FetchThreadSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
+
+ val nonNegativeGen = Gen.choose[Int](0, 1000)
+ val positiveGen = Gen.choose[Int](1, 1000)
+ val startOffsetGen = Gen.choose[Long](0L, 1000L)
+ property("FetchThread should set startOffset to iterators") {
+ forAll(nonNegativeGen, nonNegativeGen, startOffsetGen) {
+ (fetchThreshold: Int, fetchSleepMS: Int, startOffset: Long) =>
+ val topicAndPartition = mock[TopicAndPartition]
+ val consumer = mock[KafkaConsumer]
+ val createConsumer = (tp: TopicAndPartition) => consumer
+ val incomingQueue = new LinkedBlockingQueue[KafkaMessage]()
+ val fetchThread = new FetchThread(Array(topicAndPartition), createConsumer,
+ incomingQueue, fetchThreshold, fetchSleepMS)
+ fetchThread.setStartOffset(topicAndPartition, startOffset)
+ verify(consumer).setStartOffset(startOffset)
+ }
+ }
+
+ val topicAndPartitionGen = for {
+ topic <- Gen.alphaStr
+ partition <- Gen.choose[Int](0, Int.MaxValue)
+ } yield TopicAndPartition(topic, partition)
+ property("FetchThread should only fetchMessage when the number of messages in queue is below the threshold") {
+ forAll(positiveGen, nonNegativeGen, nonNegativeGen, startOffsetGen, topicAndPartitionGen) {
+ (messageNum: Int, fetchThreshold: Int, fetchSleepMS: Int,
+ startOffset: Long, topicAndPartition: TopicAndPartition) =>
+ val message = mock[KafkaMessage]
+ val consumer = mock[KafkaConsumer]
+ val createConsumer = (tp: TopicAndPartition) => consumer
+ when(consumer.hasNext).thenReturn(true)
+ when(consumer.next).thenReturn(message)
+ val incomingQueue = new LinkedBlockingQueue[KafkaMessage]()
+ val fetchThread = new FetchThread(Array(topicAndPartition), createConsumer,
+ incomingQueue, fetchThreshold, fetchSleepMS)
+
+ 0.until(messageNum) foreach { _ =>
+ fetchThread.fetchMessage
+ }
+
+ incomingQueue.size() shouldBe Math.min(messageNum, fetchThreshold)
+ }
+ }
+
+ property("FetchThread poll should try to retrieve and remove the head of incoming queue") {
+ val topicAndPartition = mock[TopicAndPartition]
+ val consumer = mock[KafkaConsumer]
+ val createConsumer = (tp: TopicAndPartition) => consumer
+ val kafkaMsg = mock[KafkaMessage]
+ val incomingQueue = new LinkedBlockingQueue[KafkaMessage]()
+ incomingQueue.put(kafkaMsg)
+ val fetchThread = new FetchThread(Array(topicAndPartition), createConsumer, incomingQueue, 0, 0)
+ fetchThread.poll shouldBe Some(kafkaMsg)
+ fetchThread.poll shouldBe None
+ }
+
+ val tpAndHasNextGen = for {
+ tp <- topicAndPartitionGen
+ hasNext <- Gen.oneOf(true, false)
+ } yield (tp, hasNext)
+ val tpHasNextMapGen = Gen.listOf[(TopicAndPartition, Boolean)](tpAndHasNextGen).map(_.toMap) suchThat (_.nonEmpty)
+ property("FetchThread fetchMessage should return false when there are no more messages from any TopicAndPartition") {
+ forAll(tpHasNextMapGen, nonNegativeGen) {
+ (tpHasNextMap: Map[TopicAndPartition, Boolean], fetchSleepMS: Int) =>
+ val createConsumer = (tp: TopicAndPartition) => {
+ val consumer = mock[KafkaConsumer]
+ val kafkaMsg = mock[KafkaMessage]
+ val hasNext = tpHasNextMap(tp)
+ when(consumer.hasNext).thenReturn(hasNext)
+ when(consumer.next).thenReturn(kafkaMsg)
+ consumer
+ }
+ val incomingQueue = new LinkedBlockingQueue[KafkaMessage]()
+ val fetchThread = new FetchThread(tpHasNextMap.keys.toArray,
+ createConsumer, incomingQueue, tpHasNextMap.size + 1, fetchSleepMS)
+ fetchThread.fetchMessage shouldBe tpHasNextMap.values.reduce(_ || _)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/9b1085cf/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/KafkaConsumerSpec.scala
----------------------------------------------------------------------
diff --git a/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/KafkaConsumerSpec.scala b/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/KafkaConsumerSpec.scala
new file mode 100644
index 0000000..2669f60
--- /dev/null
+++ b/external/kafka/src/test/scala/io/gearpump/streaming/kafka/lib/consumer/KafkaConsumerSpec.scala
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.streaming.kafka.lib.consumer
+
+import com.twitter.bijection.Injection
+import kafka.api.OffsetRequest
+import kafka.common.TopicAndPartition
+import kafka.consumer.SimpleConsumer
+import kafka.message.{Message, MessageAndOffset}
+import org.mockito.Mockito._
+import org.scalacheck.Gen
+import org.scalatest.mock.MockitoSugar
+import org.scalatest.prop.PropertyChecks
+import org.scalatest.{Matchers, PropSpec}
+
+class KafkaConsumerSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar {
+ val messageGen = Gen.alphaStr map (msg => new Message(Injection[String, Array[Byte]](msg)))
+ val messageNumGen = Gen.choose[Int](0, 1000)
+ val topicAndPartitionGen = for {
+ topic <- Gen.alphaStr
+ partition <- Gen.choose[Int](0, Int.MaxValue)
+ } yield (topic, partition)
+
+ property("KafkaConsumer should iterate MessageAndOffset calling hasNext and next") {
+ forAll(messageGen, messageNumGen, topicAndPartitionGen) {
+ (message: Message, num: Int, topicAndPartition: (String, Int)) =>
+ val (topic, partition) = topicAndPartition
+ val consumer = mock[SimpleConsumer]
+ when(consumer.earliestOrLatestOffset(TopicAndPartition(topic, partition),
+ OffsetRequest.EarliestTime, -1)).thenReturn(0)
+ val iterator = 0.until(num).map(index => MessageAndOffset(message, index.toLong)).iterator
+ val getIterator = (offset: Long) => iterator
+ val kafkaConsumer = new KafkaConsumer(consumer, topic, partition, getIterator)
+ 0.until(num).foreach { i =>
+ kafkaConsumer.hasNext shouldBe true
+ val kafkaMessage = kafkaConsumer.next
+ kafkaMessage.offset shouldBe i.toLong
+ kafkaMessage.key shouldBe None
+ }
+ kafkaConsumer.hasNext shouldBe false
+ }
+ }
+
+ val startOffsetGen = Gen.choose[Long](1L, 1000L)
+ property("KafkaConsumer setStartOffset should reset internal iterator") {
+ forAll(topicAndPartitionGen, startOffsetGen) {
+ (topicAndPartition: (String, Int), startOffset: Long) =>
+ val (topic, partition) = topicAndPartition
+ val consumer = mock[SimpleConsumer]
+ val getIterator = mock[Long => Iterator[MessageAndOffset]]
+ when(consumer.earliestOrLatestOffset(TopicAndPartition(topic, partition),
+ OffsetRequest.EarliestTime, -1)).thenReturn(0)
+ val kafkaConsumer = new KafkaConsumer(consumer, topic, partition, getIterator)
+ kafkaConsumer.setStartOffset(startOffset)
+ verify(getIterator).apply(startOffset)
+ }
+ }
+
+ property("KafkaConsumer close should close SimpleConsumer") {
+ forAll(topicAndPartitionGen) {
+ (topicAndPartition: (String, Int)) =>
+ val (topic, partition) = topicAndPartition
+ val consumer = mock[SimpleConsumer]
+ when(consumer.earliestOrLatestOffset(TopicAndPartition(topic, partition),
+ OffsetRequest.EarliestTime, -1)).thenReturn(0)
+ val getIterator = mock[Long => Iterator[MessageAndOffset]]
+ val kafkaConsumer = new KafkaConsumer(consumer, topic, partition, getIterator)
+ kafkaConsumer.close()
+ verify(consumer).close()
+ }
+ }
+}
[30/50] incubator-gearpump git commit: fix #1963,
rename CommitGuideline.md to CONTRIBUTING.md
Posted by ap...@apache.org.
fix #1963, rename CommitGuideline.md to CONTRIBUTING.md
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/0851765b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/0851765b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/0851765b
Branch: refs/heads/master
Commit: 0851765ba7ba234b2d18b0a686555a5dc8ab90a7
Parents: 9405961
Author: manuzhang <ow...@gmail.com>
Authored: Sat Feb 13 18:02:20 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Sat Feb 13 18:02:56 2016 +0800
----------------------------------------------------------------------
CONTRIBUTING.md | 92 +++++++++++++++++++++++++++++++++++++++++++++++++
CommitGuideline.md | 92 -------------------------------------------------
2 files changed, 92 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0851765b/CONTRIBUTING.md
----------------------------------------------------------------------
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..a523344
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,92 @@
+This documents are for committers which have direct write permission to intel-hadoop/gearpump
+
+Commit Guideline
+========================
+1. For all commit(except doc), must create an issue id.
+2. For all commit log message, must contain the issue id, Like this: fix #issueId, comments.
+3. For all PR, the title must contains issue Id.
+4. We use rebase and squash instead of merge to ensure the log message is clean. Check section "Pull Request merge process for Gearpump"
+5. Every commit (except doc) must have 1 guy to review before commit.
+
+Pull Request merge process for Gearpump
+========================
+1. Fork in github to create a /gearpump repo. After fork, you will have a new repo at http://github.com/<git-user_id>/gearpump.
+2. Add gearpump/gearpump as an external repo 'upstream' by following the [guide](https://help.github.com/articles/configuring-a-remote-for-a-fork/).
+
+ ```bash
+ git remote add upstream https://github.com/gearpump/gearpump.git
+ ```
+
+3. In local master branch, periodically sync the forked master with the main master with
+
+ ```
+ git pull --rebase upstream master
+ git push origin master
+ ```
+No work should ever be done in the forked master. Another way to do this is to
+
+ ```
+ git checkout master
+ git fetch upstream
+ git rebase upstream/master
+ ```
+
+4. Create a working branch
+
+ ```
+ git branch branch_issueId
+ git checkout branch_issusId
+ ```
+
+ Work on the branch, make changes, then push to your forked gearpump repo http://github.com/<git-user_id>/gearpump
+
+ ```
+ git push origin branch_issueId
+ ```
+
+5. When there is changes in upstream/master, rebase your work on upstream/master. with
+
+ ```bash
+ git checkout branch_issueId
+ git rebase -i upstream/master
+ ```
+
+ You can also use this when you want to squash(merge) multiple commits into one.
+ ```git rebase -i``` will popup a window, which allow you to squash(merge) multiple commits into one commit.
+ For example I might have 12 commits in my branch. "rebase -i upstream/master" opens a nice editor where you can mark some commits to be squashed(merged) into prior commits, and make 1 big commit (or several) out of it. In this way, I can tidy up what will be committed to the project master's history since otherwise my commit messages are like "not working" or "got it working" or "more fix" or "merged <git-user-id>/gearpump to master".
+
+6. If there is conflict, resolve the conflict, and then
+
+ ```bash
+ git rebase --continue
+ ```
+
+ After the code is successfully rebased, a window will pop up to edit the commit log, edit it then save and exit.
+7. After rebase, now you have a clean log history. push to your remote working branch
+
+ ```
+ git push origin branch_issueId
+ ```
+
+ If commits have already been pushed to <git-user-id>/gearpump fork on github, you will have to "git push --force" to overwrite them with squashed commits.
+
+ ```
+ git push origin -f branch_issueId
+ ```
+
+8. Ensure all the unit tests are passed by running command "sbt test".
+6. Open a PR, which is a one-click thing in github.com; it knows you likely are opening a PR against upstream master. [Guide](https://help.github.com/articles/creating-a-pull-request) is here.
+7. [Merge PR](https://help.github.com/articles/merging-a-pull-request), [delete branch](https://help.github.com/articles/deleting-unused-branches).
+
+
+You can skip using branches in your fork if desired, and just work out of your master. Less overhead, but, you can't pursue different unrelated changes then.
+
+The key here is using rebase, not merge. The downstream forks never need to use merge unless the work is a long-lived collaboration with someone else who is pulling from your fork.
+
+More details on Squash before pull request
+---------------------------
+More about squash, please check
+
+http://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html
+
+https://github.com/edx/edx-platform/wiki/How-to-Rebase-a-Pull-Request
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0851765b/CommitGuideline.md
----------------------------------------------------------------------
diff --git a/CommitGuideline.md b/CommitGuideline.md
deleted file mode 100644
index a523344..0000000
--- a/CommitGuideline.md
+++ /dev/null
@@ -1,92 +0,0 @@
-This documents are for committers which have direct write permission to intel-hadoop/gearpump
-
-Commit Guideline
-========================
-1. For all commit(except doc), must create an issue id.
-2. For all commit log message, must contain the issue id, Like this: fix #issueId, comments.
-3. For all PR, the title must contains issue Id.
-4. We use rebase and squash instead of merge to ensure the log message is clean. Check section "Pull Request merge process for Gearpump"
-5. Every commit (except doc) must have 1 guy to review before commit.
-
-Pull Request merge process for Gearpump
-========================
-1. Fork in github to create a /gearpump repo. After fork, you will have a new repo at http://github.com/<git-user_id>/gearpump.
-2. Add gearpump/gearpump as an external repo 'upstream' by following the [guide](https://help.github.com/articles/configuring-a-remote-for-a-fork/).
-
- ```bash
- git remote add upstream https://github.com/gearpump/gearpump.git
- ```
-
-3. In local master branch, periodically sync the forked master with the main master with
-
- ```
- git pull --rebase upstream master
- git push origin master
- ```
-No work should ever be done in the forked master. Another way to do this is to
-
- ```
- git checkout master
- git fetch upstream
- git rebase upstream/master
- ```
-
-4. Create a working branch
-
- ```
- git branch branch_issueId
- git checkout branch_issusId
- ```
-
- Work on the branch, make changes, then push to your forked gearpump repo http://github.com/<git-user_id>/gearpump
-
- ```
- git push origin branch_issueId
- ```
-
-5. When there is changes in upstream/master, rebase your work on upstream/master. with
-
- ```bash
- git checkout branch_issueId
- git rebase -i upstream/master
- ```
-
- You can also use this when you want to squash(merge) multiple commits into one.
- ```git rebase -i``` will popup a window, which allow you to squash(merge) multiple commits into one commit.
- For example I might have 12 commits in my branch. "rebase -i upstream/master" opens a nice editor where you can mark some commits to be squashed(merged) into prior commits, and make 1 big commit (or several) out of it. In this way, I can tidy up what will be committed to the project master's history since otherwise my commit messages are like "not working" or "got it working" or "more fix" or "merged <git-user-id>/gearpump to master".
-
-6. If there is conflict, resolve the conflict, and then
-
- ```bash
- git rebase --continue
- ```
-
- After the code is successfully rebased, a window will pop up to edit the commit log, edit it then save and exit.
-7. After rebase, now you have a clean log history. push to your remote working branch
-
- ```
- git push origin branch_issueId
- ```
-
- If commits have already been pushed to <git-user-id>/gearpump fork on github, you will have to "git push --force" to overwrite them with squashed commits.
-
- ```
- git push origin -f branch_issueId
- ```
-
-8. Ensure all the unit tests are passed by running command "sbt test".
-6. Open a PR, which is a one-click thing in github.com; it knows you likely are opening a PR against upstream master. [Guide](https://help.github.com/articles/creating-a-pull-request) is here.
-7. [Merge PR](https://help.github.com/articles/merging-a-pull-request), [delete branch](https://help.github.com/articles/deleting-unused-branches).
-
-
-You can skip using branches in your fork if desired, and just work out of your master. Less overhead, but, you can't pursue different unrelated changes then.
-
-The key here is using rebase, not merge. The downstream forks never need to use merge unless the work is a long-lived collaboration with someone else who is pulling from your fork.
-
-More details on Squash before pull request
----------------------------
-More about squash, please check
-
-http://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html
-
-https://github.com/edx/edx-platform/wiki/How-to-Rebase-a-Pull-Request
[17/50] incubator-gearpump git commit: bump to 0.7.6
Posted by ap...@apache.org.
bump to 0.7.6
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/2aa3f22e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/2aa3f22e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/2aa3f22e
Branch: refs/heads/master
Commit: 2aa3f22ea13d19bb91c89e63cfe2594635bffd77
Parents: e479c7e
Author: huafengw <fv...@gmail.com>
Authored: Thu Feb 4 13:04:18 2016 +0800
Committer: huafengw <fv...@gmail.com>
Committed: Thu Feb 4 13:04:18 2016 +0800
----------------------------------------------------------------------
CHANGELOG.md | 20 ++++++++++++++++++++
docs/_config.yml | 2 +-
version.sbt | 2 +-
3 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/2aa3f22e/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4f3fae0..bbec111 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,23 @@
+Gearpump 0.7.6
+===================
+Highlights:
+-------------------
+1. #1648 Support submit storm jar via dashboard
+2. Support StreamCQL over Gearpump
+
+Change log:
+-------------------
+- #1924, fix storm test spec failure
+- #1926, fix client side configuration does not work in CGroup
+- #1648, support submit storm jar via rest interface
+- #1933, Fixes some of the documentation typos, adds SBT configuration
+- #1936, allow user to config kafka source start offset time
+- #1777, Dynamic DAG failed after changing parallelism for many times
+- #1932, Fix client config overwrites critical system config
+- #1895, make GearpumpNimbus a standalone service
+- #1910, enlarge timeout in MasterClient
+- #1952, document how to use StreamCQL over Gearpump
+
Gearpump 0.7.5
===================
Highlights:
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/2aa3f22e/docs/_config.yml
----------------------------------------------------------------------
diff --git a/docs/_config.yml b/docs/_config.yml
index 0d6f7a4..12bfc4b 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -16,7 +16,7 @@ exclude:
- build_doc.sh
# These allow the documentation to be updated with newer releases of Gearpump
-GEARPUMP_VERSION: 0.7.6-SNAPSHOT
+GEARPUMP_VERSION: 0.7.6
SCALA_BINARY_VERSION: "2.11"
SCALA_VERSION: "2.11.7"
GEARPUMP_ISSUE_TRACKER_URL: https://github.com/gearpump/gearpump/issues
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/2aa3f22e/version.sbt
----------------------------------------------------------------------
diff --git a/version.sbt b/version.sbt
index 0f57559..af88c45 100644
--- a/version.sbt
+++ b/version.sbt
@@ -1 +1 @@
-version in ThisBuild := "0.7.6-SNAPSHOT"
+version in ThisBuild := "0.7.6"
[33/50] incubator-gearpump git commit: fix #1892: added instruction
text to operate network graph (2) minor tweaks of context menu
Posted by ap...@apache.org.
fix #1892: added instruction text to operate network graph (2) minor tweaks of context menu
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/3bf290e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/3bf290e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/3bf290e4
Branch: refs/heads/master
Commit: 3bf290e4f3cd117923381d88d134f44835a1f504
Parents: 9feb159
Author: Qian Xu <qi...@intel.com>
Authored: Sun Feb 14 16:29:06 2016 +0800
Committer: Qian Xu <qi...@intel.com>
Committed: Mon Feb 15 10:23:10 2016 +0800
----------------------------------------------------------------------
services/dashboard/index.html | 4 +-
services/dashboard/views/apps/apps.js | 4 +-
.../dashboard/views/apps/compose/compose.js | 6 +-
.../dashboard/views/apps/streamingapp/dag.html | 8 ++-
.../dashboard/views/apps/streamingapp/dag.js | 48 +++++++++++---
.../views/apps/streamingapp/popups/dag_edit.js | 2 +-
.../streamingapp/popups/processor_menu.html | 15 -----
.../views/apps/streamingapp/vis/network.js | 69 +++++++++++---------
.../views/widgets/metrics_period_switcher.html | 3 -
.../views/widgets/metrics_period_switcher.js | 32 ---------
.../dashboard/views/widgets/radio_group.html | 12 ----
services/dashboard/views/widgets/radio_group.js | 29 --------
.../widgets/metrics_period_switcher.html | 3 +
.../widgets/metrics_period_switcher.js | 32 +++++++++
services/dashboard/widgets/radio_group.html | 12 ++++
services/dashboard/widgets/radio_group.js | 29 ++++++++
16 files changed, 164 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/index.html
----------------------------------------------------------------------
diff --git a/services/dashboard/index.html b/services/dashboard/index.html
index 47007cc..a3bbaa9 100644
--- a/services/dashboard/index.html
+++ b/services/dashboard/index.html
@@ -77,8 +77,8 @@
<script src="services/restapi.js"></script>
<script src="services/health_check_service.js"></script>
<script src="services/login_check.js"></script>
-<script src="views/widgets/radio_group.js"></script>
-<script src="views/widgets/metrics_period_switcher.js"></script>
+<script src="widgets/radio_group.js"></script>
+<script src="widgets/metrics_period_switcher.js"></script>
<script src="views/landing/breadcrumbs.js"></script>
<script src="views/landing/header.js"></script>
<script src="views/cluster/master/master.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/apps/apps.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/apps.js b/services/dashboard/views/apps/apps.js
index 4a4f7d6..669d018 100644
--- a/services/dashboard/views/apps/apps.js
+++ b/services/dashboard/views/apps/apps.js
@@ -45,10 +45,10 @@ angular.module('dashboard')
};
$scope.composeMenuOptions = [{
- text: '<i class="glyphicon glyphicon-none"></i> <b>Submit Gearpump Application</b>',
+ text: '<i class="glyphicon glyphicon-none"></i> <b>Submit Gearpump Application...</b>',
click: $scope.openSubmitGearAppDialog
}, {
- text: '<i class="glyphicon glyphicon-none"></i> Submit Storm Application',
+ text: '<i class="glyphicon glyphicon-none"></i> Submit Storm Application...',
click: $scope.openSubmitStormAppDialog
}, {
text: '<i class="glyphicon glyphicon-pencil"></i> Compose DAG',
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/apps/compose/compose.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/compose/compose.js b/services/dashboard/views/apps/compose/compose.js
index d1ff1c6..33e9263 100644
--- a/services/dashboard/views/apps/compose/compose.js
+++ b/services/dashboard/views/apps/compose/compose.js
@@ -79,7 +79,7 @@ angular.module('dashboard')
options: $vis.newOptions(/*height=*/'400px'),
data: $vis.newData(),
events: {
- doubleClick: function(data) {
+ onDoubleClick: function(data) {
if (data.nodes.length === 1) {
var processor = $scope.visGraph.data.nodes.get(data.nodes[0]);
$scope.chooseProcessor(processor);
@@ -90,7 +90,7 @@ angular.module('dashboard')
$scope.chooseProcessor();
}
},
- oncontext: function(data) {
+ onContext: function(data) {
if (data.hasOwnProperty('node')) {
$scope.selectItemModify = function() {
$scope.chooseProcessor($scope.visGraph.data.nodes.get(data.node));
@@ -111,7 +111,7 @@ angular.module('dashboard')
var elem = document.getElementById('contextmenu');
$contextmenu.popup(elem, data.pointer.DOM);
},
- ondeletepressed: function(selection) {
+ onDeletePressed: function(selection) {
if (selection.nodes.length === 1) {
deleteProcessor(selection.nodes[0]);
} else if (selection.edges.length === 1) {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/apps/streamingapp/dag.html
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/streamingapp/dag.html b/services/dashboard/views/apps/streamingapp/dag.html
index 5b2928d..93655cb 100644
--- a/services/dashboard/views/apps/streamingapp/dag.html
+++ b/services/dashboard/views/apps/streamingapp/dag.html
@@ -1,10 +1,12 @@
<div class="col-md-8">
<div class="panel panel-sm panel-default">
- <div class="panel-heading">DAG</div>
+ <div class="panel-heading">DAG
+ <remark class="pull-right" type="info"
+ tooltip="<b>Right click (or long press on touch device)</b> on a processor will show all available operations. <b>Double click</b> a processor will show its details.">Instructions</remark>
+ </div>
<div class="panel-body">
<div class="cm-container" id="dag-node-menu"
- bs-dropdown html="true"
- template-url="views/apps/streamingapp/popups/processor_menu.html"></div>
+ bs-dropdown="processorActionMenu" html="true"></div>
<vis-network
data="visGraph.data"
options="visGraph.options"
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/apps/streamingapp/dag.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/streamingapp/dag.js b/services/dashboard/views/apps/streamingapp/dag.js
index 0c3f11e..310f555 100644
--- a/services/dashboard/views/apps/streamingapp/dag.js
+++ b/services/dashboard/views/apps/streamingapp/dag.js
@@ -21,7 +21,17 @@ angular.module('dashboard')
function($scope, $state, $modal, $contextmenu, $vis) {
'use strict';
- // todo: while metrics is not available, keep loading, otherwise the bandwidth is not calculable.
+ $scope.activeProcessorId = -1;
+ $scope.processorActionMenu = [{
+ text: '<i class="glyphicon glyphicon-stats"></i> <b>View Details</b>',
+ click: 'view()'
+ }, {
+ text: '<i class="glyphicon glyphicon-none"></i> Change Parallelism...',
+ click: 'modify({parallelism:true})'
+ }, {
+ text: '<i class="glyphicon glyphicon-none"></i> Change More...',
+ click: 'modify()'
+ }];
var editorDialog = $modal({
templateUrl: "views/apps/streamingapp/popups/dag_edit.html",
@@ -31,6 +41,11 @@ angular.module('dashboard')
controller: 'StreamingAppDagEditCtrl'
});
+ $scope.view = function() {
+ var options = {processorId: $scope.activeProcessorId};
+ $state.go('streamingapp.processor', options);
+ };
+
$scope.modify = function(options) {
$scope.modifyOptions = options;
editorDialog.$promise.then(editorDialog.show);
@@ -40,27 +55,40 @@ angular.module('dashboard')
options: $vis.newHierarchicalLayoutOptions({depth: $scope.dag.hierarchyDepth()}),
data: $vis.newData(),
events: {
- doubleClick: function(data) {
+ onDoubleClick: function(data) {
if (data.nodes.length === 1) {
var processorId = Number(data.nodes[0]);
$state.go('streamingapp.processor', {processorId: processorId});
}
},
- oncontext: function(data) {
+ onContext: function(data) {
if (data.hasOwnProperty('node')) {
- var processorId = Number(data.node);
- $scope.activeProcessor = $scope.dag.getProcessor(processorId);
- showProcessorOperationsContextMenu(processorId, data.pointer.DOM);
+ $scope.activeProcessorId = Number(data.node);
+ $scope.$apply();
+ showProcessorOperationsContextMenu(data.pointer.DOM);
+ }
+ },
+ onSelectNode: function(data) {
+ if (data.nodes.length === 1) {
+ $scope.activeProcessorId = Number(data.nodes[0]);
+ $scope.$apply();
}
+ },
+ onDeselectNode: function() {
+ $scope.activeProcessorId = -1;
+ $scope.$apply();
+ },
+ onHoverNode: function() {
+ $('html,body').css('cursor', 'pointer');
+ },
+ onBlurNode: function() {
+ $('html,body').css('cursor', 'default');
}
}
};
- function showProcessorOperationsContextMenu(processorId, position) {
+ function showProcessorOperationsContextMenu(position) {
var elem = document.getElementById('dag-node-menu');
- $scope.view = function() {
- $state.go('streamingapp.processor', {processorId: processorId});
- };
$contextmenu.popup(elem, position);
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/apps/streamingapp/popups/dag_edit.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/streamingapp/popups/dag_edit.js b/services/dashboard/views/apps/streamingapp/popups/dag_edit.js
index 5ffd376..58daeb5 100644
--- a/services/dashboard/views/apps/streamingapp/popups/dag_edit.js
+++ b/services/dashboard/views/apps/streamingapp/popups/dag_edit.js
@@ -11,7 +11,7 @@ angular.module('dashboard')
var options = $scope.modifyOptions || {};
$scope.changeParallelismOnly = options.parallelism;
- var processor = $scope.activeProcessor;
+ var processor = $scope.dag.getProcessor($scope.activeProcessorId);
$scope.processorId = processor.id;
$scope.taskClass = processor.taskClass;
$scope.description = processor.description;
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/apps/streamingapp/popups/processor_menu.html
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/streamingapp/popups/processor_menu.html b/services/dashboard/views/apps/streamingapp/popups/processor_menu.html
deleted file mode 100644
index bef7aaf..0000000
--- a/services/dashboard/views/apps/streamingapp/popups/processor_menu.html
+++ /dev/null
@@ -1,15 +0,0 @@
-<ul class="dropdown-menu dropdown-menu-sm">
- <li><a href="" ng-click="modify({parallelism:true})">
- <span class="glyphicon glyphicon-none"></span>
- Change Parallelism</a>
- </li>
- <li><a href="" ng-click="modify()">
- <span class="glyphicon glyphicon-none"></span>
- Modify</a>
- </li>
- <li role="separator" class="divider"></li>
- <li><a href="" ng-click="view()">
- <span class="glyphicon glyphicon-stats"></span>
- <b>View Details</b></a>
- </li>
-</ul>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/apps/streamingapp/vis/network.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/streamingapp/vis/network.js b/services/dashboard/views/apps/streamingapp/vis/network.js
index 184c7e7..f5c9f80 100644
--- a/services/dashboard/views/apps/streamingapp/vis/network.js
+++ b/services/dashboard/views/apps/streamingapp/vis/network.js
@@ -8,25 +8,22 @@ angular.module('dashboard')
.directive('visNetwork', [function() {
'use strict';
- function overrideOnContextEvent(network, callback) {
+ function handleOnContextEvent(network, callback) {
var rightClickHandler = function(args) {
- function handleOnContext(data) {
-
+ function handleCallback(data) {
// always disable the default context menu "Save image as, Copy Image..."
vis.util.preventDefault(args.event);
- if (callback) {
- callback(angular.merge(data, {pointer: args.pointer}));
- }
+ callback(angular.merge(data, {pointer: args.pointer}));
}
var selection = network.getNodeAt(args.pointer.DOM);
if (angular.isDefined(selection)) {
- handleOnContext({node: selection});
+ handleCallback({node: selection});
} else {
selection = network.getEdgeAt(args.pointer.DOM);
if (angular.isDefined(selection)) {
- handleOnContext({edge: selection});
+ handleCallback({edge: selection});
}
}
};
@@ -37,22 +34,19 @@ angular.module('dashboard')
network.on('hold', rightClickHandler);
}
- function overrideHoverNodeEvent(network, callback) {
- network.on('hoverNode', function(args) {
- if (callback) {
- var nodeId = parseInt(args.node);
- var radius = network.findNode(nodeId)[0].options.size;
- var position = network.getPositions([nodeId])[nodeId];
- position = network.canvasToDOM(position);
- callback({
- node: nodeId,
- radius: Math.ceil(radius),
- position: position
- });
- }
- vis.util.preventDefault(args.event);
- }
- );
+ function overrideHoverNodeCallback(network, callback) {
+ return function(args) {
+ var nodeId = parseInt(args.node);
+ var radius = network.findNode(nodeId)[0].options.size;
+ var position = network.getPositions([nodeId])[nodeId];
+ position = network.canvasToDOM(position);
+ callback({
+ node: nodeId,
+ radius: Math.ceil(radius),
+ position: position
+ });
+ vis.util.preventDefault(args.event);
+ };
}
function handleDeleteKeyPressed(network, callback) {
@@ -80,17 +74,28 @@ angular.module('dashboard')
angular.forEach(scope.events, function(callback, name) {
switch (name) {
- case 'click':
- case 'doubleClick':
- network.on(name, callback);
+ case 'onClick':
+ network.on('click', callback);
+ break;
+ case 'onDoubleClick':
+ network.on('doubleClick', callback);
+ break;
+ case 'onSelectNode':
+ network.on('selectNode', callback);
+ break;
+ case 'onDeselectNode':
+ network.on('deselectNode', callback);
+ break;
+ case 'onHoverNode':
+ network.on('hoverNode', overrideHoverNodeCallback(network, callback));
break;
- case 'oncontext':
- overrideOnContextEvent(network, callback);
+ case 'onBlurNode':
+ network.on('blurNode', overrideHoverNodeCallback(network, callback));
break;
- case 'hoverNode':
- overrideHoverNodeEvent(network, callback);
+ case 'onContext':
+ handleOnContextEvent(network, callback);
break;
- case 'ondeletepressed':
+ case 'onDeletePressed':
handleDeleteKeyPressed(network, callback);
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/widgets/metrics_period_switcher.html
----------------------------------------------------------------------
diff --git a/services/dashboard/views/widgets/metrics_period_switcher.html b/services/dashboard/views/widgets/metrics_period_switcher.html
deleted file mode 100644
index 92312a0..0000000
--- a/services/dashboard/views/widgets/metrics_period_switcher.html
+++ /dev/null
@@ -1,3 +0,0 @@
-<radio-group
- options="options" button-width="110px"
- ng-model="value"></radio-group>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/widgets/metrics_period_switcher.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/widgets/metrics_period_switcher.js b/services/dashboard/views/widgets/metrics_period_switcher.js
deleted file mode 100644
index 263ad7e..0000000
--- a/services/dashboard/views/widgets/metrics_period_switcher.js
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0
- * See accompanying LICENSE file.
- */
-
-angular.module('dashboard')
-
- .directive('metricsPeriodSwitcher', function() {
- 'use strict';
-
- return {
- restrict: 'E',
- templateUrl: 'views/widgets/metrics_period_switcher.html',
- scope: {
- pastHours: '=',
- viewCurrent: '='
- },
- link: function(scope) {
- 'use strict';
-
- scope.options = {
- current: 'Current',
- hist: 'Past %d Hours'.replace('%d', scope.pastHours)
- };
- scope.value = scope.viewCurrent ? 'current' : 'hist';
- scope.$watch('value', function(value) {
- scope.viewCurrent = String(value) === 'current';
- });
- }
- }
- })
-;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/widgets/radio_group.html
----------------------------------------------------------------------
diff --git a/services/dashboard/views/widgets/radio_group.html b/services/dashboard/views/widgets/radio_group.html
deleted file mode 100644
index 0c68574..0000000
--- a/services/dashboard/views/widgets/radio_group.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<div class="row">
- <div class="col-md-12 text-center">
- <div class="btn-group btn-group-xs">
- <button ng-repeat="(value, text) in options track by $index"
- type="button" class="btn btn-default btn-default-outline"
- ng-style="{width: buttonWidth}"
- ng-class="{'btn-primary btn-primary-outline': value == ngModel}"
- ng-bind="text"
- ng-click="toggle(value)"></button>
- </div>
- </div>
-</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/views/widgets/radio_group.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/widgets/radio_group.js b/services/dashboard/views/widgets/radio_group.js
deleted file mode 100644
index 80aea1b..0000000
--- a/services/dashboard/views/widgets/radio_group.js
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0
- * See accompanying LICENSE file.
- */
-
-angular.module('dashboard')
-
- .directive('radioGroup', function() {
- 'use strict';
-
- return {
- restrict: 'E',
- templateUrl: 'views/widgets/radio_group.html',
- replace: true,
- scope: {
- ngModel: '=',
- options: '='
- },
- link: function(scope, elem, attrs) {
- 'use strict';
-
- scope.buttonWidth = attrs.buttonWidth || '96px';
- scope.toggle = function(value) {
- scope.ngModel = value;
- };
- }
- }
- })
-;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/widgets/metrics_period_switcher.html
----------------------------------------------------------------------
diff --git a/services/dashboard/widgets/metrics_period_switcher.html b/services/dashboard/widgets/metrics_period_switcher.html
new file mode 100644
index 0000000..92312a0
--- /dev/null
+++ b/services/dashboard/widgets/metrics_period_switcher.html
@@ -0,0 +1,3 @@
+<radio-group
+ options="options" button-width="110px"
+ ng-model="value"></radio-group>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/widgets/metrics_period_switcher.js
----------------------------------------------------------------------
diff --git a/services/dashboard/widgets/metrics_period_switcher.js b/services/dashboard/widgets/metrics_period_switcher.js
new file mode 100644
index 0000000..b40aa23
--- /dev/null
+++ b/services/dashboard/widgets/metrics_period_switcher.js
@@ -0,0 +1,32 @@
+/*
+ * Licensed under the Apache License, Version 2.0
+ * See accompanying LICENSE file.
+ */
+
+angular.module('dashboard')
+
+ .directive('metricsPeriodSwitcher', function() {
+ 'use strict';
+
+ return {
+ restrict: 'E',
+ templateUrl: 'widgets/metrics_period_switcher.html',
+ scope: {
+ pastHours: '=',
+ viewCurrent: '='
+ },
+ link: function(scope) {
+ 'use strict';
+
+ scope.options = {
+ current: 'Current',
+ hist: 'Past %d Hours'.replace('%d', scope.pastHours)
+ };
+ scope.value = scope.viewCurrent ? 'current' : 'hist';
+ scope.$watch('value', function(value) {
+ scope.viewCurrent = String(value) === 'current';
+ });
+ }
+ }
+ })
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/widgets/radio_group.html
----------------------------------------------------------------------
diff --git a/services/dashboard/widgets/radio_group.html b/services/dashboard/widgets/radio_group.html
new file mode 100644
index 0000000..0c68574
--- /dev/null
+++ b/services/dashboard/widgets/radio_group.html
@@ -0,0 +1,12 @@
+<div class="row">
+ <div class="col-md-12 text-center">
+ <div class="btn-group btn-group-xs">
+ <button ng-repeat="(value, text) in options track by $index"
+ type="button" class="btn btn-default btn-default-outline"
+ ng-style="{width: buttonWidth}"
+ ng-class="{'btn-primary btn-primary-outline': value == ngModel}"
+ ng-bind="text"
+ ng-click="toggle(value)"></button>
+ </div>
+ </div>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3bf290e4/services/dashboard/widgets/radio_group.js
----------------------------------------------------------------------
diff --git a/services/dashboard/widgets/radio_group.js b/services/dashboard/widgets/radio_group.js
new file mode 100644
index 0000000..aa5427d
--- /dev/null
+++ b/services/dashboard/widgets/radio_group.js
@@ -0,0 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0
+ * See accompanying LICENSE file.
+ */
+
+angular.module('dashboard')
+
+ .directive('radioGroup', function() {
+ 'use strict';
+
+ return {
+ restrict: 'E',
+ templateUrl: 'widgets/radio_group.html',
+ replace: true,
+ scope: {
+ ngModel: '=',
+ options: '='
+ },
+ link: function(scope, elem, attrs) {
+ 'use strict';
+
+ scope.buttonWidth = attrs.buttonWidth || '96px';
+ scope.toggle = function(value) {
+ scope.ngModel = value;
+ };
+ }
+ }
+ })
+;
\ No newline at end of file
[48/50] incubator-gearpump git commit: Merge pull request #1990 from
stanleyxu2005/issue_1989
Posted by ap...@apache.org.
Merge pull request #1990 from stanleyxu2005/issue_1989
fix #1989: add confirmation dialog for kill app and restart app
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/a7bbb4c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/a7bbb4c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/a7bbb4c2
Branch: refs/heads/master
Commit: a7bbb4c2d337815ea05ead04c7e1dd07254ccecb
Parents: 81d41e5 fc91849
Author: Qian Xu <st...@users.noreply.github.com>
Authored: Mon Mar 7 13:38:57 2016 +0800
Committer: Qian Xu <st...@users.noreply.github.com>
Committed: Mon Mar 7 13:38:57 2016 +0800
----------------------------------------------------------------------
project/Build.scala | 4 ++--
services/dashboard/dashboard.js | 8 ++++++++
services/dashboard/index.html | 6 +++---
services/dashboard/styles/dashboard.css | 5 +++++
services/dashboard/views/apps/apps.html | 13 ++++++++-----
services/dashboard/views/apps/apps.js | 12 ++++++++----
6 files changed, 34 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[26/50] incubator-gearpump git commit: A few documentation points
Posted by ap...@apache.org.
A few documentation points
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/f64af570
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/f64af570
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/f64af570
Branch: refs/heads/master
Commit: f64af570ed8f5dbed8a3bb942f753b0d656642a2
Parents: 1445d12
Author: Riccardo V. Vincelli <rv...@github.com>
Authored: Tue Feb 9 13:10:13 2016 +0100
Committer: Riccardo V. Vincelli <rv...@github.com>
Committed: Tue Feb 9 13:18:26 2016 +0100
----------------------------------------------------------------------
docs/deployment-security.md | 2 +-
docs/dev-connectors.md | 17 ++++++++++++++++-
docs/dev-custom-serializer.md | 13 +++++++++++++
3 files changed, 30 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f64af570/docs/deployment-security.md
----------------------------------------------------------------------
diff --git a/docs/deployment-security.md b/docs/deployment-security.md
index 331aeba..4be1346 100644
--- a/docs/deployment-security.md
+++ b/docs/deployment-security.md
@@ -55,7 +55,7 @@ provided for the gearpump-hbase connector. Specifically, the `UserConfig` object
```scala
val principal = "gearpump/fully.qualified.domain.name@YOUR-REALM.COM"
-val keytabContent = Files.toByteArray(new File("path_to_keytab_file))
+val keytabContent = Files.toByteArray(new File("path_to_keytab_file"))
val appConfig = UserConfig.empty
.withString("gearpump.kerberos.principal", principal)
.withBytes("gearpump.keytab.file", keytabContent)
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f64af570/docs/dev-connectors.md
----------------------------------------------------------------------
diff --git a/docs/dev-connectors.md b/docs/dev-connectors.md
index 458956b..a9983fc 100644
--- a/docs/dev-connectors.md
+++ b/docs/dev-connectors.md
@@ -104,7 +104,7 @@ To use `HBaseSink` in your application, you first need to add the `gearpump-exte
To connect to HBase, you need to provide following info:
- the HBase configuration to tell which HBase service to connect
- - the table name
+ - the table name (you must create the table yourself, see the [HBase documentation](https://hbase.apache.org/book.html))
Then, you can use `HBaseSink` in your application:
@@ -123,6 +123,21 @@ Then, you can use `HBaseSink` in your application:
You can tune the connection to HBase via the HBase configuration passed in. If not passed, Gearpump will try to check local classpath to find a valid HBase configuration (`hbase-site.xml`).
+Attention, due to the issue discussed [here](http://stackoverflow.com/questions/24456484/hbase-managed-zookeeper-suddenly-trying-to-connect-to-localhost-instead-of-zooke) you may need to create additional configuration for your HBase sink:
+
+```scala
+ def hadoopConfig = {
+ val conf = new Configuration()
+ conf.set("hbase.zookeeper.quorum", "zookeeperHost")
+ conf.set("hbase.zookeeper.property.clientPort", "2181")
+ conf
+ }
+```
+
+```scala
+ val sink = HBaseSink(UserConfig.empty, tableName, hadoopConfig)
+```
+
## How to implement your own `DataSource`
To implement your own `DataSource`, you need to implement two things:
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f64af570/docs/dev-custom-serializer.md
----------------------------------------------------------------------
diff --git a/docs/dev-custom-serializer.md b/docs/dev-custom-serializer.md
index a35162b..ca0a332 100644
--- a/docs/dev-custom-serializer.md
+++ b/docs/dev-custom-serializer.md
@@ -29,6 +29,19 @@ When you decide that you want to define a custom serializer, you can do this in
Please note that Gearpump shaded the original Kryo dependency. The package name ```com.esotericsoftware``` was relocated to ```io.gearpump.esotericsoftware```. So in the following customization, you should import corresponding shaded classes, the example code will show that part.
+In general you should use the shaded version of a library whenever possible in order to avoid binary incompatibilities, eg don't use:
+
+```scala
+ import com.google.common.io.Files
+```
+
+but rather
+
+```scala
+ import io.gearpump.google.common.io.Files
+```
+
+
##### System Level Serializer
If the serializer is widely used, you can define a global serializer which is available to all applications(or worker or master) in the system.
[45/50] incubator-gearpump git commit: fix #1989: add confirmation
dialog for kill app and restart app
Posted by ap...@apache.org.
fix #1989: add confirmation dialog for kill app and restart app
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/fc918491
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/fc918491
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/fc918491
Branch: refs/heads/master
Commit: fc918491d1fc90e3d6b21198ec26e6c9ed3ce72c
Parents: 3f3d911
Author: Qian Xu <qi...@intel.com>
Authored: Fri Mar 4 14:21:49 2016 +0800
Committer: Qian Xu <qi...@intel.com>
Committed: Fri Mar 4 15:17:39 2016 +0800
----------------------------------------------------------------------
project/Build.scala | 4 ++--
services/dashboard/dashboard.js | 8 ++++++++
services/dashboard/index.html | 6 +++---
services/dashboard/styles/dashboard.css | 5 +++++
services/dashboard/views/apps/apps.html | 13 ++++++++-----
services/dashboard/views/apps/apps.js | 12 ++++++++----
6 files changed, 34 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/fc918491/project/Build.scala
----------------------------------------------------------------------
diff --git a/project/Build.scala b/project/Build.scala
index 3f91f1e..3723020 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -279,7 +279,7 @@ object Build extends sbt.Build {
"com.lihaoyi" %% "upickle" % upickleVersion,
"com.softwaremill" %% "akka-http-session" % "0.1.4",
"org.webjars" % "angularjs" % "1.4.9",
- "org.webjars.npm" % "angular-touch" % "1.5.0-rc.2", // angular 1.5 breaks ui-select, but we need ng-touch 1.5
+ "org.webjars.npm" % "angular-touch" % "1.5.0", // angular 1.5 breaks ui-select, but we need ng-touch 1.5
"org.webjars" % "angular-ui-router" % "0.2.15",
"org.webjars" % "bootstrap" % "3.3.6",
"org.webjars" % "d3js" % "3.5.6",
@@ -298,7 +298,7 @@ object Build extends sbt.Build {
"org.webjars.bower" % "vis" % "4.7.0",
"org.webjars.bower" % "clipboard.js" % "0.1.1",
"org.webjars.npm" % "dashing-deps" % "0.1.2",
- "org.webjars.npm" % "dashing" % "0.4.5"
+ "org.webjars.npm" % "dashing" % "0.4.6"
).map(_.exclude("org.scalamacros", "quasiquotes_2.10")).map(_.exclude("org.scalamacros", "quasiquotes_2.10.3")))
lazy val serviceJSSettings = Seq(
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/fc918491/services/dashboard/dashboard.js
----------------------------------------------------------------------
diff --git a/services/dashboard/dashboard.js b/services/dashboard/dashboard.js
index a9b8fc7..90f3609 100644
--- a/services/dashboard/dashboard.js
+++ b/services/dashboard/dashboard.js
@@ -61,6 +61,14 @@
});
}])
+ // configure dashing
+ .config(['dashing.i18n', function(i18n) {
+ 'use strict';
+
+ i18n.confirmationYesButtonText = 'OK';
+ i18n.confirmationNoButtonText = 'Cancel';
+ }])
+
// disable logging for production
.config(['$compileProvider', function($compileProvider) {
'use strict';
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/fc918491/services/dashboard/index.html
----------------------------------------------------------------------
diff --git a/services/dashboard/index.html b/services/dashboard/index.html
index a3bbaa9..3ba09e3 100644
--- a/services/dashboard/index.html
+++ b/services/dashboard/index.html
@@ -17,7 +17,7 @@
<link rel="stylesheet" href="webjars/ui-select/0.14.2/dist/select.min.css"/>
<link rel="stylesheet" href="webjars/angular-loading-bar/0.8.0/build/loading-bar.min.css"/>
<link rel="stylesheet" href="webjars/vis/4.7.0/dist/vis.min.css"/>
- <link rel="stylesheet" href="webjars/dashing/0.4.5/dist/dashing.min.css"/>
+ <link rel="stylesheet" href="webjars/dashing/0.4.6/dist/dashing.min.css"/>
<!-- Site styles -->
<link rel="stylesheet" href="webjars/dashing-deps/0.1.2/roboto/roboto.min.css"/>
@@ -46,7 +46,7 @@
<script src="webjars/bootstrap/3.3.6/js/bootstrap.min.js"></script>
<script src="webjars/angularjs/1.4.9/angular.min.js"></script>
<script src="webjars/angularjs/1.4.9/angular-cookies.min.js"></script>
-<script src="webjars/angular-touch/1.5.0-rc.2/angular-touch.min.js"></script>
+<script src="webjars/angular-touch/1.5.0/angular-touch.min.js"></script>
<script src="webjars/angularjs/1.4.9/angular-animate.min.js"></script>
<script src="webjars/angularjs/1.4.9/angular-sanitize.min.js"></script>
<script src="webjars/angular-loading-bar/0.8.0/build/loading-bar.min.js"></script>
@@ -62,7 +62,7 @@
<script src="webjars/ng-file-upload/5.0.9/ng-file-upload-all.min.js"></script>
<script src="webjars/clipboard.js/0.1.1/clipboard.js"></script>
<script src="webjars/dashing-deps/0.1.2/echarts/2.2.7-compact/echarts-all.min.js"></script>
-<script src="webjars/dashing/0.4.5/dist/dashing.min.js"></script>
+<script src="webjars/dashing/0.4.6/dist/dashing.min.js"></script>
<!-- Application -->
<script src="dashboard.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/fc918491/services/dashboard/styles/dashboard.css
----------------------------------------------------------------------
diff --git a/services/dashboard/styles/dashboard.css b/services/dashboard/styles/dashboard.css
index 8c17282..8062959 100644
--- a/services/dashboard/styles/dashboard.css
+++ b/services/dashboard/styles/dashboard.css
@@ -120,6 +120,11 @@ h4, table > caption {
color: #fff !important;
}
+/* Add extra margin to heading text */
+.btn-group-after-heading {
+ margin-left: 16px;
+}
+
/* Small badge for version label */
.navbar-brand > span {
font-size: .6em;
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/fc918491/services/dashboard/views/apps/apps.html
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/apps.html b/services/dashboard/views/apps/apps.html
index 30e455b..c4999a3 100644
--- a/services/dashboard/views/apps/apps.html
+++ b/services/dashboard/views/apps/apps.html
@@ -4,10 +4,11 @@
<div class="col-md-6 col-sm-6">
<span class="table-caption-ext pull-left">Applications</span>
<!-- dropdown button -->
- <div class="btn-group">
- <span class="btn btn-xs btn-primary dropdown-toggle pull-left" data-toggle="dropdown"
- style="margin-left: 6px; color: white"
- bs-dropdown="composeMenuOptions" html="true">Create
+ <div class="btn-group btn-group-after-heading">
+ <span class="btn btn-xs btn-default dropdown-toggle pull-left" data-toggle="dropdown"
+ bs-dropdown="composeMenuOptions" html="true">
+ <span class="fa fa-plus"></span>
+ Add
<span class="caret"></span>
</span>
</div>
@@ -16,7 +17,9 @@
<!--FILTER-->
</div>
<div class="col-md-3 hidden-sm hidden-xs">
- <searchbox ng-model="search" placeholder="Search Anything"></searchbox>
+ <searchbox
+ ng-show="appsTable.rows && appsTable.rows.length > 0"
+ ng-model="search" placeholder="Search Applications"></searchbox>
</div>
</div>
<!-- end of control toolbar -->
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/fc918491/services/dashboard/views/apps/apps.js
----------------------------------------------------------------------
diff --git a/services/dashboard/views/apps/apps.js b/services/dashboard/views/apps/apps.js
index 669d018..6707e36 100644
--- a/services/dashboard/views/apps/apps.js
+++ b/services/dashboard/views/apps/apps.js
@@ -22,8 +22,8 @@ angular.module('dashboard')
});
}])
- .controller('AppsCtrl', ['$scope', '$modal', '$state', '$sortableTableBuilder', 'apps0',
- function($scope, $modal, $state, $stb, apps0) {
+ .controller('AppsCtrl', ['$scope', '$modal', '$state', '$sortableTableBuilder', '$dialogs', 'apps0',
+ function($scope, $modal, $state, $stb, $dialogs, apps0) {
'use strict';
var submitWindow = $modal({
@@ -91,13 +91,17 @@ angular.module('dashboard')
kill: {
text: 'Kill', class: 'btn-xs', disabled: !app.isRunning,
click: function() {
- app.terminate();
+ $dialogs.confirm('Are you sure to kill this application?', function() {
+ app.terminate();
+ });
}
},
restart: {
text: 'Restart', class: 'btn-xs', disabled: !app.isRunning,
click: function() {
- app.restart();
+ $dialogs.confirm('Are you sure to restart this application?', function() {
+ app.restart();
+ });
}
}
};
[46/50] incubator-gearpump git commit: remove useless code
Posted by ap...@apache.org.
remove useless code
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/3b02d4a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/3b02d4a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/3b02d4a3
Branch: refs/heads/master
Commit: 3b02d4a321ee5856c78883a165f706a4c3bea256
Parents: 3f3d911
Author: darionyaphet <da...@gmail.com>
Authored: Sun Mar 6 15:14:25 2016 +0800
Committer: darionyaphet <da...@gmail.com>
Committed: Sun Mar 6 15:14:25 2016 +0800
----------------------------------------------------------------------
core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/3b02d4a3/core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala b/core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala
index e74488b..7e8a91d 100644
--- a/core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala
+++ b/core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala
@@ -48,8 +48,6 @@ object ClusterConfigSource {
config = ConfigFactory.parseResourcesAnySyntax(filePath,
ConfigParseOptions.defaults.setAllowMissing(true))
}
- config
-
new ClusterConfigSourceImpl(config)
}
}
[03/50] incubator-gearpump git commit: stand-alone nimbus
Posted by ap...@apache.org.
stand-alone nimbus
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/f83be158
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/f83be158
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/f83be158
Branch: refs/heads/master
Commit: f83be158e922bec41d6d0431230e63a39e15546a
Parents: 502dbae
Author: manuzhang <ow...@gmail.com>
Authored: Mon Feb 1 16:56:11 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Feb 1 16:56:11 2016 +0800
----------------------------------------------------------------------
.../io/gearpump/experiments/storm/main/GearpumpNimbus.scala | 4 +++-
.../io/gearpump/experiments/storm/main/GearpumpStormClient.scala | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f83be158/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
index 4204580..83328b4 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
@@ -59,7 +59,9 @@ object GearpumpNimbus extends AkkaApp with ArgumentsParser {
val system = ActorSystem("storm", akkaConf)
val clientContext = new ClientContext(akkaConf, system, null)
val stormConf = Utils.readStormConfig().asInstanceOf[JMap[AnyRef, AnyRef]]
- val thriftConf: JMap[String, String] = Map(Config.NIMBUS_THRIFT_PORT -> s"$THRIFT_PORT").asJava
+ val thriftConf: JMap[String, String] = Map(
+ Config.NIMBUS_HOST -> akkaConf.getString(Constants.GEARPUMP_HOSTNAME),
+ Config.NIMBUS_THRIFT_PORT -> s"$THRIFT_PORT").asJava
updateOutputStormConfig(thriftConf, output)
stormConf.putAll(thriftConf)
val thriftServer = createServer(clientContext, stormConf)
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f83be158/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
index 70efbf3..b813ae0 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
@@ -63,7 +63,8 @@ object GearpumpStormClient extends AkkaApp with ArgumentsParser {
private def getThriftOptions(stormConfig: String): String = {
val config = Utils.findAndReadConfigFile(stormConfig, true)
+ val host = config.get(Config.NIMBUS_HOST)
val thriftPort = config.get(Config.NIMBUS_THRIFT_PORT)
- s"${Config.NIMBUS_HOST}=127.0.0.1,${Config.NIMBUS_THRIFT_PORT}=$thriftPort"
+ s"${Config.NIMBUS_HOST}=$host,${Config.NIMBUS_THRIFT_PORT}=$thriftPort"
}
}
[16/50] incubator-gearpump git commit: Merge pull request #1955 from
manuzhang/doc_stream_cql
Posted by ap...@apache.org.
Merge pull request #1955 from manuzhang/doc_stream_cql
fix #1952, document how to use StreamCQL over Gearpump
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/e479c7e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/e479c7e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/e479c7e4
Branch: refs/heads/master
Commit: e479c7e435b59c6051fc20e370eb9425858d2a7f
Parents: 89292ac 75ad76c
Author: Sean Zhong <xi...@intel.com>
Authored: Thu Feb 4 12:30:37 2016 +0800
Committer: Sean Zhong <xi...@intel.com>
Committed: Thu Feb 4 12:30:37 2016 +0800
----------------------------------------------------------------------
docs/dev-storm.md | 57 ++++++++++++++++++--
.../experiments/storm/main/GearpumpNimbus.scala | 9 ++--
.../storm/main/GearpumpStormClient.scala | 3 +-
3 files changed, 58 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
[38/50] incubator-gearpump git commit: add Gong Yu to contributors
Posted by ap...@apache.org.
add Gong Yu to contributors
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/42806a94
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/42806a94
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/42806a94
Branch: refs/heads/master
Commit: 42806a9482d6ace7b207a78786a3167392aa353f
Parents: 0827adf
Author: ManuZhang <Ow...@gmail.com>
Authored: Wed Feb 17 19:02:57 2016 +0800
Committer: ManuZhang <Ow...@gmail.com>
Committed: Wed Feb 17 19:02:57 2016 +0800
----------------------------------------------------------------------
README.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/42806a94/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index e7f6194..32b3837 100644
--- a/README.md
+++ b/README.md
@@ -89,6 +89,7 @@ Gearpump has an integration test system which is based on Docker. Please check [
* [Stanley Xu](https://github.com/stanleyxu2005)
* [Tomasz Targonski](https://github.com/TomaszT)
* [Sun Kewei](https://github.com/skw1992)
+* [Gong Yu](https://github.com/pangolulu)
## Contacts
[31/50] incubator-gearpump git commit: Merge pull request #1964 from
manuzhang/add_contributing
Posted by ap...@apache.org.
Merge pull request #1964 from manuzhang/add_contributing
fix #1963, rename CommitGuideline.md to CONTRIBUTING.md
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/79f01ba7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/79f01ba7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/79f01ba7
Branch: refs/heads/master
Commit: 79f01ba7ce7ccac8cbf93ce855a017624a416675
Parents: 9405961 0851765
Author: Qian Xu <st...@users.noreply.github.com>
Authored: Sat Feb 13 20:05:09 2016 +0800
Committer: Qian Xu <st...@users.noreply.github.com>
Committed: Sat Feb 13 20:05:09 2016 +0800
----------------------------------------------------------------------
CONTRIBUTING.md | 92 +++++++++++++++++++++++++++++++++++++++++++++++++
CommitGuideline.md | 92 -------------------------------------------------
2 files changed, 92 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
[44/50] incubator-gearpump git commit: Merge pull request #1982 from
manuzhang/fix_kafka_util_spec
Posted by ap...@apache.org.
Merge pull request #1982 from manuzhang/fix_kafka_util_spec
fix #1983, fix KafkaUtilSpec failure
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/3f3d911e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/3f3d911e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/3f3d911e
Branch: refs/heads/master
Commit: 3f3d911e3f29582e5d45a2a457609ac89ab84ebb
Parents: c7ba3d5 ec59bbd
Author: Huafeng Wang <fv...@gmail.com>
Authored: Tue Mar 1 13:39:02 2016 +0800
Committer: Huafeng Wang <fv...@gmail.com>
Committed: Tue Mar 1 13:39:02 2016 +0800
----------------------------------------------------------------------
.../streaming/kafka/util/KafkaServerHarness.scala | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[10/50] incubator-gearpump git commit: fix #1895,
make GearpumpNimbus a standalone service
Posted by ap...@apache.org.
fix #1895, make GearpumpNimbus a standalone service
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/400f627e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/400f627e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/400f627e
Branch: refs/heads/master
Commit: 400f627ee8efa90c78a79b8a8c246773238e0909
Parents: 20c62e5
Author: manuzhang <ow...@gmail.com>
Authored: Fri Jan 29 16:55:53 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Tue Feb 2 12:39:09 2016 +0800
----------------------------------------------------------------------
docs/dev-storm.md | 40 ++-
.../gearpump/experiments/storm/Commands.scala | 36 ---
.../storm/GearpumpThriftServer.scala | 143 ----------
.../experiments/storm/StormRunner.scala | 136 ++--------
.../experiments/storm/main/GearpumpNimbus.scala | 267 +++++++++++++++++++
.../storm/main/GearpumpStormClient.scala | 70 +++++
.../storm/topology/GearpumpStormTopology.scala | 52 +---
.../experiments/storm/GearpumpNimbusSpec.scala | 68 -----
.../storm/GearpumpThriftServerSpec.scala | 48 ----
.../experiments/storm/StormRunnerSpec.scala | 58 ----
.../topology/GearpumpStormTopologySpec.scala | 20 +-
.../suites/StandaloneModeSuite.scala | 4 +-
.../integrationtest/storm/StormClient.scala | 14 +-
project/Pack.scala | 2 +-
.../io/gearpump/services/MasterService.scala | 2 +-
15 files changed, 421 insertions(+), 539 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/docs/dev-storm.md
----------------------------------------------------------------------
diff --git a/docs/dev-storm.md b/docs/dev-storm.md
index a05c4ed..a22ed6c 100644
--- a/docs/dev-storm.md
+++ b/docs/dev-storm.md
@@ -77,23 +77,43 @@ This section shows how to run an existing Storm jar in a local Gearpump cluster.
1. launch a local cluster
```
- ./target/pack/bin/local
+ bin/local
```
-2. submit a topology from storm-starter.
+2. start a local Gearpump Nimbus server
+
+ Users need server's thrift port to submit topologies later. The thrift port is written to a yaml config file set with `-output` option.
+ Users can provide an existing config file where only `nimbus.thrift.port` is overwritten. If not provided, a new file `app.yaml` is created with the config.
+
```
- bin/storm -verbose -config storm.yaml -jar storm-starter-${STORM_VERSION}.jar storm.starter.ExclamationTopology exclamation
+ bin/storm nimbus -output [conf <custom yaml config>]
```
+
+3. submit Storm applications
+
+ Users can either submit Storm applications through command line or UI.
+
+ a. submit Storm applications through command line
+
+
+ ```
+ bin/storm app -verbose -config app.yaml -jar storm-starter-${STORM_VERSION}.jar storm.starter.ExclamationTopology exclamation
+ ```
- Users are able to configure their applications through following options
+ Users are able to configure their applications through following options
- * `jar` - set the path of a storm application jar
- * `config` - submit a customized storm configuration file
+ * `jar` - set the path of a Storm application jar
+ * `config` - submit the custom configuration file generated when launching Nimbus
- That's it. Check the dashboard and you should see data flowing through your topology.
+ b. submit Storm application through UI
- *Note that submission from UI is not supported yet*.
+ 1. Click on the "Create" button on the applications page on UI.
+ 2. Click on the "Submit Storm Application" item in the pull down menu.
+ 3. In the popup console, upload the Storm application jar and the configuration file generated when launching Nimbus,
+ and fill in `storm.starter.ExclamationTopology exclamation` as arguments.
+ 4. Click on the "Submit" button
+ Either way, check the dashboard and you should see data flowing through your topology.
## How is it different from running on Storm
@@ -145,11 +165,11 @@ Gearpump has flow control between tasks such that [sender cannot flood receiver]
All Storm configurations are respected with the following priority order
```
-defaults.yaml < storm.yaml < application config < component config < custom user config
+defaults.yaml < custom file config < application config < component config
```
where
* application config is submit from Storm application along with the topology
* component config is set in spout / bolt with `getComponentConfiguration`
-* custom user config is specified with the `-config` option when submitting Storm application from command line
\ No newline at end of file
+* custom file config is specified with the `-config` option when submitting Storm application from command line or uploaded from UI
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala
deleted file mode 100644
index 1d5b903..0000000
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.experiments.storm
-
-import backtype.storm.generated.{KillOptions, StormTopology, SubmitOptions}
-
-object Commands {
-
- case class Submit(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology, options: SubmitOptions)
-
- case class AppSubmitted(name: String, appId: Int)
-
- case class Kill(name: String, option: KillOptions)
-
- case class AppKilled(name: String, appId: Int)
-
- case object GetClusterInfo
-
- case class GetTopology(id: String)
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala
deleted file mode 100644
index 640c20c..0000000
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.experiments.storm
-
-import java.nio.ByteBuffer
-import java.util.concurrent.TimeUnit
-import java.util.{HashMap => JHashMap, Map => JMap}
-
-import akka.actor.ActorRef
-import backtype.storm.Config
-import backtype.storm.generated._
-import backtype.storm.security.auth.{ThriftConnectionType, ThriftServer}
-import backtype.storm.utils.Utils
-import io.gearpump.experiments.storm.Commands.{GetClusterInfo, _}
-import io.gearpump.experiments.storm.util.StormUtil
-import io.gearpump.util.ActorUtil.askActor
-
-import scala.concurrent.Await
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-object GearpumpThriftServer {
- val THRIFT_PORT = StormUtil.getThriftPort
- implicit val timeOut = akka.util.Timeout(3, TimeUnit.SECONDS)
-
- private def createServer(handler: ActorRef): ThriftServer = {
- val processor = new Nimbus.Processor[GearpumpNimbus](new GearpumpNimbus(handler))
- val connectionType = ThriftConnectionType.NIMBUS
- val config = Utils.readDefaultConfig().asInstanceOf[JMap[AnyRef, AnyRef]]
- config.put(Config.NIMBUS_THRIFT_PORT, s"$THRIFT_PORT")
- new ThriftServer(config, processor, connectionType)
- }
-
- def apply(handler: ActorRef): GearpumpThriftServer = {
- new GearpumpThriftServer(createServer(handler))
- }
-
- class GearpumpNimbus(handler: ActorRef) extends Nimbus.Iface {
-
- override def submitTopology(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology): Unit = {
- val ask = askActor(handler, Submit(name, uploadedJarLocation, jsonConf, topology, new SubmitOptions(TopologyInitialStatus.ACTIVE)))
- Await.result(ask, 30 seconds)
- }
-
- override def killTopologyWithOpts(name: String, options: KillOptions): Unit = {
- Await.result(askActor(handler,Kill(name, options)), 10 seconds)
- }
-
- override def submitTopologyWithOpts(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology, options: SubmitOptions): Unit = {
- Await.result(askActor(handler,Submit(name, uploadedJarLocation, jsonConf, topology, options)), 10 seconds)
- }
-
- override def uploadChunk(location: String, chunk: ByteBuffer): Unit = {
- }
-
- override def getNimbusConf: String = {
- throw new UnsupportedOperationException
- }
-
- override def getTopology(id: String): StormTopology = {
- Await.result(askActor[StormTopology](handler, GetTopology(id)), 10 seconds)
- }
-
- override def getTopologyConf(id: String): String = {
- throw new UnsupportedOperationException
- }
-
- override def beginFileDownload(file: String): String = {
- throw new UnsupportedOperationException
- }
-
- override def getUserTopology(id: String): StormTopology = getTopology(id)
-
- override def activate(name: String): Unit = {
- throw new UnsupportedOperationException
- }
-
- override def rebalance(name: String, options: RebalanceOptions): Unit = {
- throw new UnsupportedOperationException
- }
-
- override def deactivate(name: String): Unit = {
- throw new UnsupportedOperationException
- }
-
- override def getTopologyInfo(id: String): TopologyInfo = {
- throw new UnsupportedOperationException
- }
-
- override def getTopologyInfoWithOpts(s: String, getInfoOptions: GetInfoOptions): TopologyInfo = {
- throw new UnsupportedOperationException
- }
-
- override def killTopology(name: String): Unit = killTopologyWithOpts(name, new KillOptions())
-
- override def downloadChunk(id: String): ByteBuffer = {
- throw new UnsupportedOperationException
- }
-
- override def beginFileUpload(): String = {
- "local thrift server"
- }
-
- override def getClusterInfo: ClusterSummary = {
- Await.result(askActor[ClusterSummary](handler, GetClusterInfo), 10 seconds)
- }
-
- override def finishFileUpload(location: String): Unit = {
- }
-
- override def uploadNewCredentials(s: String, credentials: Credentials): Unit = {
- throw new UnsupportedOperationException
- }
- }
-}
-
-class GearpumpThriftServer(server: ThriftServer) extends Thread {
-
- override def run(): Unit = {
- server.serve()
- }
-
- def close(): Unit = {
- server.stop()
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
index 84d9460..daa7aa5 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
@@ -18,131 +18,35 @@
package io.gearpump.experiments.storm
-import java.io.File
-import java.util.{Map => JMap}
+import io.gearpump.experiments.storm.main.{GearpumpNimbus, GearpumpStormClient}
+import io.gearpump.util.LogUtil
+import org.slf4j.Logger
-import akka.actor.{Actor, ActorSystem, Props}
-import backtype.storm.Config
-import backtype.storm.generated.{ClusterSummary, StormTopology, SupervisorSummary, TopologySummary}
-import com.typesafe.config.ConfigValueFactory
-import io.gearpump.cluster.UserConfig
-import io.gearpump.cluster.client.ClientContext
-import io.gearpump.cluster.main.{ArgumentsParser, CLIOption}
-import io.gearpump.experiments.storm.Commands.{GetClusterInfo, _}
-import io.gearpump.experiments.storm.topology.GearpumpStormTopology
-import io.gearpump.experiments.storm.util.{GraphBuilder, StormConstants}
-import io.gearpump.streaming.StreamApplication
-import io.gearpump.util.Constants._
-import io.gearpump.util.{AkkaApp, Constants, LogUtil, Util}
+object StormRunner {
+ private val LOG: Logger = LogUtil.getLogger(getClass)
-import scala.collection.JavaConverters._
+ private val commands = Map("nimbus" -> GearpumpNimbus, "app" -> GearpumpStormClient)
-object StormRunner extends AkkaApp with ArgumentsParser {
- override val options: Array[(String, CLIOption[Any])] = Array(
- "jar" -> CLIOption[String]("<storm jar>", required = true),
- "config" -> CLIOption[String]("<storm config path>", required = false),
- "verbose" -> CLIOption("<print verbose log on console>", required = false, defaultValue = Some(false)))
-
- override val remainArgs = Array("topology_name")
-
- override def main(inputAkkaConf: Config, args: Array[String]): Unit = {
-
- val akkaConf = updateConfig(inputAkkaConf)
- val config = parse(args)
-
- val verbose = config.getBoolean("verbose")
- if (verbose) {
- LogUtil.verboseLogToConsole
- }
-
- val jar = config.getString("jar")
- val stormConfig = config.getString("config")
- val topology = config.remainArgs(0)
- val stormArgs = config.remainArgs.drop(1)
-
- val system = ActorSystem("storm", akkaConf)
- val clientContext = new ClientContext(akkaConf, system, null)
-
- val gearpumpNimbus = system.actorOf(Props(new Handler(clientContext, jar, stormConfig)))
- val thriftServer = GearpumpThriftServer(gearpumpNimbus)
- thriftServer.start()
-
- val stormOptions = Array("-Dstorm.options=" +
- s"${Config.NIMBUS_HOST}=127.0.0.1,${Config.NIMBUS_THRIFT_PORT}=${GearpumpThriftServer.THRIFT_PORT}",
- "-Dstorm.jar=" + jar,
- s"-D${PREFER_IPV4}=true"
- )
-
- val classPath = Array(System.getProperty("java.class.path"), jar)
- val process = Util.startProcess(stormOptions, classPath, topology, stormArgs)
-
- // wait till the process exit
- val exit = process.exitValue()
-
- thriftServer.close()
- clientContext.close()
- system.shutdown()
- system.awaitTermination()
-
- if (exit != 0) {
- throw new Exception(s"failed to submit jar, exit code $exit, error summary: ${process.logger.error}")
- }
+ private def usage: Unit = {
+ val keys = commands.keys.toList.sorted
+ Console.err.println("Usage: " + "<" + keys.mkString("|") + ">")
}
- import Constants._
- private def updateConfig(config: Config): Config = {
- val storm = s"<${GEARPUMP_HOME}>/lib/storm/*"
- val appClassPath = s"$storm${File.pathSeparator}" + config.getString(GEARPUMP_APPMASTER_EXTRA_CLASSPATH)
- val executorClassPath = s"$storm${File.pathSeparator}" + config.getString(Constants.GEARPUMP_EXECUTOR_EXTRA_CLASSPATH)
-
- val updated = config.withValue(GEARPUMP_APPMASTER_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(appClassPath))
- .withValue(GEARPUMP_EXECUTOR_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(executorClassPath))
-
- if (config.hasPath(StormConstants.STORM_SERIALIZATION_FRAMEWORK)) {
- val serializerConfig = ConfigValueFactory.fromAnyRef(config.getString(StormConstants.STORM_SERIALIZATION_FRAMEWORK))
- updated.withValue(GEARPUMP_SERIALIZER_POOL, serializerConfig)
+ private def executeCommand(command : String, commandArgs : Array[String]): Unit = {
+ if (!commands.contains(command)) {
+ usage
} else {
- updated
+ commands(command).main(commandArgs)
}
}
-
- class Handler(clientContext: ClientContext, jar: String, fileConfig: String) extends Actor {
- private var applications = Map.empty[String, Int]
- private var topologies = Map.empty[String, StormTopology]
- private val LOG = LogUtil.getLogger(classOf[Handler])
-
- implicit val system = context.system
-
- def receive: Receive = {
- case Kill(name, option) =>
- topologies -= name
- clientContext.shutdown(applications.getOrElse(name, throw new RuntimeException(s"topology $name not found")))
- val appId = applications(name)
- applications -= name
- LOG.info(s"Killed topology $name")
- sender ! AppKilled(name, appId)
- case Submit(name, uploadedJarLocation, appConfig, topology, option) =>
- topologies += name -> topology
-
- val gearpumpStormTopology = GearpumpStormTopology(topology, appConfig, fileConfig)
- val stormConfig = gearpumpStormTopology.getStormConfig
- val processorGraph = GraphBuilder.build(gearpumpStormTopology)
- val config = UserConfig.empty
- .withValue[StormTopology](StormConstants.STORM_TOPOLOGY, topology)
- .withValue[JMap[AnyRef, AnyRef]](StormConstants.STORM_CONFIG, stormConfig)
- val app = StreamApplication(name, processorGraph, config)
- val appId = clientContext.submit(app, jar)
- applications += name -> appId
- LOG.info(s"Storm Application $appId submitted")
- sender ! AppSubmitted(name, appId)
- case GetClusterInfo =>
- val topologySummaryList = topologies.map { case (name, _) =>
- new TopologySummary(name, name, 0, 0, 0, 0, "")
- }.toSeq
- sender ! new ClusterSummary(List[SupervisorSummary]().asJava, 0, topologySummaryList.asJava)
- case GetTopology(id) =>
- sender ! topologies(id)
+ def main(args: Array[String]) = {
+ if (args.length == 0) {
+ usage
+ } else {
+ val command = args(0)
+ val commandArgs = args.drop(1)
+ executeCommand(command, commandArgs)
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
new file mode 100644
index 0000000..96c12ae
--- /dev/null
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.experiments.storm.main
+
+import java.io.{File, FileOutputStream, FileWriter}
+import java.nio.ByteBuffer
+import java.nio.channels.{Channels, WritableByteChannel}
+import java.util.{Map => JMap, HashMap => JHashMap, UUID}
+
+import akka.actor.ActorSystem
+import backtype.storm.Config
+import backtype.storm.generated._
+import backtype.storm.security.auth.{ThriftConnectionType, ThriftServer}
+import backtype.storm.utils.TimeCacheMap.ExpiredCallback
+import backtype.storm.utils.{TimeCacheMap, Utils}
+import com.typesafe.config.ConfigValueFactory
+import io.gearpump.cluster.{MasterToAppMaster, UserConfig}
+import io.gearpump.cluster.client.ClientContext
+import io.gearpump.cluster.main.{ArgumentsParser, CLIOption}
+import io.gearpump.experiments.storm.topology.GearpumpStormTopology
+import io.gearpump.experiments.storm.util.{GraphBuilder, StormConstants, StormUtil}
+import io.gearpump.streaming.StreamApplication
+import io.gearpump.util.{AkkaApp, Constants, LogUtil}
+import org.apache.storm.shade.org.json.simple.JSONValue
+import org.apache.storm.shade.org.yaml.snakeyaml.Yaml
+import org.slf4j.Logger
+
+import scala.collection.JavaConverters._
+
+object GearpumpNimbus extends AkkaApp with ArgumentsParser {
+ private val THRIFT_PORT = StormUtil.getThriftPort
+ private val OUTPUT = "output"
+
+ override val options: Array[(String, CLIOption[Any])] = Array(
+ OUTPUT -> CLIOption[String]("<output path for configuration file>", required = false, defaultValue = Some("app.yaml"))
+ )
+
+ override def main(inputAkkaConf: Config, args: Array[String]): Unit = {
+ val parsed = parse(args)
+ val output = parsed.getString(OUTPUT)
+ val akkaConf = updateClientConfig(inputAkkaConf)
+ val system = ActorSystem("storm", akkaConf)
+ val clientContext = new ClientContext(akkaConf, system, null)
+ val stormConf = Utils.readStormConfig().asInstanceOf[JMap[AnyRef, AnyRef]]
+ val thriftConf: JMap[String, String] = Map(
+ Config.NIMBUS_HOST -> ("" + akkaConf.getString(Constants.GEARPUMP_HOSTNAME)),
+ Config.NIMBUS_THRIFT_PORT -> s"$THRIFT_PORT").asJava
+ updateStormConfig(thriftConf, output)
+ stormConf.putAll(thriftConf)
+ val thriftServer = createServer(clientContext, stormConf)
+ thriftServer.serve()
+ system.awaitTermination()
+ }
+
+ private def createServer(clientContext: ClientContext, stormConf: JMap[AnyRef, AnyRef]): ThriftServer = {
+ val processor = new Nimbus.Processor[GearpumpNimbus](new GearpumpNimbus(clientContext, stormConf))
+ val connectionType = ThriftConnectionType.NIMBUS
+ new ThriftServer(stormConf, processor, connectionType)
+ }
+
+ private def updateStormConfig(thriftConfig: JMap[String, String], output: String): Unit = {
+ val updatedConfig: JMap[AnyRef, AnyRef] = new JHashMap[AnyRef, AnyRef]
+ val outputConfig = Utils.findAndReadConfigFile(output, false).asInstanceOf[JMap[AnyRef, AnyRef]]
+ updatedConfig.putAll(outputConfig)
+ updatedConfig.putAll(thriftConfig)
+ // workaround that yaml dumps {nimbus.host: 127.0.0.1} instead of {nimbus.host: '127.0.0.1'}
+ // which can not be loaded
+ val nimbusHost = s"${Config.NIMBUS_HOST}: '${thriftConfig.get(Config.NIMBUS_HOST)}'"
+ updatedConfig.remove(Config.NIMBUS_HOST)
+ val yaml = new Yaml
+ val serialized = yaml.dumpAsMap(updatedConfig) ++ s"$nimbusHost\n"
+ val writer = new FileWriter(new File(output))
+ try {
+ writer.write(serialized)
+ } catch {
+ case e: Exception => throw e
+ } finally {
+ writer.close()
+ }
+ }
+
+ import Constants._
+ private def updateClientConfig(config: Config): Config = {
+ val storm = s"<${GEARPUMP_HOME}>/lib/storm/*"
+ val appClassPath = s"$storm${File.pathSeparator}" + config.getString(GEARPUMP_APPMASTER_EXTRA_CLASSPATH)
+ val executorClassPath = s"$storm${File.pathSeparator}" + config.getString(Constants.GEARPUMP_EXECUTOR_EXTRA_CLASSPATH)
+
+ val updated = config.withValue(GEARPUMP_APPMASTER_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(appClassPath))
+ .withValue(GEARPUMP_EXECUTOR_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(executorClassPath))
+
+ if (config.hasPath(StormConstants.STORM_SERIALIZATION_FRAMEWORK)) {
+ val serializerConfig = ConfigValueFactory.fromAnyRef(config.getString(StormConstants.STORM_SERIALIZATION_FRAMEWORK))
+ updated.withValue(GEARPUMP_SERIALIZER_POOL, serializerConfig)
+ } else {
+ updated
+ }
+ }
+
+}
+
+class GearpumpNimbus(clientContext: ClientContext, stormConf: JMap[AnyRef, AnyRef]) extends Nimbus.Iface {
+ private val LOG: Logger = LogUtil.getLogger(classOf[GearpumpNimbus])
+ private var applications = Map.empty[String, Int]
+ private var topologies = Map.empty[String, TopologyData]
+ private val expireSeconds = StormUtil.getInt(stormConf, Config.NIMBUS_FILE_COPY_EXPIRATION_SECS).get
+ private val expiredCallback = new ExpiredCallback[String, WritableByteChannel] {
+ override def expire(k: String, v: WritableByteChannel): Unit = {
+ v.close()
+ }
+ }
+ private val fileCacheMap = new TimeCacheMap[String, WritableByteChannel](expireSeconds, expiredCallback)
+
+ override def submitTopology(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology): Unit = {
+ submitTopologyWithOpts(name, uploadedJarLocation, jsonConf, topology, new SubmitOptions(TopologyInitialStatus.ACTIVE))
+ }
+
+ override def submitTopologyWithOpts(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology, options: SubmitOptions): Unit = {
+
+ implicit val system = clientContext.system
+ val gearpumpStormTopology = GearpumpStormTopology(name, topology, jsonConf)
+ val stormConfig = gearpumpStormTopology.getStormConfig
+ val processorGraph = GraphBuilder.build(gearpumpStormTopology)
+ val config = UserConfig.empty
+ .withValue[StormTopology](StormConstants.STORM_TOPOLOGY, topology)
+ .withValue[JMap[AnyRef, AnyRef]](StormConstants.STORM_CONFIG, stormConfig)
+ val app = StreamApplication(name, processorGraph, config)
+ LOG.info(s"jar file uploaded to $uploadedJarLocation")
+ val appId = clientContext.submit(app, uploadedJarLocation)
+ applications += name -> appId
+ topologies += name -> TopologyData(topology, stormConfig, uploadedJarLocation)
+ LOG.info(s"Storm Application $appId submitted")
+ }
+
+ override def killTopologyWithOpts(name: String, options: KillOptions): Unit = {
+ if (applications.contains(name)) {
+ clientContext.shutdown(applications(name))
+ removeTopology(name)
+ LOG.info(s"Killed topology $name")
+ } else {
+ throw new RuntimeException(s"topology $name not found")
+ }
+ }
+
+ override def getNimbusConf: String = {
+ JSONValue.toJSONString(stormConf)
+ }
+
+ override def getTopology(name: String): StormTopology = {
+ updateApps
+ topologies.getOrElse(name,
+ throw new RuntimeException(s"topology $name not found")).topology
+ }
+
+ override def getTopologyConf(name: String): String = {
+ updateApps
+ JSONValue.toJSONString(topologies.getOrElse(name,
+ throw new RuntimeException(s"topology $name not found")).config)
+ }
+
+ override def getUserTopology(id: String): StormTopology = getTopology(id)
+
+ override def beginFileUpload(): String = {
+ val file = File.createTempFile(s"storm-jar-${UUID.randomUUID()}", ".jar")
+ val location = file.getAbsolutePath
+ val channel = Channels.newChannel(new FileOutputStream(location))
+ fileCacheMap.put(location, channel)
+ LOG.info(s"Uploading file from client to $location")
+ location
+ }
+
+ override def uploadChunk(location: String, chunk: ByteBuffer): Unit = {
+ if (!fileCacheMap.containsKey(location)) {
+ throw new RuntimeException(s"File for $location does not exist (or timed out)")
+ } else {
+ val channel = fileCacheMap.get(location)
+ channel.write(chunk)
+ fileCacheMap.put(location, channel)
+ }
+ }
+
+ override def finishFileUpload(location: String): Unit = {
+ if (!fileCacheMap.containsKey(location)) {
+ throw new RuntimeException(s"File for $location does not exist (or timed out)")
+ } else {
+ val channel = fileCacheMap.get(location)
+ channel.close()
+ fileCacheMap.remove(location)
+ }
+ }
+
+ override def getClusterInfo: ClusterSummary = {
+ updateApps
+ val topologySummaryList = topologies.map { case (name, _) =>
+ new TopologySummary(name, name, 0, 0, 0, 0, "")
+ }.toSeq
+ new ClusterSummary(List[SupervisorSummary]().asJava, 0, topologySummaryList.asJava)
+ }
+
+ override def beginFileDownload(file: String): String = {
+ throw new UnsupportedOperationException
+ }
+
+ override def uploadNewCredentials(s: String, credentials: Credentials): Unit = {
+ throw new UnsupportedOperationException
+ }
+ override def activate(name: String): Unit = {
+ throw new UnsupportedOperationException
+ }
+
+ override def rebalance(name: String, options: RebalanceOptions): Unit = {
+ throw new UnsupportedOperationException
+ }
+
+ override def deactivate(name: String): Unit = {
+ throw new UnsupportedOperationException
+ }
+
+ override def getTopologyInfo(name: String): TopologyInfo = {
+ throw new UnsupportedOperationException
+ }
+
+ override def getTopologyInfoWithOpts(s: String, getInfoOptions: GetInfoOptions): TopologyInfo = {
+ throw new UnsupportedOperationException
+ }
+
+ override def killTopology(name: String): Unit = killTopologyWithOpts(name, new KillOptions())
+
+ override def downloadChunk(name: String): ByteBuffer = {
+ throw new UnsupportedOperationException
+ }
+
+ private def updateApps: Unit = {
+ clientContext.listApps.appMasters.foreach { app =>
+ val name = app.appName
+ if (applications.contains(name)) {
+ if (app.status != MasterToAppMaster.AppMasterActive) {
+ removeTopology(name)
+ }
+ }
+ }
+ }
+
+ private def removeTopology(name: String): Unit = {
+ applications -= name
+ val jar = topologies(name).jar
+ new File(jar).delete()
+ topologies -= name
+ }
+}
+
+case class TopologyData(topology: StormTopology, config: JMap[AnyRef, AnyRef], jar: String)
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
new file mode 100644
index 0000000..b813ae0
--- /dev/null
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.experiments.storm.main
+
+import backtype.storm.Config
+import backtype.storm.utils.Utils
+import io.gearpump.cluster.main.{ArgumentsParser, CLIOption}
+import io.gearpump.util.Constants._
+import io.gearpump.util.{AkkaApp, LogUtil, Util}
+
+object GearpumpStormClient extends AkkaApp with ArgumentsParser {
+
+ override val options: Array[(String, CLIOption[Any])] = Array(
+ "jar" -> CLIOption[String]("<storm jar>", required = true),
+ "config" -> CLIOption[Int]("<storm config file>", required = true),
+ "verbose" -> CLIOption("<print verbose log on console>", required = false, defaultValue = Some(false)))
+
+ override def main(inputAkkaConf: Config, args: Array[String]): Unit = {
+ val config = parse(args)
+
+ val verbose = config.getBoolean("verbose")
+ if (verbose) {
+ LogUtil.verboseLogToConsole
+ }
+
+ val jar = config.getString("jar")
+ val stormConfig = config.getString("config")
+ val topology = config.remainArgs(0)
+ val stormArgs = config.remainArgs.drop(1)
+ val stormOptions = Array(
+ s"-Dstorm.options=${getThriftOptions(stormConfig)}",
+ s"-Dstorm.jar=$jar",
+ s"-Dstorm.config.file=$stormConfig",
+ s"-D${PREFER_IPV4}=true"
+ )
+
+ val classPath = Array(s"${System.getProperty(GEARPUMP_HOME)}/lib/storm/*", jar)
+ val process = Util.startProcess(stormOptions, classPath, topology, stormArgs)
+
+ // wait till the process exit
+ val exit = process.exitValue()
+
+ if (exit != 0) {
+ throw new Exception(s"failed to submit jar, exit code $exit, error summary: ${process.logger.error}")
+ }
+ }
+
+ private def getThriftOptions(stormConfig: String): String = {
+ val config = Utils.findAndReadConfigFile(stormConfig, true)
+ val host = config.get(Config.NIMBUS_HOST)
+ val thriftPort = config.get(Config.NIMBUS_THRIFT_PORT)
+ s"${Config.NIMBUS_HOST}=$host,${Config.NIMBUS_THRIFT_PORT}=$thriftPort"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
index 996498b..b88a200 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
@@ -18,7 +18,6 @@
package io.gearpump.experiments.storm.topology
-import java.io._
import java.lang.{Iterable => JIterable}
import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap}
@@ -35,8 +34,6 @@ import io.gearpump.experiments.storm.util.StormUtil._
import io.gearpump.streaming.Processor
import io.gearpump.streaming.task.Task
import io.gearpump.util.LogUtil
-import org.apache.storm.shade.org.yaml.snakeyaml.Yaml
-import org.apache.storm.shade.org.yaml.snakeyaml.constructor.SafeConstructor
import org.slf4j.Logger
import scala.collection.JavaConversions._
@@ -45,43 +42,18 @@ object GearpumpStormTopology {
private val LOG: Logger = LogUtil.getLogger(classOf[GearpumpStormTopology])
def apply(
+ name: String,
topology: StormTopology,
- appConfigInJson: String,
- fileConfig: String)(implicit system: ActorSystem): GearpumpStormTopology = {
+ appConfigInJson: String)(implicit system: ActorSystem): GearpumpStormTopology = {
new GearpumpStormTopology(
+ name,
topology,
Utils.readStormConfig().asInstanceOf[JMap[AnyRef, AnyRef]],
- parseJsonStringToMap(appConfigInJson),
- readStormConfig(fileConfig)
+ parseJsonStringToMap(appConfigInJson)
)
}
- /**
- * @param configFile user provided local config file
- * @return a config Map loaded from config file
- */
- private def readStormConfig(configFile: String): JMap[AnyRef, AnyRef] = {
- var ret: JMap[AnyRef, AnyRef] = new JHashMap[AnyRef, AnyRef]
- try {
- val yaml = new Yaml(new SafeConstructor)
- val input: InputStream = new FileInputStream(configFile)
- try {
- ret = yaml.load(new InputStreamReader(input)).asInstanceOf[JMap[AnyRef, AnyRef]]
- } catch {
- case e: IOException =>
- LOG.warn(s"failed to load config file $configFile")
- } finally {
- input.close()
- }
- } catch {
- case e: FileNotFoundException =>
- LOG.warn(s"failed to find config file $configFile")
- case t: Throwable =>
- LOG.error(t.getMessage)
- }
- ret
- }
}
/**
@@ -91,21 +63,20 @@ object GearpumpStormTopology {
* 3. provides interface for Gearpump applications to use Storm topology
*
* an implicit ActorSystem is required to create Gearpump processors
+ * @param name topology name
* @param topology Storm topology
- * @param sysConfig configs from "defaults.yaml" and "storm.yaml"
+ * @param sysConfig configs from "defaults.yaml" and custom config file
* @param appConfig config submitted from user application
- * @param fileConfig custom file config set by user in command line
*/
private[storm] class GearpumpStormTopology(
+ name: String,
topology: StormTopology,
sysConfig: JMap[AnyRef, AnyRef],
- appConfig: JMap[AnyRef, AnyRef],
- fileConfig: JMap[AnyRef, AnyRef])(implicit system: ActorSystem) {
- import io.gearpump.experiments.storm.topology.GearpumpStormTopology._
+ appConfig: JMap[AnyRef, AnyRef])(implicit system: ActorSystem) {
private val spouts = topology.get_spouts()
private val bolts = topology.get_bolts()
- private val stormConfig = mergeConfigs(sysConfig, appConfig, fileConfig, getComponentConfigs(spouts, bolts))
+ private val stormConfig = mergeConfigs(sysConfig, appConfig, getComponentConfigs(spouts, bolts))
private val spoutProcessors = spouts.map { case (id, spout) =>
id -> spoutToProcessor(id, spout, stormConfig.toMap) }.toMap
private val boltProcessors = bolts.map { case (id, bolt) =>
@@ -114,7 +85,7 @@ private[storm] class GearpumpStormTopology(
/**
* @return merged Storm config with priority
- * defaults.yaml < storm.yaml < application config < component config < custom file config
+ * defaults.yaml < custom file config < application config < component config
*/
def getStormConfig: JMap[AnyRef, AnyRef] = stormConfig
@@ -139,13 +110,12 @@ private[storm] class GearpumpStormTopology(
private def mergeConfigs(
sysConfig: JMap[AnyRef, AnyRef],
appConfig: JMap[AnyRef, AnyRef],
- fileConfig: JMap[AnyRef, AnyRef],
componentConfigs: Iterable[JMap[AnyRef, AnyRef]]): JMap[AnyRef, AnyRef] = {
val allConfig = new JHashMap[AnyRef, AnyRef]
allConfig.putAll(sysConfig)
allConfig.putAll(appConfig)
allConfig.putAll(getMergedComponentConfig(componentConfigs, allConfig.toMap))
- allConfig.putAll(fileConfig)
+ allConfig.put(Config.TOPOLOGY_NAME, name)
allConfig
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala
deleted file mode 100644
index 20b63b3..0000000
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.experiments.storm
-
-import akka.actor.ActorSystem
-import akka.testkit.TestProbe
-import backtype.storm.generated.StormTopology
-import io.gearpump.experiments.storm.util.TopologyUtil
-import io.gearpump.cluster.TestUtil
-import Commands.{GetTopology, Kill, Submit}
-import GearpumpThriftServer.GearpumpNimbus
-import org.scalatest.mock.MockitoSugar
-import org.scalatest.{Matchers, WordSpec}
-
-import scala.concurrent.Future
-
-class GearpumpNimbusSpec extends WordSpec with Matchers with MockitoSugar {
-
- "GearpumpNimbus" should {
- "submit and kill topology through ClientContext" in {
-
- implicit val system = ActorSystem("storm-test", TestUtil.DEFAULT_CONFIG)
- implicit val dispatcher = system.dispatcher
-
- val handler = TestProbe()
- val gearpumpNimbus = new GearpumpNimbus(handler.ref)
-
- val appId = 0
- val name = "test"
- val uploadedJarLocation = "local"
- val jsonConf = "storm_json_conf"
- val topology = TopologyUtil.getTestTopology
-
- Future(gearpumpNimbus.submitTopology(name, uploadedJarLocation, jsonConf, topology))
- handler.expectMsgType[Submit]
-
- Future(gearpumpNimbus.getTopology(name))
- handler.expectMsgType[GetTopology]
- handler.reply(new StormTopology)
-
- Future(gearpumpNimbus.getUserTopology(name))
- handler.expectMsgType[GetTopology]
- handler.reply(new StormTopology)
-
- Future(gearpumpNimbus.killTopology(name))
- handler.expectMsgType[Kill]
-
- system.shutdown()
- system.awaitTermination()
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala
deleted file mode 100644
index b1736f0..0000000
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.experiments.storm
-
-import backtype.storm.security.auth.ThriftServer
-import org.mockito.Mockito._
-import org.scalatest.WordSpec
-import org.scalatest.mock.MockitoSugar
-
-class GearpumpThriftServerSpec extends WordSpec with MockitoSugar {
-
- "GearpumpThriftServer" should {
- "run ThriftServer.serve" in {
- val tServer = mock[ThriftServer]
- val thriftServer = new GearpumpThriftServer(tServer)
-
- thriftServer.run()
-
- verify(tServer).serve()
- }
-
- "stop ThriftServer" in {
- val tServer = mock[ThriftServer]
- val thriftServer = new GearpumpThriftServer(tServer)
-
- thriftServer.close()
-
- verify(tServer).stop()
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala
deleted file mode 100644
index 95cf279..0000000
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package io.gearpump.experiments.storm
-
-import akka.actor.{ActorSystem, Props}
-import akka.testkit.TestProbe
-import io.gearpump.cluster.TestUtil
-import io.gearpump.cluster.client.ClientContext
-import io.gearpump.experiments.storm.Commands.{AppKilled, AppSubmitted, Kill, Submit}
-import io.gearpump.experiments.storm.StormRunner.Handler
-import io.gearpump.experiments.storm.util.TopologyUtil
-import org.mockito.Matchers._
-import org.mockito.Mockito
-import org.scalatest.{FlatSpec, Matchers}
-
-class StormRunnerSpec extends FlatSpec with Matchers {
- it should "handle submit/kill correctly" in {
- val conf = TestUtil.DEFAULT_CONFIG
- implicit val system = ActorSystem("storm-test", conf)
-
- val uploadedJarLocation = "local"
- val jsonConf = "storm_json_conf"
- val topology = TopologyUtil.getTestTopology
-
- implicit val dispatcher = system.dispatcher
- val clientContext = Mockito.mock(classOf[ClientContext])
- Mockito.when(clientContext.submit(any(), any())).thenReturn(0)
- val handler = system.actorOf(Props(new Handler(clientContext, "jar", "user_config")))
- val client = TestProbe()
-
- client.send(handler, Submit("app", uploadedJarLocation, jsonConf, topology, null))
- client.expectMsg(AppSubmitted("app", 0))
-
-
- client.send(handler, Kill("app", null))
- client.expectMsg(AppKilled("app", 0))
-
- system.shutdown()
- system.awaitTermination()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
index d1c330b..8f10886 100644
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
+++ b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
@@ -20,6 +20,7 @@ package io.gearpump.experiments.storm.topology
import java.util.{HashMap => JHashMap, Map => JMap}
+import backtype.storm.Config
import io.gearpump.experiments.storm.processor.StormProcessor
import io.gearpump.experiments.storm.producer.StormProducer
import io.gearpump.experiments.storm.util.TopologyUtil
@@ -40,28 +41,27 @@ class GearpumpStormTopologySpec extends WordSpec with Matchers with MockitoSugar
val sysConfig = newJavaConfig(name, sysVal)
val appVal = "app"
val appConfig = newJavaConfig(name, appVal)
- val fileVal = "file"
- val fileConfig = newJavaConfig(name, fileVal)
implicit val system = MockUtil.system
- val topology1 = new GearpumpStormTopology(stormTopology, newEmptyConfig, newEmptyConfig, newEmptyConfig)
- topology1.getStormConfig shouldBe empty
+ val topology1 = new GearpumpStormTopology("topology1", stormTopology, newEmptyConfig, newEmptyConfig)
+ topology1.getStormConfig(Config.TOPOLOGY_NAME) shouldBe "topology1"
+ topology1.getStormConfig should not contain name
- val topology2 = new GearpumpStormTopology(stormTopology, sysConfig, newEmptyConfig, newEmptyConfig)
+ val topology2 = new GearpumpStormTopology("topology2", stormTopology, sysConfig, newEmptyConfig)
+ topology2.getStormConfig(Config.TOPOLOGY_NAME) shouldBe "topology2"
topology2.getStormConfig.get(name) shouldBe sysVal
- val topology3 = new GearpumpStormTopology(stormTopology, sysConfig, appConfig, newEmptyConfig)
+ val topology3 = new GearpumpStormTopology("topology3", stormTopology, sysConfig, appConfig)
+ topology3.getStormConfig(Config.TOPOLOGY_NAME) shouldBe "topology3"
topology3.getStormConfig.get(name) shouldBe appVal
- val topology4 = new GearpumpStormTopology(stormTopology, sysConfig, appConfig, fileConfig)
- topology4.getStormConfig.get(name) shouldBe fileVal
}
"create Gearpump processors from Storm topology" in {
val stormTopology = TopologyUtil.getTestTopology
implicit val system = MockUtil.system
val gearpumpStormTopology =
- GearpumpStormTopology(stormTopology, null, "")
+ GearpumpStormTopology("app", stormTopology, null)
val processors = gearpumpStormTopology.getProcessors
stormTopology.get_spouts().foreach { case (spoutId, _) =>
val processor = processors(spoutId)
@@ -80,7 +80,7 @@ class GearpumpStormTopologySpec extends WordSpec with Matchers with MockitoSugar
implicit val system = MockUtil.system
val sysConfig = new JHashMap[AnyRef, AnyRef]
val gearpumpStormTopology =
- GearpumpStormTopology(stormTopology, null, "")
+ GearpumpStormTopology("app", stormTopology, null)
val targets0 = gearpumpStormTopology.getTargets("1")
targets0 should contain key "3"
targets0 should contain key "4"
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/integrationtest/core/src/it/scala/io/gearpump/integrationtest/suites/StandaloneModeSuite.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/it/scala/io/gearpump/integrationtest/suites/StandaloneModeSuite.scala b/integrationtest/core/src/it/scala/io/gearpump/integrationtest/suites/StandaloneModeSuite.scala
index c7a2c3e..944f542 100644
--- a/integrationtest/core/src/it/scala/io/gearpump/integrationtest/suites/StandaloneModeSuite.scala
+++ b/integrationtest/core/src/it/scala/io/gearpump/integrationtest/suites/StandaloneModeSuite.scala
@@ -32,8 +32,8 @@ class StandaloneModeSuite extends Suites(
new RestServiceSpec,
new ExampleSpec,
new DynamicDagSpec,
- new StabilitySpec,
- new StormCompatibilitySpec
+ new StabilitySpec/*,
+ new StormCompatibilitySpec*/
) with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
index ed792f4..15acf4e 100644
--- a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
+++ b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
@@ -20,7 +20,6 @@ package io.gearpump.integrationtest.storm
import backtype.storm.utils.{Utils, DRPCClient}
-import io.gearpump.experiments.storm.GearpumpThriftServer
import io.gearpump.integrationtest.Docker
import io.gearpump.integrationtest.minicluster.BaseContainer
import org.apache.log4j.Logger
@@ -30,23 +29,28 @@ class StormClient(masterAddrs: Seq[(String, Int)]) {
private val LOG = Logger.getLogger(getClass)
private val STORM_HOST = "storm0"
- private val STORM_CMD = "/opt/start storm"
+ private val STORM_NIMBUS = "/opt/start storm nimbus"
+ private val STORM_APP = "/opt/start storm app"
private val STORM_DRPC = "storm-drpc"
private val CONFIG_FILE = "storm.yaml"
- private val NIMBUS_THRIFT_PORT = GearpumpThriftServer.THRIFT_PORT
private val DRPC_PORT = 3772
private val DRPC_INVOCATIONS_PORT = 3773
private val container = new BaseContainer(STORM_HOST, STORM_DRPC, masterAddrs,
- tunnelPorts = Set(NIMBUS_THRIFT_PORT, DRPC_PORT, DRPC_INVOCATIONS_PORT))
+ tunnelPorts = Set(DRPC_PORT, DRPC_INVOCATIONS_PORT))
def start(): Unit = {
container.createAndStart()
+ startNimbus
+ }
+
+ private def startNimbus: String = {
+ Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_NIMBUS -output $CONFIG_FILE")
}
def submitStormApp(jar: String, mainClass: String, args: String = ""): Int = {
try {
- Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_CMD -config $CONFIG_FILE " +
+ Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_APP -config $CONFIG_FILE " +
s"-jar $jar $mainClass $args").split("\n")
.filter(_.contains("The application id is ")).head.split(" ").last.toInt
} catch {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/project/Pack.scala
----------------------------------------------------------------------
diff --git a/project/Pack.scala b/project/Pack.scala
index b242b36..a3852c4 100644
--- a/project/Pack.scala
+++ b/project/Pack.scala
@@ -59,7 +59,7 @@ object Pack extends sbt.Build {
"worker" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-DlogFilename=worker", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"),
"services" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"),
"yarnclient" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"),
- "storm" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost")
+ "storm" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}")
),
packLibDir := Map(
"lib" -> new ProjectsToPack(core.id, streaming.id),
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/400f627e/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
----------------------------------------------------------------------
diff --git a/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala b/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
index e2b2c47..aa90726 100644
--- a/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
+++ b/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
@@ -221,7 +221,7 @@ object MasterService {
*/
def submitStormApp(jar: Option[File], stormConf: Option[File], args: String, systemConfig: Config): Boolean = {
submitAndDeleteTempFiles(
- "io.gearpump.experiments.storm.StormRunner",
+ "io.gearpump.experiments.storm.main.GearpumpStormClient",
argsArray = spaceSeparatedArgumentsToArray(args),
fileMap = Map("jar" -> jar, "config" -> stormConf).filter(_._2.isDefined).mapValues(_.get),
classPath = getStormApplicationClassPath,
[32/50] incubator-gearpump git commit: update readme
Posted by ap...@apache.org.
update readme
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/9feb1598
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/9feb1598
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/9feb1598
Branch: refs/heads/master
Commit: 9feb159834b67a7fc0fdfa89f30956b14b4d65c3
Parents: 79f01ba
Author: Qian Xu <qi...@intel.com>
Authored: Sun Feb 14 11:36:28 2016 +0800
Committer: Qian Xu <qi...@intel.com>
Committed: Sun Feb 14 11:36:28 2016 +0800
----------------------------------------------------------------------
README.md | 81 +++++++++++++++-------------------------------------------
1 file changed, 20 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/9feb1598/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index f3a223e..e7f6194 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,7 @@
-## Gearpump [](http://www.gearpump.io/download.html) [](https://github.com/gearpump/gearpump/blob/master/LICENSE)
+## Gearpump [](http://www.gearpump.io/download.html) [](https://github.com/gearpump/gearpump/blob/master/LICENSE) [](https://travis-ci.org/gearpump/gearpump?branch=master) [](https://codecov.io/github/gearpump/gearpump?branch=master)
-[](https://travis-ci.org/gearpump/gearpump?branch=master) [](https://codecov.io/github/gearpump/gearpump?branch=master)
-
-Web site: http://gearpump.io
+Online Demo Site: http://demo.gearpump.io/
[](http://www.gearpump.io/download.html)
@@ -11,8 +9,7 @@ Gearpump is a lightweight real-time big data streaming engine. It is inspired by
-The name Gearpump is a reference to the engineering term “gear pump,” which is a super simple
-pump that consists of only two gears, but is very powerful at streaming water.
+The name Gearpump is a reference to the engineering term "gear pump", which is a super simple pump that consists of only two gears, but is very powerful at streaming water.
@@ -24,19 +21,18 @@ Per initial benchmarks we are able to process near 18 million messages/second (1
-For steps to reproduce the performance test, please check [Performance benchmark](http://www.gearpump.io/releases/latest/performance-report.html)
-
-## Design Document
-
-There is a 20 pages technical paper on typesafe blog, with technical highlights https://typesafe.com/blog/gearpump-real-time-streaming-engine-using-akka
+For steps to reproduce the performance test, please check [Performance benchmark](http://www.gearpump.io/releases/latest/performance-report.html).
-## Introduction and Motivation
+## Useful Resources
-Check [Introduction](http://www.gearpump.io/overview.html)
-
-## Getting Started
-
-Check guide [Get Started](http://www.gearpump.io/releases/latest/submit-your-1st-application.html)
+* Read the [Introduction on TypeSafe's Blog](https://typesafe.com/blog/gearpump-real-time-streaming-engine-using-akka)
+* Learn the [Basic Concepts](http://www.gearpump.io/releases/latest/basic-concepts.html)
+* How to [Develop your first application](http://www.gearpump.io/releases/latest/dev-write-1st-app.html)
+* How to [Submit your first application](http://www.gearpump.io/releases/latest/submit-your-1st-application.html)
+* Explore the [Maven dependencies](http://www.gearpump.io/releases/latest/maven-setting.html)
+* Explore the [Document site](http://gearpump.io)
+* Explore the [User List](https://groups.google.com/forum/#!forum/gearpump-user)
+* Report an [issue](https://github.com/gearpump/gearpump/issues)
## How to Build
@@ -79,43 +75,8 @@ export HTTP_PROXY=http://host:port
export HTTPS_PROXY= http://host:port
```
-
## How to run Gearpump integration test
-Gearpump has an integration test system which is based on Docker.
-
-### Prerequisite of integration test
-To run Gearpump integration test, we first must have Docker installed in your PATH.
-
-And you need to have Docker well configured if you are behind firewall.
-You can run command `docker pull stanleyxu2005/gpct-jdk8` to check whether you have Docker installed and well configured.
-
-#### Set docker proxy if you behind a firewall
-In ubuntu, you can modify file `/etc/default/docker` and enable the http_proxy setting.
-
-### Run integration test
-After Docker well configured, you can run the whole integration test via command:
-```bash
-sbt it:test
-```
-
-## Concepts
-
-Check guide [Concepts](http://www.gearpump.io/releases/latest/basic-concepts.html)
-
-## How to write a Gearpump Application
-
-Check guide [Streaming Application Developer Guide](http://www.gearpump.io/releases/latest/dev-write-1st-app.html)
-
-
-# Maven dependencies
-
-Check [Maven settings](http://www.gearpump.io/releases/latest/maven-setting.html)
-
-## Further information
-
-- Document site [gearpump.io](http://gearpump.io)
-- User List: [gearpump-user](https://groups.google.com/forum/#!forum/gearpump-user).
-- Report issues: [issue tracker](https://github.com/gearpump/gearpump/issues)
+Gearpump has an integration test system which is based on Docker. Please check [the instructions](integrationtest/README.md).
## Contributors (time order)
@@ -129,17 +90,15 @@ Check [Maven settings](http://www.gearpump.io/releases/latest/maven-setting.html
* [Tomasz Targonski](https://github.com/TomaszT)
* [Sun Kewei](https://github.com/skw1992)
-## Contacts:
+## Contacts
-Please use the google user list if possible. For things that are not OK to be shared in maillist, please contact:
-xiang.zhong@intel.com
-kam.d.kasravi@intel.com
-weihua.jiang@intel.com
+Please use the google user list if possible. For things that are not OK to be shared in mailing list, please contact:
+[Sean Zhong](mailto:xiang.zhong@intel.com), [Kam Kasravi](mailto:kam.d.kasravi@intel.com) or [Weihua Jiang](mailto:weihua.jiang@intel.com).
## License
-Gearpump itself is licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
-For library it used, please see [/LICENSE](https://github.com/gearpump/gearpump/blob/master/LICENSE)
+Gearpump itself is licensed under the [Apache License (2.0)](http://www.apache.org/licenses/LICENSE-2.0).
+For library it used, please see [LICENSE](https://github.com/gearpump/gearpump/blob/master/LICENSE).
## Acknowledgement
@@ -147,4 +106,4 @@ The netty transport code work is based on [Apache Storm](http://storm.apache.org
The cgroup code work is based on [JStorm](https://github.com/alibaba/jstorm). Thanks JStorm contributors.
-Thanks to Jetbrains for providing [IntelliJ IDEA Free Open Source License](https://www.jetbrains.com/buy/opensource/?product=idea).
+Thanks to Jetbrains for providing a [IntelliJ IDEA Free Open Source License](https://www.jetbrains.com/buy/opensource/?product=idea).
[18/50] incubator-gearpump git commit: bump to 0.7.7-SNAPSHOT
Posted by ap...@apache.org.
bump to 0.7.7-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/afcd6e0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/afcd6e0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/afcd6e0a
Branch: refs/heads/master
Commit: afcd6e0aa3d387e225ce80d9d172158b4c3641ff
Parents: 2aa3f22
Author: huafengw <fv...@gmail.com>
Authored: Thu Feb 4 13:48:40 2016 +0800
Committer: huafengw <fv...@gmail.com>
Committed: Thu Feb 4 13:48:40 2016 +0800
----------------------------------------------------------------------
docs/_config.yml | 2 +-
version.sbt | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/afcd6e0a/docs/_config.yml
----------------------------------------------------------------------
diff --git a/docs/_config.yml b/docs/_config.yml
index 12bfc4b..84b9ab4 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -16,7 +16,7 @@ exclude:
- build_doc.sh
# These allow the documentation to be updated with newer releases of Gearpump
-GEARPUMP_VERSION: 0.7.6
+GEARPUMP_VERSION: 0.7.7-SNAPSHOT
SCALA_BINARY_VERSION: "2.11"
SCALA_VERSION: "2.11.7"
GEARPUMP_ISSUE_TRACKER_URL: https://github.com/gearpump/gearpump/issues
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/afcd6e0a/version.sbt
----------------------------------------------------------------------
diff --git a/version.sbt b/version.sbt
index af88c45..f6b7b20 100644
--- a/version.sbt
+++ b/version.sbt
@@ -1 +1 @@
-version in ThisBuild := "0.7.6"
+version in ThisBuild := "0.7.7-SNAPSHOT"
[23/50] incubator-gearpump git commit: Merge pull request #1957 from
stanleyxu2005/issue_1906
Posted by ap...@apache.org.
Merge pull request #1957 from stanleyxu2005/issue_1906
fix #1906: some visual glichtes found in mobile view
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/058bbf56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/058bbf56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/058bbf56
Branch: refs/heads/master
Commit: 058bbf5673a13fb2ca4ab872ae263426466df221
Parents: 34c62fc 25f82ed
Author: Sean Zhong <xi...@intel.com>
Authored: Fri Feb 5 13:44:13 2016 +0800
Committer: Sean Zhong <xi...@intel.com>
Committed: Fri Feb 5 13:44:13 2016 +0800
----------------------------------------------------------------------
project/Build.scala | 7 ++++---
services/dashboard/index.html | 18 +++++++++---------
.../dashboard/views/apps/streamingapp/executor.js | 2 +-
.../views/apps/streamingapp/metric_selector.js | 2 +-
services/dashboard/views/cluster/master/master.js | 2 +-
.../views/cluster/workers/worker/worker.js | 2 +-
.../views/cluster/workers/workers_listview.js | 4 +---
7 files changed, 18 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
[24/50] incubator-gearpump git commit: fix #1958: better test report
organizing
Posted by ap...@apache.org.
fix #1958: better test report organizing
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/f048d6f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/f048d6f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/f048d6f2
Branch: refs/heads/master
Commit: f048d6f2112ab1ee1024def9f1ceec9dc347cc42
Parents: 058bbf5
Author: Qian Xu <qi...@intel.com>
Authored: Fri Feb 5 15:23:53 2016 +0800
Committer: Qian Xu <qi...@intel.com>
Committed: Fri Feb 5 15:33:00 2016 +0800
----------------------------------------------------------------------
integrationtest/autorun/autorun.sh | 17 ++++-----
.../minicluster/BaseContainer.scala | 14 ++++++--
.../minicluster/RestClient.scala | 38 ++++++++++++--------
3 files changed, 43 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f048d6f2/integrationtest/autorun/autorun.sh
----------------------------------------------------------------------
diff --git a/integrationtest/autorun/autorun.sh b/integrationtest/autorun/autorun.sh
index e9944e0..4963995 100644
--- a/integrationtest/autorun/autorun.sh
+++ b/integrationtest/autorun/autorun.sh
@@ -34,6 +34,8 @@ git checkout -q master
# Define the variables
DIST_DIR=output/target/pack
+LOG_DIR=/tmp/gearpump
+ATTACHMENT=everything.zip
REPORT_DIR=report
COMMIT_REV=''
@@ -51,7 +53,7 @@ function run_test_if_new_commit_found {
COMMIT_REV=$NEW_COMMIT_REV
COMMIT_LOG=$(git log -1)
- rm -rf $DIST_DIR
+ rm -Rf $DIST_DIR $LOG_DIR $ATTACHMENT
echo "Rebuild the project ..."
sbt clean assembly packArchiveZip
if [ $? -ne 0 ]; then
@@ -62,9 +64,9 @@ function run_test_if_new_commit_found {
fi
echo "Run tests ... (it will take couple of minutes or longer)"
- sbt "it:test-only *Suite* -- -h $REPORT_DIR" | tee console.log
+ sbt "it:test-only *Suite* -- -h $REPORT_DIR" | tee $REPORT_DIR/console.out
if [ $? -eq 0 ]; then
- grep -q "sbt.TestsFailedException: Tests unsuccessful" console.log
+ grep -q "sbt.TestsFailedException: Tests unsuccessful" $REPORT_DIR/console.out
if [ $? -ne 0 ]; then
mail_to \
"Gearpump test passed $COMMIT_REV" \
@@ -74,15 +76,14 @@ function run_test_if_new_commit_found {
fi
echo "Copy test logs into report directory ..."
- SELECTION="$REPORT_DIR $DIST_DIR/logs console.log"
- ZIP_FILES="$NEW_COMMIT_REV.zip"
- zip -q -r $ZIP_FILES "$SELECTION"
- rm -rf "$SELECTION"
+ SELECTION="$REPORT_DIR $LOG_DIR"
+ zip -q -r $ATTACHMENT $SELECTION
+ rm -Rf $SELECTION
mail_to \
"Gearpump test failed $COMMIT_REV" \
"Integration test failed. Please check attached files.\n\n$COMMIT_LOG" \
- -a $ZIP_FILES
+ -a $ATTACHMENT
}
while true; do
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f048d6f2/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/BaseContainer.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/BaseContainer.scala b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/BaseContainer.scala
index b283295..1d7ddd9 100644
--- a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/BaseContainer.scala
+++ b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/BaseContainer.scala
@@ -29,8 +29,14 @@ class BaseContainer(val host: String, command: String,
tunnelPorts: Set[Int] = Set.empty) {
private val IMAGE_NAME = "stanleyxu2005/gearpump-launcher"
- private val SUT_LOCAL_PATH = "pwd".!!.trim + "/output/target/pack"
- val SUT_HOME = "/opt/gearpump"
+ private val IMAGE_SUT_HOME = "/opt/gearpump"
+ private val IMAGE_LOG_HOME = "/var/log/gearpump"
+ private val LOCAL_SUT_HOME = "pwd".!!.trim + "/output/target/pack"
+ private val LOCAL_LOG_HOME = {
+ val dir = "/tmp/gearpump"
+ s"mkdir -p $dir".!!
+ s"mktemp -p $dir -d".!!.trim
+ }
private val CLUSTER_OPTS = {
masterAddrs.zipWithIndex.map { case (hostPort, index) =>
@@ -41,7 +47,9 @@ class BaseContainer(val host: String, command: String,
def createAndStart(): String = {
Docker.createAndStartContainer(host, IMAGE_NAME, command,
environ = Map("JAVA_OPTS" -> CLUSTER_OPTS),
- volumes = Map(SUT_LOCAL_PATH -> SUT_HOME),
+ volumes = Map(
+ LOCAL_SUT_HOME -> IMAGE_SUT_HOME,
+ LOCAL_LOG_HOME -> IMAGE_LOG_HOME),
knownHosts = masterAddrs.map(_._1).filter(_ != host).toSet,
tunnelPorts = tunnelPorts)
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f048d6f2/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/RestClient.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/RestClient.scala b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/RestClient.scala
index 07e9975..2b3003c 100644
--- a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/RestClient.scala
+++ b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/RestClient.scala
@@ -53,13 +53,21 @@ class RestClient(host: String, port: Int) {
Graph(vertexList, edgeList)
}
+ private def decodeAs[T: upickle.default.Reader](expr: String): T = try {
+ read[T](expr)
+ } catch {
+ case ex: Throwable =>
+ LOG.error(ex)
+ throw ex
+ }
+
def queryVersion(): String = {
callFromRoot("version")
}
def listWorkers(): Array[WorkerSummary] = {
val resp = callApi("master/workerlist")
- read[List[WorkerSummary]](resp).toArray
+ decodeAs[List[WorkerSummary]](resp).toArray
}
def listRunningWorkers(): Array[WorkerSummary] = {
@@ -68,7 +76,7 @@ class RestClient(host: String, port: Int) {
def listApps(): Array[AppMasterData] = {
val resp = callApi("master/applist")
- read[AppMastersData](resp).appMasters.toArray
+ decodeAs[AppMastersData](resp).appMasters.toArray
}
def listRunningApps(): Array[AppMasterData] = {
@@ -89,7 +97,7 @@ class RestClient(host: String, port: Int) {
options :+= s"conf=@$config"
}
val resp = callApi(endpoint, options.map("-F " + _).mkString(" "))
- val result = read[AppSubmissionResult](resp)
+ val result = decodeAs[AppSubmissionResult](resp)
assert(result.success)
true
} catch {
@@ -100,7 +108,7 @@ class RestClient(host: String, port: Int) {
def queryApp(appId: Int): AppMasterData = {
val resp = callApi(s"appmaster/$appId")
- read[AppMasterData](resp)
+ decodeAs[AppMasterData](resp)
}
def queryAppMasterConfig(appId: Int): Config = {
@@ -110,18 +118,18 @@ class RestClient(host: String, port: Int) {
def queryStreamingAppDetail(appId: Int): StreamAppMasterSummary = {
val resp = callApi(s"appmaster/$appId?detail=true")
- upickle.default.read[StreamAppMasterSummary](resp)
+ decodeAs[StreamAppMasterSummary](resp)
}
def queryStreamingAppMetrics(appId: Int, current: Boolean, path: String = "processor*"): HistoryMetrics = {
val args = if (current) "?readLatest=true" else ""
val resp = callApi(s"appmaster/$appId/metrics/app$appId.$path$args")
- upickle.default.read[HistoryMetrics](resp)
+ decodeAs[HistoryMetrics](resp)
}
def queryExecutorSummary(appId: Int, executorId: Int): ExecutorSummary = {
val resp = callApi(s"appmaster/$appId/executor/$executorId")
- upickle.default.read[ExecutorSummary](resp)
+ decodeAs[ExecutorSummary](resp)
}
def queryExecutorBrief(appId: Int): Array[ExecutorBrief] = {
@@ -131,7 +139,7 @@ class RestClient(host: String, port: Int) {
def queryExecutorMetrics(appId: Int, current: Boolean): HistoryMetrics = {
val args = if (current) "?readLatest=true" else ""
val resp = callApi(s"appmaster/$appId/metrics/app$appId.executor*$args")
- upickle.default.read[HistoryMetrics](resp)
+ decodeAs[HistoryMetrics](resp)
}
def queryExecutorConfig(appId: Int, executorId: Int): Config = {
@@ -141,13 +149,13 @@ class RestClient(host: String, port: Int) {
def queryMaster(): MasterSummary = {
val resp = callApi("master")
- read[MasterData](resp).masterDescription
+ decodeAs[MasterData](resp).masterDescription
}
def queryMasterMetrics(current: Boolean): HistoryMetrics = {
val args = if (current) "?readLatest=true" else ""
val resp = callApi(s"master/metrics/master?$args")
- upickle.default.read[HistoryMetrics](resp)
+ decodeAs[HistoryMetrics](resp)
}
def queryMasterConfig(): Config = {
@@ -158,7 +166,7 @@ class RestClient(host: String, port: Int) {
def queryWorkerMetrics(workerId: Int, current: Boolean): HistoryMetrics = {
val args = if (current) "?readLatest=true" else ""
val resp = callApi(s"worker/$workerId/metrics/worker$workerId?$args")
- upickle.default.read[HistoryMetrics](resp)
+ decodeAs[HistoryMetrics](resp)
}
def queryWorkerConfig(workerId: Int): Config = {
@@ -168,19 +176,19 @@ class RestClient(host: String, port: Int) {
def queryBuiltInPartitioners(): Array[String] = {
val resp = callApi("master/partitioners")
- upickle.default.read[BuiltinPartitioners](resp).partitioners
+ decodeAs[BuiltinPartitioners](resp).partitioners
}
def uploadJar(localFilePath: String): AppJar = {
val resp = callApi(s"master/uploadjar -F jar=@$localFilePath", CRUD_POST)
- upickle.default.read[AppJar](resp)
+ decodeAs[AppJar](resp)
}
def replaceStreamingAppProcessor(appId: Int, replaceMe: ProcessorDescription): Boolean = try {
val replaceOperation = new ReplaceProcessor(replaceMe.id, replaceMe)
val args = upickle.default.write(replaceOperation)
val resp = callApi(s"appmaster/$appId/dynamicdag?args=" + Util.encodeUriComponent(args), CRUD_POST)
- upickle.default.read[DAGOperationResult](resp)
+ decodeAs[DAGOperationResult](resp)
true
} catch {
case ex: Throwable =>
@@ -214,7 +222,7 @@ class RestClient(host: String, port: Int) {
def restartApp(appId: Int): Boolean = try {
val resp = callApi(s"appmaster/$appId/restart", CRUD_POST)
- upickle.default.read[Status](resp).success
+ decodeAs[Status](resp).success
} catch {
case ex: Throwable =>
LOG.warn(s"swallowed an exception: $ex")
[47/50] incubator-gearpump git commit: Merge pull request #1993 from
darionyaphet/branch_1992
Posted by ap...@apache.org.
Merge pull request #1993 from darionyaphet/branch_1992
[fix#1992] remove useless code
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/81d41e52
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/81d41e52
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/81d41e52
Branch: refs/heads/master
Commit: 81d41e52c0fc638110de620fd9eddc5539e19de0
Parents: 3f3d911 3b02d4a
Author: Qian Xu <st...@users.noreply.github.com>
Authored: Sun Mar 6 18:02:07 2016 +0800
Committer: Qian Xu <st...@users.noreply.github.com>
Committed: Sun Mar 6 18:02:07 2016 +0800
----------------------------------------------------------------------
core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
[36/50] incubator-gearpump git commit: fix #1966 make Partitioner API
Java compatible
Posted by ap...@apache.org.
fix #1966 make Partitioner API Java compatible
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/25aacb29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/25aacb29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/25aacb29
Branch: refs/heads/master
Commit: 25aacb29c73e0fccca2cfbdc4b13aae9204b2aa4
Parents: 9feb159
Author: huafengw <fv...@gmail.com>
Authored: Tue Feb 16 15:42:25 2016 +0800
Committer: huafengw <fv...@gmail.com>
Committed: Wed Feb 17 15:25:53 2016 +0800
----------------------------------------------------------------------
.../partitioner/BroadcastPartitioner.scala | 11 +++++++--
.../io/gearpump/partitioner/Partitioner.scala | 4 +--
.../streaming/examples/stock/QueryServer.scala | 26 ++++++++++----------
.../examples/transport/QueryServer.scala | 18 +++++++-------
.../storm/partitioner/StormPartitioner.scala | 6 ++---
.../storm/topology/GearpumpTuple.scala | 2 +-
.../experiments/storm/util/Grouper.scala | 25 ++++++++++---------
.../storm/util/StormOutputCollector.scala | 10 ++++----
.../partitioner/StormPartitionerSpec.scala | 6 ++---
.../storm/util/StormOutputCollectorSpec.scala | 10 ++++----
.../io/gearpump/services/MasterService.scala | 22 +++++++----------
.../streaming/appmaster/TaskManagerSpec.scala | 18 ++++----------
12 files changed, 77 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/core/src/main/scala/io/gearpump/partitioner/BroadcastPartitioner.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/partitioner/BroadcastPartitioner.scala b/core/src/main/scala/io/gearpump/partitioner/BroadcastPartitioner.scala
index 07ce399..dba02ee 100644
--- a/core/src/main/scala/io/gearpump/partitioner/BroadcastPartitioner.scala
+++ b/core/src/main/scala/io/gearpump/partitioner/BroadcastPartitioner.scala
@@ -21,7 +21,14 @@ package io.gearpump.partitioner
import io.gearpump.Message
class BroadcastPartitioner extends MulticastPartitioner {
- override def getPartitions(msg: Message, partitionNum: Int, currentPartitionId: Int): List[Int] = {
- (0 until partitionNum).toList
+ private var lastPartitionNum = -1
+ private var partitions = Array.empty[Int]
+
+ override def getPartitions(msg: Message, partitionNum: Int, currentPartitionId: Int): Array[Int] = {
+ if (partitionNum != lastPartitionNum) {
+ partitions = (0 until partitionNum).toArray
+ lastPartitionNum = partitionNum
+ }
+ partitions
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/core/src/main/scala/io/gearpump/partitioner/Partitioner.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/io/gearpump/partitioner/Partitioner.scala b/core/src/main/scala/io/gearpump/partitioner/Partitioner.scala
index 2478d3a..6285bb7 100644
--- a/core/src/main/scala/io/gearpump/partitioner/Partitioner.scala
+++ b/core/src/main/scala/io/gearpump/partitioner/Partitioner.scala
@@ -41,9 +41,9 @@ trait UnicastPartitioner extends Partitioner {
}
trait MulticastPartitioner extends Partitioner {
- def getPartitions(msg: Message, partitionNum: Int, currentPartitionId: Int): List[Int]
+ def getPartitions(msg: Message, partitionNum: Int, currentPartitionId: Int): Array[Int]
- def getPartitions(msg: Message, partitionNum: Int): List[Int] = {
+ def getPartitions(msg: Message, partitionNum: Int): Array[Int] = {
getPartitions(msg, partitionNum, Partitioner.UNKNOWN_PARTITION_ID)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/examples/streaming/stockcrawler/src/main/scala/io/gearpump/streaming/examples/stock/QueryServer.scala
----------------------------------------------------------------------
diff --git a/examples/streaming/stockcrawler/src/main/scala/io/gearpump/streaming/examples/stock/QueryServer.scala b/examples/streaming/stockcrawler/src/main/scala/io/gearpump/streaming/examples/stock/QueryServer.scala
index a705000..aa8541a 100644
--- a/examples/streaming/stockcrawler/src/main/scala/io/gearpump/streaming/examples/stock/QueryServer.scala
+++ b/examples/streaming/stockcrawler/src/main/scala/io/gearpump/streaming/examples/stock/QueryServer.scala
@@ -21,29 +21,29 @@ package io.gearpump.streaming.examples.stock
import java.util.concurrent.TimeUnit
-import akka.actor.{Props, Actor}
-import akka.io.IO
-import io.gearpump.streaming.ProcessorId
-import io.gearpump.streaming.appmaster.{ProcessorSummary, StreamAppMasterSummary, AppMaster}
-import AppMaster.{TaskActorRef, LookupTaskActorRef}
import akka.actor.Actor._
-import io.gearpump.streaming.task.{StartTime, Task, TaskContext, TaskId}
+import akka.actor.{Actor, Props}
+import akka.io.IO
+import akka.pattern.ask
import io.gearpump.Message
import io.gearpump.cluster.MasterToAppMaster.AppMasterDataDetailRequest
import io.gearpump.cluster.UserConfig
-import QueryServer.WebServer
-import akka.pattern.ask
+import io.gearpump.streaming.ProcessorId
+import io.gearpump.streaming.appmaster.AppMaster.{LookupTaskActorRef, TaskActorRef}
+import io.gearpump.streaming.appmaster.{AppMaster, ProcessorSummary, StreamAppMasterSummary}
+import io.gearpump.streaming.examples.stock.QueryServer.WebServer
+import io.gearpump.streaming.task.{StartTime, Task, TaskContext, TaskId}
import spray.can.Http
-import spray.http.{StatusCodes}
-import spray.routing.HttpService
-import upickle.default.{read, write}
+import spray.http.StatusCodes
import spray.json._
+import spray.routing.HttpService
+import upickle.default.write
-import scala.concurrent.{Future, ExecutionContext}
+import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success}
class QueryServer(taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, conf){
- import taskContext.{appMaster, appId}
+ import taskContext.{appId, appMaster}
import ExecutionContext.Implicits.global
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/examples/streaming/transport/src/main/scala/io/gearpump/streaming/examples/transport/QueryServer.scala
----------------------------------------------------------------------
diff --git a/examples/streaming/transport/src/main/scala/io/gearpump/streaming/examples/transport/QueryServer.scala b/examples/streaming/transport/src/main/scala/io/gearpump/streaming/examples/transport/QueryServer.scala
index ce22648..5f91883 100644
--- a/examples/streaming/transport/src/main/scala/io/gearpump/streaming/examples/transport/QueryServer.scala
+++ b/examples/streaming/transport/src/main/scala/io/gearpump/streaming/examples/transport/QueryServer.scala
@@ -19,31 +19,31 @@ package io.gearpump.streaming.examples.transport
import java.util.concurrent.TimeUnit
-import akka.actor.{Props, Actor}
import akka.actor.Actor._
+import akka.actor.{Actor, Props}
import akka.io.IO
import akka.pattern.ask
-import io.gearpump.streaming.{ProcessorId, StreamApplication, ProcessorDescription, DAG}
-import io.gearpump.streaming.appmaster.AppMaster
-import io.gearpump.streaming.task.{StartTime, Task, TaskContext, TaskId}
import io.gearpump.Message
import io.gearpump.cluster.UserConfig
-import QueryServer.{GetAllRecords, WebServer}
import io.gearpump.partitioner.PartitionerDescription
-import AppMaster.{TaskActorRef, LookupTaskActorRef}
+import io.gearpump.streaming.appmaster.AppMaster
+import io.gearpump.streaming.appmaster.AppMaster.{LookupTaskActorRef, TaskActorRef}
+import io.gearpump.streaming.examples.transport.QueryServer.{GetAllRecords, WebServer}
+import io.gearpump.streaming.task.{StartTime, Task, TaskContext, TaskId}
+import io.gearpump.streaming.{DAG, ProcessorDescription, ProcessorId, StreamApplication}
import io.gearpump.util.Graph
import spray.can.Http
import spray.http.StatusCodes
-import spray.routing.HttpService
-import upickle.default.{read, write}
import spray.json._
+import spray.routing.HttpService
+import upickle.default.write
import scala.concurrent.Future
import scala.util.{Failure, Success}
class QueryServer(taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, conf){
- import taskContext.{appMaster, appId}
import system.dispatcher
+ import taskContext.appMaster
var inspector: (ProcessorId, ProcessorDescription) = null
implicit val timeOut = akka.util.Timeout(3, TimeUnit.SECONDS)
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/experiments/storm/src/main/scala/io/gearpump/experiments/storm/partitioner/StormPartitioner.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/partitioner/StormPartitioner.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/partitioner/StormPartitioner.scala
index d2071ae..04ccb91 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/partitioner/StormPartitioner.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/partitioner/StormPartitioner.scala
@@ -21,7 +21,7 @@ package io.gearpump.experiments.storm.partitioner
import io.gearpump.Message
import io.gearpump.experiments.storm.topology.GearpumpTuple
import io.gearpump.experiments.storm.util.StormOutputCollector
-import io.gearpump.partitioner.{MulticastPartitioner, Partitioner}
+import io.gearpump.partitioner.{Partitioner, MulticastPartitioner}
/**
* this is a partitioner bound to a target Storm component
@@ -38,9 +38,9 @@ import io.gearpump.partitioner.{MulticastPartitioner, Partitioner}
*/
private[storm] class StormPartitioner(target: String) extends MulticastPartitioner {
- override def getPartitions(msg: Message, partitionNum: Int, currentPartitionId: Int): List[Int] = {
+ override def getPartitions(msg: Message, partitionNum: Int, currentPartitionId: Int): Array[Int] = {
val stormTuple = msg.msg.asInstanceOf[GearpumpTuple]
- stormTuple.targetPartitions.getOrElse(target, List(Partitioner.UNKNOWN_PARTITION_ID))
+ stormTuple.targetPartitions.getOrElse(target, Array(Partitioner.UNKNOWN_PARTITION_ID))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpTuple.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpTuple.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpTuple.scala
index a52c9c7..7662f36 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpTuple.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpTuple.scala
@@ -34,7 +34,7 @@ private[storm] class GearpumpTuple(
val values: JList[AnyRef],
val sourceTaskId: Integer,
val sourceStreamId: String,
- @transient val targetPartitions: Map[String, List[Int]]) extends Serializable {
+ @transient val targetPartitions: Map[String, Array[Int]]) extends Serializable {
/**
* creates a Storm [[Tuple]] to be passed to a Storm component
* this is needed for each incoming message
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/experiments/storm/src/main/scala/io/gearpump/experiments/storm/util/Grouper.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/util/Grouper.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/util/Grouper.scala
index 1ac5f71..d727b7a 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/util/Grouper.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/util/Grouper.scala
@@ -38,14 +38,14 @@ sealed trait Grouper {
* @param values storm tuple values
* @return a list of gearpump partitions
*/
- def getPartitions(taskId: Int, values: JList[AnyRef]): List[Int]
+ def getPartitions(taskId: Int, values: JList[AnyRef]): Array[Int]
}
/**
* GlobalGrouper always returns partition 0
*/
class GlobalGrouper extends Grouper {
- override def getPartitions(taskId: Int, values: JList[AnyRef]): List[Int] = List(0)
+ override def getPartitions(taskId: Int, values: JList[AnyRef]): Array[Int] = Array(0)
}
/**
@@ -55,9 +55,9 @@ class GlobalGrouper extends Grouper {
class NoneGrouper(numTasks: Int) extends Grouper {
private val random = new Random
- override def getPartitions(taskId: Int, values: JList[AnyRef]): List[Int] = {
+ override def getPartitions(taskId: Int, values: JList[AnyRef]): Array[Int] = {
val partition = StormUtil.mod(random.nextInt, numTasks)
- List(partition)
+ Array(partition)
}
}
@@ -71,7 +71,7 @@ class ShuffleGrouper(numTasks: Int) extends Grouper {
private var index = -1
private var partitions = List.empty[Int]
- override def getPartitions(taskId: Int, values: JList[AnyRef]): List[Int] = {
+ override def getPartitions(taskId: Int, values: JList[AnyRef]): Array[Int] = {
index += 1
if (partitions.isEmpty) {
partitions = 0.until(numTasks).toList
@@ -80,7 +80,7 @@ class ShuffleGrouper(numTasks: Int) extends Grouper {
index = 0
partitions = random.shuffle(partitions)
}
- List(partitions(index))
+ Array(partitions(index))
}
}
@@ -92,10 +92,10 @@ class ShuffleGrouper(numTasks: Int) extends Grouper {
*/
class FieldsGrouper(outFields: Fields, groupFields: Fields, numTasks: Int) extends Grouper {
- override def getPartitions(taskId: Int, values: JList[AnyRef]): List[Int] = {
+ override def getPartitions(taskId: Int, values: JList[AnyRef]): Array[Int] = {
val hash = outFields.select(groupFields, values).hashCode()
val partition = StormUtil.mod(hash, numTasks)
- List(partition)
+ Array(partition)
}
}
@@ -104,9 +104,10 @@ class FieldsGrouper(outFields: Fields, groupFields: Fields, numTasks: Int) exten
* @param numTasks number of target tasks
*/
class AllGrouper(numTasks: Int) extends Grouper {
+ val partitions = (0 until numTasks).toArray
- override def getPartitions(taskId: Int, values: JList[AnyRef]): List[Int] = {
- (0 until numTasks).toList
+ override def getPartitions(taskId: Int, values: JList[AnyRef]): Array[Int] = {
+ partitions
}
}
@@ -120,8 +121,8 @@ class CustomGrouper(grouping: CustomStreamGrouping) extends Grouper {
grouping.prepare(topologyContext, globalStreamId, targetTasks)
}
- override def getPartitions(taskId: Int, values: JList[AnyRef]): List[Int] = {
- grouping.chooseTasks(taskId, values).map(StormUtil.stormTaskIdToGearpump(_).index).toList
+ override def getPartitions(taskId: Int, values: JList[AnyRef]): Array[Int] = {
+ grouping.chooseTasks(taskId, values).map(StormUtil.stormTaskIdToGearpump(_).index).toArray
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/experiments/storm/src/main/scala/io/gearpump/experiments/storm/util/StormOutputCollector.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/util/StormOutputCollector.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/util/StormOutputCollector.scala
index 1abc246..c1df63b 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/util/StormOutputCollector.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/util/StormOutputCollector.scala
@@ -66,12 +66,12 @@ object StormOutputCollector {
targets: JMap[String, JMap[String, Grouping]],
streamGroupers: Map[String, Grouper],
componentToProcessorId: Map[String, ProcessorId],
- values: JList[AnyRef]): (Map[String, List[Int]], JList[Integer]) ={
+ values: JList[AnyRef]): (Map[String, Array[Int]], JList[Integer]) ={
val ret: JList[Integer] = new JArrayList[Integer](targets.size)
@annotation.tailrec
def getRecur(iter: JIterator[String],
- accum: Map[String, List[Int]]): Map[String, List[Int]] = {
+ accum: Map[String, Array[Int]]): Map[String, Array[Int]] = {
if (iter.hasNext) {
val target = iter.next
val grouper = streamGroupers(streamId)
@@ -85,7 +85,7 @@ object StormOutputCollector {
accum
}
}
- val targetPartitions = getRecur(targets.get(streamId).keySet().iterator, Map.empty[String, List[Int]])
+ val targetPartitions = getRecur(targets.get(streamId).keySet().iterator, Map.empty[String, Array[Int]])
(targetPartitions, ret)
}
@@ -156,7 +156,7 @@ class StormOutputCollector(
stormTaskId: Int,
taskToComponent: JMap[Integer, String],
targets: JMap[String, JMap[String, Grouping]],
- getTargetPartitionsFn: (String, JList[AnyRef]) => (Map[String, List[Int]], JList[Integer]),
+ getTargetPartitionsFn: (String, JList[AnyRef]) => (Map[String, Array[Int]], JList[Integer]),
val taskContext: TaskContext,
private var timestamp: TimeStamp) {
import io.gearpump.experiments.storm.util.StormOutputCollector._
@@ -196,7 +196,7 @@ class StormOutputCollector(
if (targets.containsKey(streamId)) {
val target = taskToComponent.get(id)
val partition = stormTaskIdToGearpump(id).index
- val targetPartitions = Map(target -> List(partition))
+ val targetPartitions = Map(target -> Array(partition))
val tuple = new GearpumpTuple(values, stormTaskId, streamId, targetPartitions)
taskContext.output(Message(tuple, timestamp))
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/experiments/storm/src/test/scala/io/gearpump/experiments/storm/partitioner/StormPartitionerSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/partitioner/StormPartitionerSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/partitioner/StormPartitionerSpec.scala
index ef66937..200807c 100644
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/partitioner/StormPartitionerSpec.scala
+++ b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/partitioner/StormPartitionerSpec.scala
@@ -32,17 +32,17 @@ class StormPartitionerSpec extends PropSpec with PropertyChecks with Matchers {
property("StormPartitioner should get partitions directed by message and target") {
val idGen = Gen.chooseNum[Int](0, Int.MaxValue)
val componentsGen = Gen.listOf[String](Gen.alphaStr).map(_.distinct).suchThat(_.size > 1)
- val partitionsGen = Gen.listOf[Int](idGen).suchThat(_.nonEmpty).map(_.distinct.sorted)
+ val partitionsGen = Gen.listOf[Int](idGen).suchThat(_.nonEmpty).map(_.distinct.sorted.toArray)
val tupleFactoryGen = for {
values <- Gen.listOf[String](Gen.alphaStr).map(_.asJava.asInstanceOf[JList[AnyRef]])
sourceTaskId <- idGen
sourceStreamId <- Gen.alphaStr
- } yield (targetPartitions: Map[String, List[Int]]) => {
+ } yield (targetPartitions: Map[String, Array[Int]]) => {
new GearpumpTuple(values, new Integer(sourceTaskId), sourceStreamId, targetPartitions)
}
forAll(tupleFactoryGen, idGen, componentsGen, partitionsGen) {
- (tupleFactory: Map[String, List[Int]] => GearpumpTuple, id: Int, components: List[String], partitions: List[Int]) =>
+ (tupleFactory: Map[String, Array[Int]] => GearpumpTuple, id: Int, components: List[String], partitions: Array[Int]) =>
val currentPartitionId = id
val targetPartitions = components.init.map(c => (c, partitions)).toMap
val tuple = tupleFactory(targetPartitions)
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/experiments/storm/src/test/scala/io/gearpump/experiments/storm/util/StormOutputCollectorSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/util/StormOutputCollectorSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/util/StormOutputCollectorSpec.scala
index 6f16eb1..c75e92c 100644
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/util/StormOutputCollectorSpec.scala
+++ b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/util/StormOutputCollectorSpec.scala
@@ -43,8 +43,8 @@ class StormOutputCollectorSpec extends PropSpec with PropertyChecks with Matcher
(timestamp: TimeStamp, streamId: String, values: JList[AnyRef]) =>
val targets = mock[JMap[String, JMap[String, Grouping]]]
val taskToComponent = mock[JMap[Integer, String]]
- val getTargetPartitionsFn = mock[(String, JList[AnyRef]) => (Map[String, List[Int]], JList[Integer])]
- val targetPartitions = mock[Map[String, List[Int]]]
+ val getTargetPartitionsFn = mock[(String, JList[AnyRef]) => (Map[String, Array[Int]], JList[Integer])]
+ val targetPartitions = mock[Map[String, Array[Int]]]
val targetStormTaskIds = mock[JList[Integer]]
when(getTargetPartitionsFn(streamId, values)).thenReturn((targetPartitions, targetStormTaskIds))
val taskContext = MockUtil.mockTaskContext
@@ -75,8 +75,8 @@ class StormOutputCollectorSpec extends PropSpec with PropertyChecks with Matcher
val targets = mock[JMap[String, JMap[String, Grouping]]]
val taskToComponent = mock[JMap[Integer, String]]
when(taskToComponent.get(id)).thenReturn(target)
- val getTargetPartitionsFn = mock[(String, JList[AnyRef]) => (Map[String, List[Int]], JList[Integer])]
- val targetPartitions = mock[Map[String, List[Int]]]
+ val getTargetPartitionsFn = mock[(String, JList[AnyRef]) => (Map[String, Array[Int]], JList[Integer])]
+ val targetPartitions = mock[Map[String, Array[Int]]]
val targetStormTaskIds = mock[JList[Integer]]
when(getTargetPartitionsFn(streamId, values)).thenReturn((targetPartitions, targetStormTaskIds))
val taskContext = MockUtil.mockTaskContext
@@ -89,7 +89,7 @@ class StormOutputCollectorSpec extends PropSpec with PropertyChecks with Matcher
when(targets.containsKey(streamId)).thenReturn(true)
stormOutputCollector.setTimestamp(timestamp)
stormOutputCollector.emitDirect(id, streamId, values)
- val partitions = List(StormUtil.stormTaskIdToGearpump(id).index)
+ val partitions = Array(StormUtil.stormTaskIdToGearpump(id).index)
verify(taskContext, times(1)).output(MockUtil.argMatch[Message]({
case Message(tuple: GearpumpTuple, t) =>
val expected = new GearpumpTuple(values, stormTaskId, streamId, Map(target -> partitions))
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
----------------------------------------------------------------------
diff --git a/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala b/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
index aa90726..839c0ae 100644
--- a/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
+++ b/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
@@ -23,32 +23,28 @@ import java.io.{File, IOException}
import akka.actor.{ActorRef, ActorSystem}
import akka.http.scaladsl.server.Directives._
-import akka.http.scaladsl.server.Route
import akka.http.scaladsl.server.directives.ParameterDirectives.ParamMagnet
import akka.http.scaladsl.unmarshalling.Unmarshaller._
-import akka.stream.{Materializer, ActorMaterializer}
-import com.typesafe.config.{ConfigRenderOptions, Config}
+import akka.stream.Materializer
+import com.typesafe.config.Config
import io.gearpump.cluster.AppMasterToMaster.{GetAllWorkers, GetMasterData, GetWorkerData, MasterData, WorkerData}
-import io.gearpump.cluster.ClientToMaster.{ReadOption, QueryHistoryMetrics, QueryMasterConfig}
+import io.gearpump.cluster.ClientToMaster.{QueryHistoryMetrics, QueryMasterConfig, ReadOption}
import io.gearpump.cluster.MasterToAppMaster.{AppMastersData, AppMastersDataRequest, WorkerList}
import io.gearpump.cluster.MasterToClient.{HistoryMetrics, MasterConfig, SubmitApplicationResultValue}
-import io.gearpump.cluster.{ClusterConfig, UserConfig}
import io.gearpump.cluster.client.ClientContext
-import io.gearpump.cluster.main.AppSubmitter
import io.gearpump.cluster.worker.WorkerSummary
+import io.gearpump.cluster.{ClusterConfig, UserConfig}
import io.gearpump.jarstore.JarStoreService
+import io.gearpump.services.MasterService.{BuiltinPartitioners, SubmitApplicationRequest}
import io.gearpump.partitioner.{PartitionerByClassName, PartitionerDescription}
import io.gearpump.streaming.{ProcessorDescription, ProcessorId, StreamApplication}
-import io.gearpump.util.ActorUtil.{askActor, _}
-import io.gearpump.util.Constants._
-import io.gearpump.util.FileDirective._
-import io.gearpump.util.{Graph, Constants, Util, FileUtils}
import io.gearpump.util.ActorUtil._
-import io.gearpump.services.MasterService.{SubmitApplicationRequest, BuiltinPartitioners}
+import io.gearpump.util.FileDirective._
+import io.gearpump.util.{Constants, Graph, Util}
import scala.collection.JavaConversions._
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Failure, Success, Try}
+import scala.concurrent.Future
+import scala.util.{Failure, Success}
class MasterService(val master: ActorRef,
val jarStore: JarStoreService, override val system: ActorSystem)
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/25aacb29/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskManagerSpec.scala
----------------------------------------------------------------------
diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskManagerSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskManagerSpec.scala
index 4e4e96b..621455d 100644
--- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskManagerSpec.scala
+++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskManagerSpec.scala
@@ -20,34 +20,26 @@ package io.gearpump.streaming.appmaster
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.testkit.TestProbe
-import io.gearpump.streaming.AppMasterToExecutor.{StartAllTasks, TaskLocationsReceived, StartDynamicDag, TaskLocationsReady, TaskRegistered, LaunchTasks}
-import io.gearpump.streaming.{ProcessorId, DAG, LifeTime, ProcessorDescription}
-import io.gearpump.streaming.ExecutorToAppMaster.RegisterTask
-import io.gearpump.streaming.appmaster.AppMaster.AllocateResourceTimeOut
-import io.gearpump.streaming.appmaster.ClockService.{ChangeToNewDAGSuccess, ChangeToNewDAG}
-import io.gearpump.streaming.appmaster.DagManager.{TaskLaunchData, GetLatestDAG, NewDAGDeployed, WatchChange}
-import io.gearpump.streaming.appmaster.ExecutorManager.{ExecutorResourceUsageSummary, SetTaskManager, StartExecutorsTimeOut, StartExecutors}
-import io.gearpump.streaming.appmaster.TaskRegistry.TaskLocations
-import io.gearpump.streaming.task.{StartTime, TaskContext, GetStartClock, Subscriber}
-import io.gearpump.{TimeStamp, Message}
import io.gearpump.cluster.MasterToAppMaster.ReplayFromTimestampWindowTrailingEdge
import io.gearpump.cluster.scheduler.{Resource, ResourceRequest}
import io.gearpump.cluster.{AppJar, TestUtil, UserConfig}
import io.gearpump.jarstore.FilePath
import io.gearpump.partitioner.{HashPartitioner, Partitioner, PartitionerDescription}
+import io.gearpump.streaming.AppMasterToExecutor.{LaunchTasks, StartAllTasks, StartDynamicDag, TaskLocationsReady, TaskLocationsReceived, TaskRegistered}
import io.gearpump.streaming.ExecutorToAppMaster.RegisterTask
import io.gearpump.streaming.appmaster.AppMaster.AllocateResourceTimeOut
import io.gearpump.streaming.appmaster.ClockService.{ChangeToNewDAG, ChangeToNewDAGSuccess}
import io.gearpump.streaming.appmaster.DagManager.{GetLatestDAG, GetTaskLaunchData, LatestDAG, NewDAGDeployed, TaskLaunchData, WatchChange}
-import io.gearpump.streaming.appmaster.ExecutorManager._
+import io.gearpump.streaming.appmaster.ExecutorManager.{ExecutorResourceUsageSummary, SetTaskManager, StartExecutors, _}
import io.gearpump.streaming.appmaster.JarScheduler.ResourceRequestDetail
import io.gearpump.streaming.appmaster.TaskManagerSpec.{Env, Task1, Task2}
-import io.gearpump.streaming.appmaster.TaskRegistry.TaskLocations
import io.gearpump.streaming.executor.Executor.RestartTasks
-import io.gearpump.streaming.task._
+import io.gearpump.streaming.task.{StartTime, TaskContext, _}
+import io.gearpump.streaming.{DAG, LifeTime, ProcessorDescription, ProcessorId}
import io.gearpump.transport.HostPort
import io.gearpump.util.Graph
import io.gearpump.util.Graph._
+import io.gearpump.{Message, TimeStamp}
import org.mockito.Mockito._
import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers}
[06/50] incubator-gearpump git commit: Revert "fix #1895,
make GearpumpNimbus a standalone service"
Posted by ap...@apache.org.
Revert "fix #1895, make GearpumpNimbus a standalone service"
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/b8f8bb12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/b8f8bb12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/b8f8bb12
Branch: refs/heads/master
Commit: b8f8bb12bd0fe51d45fc63962579ef5b71f7cbe1
Parents: 7f48c12
Author: ManuZhang <Ow...@gmail.com>
Authored: Tue Feb 2 07:54:53 2016 +0800
Committer: ManuZhang <Ow...@gmail.com>
Committed: Tue Feb 2 07:54:53 2016 +0800
----------------------------------------------------------------------
docs/dev-storm.md | 40 +--
.../gearpump/experiments/storm/Commands.scala | 36 +++
.../storm/GearpumpThriftServer.scala | 143 +++++++++++
.../experiments/storm/StormRunner.scala | 137 ++++++++--
.../experiments/storm/main/GearpumpNimbus.scala | 247 -------------------
.../storm/main/GearpumpStormClient.scala | 70 ------
.../storm/topology/GearpumpStormTopology.scala | 52 +++-
.../experiments/storm/GearpumpNimbusSpec.scala | 68 +++++
.../storm/GearpumpThriftServerSpec.scala | 48 ++++
.../experiments/storm/StormRunnerSpec.scala | 58 +++++
.../topology/GearpumpStormTopologySpec.scala | 20 +-
.../integrationtest/storm/StormClient.scala | 14 +-
project/Pack.scala | 2 +-
.../io/gearpump/services/MasterService.scala | 2 +-
14 files changed, 537 insertions(+), 400 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/docs/dev-storm.md
----------------------------------------------------------------------
diff --git a/docs/dev-storm.md b/docs/dev-storm.md
index a22ed6c..a05c4ed 100644
--- a/docs/dev-storm.md
+++ b/docs/dev-storm.md
@@ -77,43 +77,23 @@ This section shows how to run an existing Storm jar in a local Gearpump cluster.
1. launch a local cluster
```
- bin/local
+ ./target/pack/bin/local
```
-2. start a local Gearpump Nimbus server
-
- Users need server's thrift port to submit topologies later. The thrift port is written to a yaml config file set with `-output` option.
- Users can provide an existing config file where only `nimbus.thrift.port` is overwritten. If not provided, a new file `app.yaml` is created with the config.
-
+2. submit a topology from storm-starter.
```
- bin/storm nimbus -output [conf <custom yaml config>]
+ bin/storm -verbose -config storm.yaml -jar storm-starter-${STORM_VERSION}.jar storm.starter.ExclamationTopology exclamation
```
-
-3. submit Storm applications
-
- Users can either submit Storm applications through command line or UI.
-
- a. submit Storm applications through command line
-
-
- ```
- bin/storm app -verbose -config app.yaml -jar storm-starter-${STORM_VERSION}.jar storm.starter.ExclamationTopology exclamation
- ```
- Users are able to configure their applications through following options
+ Users are able to configure their applications through following options
- * `jar` - set the path of a Storm application jar
- * `config` - submit the custom configuration file generated when launching Nimbus
+ * `jar` - set the path of a storm application jar
+ * `config` - submit a customized storm configuration file
- b. submit Storm application through UI
+ That's it. Check the dashboard and you should see data flowing through your topology.
- 1. Click on the "Create" button on the applications page on UI.
- 2. Click on the "Submit Storm Application" item in the pull down menu.
- 3. In the popup console, upload the Storm application jar and the configuration file generated when launching Nimbus,
- and fill in `storm.starter.ExclamationTopology exclamation` as arguments.
- 4. Click on the "Submit" button
+ *Note that submission from UI is not supported yet*.
- Either way, check the dashboard and you should see data flowing through your topology.
## How is it different from running on Storm
@@ -165,11 +145,11 @@ Gearpump has flow control between tasks such that [sender cannot flood receiver]
All Storm configurations are respected with the following priority order
```
-defaults.yaml < custom file config < application config < component config
+defaults.yaml < storm.yaml < application config < component config < custom user config
```
where
* application config is submit from Storm application along with the topology
* component config is set in spout / bolt with `getComponentConfiguration`
-* custom file config is specified with the `-config` option when submitting Storm application from command line or uploaded from UI
\ No newline at end of file
+* custom user config is specified with the `-config` option when submitting Storm application from command line
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala
new file mode 100644
index 0000000..1d5b903
--- /dev/null
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/Commands.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.experiments.storm
+
+import backtype.storm.generated.{KillOptions, StormTopology, SubmitOptions}
+
+object Commands {
+
+ case class Submit(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology, options: SubmitOptions)
+
+ case class AppSubmitted(name: String, appId: Int)
+
+ case class Kill(name: String, option: KillOptions)
+
+ case class AppKilled(name: String, appId: Int)
+
+ case object GetClusterInfo
+
+ case class GetTopology(id: String)
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala
new file mode 100644
index 0000000..640c20c
--- /dev/null
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/GearpumpThriftServer.scala
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.experiments.storm
+
+import java.nio.ByteBuffer
+import java.util.concurrent.TimeUnit
+import java.util.{HashMap => JHashMap, Map => JMap}
+
+import akka.actor.ActorRef
+import backtype.storm.Config
+import backtype.storm.generated._
+import backtype.storm.security.auth.{ThriftConnectionType, ThriftServer}
+import backtype.storm.utils.Utils
+import io.gearpump.experiments.storm.Commands.{GetClusterInfo, _}
+import io.gearpump.experiments.storm.util.StormUtil
+import io.gearpump.util.ActorUtil.askActor
+
+import scala.concurrent.Await
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+object GearpumpThriftServer {
+ val THRIFT_PORT = StormUtil.getThriftPort
+ implicit val timeOut = akka.util.Timeout(3, TimeUnit.SECONDS)
+
+ private def createServer(handler: ActorRef): ThriftServer = {
+ val processor = new Nimbus.Processor[GearpumpNimbus](new GearpumpNimbus(handler))
+ val connectionType = ThriftConnectionType.NIMBUS
+ val config = Utils.readDefaultConfig().asInstanceOf[JMap[AnyRef, AnyRef]]
+ config.put(Config.NIMBUS_THRIFT_PORT, s"$THRIFT_PORT")
+ new ThriftServer(config, processor, connectionType)
+ }
+
+ def apply(handler: ActorRef): GearpumpThriftServer = {
+ new GearpumpThriftServer(createServer(handler))
+ }
+
+ class GearpumpNimbus(handler: ActorRef) extends Nimbus.Iface {
+
+ override def submitTopology(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology): Unit = {
+ val ask = askActor(handler, Submit(name, uploadedJarLocation, jsonConf, topology, new SubmitOptions(TopologyInitialStatus.ACTIVE)))
+ Await.result(ask, 30 seconds)
+ }
+
+ override def killTopologyWithOpts(name: String, options: KillOptions): Unit = {
+ Await.result(askActor(handler,Kill(name, options)), 10 seconds)
+ }
+
+ override def submitTopologyWithOpts(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology, options: SubmitOptions): Unit = {
+ Await.result(askActor(handler,Submit(name, uploadedJarLocation, jsonConf, topology, options)), 10 seconds)
+ }
+
+ override def uploadChunk(location: String, chunk: ByteBuffer): Unit = {
+ }
+
+ override def getNimbusConf: String = {
+ throw new UnsupportedOperationException
+ }
+
+ override def getTopology(id: String): StormTopology = {
+ Await.result(askActor[StormTopology](handler, GetTopology(id)), 10 seconds)
+ }
+
+ override def getTopologyConf(id: String): String = {
+ throw new UnsupportedOperationException
+ }
+
+ override def beginFileDownload(file: String): String = {
+ throw new UnsupportedOperationException
+ }
+
+ override def getUserTopology(id: String): StormTopology = getTopology(id)
+
+ override def activate(name: String): Unit = {
+ throw new UnsupportedOperationException
+ }
+
+ override def rebalance(name: String, options: RebalanceOptions): Unit = {
+ throw new UnsupportedOperationException
+ }
+
+ override def deactivate(name: String): Unit = {
+ throw new UnsupportedOperationException
+ }
+
+ override def getTopologyInfo(id: String): TopologyInfo = {
+ throw new UnsupportedOperationException
+ }
+
+ override def getTopologyInfoWithOpts(s: String, getInfoOptions: GetInfoOptions): TopologyInfo = {
+ throw new UnsupportedOperationException
+ }
+
+ override def killTopology(name: String): Unit = killTopologyWithOpts(name, new KillOptions())
+
+ override def downloadChunk(id: String): ByteBuffer = {
+ throw new UnsupportedOperationException
+ }
+
+ override def beginFileUpload(): String = {
+ "local thrift server"
+ }
+
+ override def getClusterInfo: ClusterSummary = {
+ Await.result(askActor[ClusterSummary](handler, GetClusterInfo), 10 seconds)
+ }
+
+ override def finishFileUpload(location: String): Unit = {
+ }
+
+ override def uploadNewCredentials(s: String, credentials: Credentials): Unit = {
+ throw new UnsupportedOperationException
+ }
+ }
+}
+
+class GearpumpThriftServer(server: ThriftServer) extends Thread {
+
+ override def run(): Unit = {
+ server.serve()
+ }
+
+ def close(): Unit = {
+ server.stop()
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
index dd8d782..84d9460 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/StormRunner.scala
@@ -18,36 +18,131 @@
package io.gearpump.experiments.storm
-import io.gearpump.experiments.storm.main.{GearpumpNimbus, GearpumpStormClient}
-import io.gearpump.util.LogUtil
-import org.slf4j.Logger
+import java.io.File
+import java.util.{Map => JMap}
-object StormRunner {
- private val LOG: Logger = LogUtil.getLogger(getClass)
+import akka.actor.{Actor, ActorSystem, Props}
+import backtype.storm.Config
+import backtype.storm.generated.{ClusterSummary, StormTopology, SupervisorSummary, TopologySummary}
+import com.typesafe.config.ConfigValueFactory
+import io.gearpump.cluster.UserConfig
+import io.gearpump.cluster.client.ClientContext
+import io.gearpump.cluster.main.{ArgumentsParser, CLIOption}
+import io.gearpump.experiments.storm.Commands.{GetClusterInfo, _}
+import io.gearpump.experiments.storm.topology.GearpumpStormTopology
+import io.gearpump.experiments.storm.util.{GraphBuilder, StormConstants}
+import io.gearpump.streaming.StreamApplication
+import io.gearpump.util.Constants._
+import io.gearpump.util.{AkkaApp, Constants, LogUtil, Util}
- private val commands = Map("nimbus" -> GearpumpNimbus, "app" -> GearpumpStormClient)
+import scala.collection.JavaConverters._
- private def usage: Unit = {
- println(commands)
- val keys = commands.keys.toList.sorted
- Console.err.println("Usage: " + "<" + keys.mkString("|") + ">")
+object StormRunner extends AkkaApp with ArgumentsParser {
+ override val options: Array[(String, CLIOption[Any])] = Array(
+ "jar" -> CLIOption[String]("<storm jar>", required = true),
+ "config" -> CLIOption[String]("<storm config path>", required = false),
+ "verbose" -> CLIOption("<print verbose log on console>", required = false, defaultValue = Some(false)))
+
+ override val remainArgs = Array("topology_name")
+
+ override def main(inputAkkaConf: Config, args: Array[String]): Unit = {
+
+ val akkaConf = updateConfig(inputAkkaConf)
+ val config = parse(args)
+
+ val verbose = config.getBoolean("verbose")
+ if (verbose) {
+ LogUtil.verboseLogToConsole
+ }
+
+ val jar = config.getString("jar")
+ val stormConfig = config.getString("config")
+ val topology = config.remainArgs(0)
+ val stormArgs = config.remainArgs.drop(1)
+
+ val system = ActorSystem("storm", akkaConf)
+ val clientContext = new ClientContext(akkaConf, system, null)
+
+ val gearpumpNimbus = system.actorOf(Props(new Handler(clientContext, jar, stormConfig)))
+ val thriftServer = GearpumpThriftServer(gearpumpNimbus)
+ thriftServer.start()
+
+ val stormOptions = Array("-Dstorm.options=" +
+ s"${Config.NIMBUS_HOST}=127.0.0.1,${Config.NIMBUS_THRIFT_PORT}=${GearpumpThriftServer.THRIFT_PORT}",
+ "-Dstorm.jar=" + jar,
+ s"-D${PREFER_IPV4}=true"
+ )
+
+ val classPath = Array(System.getProperty("java.class.path"), jar)
+ val process = Util.startProcess(stormOptions, classPath, topology, stormArgs)
+
+ // wait till the process exit
+ val exit = process.exitValue()
+
+ thriftServer.close()
+ clientContext.close()
+ system.shutdown()
+ system.awaitTermination()
+
+ if (exit != 0) {
+ throw new Exception(s"failed to submit jar, exit code $exit, error summary: ${process.logger.error}")
+ }
}
- private def executeCommand(command : String, commandArgs : Array[String]): Unit = {
- if (!commands.contains(command)) {
- usage
+ import Constants._
+ private def updateConfig(config: Config): Config = {
+ val storm = s"<${GEARPUMP_HOME}>/lib/storm/*"
+ val appClassPath = s"$storm${File.pathSeparator}" + config.getString(GEARPUMP_APPMASTER_EXTRA_CLASSPATH)
+ val executorClassPath = s"$storm${File.pathSeparator}" + config.getString(Constants.GEARPUMP_EXECUTOR_EXTRA_CLASSPATH)
+
+ val updated = config.withValue(GEARPUMP_APPMASTER_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(appClassPath))
+ .withValue(GEARPUMP_EXECUTOR_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(executorClassPath))
+
+ if (config.hasPath(StormConstants.STORM_SERIALIZATION_FRAMEWORK)) {
+ val serializerConfig = ConfigValueFactory.fromAnyRef(config.getString(StormConstants.STORM_SERIALIZATION_FRAMEWORK))
+ updated.withValue(GEARPUMP_SERIALIZER_POOL, serializerConfig)
} else {
- commands(command).main(commandArgs)
+ updated
}
}
- def main(args: Array[String]) = {
- if (args.length == 0) {
- usage
- } else {
- val command = args(0)
- val commandArgs = args.drop(1)
- executeCommand(command, commandArgs)
+
+ class Handler(clientContext: ClientContext, jar: String, fileConfig: String) extends Actor {
+ private var applications = Map.empty[String, Int]
+ private var topologies = Map.empty[String, StormTopology]
+ private val LOG = LogUtil.getLogger(classOf[Handler])
+
+ implicit val system = context.system
+
+ def receive: Receive = {
+ case Kill(name, option) =>
+ topologies -= name
+ clientContext.shutdown(applications.getOrElse(name, throw new RuntimeException(s"topology $name not found")))
+ val appId = applications(name)
+ applications -= name
+ LOG.info(s"Killed topology $name")
+ sender ! AppKilled(name, appId)
+ case Submit(name, uploadedJarLocation, appConfig, topology, option) =>
+ topologies += name -> topology
+
+ val gearpumpStormTopology = GearpumpStormTopology(topology, appConfig, fileConfig)
+ val stormConfig = gearpumpStormTopology.getStormConfig
+ val processorGraph = GraphBuilder.build(gearpumpStormTopology)
+ val config = UserConfig.empty
+ .withValue[StormTopology](StormConstants.STORM_TOPOLOGY, topology)
+ .withValue[JMap[AnyRef, AnyRef]](StormConstants.STORM_CONFIG, stormConfig)
+ val app = StreamApplication(name, processorGraph, config)
+ val appId = clientContext.submit(app, jar)
+ applications += name -> appId
+ LOG.info(s"Storm Application $appId submitted")
+ sender ! AppSubmitted(name, appId)
+ case GetClusterInfo =>
+ val topologySummaryList = topologies.map { case (name, _) =>
+ new TopologySummary(name, name, 0, 0, 0, 0, "")
+ }.toSeq
+ sender ! new ClusterSummary(List[SupervisorSummary]().asJava, 0, topologySummaryList.asJava)
+ case GetTopology(id) =>
+ sender ! topologies(id)
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
deleted file mode 100644
index 83328b4..0000000
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.experiments.storm.main
-
-import java.io.{File, FileOutputStream, FileWriter}
-import java.nio.ByteBuffer
-import java.nio.channels.{Channels, WritableByteChannel}
-import java.util.{Map => JMap, UUID}
-
-import akka.actor.ActorSystem
-import backtype.storm.Config
-import backtype.storm.generated._
-import backtype.storm.security.auth.{ThriftConnectionType, ThriftServer}
-import backtype.storm.utils.TimeCacheMap.ExpiredCallback
-import backtype.storm.utils.{TimeCacheMap, Utils}
-import com.typesafe.config.ConfigValueFactory
-import io.gearpump.cluster.{MasterToAppMaster, UserConfig}
-import io.gearpump.cluster.client.ClientContext
-import io.gearpump.cluster.main.{ArgumentsParser, CLIOption}
-import io.gearpump.experiments.storm.topology.GearpumpStormTopology
-import io.gearpump.experiments.storm.util.{GraphBuilder, StormConstants, StormUtil}
-import io.gearpump.streaming.StreamApplication
-import io.gearpump.util.{FileUtils, AkkaApp, Constants, LogUtil}
-import org.apache.storm.shade.org.json.simple.JSONValue
-import org.apache.storm.shade.org.yaml.snakeyaml.Yaml
-import org.apache.storm.shade.org.yaml.snakeyaml.constructor.SafeConstructor
-import org.slf4j.Logger
-
-import scala.collection.JavaConverters._
-
-object GearpumpNimbus extends AkkaApp with ArgumentsParser {
- private val THRIFT_PORT = StormUtil.getThriftPort
- private val OUTPUT = "output"
-
- override val options: Array[(String, CLIOption[Any])] = Array(
- OUTPUT -> CLIOption[String]("<output path for configuration file>", required = false, defaultValue = Some("app.yaml"))
- )
-
- override def main(inputAkkaConf: Config, args: Array[String]): Unit = {
- val parsed = parse(args)
- val output = parsed.getString(OUTPUT)
- val akkaConf = updateClientConfig(inputAkkaConf)
- val system = ActorSystem("storm", akkaConf)
- val clientContext = new ClientContext(akkaConf, system, null)
- val stormConf = Utils.readStormConfig().asInstanceOf[JMap[AnyRef, AnyRef]]
- val thriftConf: JMap[String, String] = Map(
- Config.NIMBUS_HOST -> akkaConf.getString(Constants.GEARPUMP_HOSTNAME),
- Config.NIMBUS_THRIFT_PORT -> s"$THRIFT_PORT").asJava
- updateOutputStormConfig(thriftConf, output)
- stormConf.putAll(thriftConf)
- val thriftServer = createServer(clientContext, stormConf)
- thriftServer.serve()
- system.awaitTermination()
- }
-
- private def createServer(clientContext: ClientContext, stormConf: JMap[AnyRef, AnyRef]): ThriftServer = {
- val processor = new Nimbus.Processor[GearpumpNimbus](new GearpumpNimbus(clientContext, stormConf))
- val connectionType = ThriftConnectionType.NIMBUS
- new ThriftServer(stormConf, processor, connectionType)
- }
-
- private def updateOutputStormConfig(conf: JMap[String, String], output: String): Unit = {
- // read existing config
- val outputConfig = Utils.findAndReadConfigFile(output, false).asInstanceOf[JMap[AnyRef, AnyRef]]
- outputConfig.putAll(conf)
- val yaml = new Yaml(new SafeConstructor)
- val writer = new FileWriter(new File(output))
- yaml.dump(outputConfig, writer)
- }
-
- import Constants._
- private def updateClientConfig(config: Config): Config = {
- val storm = s"<${GEARPUMP_HOME}>/lib/storm/*"
- val appClassPath = s"$storm${File.pathSeparator}" + config.getString(GEARPUMP_APPMASTER_EXTRA_CLASSPATH)
- val executorClassPath = s"$storm${File.pathSeparator}" + config.getString(Constants.GEARPUMP_EXECUTOR_EXTRA_CLASSPATH)
-
- val updated = config.withValue(GEARPUMP_APPMASTER_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(appClassPath))
- .withValue(GEARPUMP_EXECUTOR_EXTRA_CLASSPATH, ConfigValueFactory.fromAnyRef(executorClassPath))
-
- if (config.hasPath(StormConstants.STORM_SERIALIZATION_FRAMEWORK)) {
- val serializerConfig = ConfigValueFactory.fromAnyRef(config.getString(StormConstants.STORM_SERIALIZATION_FRAMEWORK))
- updated.withValue(GEARPUMP_SERIALIZER_POOL, serializerConfig)
- } else {
- updated
- }
- }
-
-}
-
-class GearpumpNimbus(clientContext: ClientContext, stormConf: JMap[AnyRef, AnyRef]) extends Nimbus.Iface {
- private val LOG: Logger = LogUtil.getLogger(classOf[GearpumpNimbus])
- private var applications = Map.empty[String, Int]
- private var topologies = Map.empty[String, (StormTopology, JMap[AnyRef, AnyRef])]
- private val expireSeconds = StormUtil.getInt(stormConf, Config.NIMBUS_FILE_COPY_EXPIRATION_SECS).get
- private val expiredCallback = new ExpiredCallback[String, WritableByteChannel] {
- override def expire(k: String, v: WritableByteChannel): Unit = {
- v.close()
- }
- }
- private val fileCacheMap = new TimeCacheMap[String, WritableByteChannel](expireSeconds, expiredCallback)
-
- override def submitTopology(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology): Unit = {
- submitTopologyWithOpts(name, uploadedJarLocation, jsonConf, topology, new SubmitOptions(TopologyInitialStatus.ACTIVE))
- }
-
- override def submitTopologyWithOpts(name: String, uploadedJarLocation: String, jsonConf: String, topology: StormTopology, options: SubmitOptions): Unit = {
-
- implicit val system = clientContext.system
- val gearpumpStormTopology = GearpumpStormTopology(name, topology, jsonConf)
- val stormConfig = gearpumpStormTopology.getStormConfig
- val processorGraph = GraphBuilder.build(gearpumpStormTopology)
- val config = UserConfig.empty
- .withValue[StormTopology](StormConstants.STORM_TOPOLOGY, topology)
- .withValue[JMap[AnyRef, AnyRef]](StormConstants.STORM_CONFIG, stormConfig)
- val app = StreamApplication(name, processorGraph, config)
- LOG.info(s"jar file uploaded to $uploadedJarLocation")
- val appId = clientContext.submit(app, uploadedJarLocation)
- applications += name -> appId
- topologies += name -> (topology, stormConfig)
- LOG.info(s"Storm Application $appId submitted")
- }
-
- override def killTopologyWithOpts(name: String, options: KillOptions): Unit = {
- clientContext.shutdown(applications.getOrElse(name, throw new RuntimeException(s"topology $name not found")))
- applications -= name
- topologies -= name
- LOG.info(s"Killed topology $name")
- }
-
- override def getNimbusConf: String = {
- JSONValue.toJSONString(stormConf)
- }
-
- override def getTopology(name: String): StormTopology = {
- updateApps
- topologies.getOrElse(name,
- throw new RuntimeException(s"topology $name not found"))._1
- }
-
- override def getTopologyConf(name: String): String = {
- updateApps
- JSONValue.toJSONString(topologies.getOrElse(name,
- throw new RuntimeException(s"topology $name not found"))._2)
- }
-
- override def getUserTopology(id: String): StormTopology = getTopology(id)
-
- override def beginFileUpload(): String = {
- val location = s"stormjar-${UUID.randomUUID()}.jar"
- val channel = Channels.newChannel(new FileOutputStream(location))
- fileCacheMap.put(location, channel)
- LOG.info(s"Uploading file from client to $location")
- location
- }
-
- override def uploadChunk(location: String, chunk: ByteBuffer): Unit = {
- if (!fileCacheMap.containsKey(location)) {
- throw new RuntimeException(s"File for $location does not exist (or timed out)")
- } else {
- val channel = fileCacheMap.get(location)
- channel.write(chunk)
- fileCacheMap.put(location, channel)
- }
- }
-
- override def finishFileUpload(location: String): Unit = {
- if (!fileCacheMap.containsKey(location)) {
- throw new RuntimeException(s"File for $location does not exist (or timed out)")
- } else {
- val channel = fileCacheMap.get(location)
- channel.close()
- fileCacheMap.remove(location)
- new File(location).delete()
- }
- }
-
- override def getClusterInfo: ClusterSummary = {
- updateApps
- val topologySummaryList = topologies.map { case (name, _) =>
- new TopologySummary(name, name, 0, 0, 0, 0, "")
- }.toSeq
- new ClusterSummary(List[SupervisorSummary]().asJava, 0, topologySummaryList.asJava)
- }
-
- override def beginFileDownload(file: String): String = {
- throw new UnsupportedOperationException
- }
-
- override def uploadNewCredentials(s: String, credentials: Credentials): Unit = {
- throw new UnsupportedOperationException
- }
- override def activate(name: String): Unit = {
- throw new UnsupportedOperationException
- }
-
- override def rebalance(name: String, options: RebalanceOptions): Unit = {
- throw new UnsupportedOperationException
- }
-
- override def deactivate(name: String): Unit = {
- throw new UnsupportedOperationException
- }
-
- override def getTopologyInfo(name: String): TopologyInfo = {
- throw new UnsupportedOperationException
- }
-
- override def getTopologyInfoWithOpts(s: String, getInfoOptions: GetInfoOptions): TopologyInfo = {
- throw new UnsupportedOperationException
- }
-
- override def killTopology(name: String): Unit = killTopologyWithOpts(name, new KillOptions())
-
- override def downloadChunk(name: String): ByteBuffer = {
- throw new UnsupportedOperationException
- }
-
- private def updateApps: Unit = {
- clientContext.listApps.appMasters.foreach { app =>
- val name = app.appName
- if (applications.contains(app.appName)) {
- if (app.status != MasterToAppMaster.AppMasterActive) {
- applications -= name
- topologies -= name
- }
- }
- }
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
deleted file mode 100644
index b813ae0..0000000
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.gearpump.experiments.storm.main
-
-import backtype.storm.Config
-import backtype.storm.utils.Utils
-import io.gearpump.cluster.main.{ArgumentsParser, CLIOption}
-import io.gearpump.util.Constants._
-import io.gearpump.util.{AkkaApp, LogUtil, Util}
-
-object GearpumpStormClient extends AkkaApp with ArgumentsParser {
-
- override val options: Array[(String, CLIOption[Any])] = Array(
- "jar" -> CLIOption[String]("<storm jar>", required = true),
- "config" -> CLIOption[Int]("<storm config file>", required = true),
- "verbose" -> CLIOption("<print verbose log on console>", required = false, defaultValue = Some(false)))
-
- override def main(inputAkkaConf: Config, args: Array[String]): Unit = {
- val config = parse(args)
-
- val verbose = config.getBoolean("verbose")
- if (verbose) {
- LogUtil.verboseLogToConsole
- }
-
- val jar = config.getString("jar")
- val stormConfig = config.getString("config")
- val topology = config.remainArgs(0)
- val stormArgs = config.remainArgs.drop(1)
- val stormOptions = Array(
- s"-Dstorm.options=${getThriftOptions(stormConfig)}",
- s"-Dstorm.jar=$jar",
- s"-Dstorm.config.file=$stormConfig",
- s"-D${PREFER_IPV4}=true"
- )
-
- val classPath = Array(s"${System.getProperty(GEARPUMP_HOME)}/lib/storm/*", jar)
- val process = Util.startProcess(stormOptions, classPath, topology, stormArgs)
-
- // wait till the process exit
- val exit = process.exitValue()
-
- if (exit != 0) {
- throw new Exception(s"failed to submit jar, exit code $exit, error summary: ${process.logger.error}")
- }
- }
-
- private def getThriftOptions(stormConfig: String): String = {
- val config = Utils.findAndReadConfigFile(stormConfig, true)
- val host = config.get(Config.NIMBUS_HOST)
- val thriftPort = config.get(Config.NIMBUS_THRIFT_PORT)
- s"${Config.NIMBUS_HOST}=$host,${Config.NIMBUS_THRIFT_PORT}=$thriftPort"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
index b88a200..996498b 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopology.scala
@@ -18,6 +18,7 @@
package io.gearpump.experiments.storm.topology
+import java.io._
import java.lang.{Iterable => JIterable}
import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap}
@@ -34,6 +35,8 @@ import io.gearpump.experiments.storm.util.StormUtil._
import io.gearpump.streaming.Processor
import io.gearpump.streaming.task.Task
import io.gearpump.util.LogUtil
+import org.apache.storm.shade.org.yaml.snakeyaml.Yaml
+import org.apache.storm.shade.org.yaml.snakeyaml.constructor.SafeConstructor
import org.slf4j.Logger
import scala.collection.JavaConversions._
@@ -42,18 +45,43 @@ object GearpumpStormTopology {
private val LOG: Logger = LogUtil.getLogger(classOf[GearpumpStormTopology])
def apply(
- name: String,
topology: StormTopology,
- appConfigInJson: String)(implicit system: ActorSystem): GearpumpStormTopology = {
+ appConfigInJson: String,
+ fileConfig: String)(implicit system: ActorSystem): GearpumpStormTopology = {
new GearpumpStormTopology(
- name,
topology,
Utils.readStormConfig().asInstanceOf[JMap[AnyRef, AnyRef]],
- parseJsonStringToMap(appConfigInJson)
+ parseJsonStringToMap(appConfigInJson),
+ readStormConfig(fileConfig)
)
}
+ /**
+ * @param configFile user provided local config file
+ * @return a config Map loaded from config file
+ */
+ private def readStormConfig(configFile: String): JMap[AnyRef, AnyRef] = {
+ var ret: JMap[AnyRef, AnyRef] = new JHashMap[AnyRef, AnyRef]
+ try {
+ val yaml = new Yaml(new SafeConstructor)
+ val input: InputStream = new FileInputStream(configFile)
+ try {
+ ret = yaml.load(new InputStreamReader(input)).asInstanceOf[JMap[AnyRef, AnyRef]]
+ } catch {
+ case e: IOException =>
+ LOG.warn(s"failed to load config file $configFile")
+ } finally {
+ input.close()
+ }
+ } catch {
+ case e: FileNotFoundException =>
+ LOG.warn(s"failed to find config file $configFile")
+ case t: Throwable =>
+ LOG.error(t.getMessage)
+ }
+ ret
+ }
}
/**
@@ -63,20 +91,21 @@ object GearpumpStormTopology {
* 3. provides interface for Gearpump applications to use Storm topology
*
* an implicit ActorSystem is required to create Gearpump processors
- * @param name topology name
* @param topology Storm topology
- * @param sysConfig configs from "defaults.yaml" and custom config file
+ * @param sysConfig configs from "defaults.yaml" and "storm.yaml"
* @param appConfig config submitted from user application
+ * @param fileConfig custom file config set by user in command line
*/
private[storm] class GearpumpStormTopology(
- name: String,
topology: StormTopology,
sysConfig: JMap[AnyRef, AnyRef],
- appConfig: JMap[AnyRef, AnyRef])(implicit system: ActorSystem) {
+ appConfig: JMap[AnyRef, AnyRef],
+ fileConfig: JMap[AnyRef, AnyRef])(implicit system: ActorSystem) {
+ import io.gearpump.experiments.storm.topology.GearpumpStormTopology._
private val spouts = topology.get_spouts()
private val bolts = topology.get_bolts()
- private val stormConfig = mergeConfigs(sysConfig, appConfig, getComponentConfigs(spouts, bolts))
+ private val stormConfig = mergeConfigs(sysConfig, appConfig, fileConfig, getComponentConfigs(spouts, bolts))
private val spoutProcessors = spouts.map { case (id, spout) =>
id -> spoutToProcessor(id, spout, stormConfig.toMap) }.toMap
private val boltProcessors = bolts.map { case (id, bolt) =>
@@ -85,7 +114,7 @@ private[storm] class GearpumpStormTopology(
/**
* @return merged Storm config with priority
- * defaults.yaml < custom file config < application config < component config
+ * defaults.yaml < storm.yaml < application config < component config < custom file config
*/
def getStormConfig: JMap[AnyRef, AnyRef] = stormConfig
@@ -110,12 +139,13 @@ private[storm] class GearpumpStormTopology(
private def mergeConfigs(
sysConfig: JMap[AnyRef, AnyRef],
appConfig: JMap[AnyRef, AnyRef],
+ fileConfig: JMap[AnyRef, AnyRef],
componentConfigs: Iterable[JMap[AnyRef, AnyRef]]): JMap[AnyRef, AnyRef] = {
val allConfig = new JHashMap[AnyRef, AnyRef]
allConfig.putAll(sysConfig)
allConfig.putAll(appConfig)
allConfig.putAll(getMergedComponentConfig(componentConfigs, allConfig.toMap))
- allConfig.put(Config.TOPOLOGY_NAME, name)
+ allConfig.putAll(fileConfig)
allConfig
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala
new file mode 100644
index 0000000..20b63b3
--- /dev/null
+++ b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpNimbusSpec.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.experiments.storm
+
+import akka.actor.ActorSystem
+import akka.testkit.TestProbe
+import backtype.storm.generated.StormTopology
+import io.gearpump.experiments.storm.util.TopologyUtil
+import io.gearpump.cluster.TestUtil
+import Commands.{GetTopology, Kill, Submit}
+import GearpumpThriftServer.GearpumpNimbus
+import org.scalatest.mock.MockitoSugar
+import org.scalatest.{Matchers, WordSpec}
+
+import scala.concurrent.Future
+
+class GearpumpNimbusSpec extends WordSpec with Matchers with MockitoSugar {
+
+ "GearpumpNimbus" should {
+ "submit and kill topology through ClientContext" in {
+
+ implicit val system = ActorSystem("storm-test", TestUtil.DEFAULT_CONFIG)
+ implicit val dispatcher = system.dispatcher
+
+ val handler = TestProbe()
+ val gearpumpNimbus = new GearpumpNimbus(handler.ref)
+
+ val appId = 0
+ val name = "test"
+ val uploadedJarLocation = "local"
+ val jsonConf = "storm_json_conf"
+ val topology = TopologyUtil.getTestTopology
+
+ Future(gearpumpNimbus.submitTopology(name, uploadedJarLocation, jsonConf, topology))
+ handler.expectMsgType[Submit]
+
+ Future(gearpumpNimbus.getTopology(name))
+ handler.expectMsgType[GetTopology]
+ handler.reply(new StormTopology)
+
+ Future(gearpumpNimbus.getUserTopology(name))
+ handler.expectMsgType[GetTopology]
+ handler.reply(new StormTopology)
+
+ Future(gearpumpNimbus.killTopology(name))
+ handler.expectMsgType[Kill]
+
+ system.shutdown()
+ system.awaitTermination()
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala
new file mode 100644
index 0000000..b1736f0
--- /dev/null
+++ b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/GearpumpThriftServerSpec.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.gearpump.experiments.storm
+
+import backtype.storm.security.auth.ThriftServer
+import org.mockito.Mockito._
+import org.scalatest.WordSpec
+import org.scalatest.mock.MockitoSugar
+
+class GearpumpThriftServerSpec extends WordSpec with MockitoSugar {
+
+ "GearpumpThriftServer" should {
+ "run ThriftServer.serve" in {
+ val tServer = mock[ThriftServer]
+ val thriftServer = new GearpumpThriftServer(tServer)
+
+ thriftServer.run()
+
+ verify(tServer).serve()
+ }
+
+ "stop ThriftServer" in {
+ val tServer = mock[ThriftServer]
+ val thriftServer = new GearpumpThriftServer(tServer)
+
+ thriftServer.close()
+
+ verify(tServer).stop()
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala
new file mode 100644
index 0000000..95cf279
--- /dev/null
+++ b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/StormRunnerSpec.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package io.gearpump.experiments.storm
+
+import akka.actor.{ActorSystem, Props}
+import akka.testkit.TestProbe
+import io.gearpump.cluster.TestUtil
+import io.gearpump.cluster.client.ClientContext
+import io.gearpump.experiments.storm.Commands.{AppKilled, AppSubmitted, Kill, Submit}
+import io.gearpump.experiments.storm.StormRunner.Handler
+import io.gearpump.experiments.storm.util.TopologyUtil
+import org.mockito.Matchers._
+import org.mockito.Mockito
+import org.scalatest.{FlatSpec, Matchers}
+
+class StormRunnerSpec extends FlatSpec with Matchers {
+ it should "handle submit/kill correctly" in {
+ val conf = TestUtil.DEFAULT_CONFIG
+ implicit val system = ActorSystem("storm-test", conf)
+
+ val uploadedJarLocation = "local"
+ val jsonConf = "storm_json_conf"
+ val topology = TopologyUtil.getTestTopology
+
+ implicit val dispatcher = system.dispatcher
+ val clientContext = Mockito.mock(classOf[ClientContext])
+ Mockito.when(clientContext.submit(any(), any())).thenReturn(0)
+ val handler = system.actorOf(Props(new Handler(clientContext, "jar", "user_config")))
+ val client = TestProbe()
+
+ client.send(handler, Submit("app", uploadedJarLocation, jsonConf, topology, null))
+ client.expectMsg(AppSubmitted("app", 0))
+
+
+ client.send(handler, Kill("app", null))
+ client.expectMsg(AppKilled("app", 0))
+
+ system.shutdown()
+ system.awaitTermination()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
index 8f10886..d1c330b 100644
--- a/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
+++ b/experiments/storm/src/test/scala/io/gearpump/experiments/storm/topology/GearpumpStormTopologySpec.scala
@@ -20,7 +20,6 @@ package io.gearpump.experiments.storm.topology
import java.util.{HashMap => JHashMap, Map => JMap}
-import backtype.storm.Config
import io.gearpump.experiments.storm.processor.StormProcessor
import io.gearpump.experiments.storm.producer.StormProducer
import io.gearpump.experiments.storm.util.TopologyUtil
@@ -41,27 +40,28 @@ class GearpumpStormTopologySpec extends WordSpec with Matchers with MockitoSugar
val sysConfig = newJavaConfig(name, sysVal)
val appVal = "app"
val appConfig = newJavaConfig(name, appVal)
+ val fileVal = "file"
+ val fileConfig = newJavaConfig(name, fileVal)
implicit val system = MockUtil.system
- val topology1 = new GearpumpStormTopology("topology1", stormTopology, newEmptyConfig, newEmptyConfig)
- topology1.getStormConfig(Config.TOPOLOGY_NAME) shouldBe "topology1"
- topology1.getStormConfig should not contain name
+ val topology1 = new GearpumpStormTopology(stormTopology, newEmptyConfig, newEmptyConfig, newEmptyConfig)
+ topology1.getStormConfig shouldBe empty
- val topology2 = new GearpumpStormTopology("topology2", stormTopology, sysConfig, newEmptyConfig)
- topology2.getStormConfig(Config.TOPOLOGY_NAME) shouldBe "topology2"
+ val topology2 = new GearpumpStormTopology(stormTopology, sysConfig, newEmptyConfig, newEmptyConfig)
topology2.getStormConfig.get(name) shouldBe sysVal
- val topology3 = new GearpumpStormTopology("topology3", stormTopology, sysConfig, appConfig)
- topology3.getStormConfig(Config.TOPOLOGY_NAME) shouldBe "topology3"
+ val topology3 = new GearpumpStormTopology(stormTopology, sysConfig, appConfig, newEmptyConfig)
topology3.getStormConfig.get(name) shouldBe appVal
+ val topology4 = new GearpumpStormTopology(stormTopology, sysConfig, appConfig, fileConfig)
+ topology4.getStormConfig.get(name) shouldBe fileVal
}
"create Gearpump processors from Storm topology" in {
val stormTopology = TopologyUtil.getTestTopology
implicit val system = MockUtil.system
val gearpumpStormTopology =
- GearpumpStormTopology("app", stormTopology, null)
+ GearpumpStormTopology(stormTopology, null, "")
val processors = gearpumpStormTopology.getProcessors
stormTopology.get_spouts().foreach { case (spoutId, _) =>
val processor = processors(spoutId)
@@ -80,7 +80,7 @@ class GearpumpStormTopologySpec extends WordSpec with Matchers with MockitoSugar
implicit val system = MockUtil.system
val sysConfig = new JHashMap[AnyRef, AnyRef]
val gearpumpStormTopology =
- GearpumpStormTopology("app", stormTopology, null)
+ GearpumpStormTopology(stormTopology, null, "")
val targets0 = gearpumpStormTopology.getTargets("1")
targets0 should contain key "3"
targets0 should contain key "4"
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
----------------------------------------------------------------------
diff --git a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
index 15acf4e..ed792f4 100644
--- a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
+++ b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/storm/StormClient.scala
@@ -20,6 +20,7 @@ package io.gearpump.integrationtest.storm
import backtype.storm.utils.{Utils, DRPCClient}
+import io.gearpump.experiments.storm.GearpumpThriftServer
import io.gearpump.integrationtest.Docker
import io.gearpump.integrationtest.minicluster.BaseContainer
import org.apache.log4j.Logger
@@ -29,28 +30,23 @@ class StormClient(masterAddrs: Seq[(String, Int)]) {
private val LOG = Logger.getLogger(getClass)
private val STORM_HOST = "storm0"
- private val STORM_NIMBUS = "/opt/start storm nimbus"
- private val STORM_APP = "/opt/start storm app"
+ private val STORM_CMD = "/opt/start storm"
private val STORM_DRPC = "storm-drpc"
private val CONFIG_FILE = "storm.yaml"
+ private val NIMBUS_THRIFT_PORT = GearpumpThriftServer.THRIFT_PORT
private val DRPC_PORT = 3772
private val DRPC_INVOCATIONS_PORT = 3773
private val container = new BaseContainer(STORM_HOST, STORM_DRPC, masterAddrs,
- tunnelPorts = Set(DRPC_PORT, DRPC_INVOCATIONS_PORT))
+ tunnelPorts = Set(NIMBUS_THRIFT_PORT, DRPC_PORT, DRPC_INVOCATIONS_PORT))
def start(): Unit = {
container.createAndStart()
- startNimbus
- }
-
- private def startNimbus: String = {
- Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_NIMBUS -output $CONFIG_FILE")
}
def submitStormApp(jar: String, mainClass: String, args: String = ""): Int = {
try {
- Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_APP -config $CONFIG_FILE " +
+ Docker.execAndCaptureOutput(STORM_HOST, s"$STORM_CMD -config $CONFIG_FILE " +
s"-jar $jar $mainClass $args").split("\n")
.filter(_.contains("The application id is ")).head.split(" ").last.toInt
} catch {
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/project/Pack.scala
----------------------------------------------------------------------
diff --git a/project/Pack.scala b/project/Pack.scala
index a3852c4..b242b36 100644
--- a/project/Pack.scala
+++ b/project/Pack.scala
@@ -59,7 +59,7 @@ object Pack extends sbt.Build {
"worker" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-DlogFilename=worker", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"),
"services" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"),
"yarnclient" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost"),
- "storm" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}")
+ "storm" -> Seq("-server", "-Djava.net.preferIPv4Stack=true", "-Dgearpump.home=${PROG_HOME}", "-Djava.rmi.server.hostname=localhost")
),
packLibDir := Map(
"lib" -> new ProjectsToPack(core.id, streaming.id),
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/b8f8bb12/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
----------------------------------------------------------------------
diff --git a/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala b/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
index aa90726..e2b2c47 100644
--- a/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
+++ b/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala
@@ -221,7 +221,7 @@ object MasterService {
*/
def submitStormApp(jar: Option[File], stormConf: Option[File], args: String, systemConfig: Config): Boolean = {
submitAndDeleteTempFiles(
- "io.gearpump.experiments.storm.main.GearpumpStormClient",
+ "io.gearpump.experiments.storm.StormRunner",
argsArray = spaceSeparatedArgumentsToArray(args),
fileMap = Map("jar" -> jar, "config" -> stormConf).filter(_._2.isDefined).mapValues(_.get),
classPath = getStormApplicationClassPath,
[29/50] incubator-gearpump git commit: Merge pull request #1962 from
kimutansk/doc_1stapp_fix
Posted by ap...@apache.org.
Merge pull request #1962 from kimutansk/doc_1stapp_fix
Fix document "Submit Your 1st Gearpump Application command" has not been updated
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/9405961d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/9405961d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/9405961d
Branch: refs/heads/master
Commit: 9405961dd0e800f6cd62a3de40642fd7dfb45eb8
Parents: d1a8c0a bfd510b
Author: Kam Kasravi <ka...@yahoo.com>
Authored: Thu Feb 11 07:07:26 2016 -0800
Committer: Kam Kasravi <ka...@yahoo.com>
Committed: Thu Feb 11 07:07:26 2016 -0800
----------------------------------------------------------------------
docs/submit-your-1st-application.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[37/50] incubator-gearpump git commit: Merge pull request #1968 from
huafengw/fix#1966
Posted by ap...@apache.org.
Merge pull request #1968 from huafengw/fix#1966
fix #1966 make Partitioner API Java compatible
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/0827adf3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/0827adf3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/0827adf3
Branch: refs/heads/master
Commit: 0827adf3bec894e026d68f8851cb8c75dc769445
Parents: aff1fe3 25aacb2
Author: Huafeng Wang <fv...@gmail.com>
Authored: Wed Feb 17 16:56:31 2016 +0800
Committer: Huafeng Wang <fv...@gmail.com>
Committed: Wed Feb 17 16:56:31 2016 +0800
----------------------------------------------------------------------
.../partitioner/BroadcastPartitioner.scala | 11 +++++++--
.../io/gearpump/partitioner/Partitioner.scala | 4 +--
.../streaming/examples/stock/QueryServer.scala | 26 ++++++++++----------
.../examples/transport/QueryServer.scala | 18 +++++++-------
.../storm/partitioner/StormPartitioner.scala | 6 ++---
.../storm/topology/GearpumpTuple.scala | 2 +-
.../experiments/storm/util/Grouper.scala | 25 ++++++++++---------
.../storm/util/StormOutputCollector.scala | 10 ++++----
.../partitioner/StormPartitionerSpec.scala | 6 ++---
.../storm/util/StormOutputCollectorSpec.scala | 10 ++++----
.../io/gearpump/services/MasterService.scala | 22 +++++++----------
.../streaming/appmaster/TaskManagerSpec.scala | 18 ++++----------
12 files changed, 77 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
[19/50] incubator-gearpump git commit: update README
Posted by ap...@apache.org.
update README
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/0d5e117f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/0d5e117f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/0d5e117f
Branch: refs/heads/master
Commit: 0d5e117f77c2886caf1cecf1d80bdbfae6b94159
Parents: afcd6e0
Author: Huafeng Wang <fv...@gmail.com>
Authored: Thu Feb 4 15:24:51 2016 +0800
Committer: Huafeng Wang <fv...@gmail.com>
Committed: Thu Feb 4 15:24:51 2016 +0800
----------------------------------------------------------------------
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/0d5e117f/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 85a5de7..f3a223e 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ Web site: http://gearpump.io
Gearpump is a lightweight real-time big data streaming engine. It is inspired by recent advances in the [Akka](https://github.com/akka/akka) framework and a desire to improve on existing streaming frameworks.
-
+
The name Gearpump is a reference to the engineering term “gear pump,” which is a super simple
pump that consists of only two gears, but is very powerful at streaming water.
[15/50] incubator-gearpump git commit: fix #1952,
document how to use StreamCQL over Gearpump
Posted by ap...@apache.org.
fix #1952, document how to use StreamCQL over Gearpump
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/75ad76cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/75ad76cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/75ad76cb
Branch: refs/heads/master
Commit: 75ad76cb386c9d95dcf907b141722aeb207f1a81
Parents: 89292ac
Author: manuzhang <ow...@gmail.com>
Authored: Thu Feb 4 11:39:02 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Thu Feb 4 12:10:54 2016 +0800
----------------------------------------------------------------------
docs/dev-storm.md | 57 ++++++++++++++++++--
.../experiments/storm/main/GearpumpNimbus.scala | 9 ++--
.../storm/main/GearpumpStormClient.scala | 3 +-
3 files changed, 58 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/75ad76cb/docs/dev-storm.md
----------------------------------------------------------------------
diff --git a/docs/dev-storm.md b/docs/dev-storm.md
index a22ed6c..0ff1127 100644
--- a/docs/dev-storm.md
+++ b/docs/dev-storm.md
@@ -80,10 +80,10 @@ This section shows how to run an existing Storm jar in a local Gearpump cluster.
bin/local
```
-2. start a local Gearpump Nimbus server
+2. start a Gearpump Nimbus server
- Users need server's thrift port to submit topologies later. The thrift port is written to a yaml config file set with `-output` option.
- Users can provide an existing config file where only `nimbus.thrift.port` is overwritten. If not provided, a new file `app.yaml` is created with the config.
+ Users need server's address(`nimbus.host` and `nimbus.thrift.port`) to submit topologies later. The address is written to a yaml config file set with `-output` option.
+ Users can provide an existing config file where only the address will be overwritten. If not provided, a new file `app.yaml` is created with the config.
```
bin/storm nimbus -output [conf <custom yaml config>]
@@ -172,4 +172,53 @@ where
* application config is submit from Storm application along with the topology
* component config is set in spout / bolt with `getComponentConfiguration`
-* custom file config is specified with the `-config` option when submitting Storm application from command line or uploaded from UI
\ No newline at end of file
+* custom file config is specified with the `-config` option when submitting Storm application from command line or uploaded from UI
+
+## StreamCQL Support
+
+[StreamCQL](https://github.com/HuaweiBigData/StreamCQL) is a Continuous Query Language on RealTime Computation System open sourced by Huawei.
+Since StreamCQL already supports Storm, it's straightforward to run StreamCQL over Gearpump.
+
+1. Install StreamCQL as in the official [README](https://github.com/HuaweiBigData/StreamCQL#install-streamcql)
+
+2. Launch Gearpump Nimbus Server as before
+
+3. Go to the installed stream-cql-binary, and change following settings in `conf/streaming-site.xml` with the output Nimbus configs in Step 2.
+
+ ```xml
+ <property>
+ <name>streaming.storm.nimbus.host</name>
+ <value>${nimbus.host}</value>
+ </property>
+ <property>
+ <name>streaming.storm.nimbus.port</name>
+ <value>${nimbus.thrift.port}</value>
+ </property>
+ ```
+
+4. Open CQL client shell and execute a simple cql example
+
+ ```
+ bin/cql
+ ```
+
+ ```
+ Streaming> CREATE INPUT STREAM s
+ (id INT, name STRING, type INT)
+ SOURCE randomgen
+ PROPERTIES ( timeUnit = "SECONDS", period = "1",
+ eventNumPerperiod = "1", isSchedule = "true" );
+
+ CREATE OUTPUT STREAM rs
+ (type INT, cc INT)
+ SINK consoleOutput;
+
+ INSERT INTO STREAM rs SELECT type, COUNT(id) as cc
+ FROM s[RANGE 20 SECONDS BATCH]
+ WHERE id > 5 GROUP BY type;
+
+ SUBMIT APPLICATION example;
+ ```
+
+5. Check the dashboard and you should see data flowing through a topology of 3 components
+
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/75ad76cb/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
index 96c12ae..6d5b0aa 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpNimbus.scala
@@ -39,6 +39,7 @@ import io.gearpump.streaming.StreamApplication
import io.gearpump.util.{AkkaApp, Constants, LogUtil}
import org.apache.storm.shade.org.json.simple.JSONValue
import org.apache.storm.shade.org.yaml.snakeyaml.Yaml
+import org.apache.storm.shade.org.yaml.snakeyaml.constructor.SafeConstructor
import org.slf4j.Logger
import scala.collection.JavaConverters._
@@ -79,12 +80,8 @@ object GearpumpNimbus extends AkkaApp with ArgumentsParser {
val outputConfig = Utils.findAndReadConfigFile(output, false).asInstanceOf[JMap[AnyRef, AnyRef]]
updatedConfig.putAll(outputConfig)
updatedConfig.putAll(thriftConfig)
- // workaround that yaml dumps {nimbus.host: 127.0.0.1} instead of {nimbus.host: '127.0.0.1'}
- // which can not be loaded
- val nimbusHost = s"${Config.NIMBUS_HOST}: '${thriftConfig.get(Config.NIMBUS_HOST)}'"
- updatedConfig.remove(Config.NIMBUS_HOST)
- val yaml = new Yaml
- val serialized = yaml.dumpAsMap(updatedConfig) ++ s"$nimbusHost\n"
+ val yaml = new Yaml(new SafeConstructor)
+ val serialized = yaml.dumpAsMap(updatedConfig)
val writer = new FileWriter(new File(output))
try {
writer.write(serialized)
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/75ad76cb/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
----------------------------------------------------------------------
diff --git a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
index b813ae0..643e926 100644
--- a/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
+++ b/experiments/storm/src/main/scala/io/gearpump/experiments/storm/main/GearpumpStormClient.scala
@@ -50,7 +50,8 @@ object GearpumpStormClient extends AkkaApp with ArgumentsParser {
s"-D${PREFER_IPV4}=true"
)
- val classPath = Array(s"${System.getProperty(GEARPUMP_HOME)}/lib/storm/*", jar)
+ val gearpumpHome = System.getProperty(GEARPUMP_HOME)
+ val classPath = Array(s"$gearpumpHome/lib/*", s"$gearpumpHome/lib/storm/*", jar)
val process = Util.startProcess(stormOptions, classPath, topology, stormArgs)
// wait till the process exit