You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gearpump.apache.org by ma...@apache.org on 2017/07/26 02:29:09 UTC
incubator-gearpump git commit: [GEARPUMP-330] Allow examples to run
in `sbt run` and Intellij
Repository: incubator-gearpump
Updated Branches:
refs/heads/master a47e1a6db -> ac8ac0392
[GEARPUMP-330] Allow examples to run in `sbt run` and Intellij
Author: manuzhang <ow...@gmail.com>
Closes #200 from manuzhang/run_examples_in_ide.
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/ac8ac039
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/ac8ac039
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/ac8ac039
Branch: refs/heads/master
Commit: ac8ac039217725f9f924ec901fdf7adfd9f51657
Parents: a47e1a6
Author: manuzhang <ow...@gmail.com>
Authored: Wed Jul 26 10:28:25 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Wed Jul 26 10:28:41 2017 +0800
----------------------------------------------------------------------
docs/contents/introduction/quick-start.md | 34 ++++++++++
docs/mkdocs.yml | 3 +-
.../examples/wordcount/dsl/WordCount.scala | 8 ++-
project/BuildExamples.scala | 69 ++++++++++++--------
project/BuildExperiments.scala | 5 +-
project/plugins.sbt | 4 +-
6 files changed, 88 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/docs/contents/introduction/quick-start.md
----------------------------------------------------------------------
diff --git a/docs/contents/introduction/quick-start.md b/docs/contents/introduction/quick-start.md
new file mode 100644
index 0000000..c38e50f
--- /dev/null
+++ b/docs/contents/introduction/quick-start.md
@@ -0,0 +1,34 @@
+This quick start will walk you through executing your first Gearpump pipeline to run WordCount written in Stream DSL.
+
+### Set up development environment
+1. Download and install [Java Development Kit(JDK)](http://www.oracle.com/technetwork/java/javase/downloads/index.html) 1.8. Verify that [JAVA_HOME](https://docs.oracle.com/javase/8/docs/technotes/guides/troubleshoot/envvars001.html) is set and points to your JDK installation.
+2. Download and install [sbt](http://www.scala-sbt.org/download.html) by following [sbt's installation guide](http://www.scala-sbt.org/0.13/docs/Setup.html) for your specific operating system.
+
+### Run WordCount
+
+1. Run WordCount example with sbt
+
+
+ :::bash
+ sbt "project gearpump-examples-wordcount" run
+
+
+2. Select the third main class in the sbt console
+
+ :::bash
+ Multiple main classes detected, select one to run:
+ [1] org.apache.gearpump.streaming.examples.wordcount.WordCount
+ [2] org.apache.gearpump.streaming.examples.wordcount.dsl.WindowedWordCount
+ [3] org.apache.gearpump.streaming.examples.wordcount.dsl.WordCount
+
+ Enter number: 3
+
+ If everything goes fine, the following output is expected
+
+ :::bash
+ (is,1)(bingo!!,2)(a,1)(good,1)(This,1)(start,,1)
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/docs/mkdocs.yml
----------------------------------------------------------------------
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 7ce5287..ae5cf1b 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -12,7 +12,8 @@ docs_dir: tmp
pages:
- Overview: 'index.md'
-- Introduction:
+- Introduction:
+ - 'Quick Start': 'introduction/quick-start.md'
- 'Submit Your 1st Application': 'introduction/submit-your-1st-application.md'
- 'Client Command Line': 'introduction/commandline.md'
- 'Basic Concepts': 'introduction/basic-concepts.md'
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/examples/streaming/wordcount/src/main/scala/org/apache/gearpump/streaming/examples/wordcount/dsl/WordCount.scala
----------------------------------------------------------------------
diff --git a/examples/streaming/wordcount/src/main/scala/org/apache/gearpump/streaming/examples/wordcount/dsl/WordCount.scala b/examples/streaming/wordcount/src/main/scala/org/apache/gearpump/streaming/examples/wordcount/dsl/WordCount.scala
index 5d1e607..65f63d2 100644
--- a/examples/streaming/wordcount/src/main/scala/org/apache/gearpump/streaming/examples/wordcount/dsl/WordCount.scala
+++ b/examples/streaming/wordcount/src/main/scala/org/apache/gearpump/streaming/examples/wordcount/dsl/WordCount.scala
@@ -19,6 +19,7 @@
package org.apache.gearpump.streaming.examples.wordcount.dsl
import org.apache.gearpump.cluster.client.ClientContext
+import org.apache.gearpump.cluster.embedded.EmbeddedCluster
import org.apache.gearpump.cluster.main.{ArgumentsParser, CLIOption}
import org.apache.gearpump.streaming.dsl.scalaapi.StreamApp
import org.apache.gearpump.streaming.dsl.scalaapi.StreamApp._
@@ -30,16 +31,19 @@ object WordCount extends AkkaApp with ArgumentsParser {
override val options: Array[(String, CLIOption[Any])] = Array.empty
override def main(akkaConf: Config, args: Array[String]): Unit = {
- val context = ClientContext(akkaConf)
+ val cluster = new EmbeddedCluster(akkaConf)
+ cluster.start()
+ val context: ClientContext = cluster.newClientContext
val app = StreamApp("dsl", context)
val data = "This is a good start, bingo!! bingo!!"
app.source(data.lines.toList, 1, "source").
// word => (word, count)
flatMap(line => line.split("[\\s]+")).map((_, 1)).
// (word, count1), (word, count2) => (word, count1 + count2)
- groupByKey().sum.log
+ groupByKey().sum.map(print)
context.submit(app).waitUntilFinish()
context.close()
+ cluster.stop()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/project/BuildExamples.scala
----------------------------------------------------------------------
diff --git a/project/BuildExamples.scala b/project/BuildExamples.scala
index d7390b5..00affb0 100644
--- a/project/BuildExamples.scala
+++ b/project/BuildExamples.scala
@@ -42,7 +42,7 @@ object BuildExamples extends sbt.Build {
lazy val example_hbase = Project(
id = "gearpump-examples-hbase",
base = file("examples/streaming/hbase"),
- settings = commonSettings ++ noPublish ++ myAssemblySettings ++
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.hbase.HBaseConn") ++
Seq(
libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-common" % hadoopVersion
@@ -50,38 +50,37 @@ object BuildExamples extends sbt.Build {
exclude("commons-beanutils", "commons-beanutils")
exclude("asm", "asm")
exclude("org.ow2.asm", "asm")
- ),
- mainClass in(Compile, packageBin) :=
- Some("org.apache.gearpump.streaming.examples.hbase.HBaseConn"),
-
- target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
- CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ) dependsOn(streaming % "test->test; provided", core % "provided", external_hbase)
+ )
+ ) ++ include("examples/streaming/hbase", "external/hbase")
+ ) dependsOn(core, streaming % "compile; test->test", external_hbase)
lazy val wordcountJava = Project(
id = "gearpump-examples-wordcountjava",
base = file("examples/streaming/wordcount-java"),
- settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcountjava.WordCount")
- ).dependsOn(core % "provided", streaming % "test->test; provided")
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcountjava.WordCount") ++
+ include("examples/streaming/wordcount-java")
+ ).dependsOn(core, streaming % "compile; test->test")
lazy val wordcount = Project(
id = "gearpump-examples-wordcount",
base = file("examples/streaming/wordcount"),
- settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcount.WordCount")
- ).dependsOn(core % "provided", streaming % "test->test; provided")
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.wordcount.dsl.WordCount") ++
+ include("examples/streaming/wordcount")
+ ).dependsOn(core, streaming % "compile; test->test")
lazy val sol = Project(
id = "gearpump-examples-sol",
base = file("examples/streaming/sol"),
- settings = exampleSettings("org.apache.gearpump.streaming.examples.sol.SOL")
- ).dependsOn(core % "provided", streaming % "test->test; provided")
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.sol.SOL") ++
+ include("examples/streaming/sol")
+ ).dependsOn(core, streaming % "compile; test->test")
lazy val complexdag = Project(
id = "gearpump-examples-complexdag",
base = file("examples/streaming/complexdag"),
- settings = exampleSettings("org.apache.gearpump.streaming.examples.complexdag.Dag")
- ).dependsOn(core % "provided", streaming % "test->test; provided")
+ settings = exampleSettings("org.apache.gearpump.streaming.examples.complexdag.Dag") ++
+ include("examples/streaming/complexdag")
+ ).dependsOn(core, streaming % "compile; test->test")
lazy val distributedshell = Project(
id = "gearpump-examples-distributedshell",
@@ -91,8 +90,8 @@ object BuildExamples extends sbt.Build {
Some("org.apache.gearpump.examples.distributedshell.DistributedShell"),
target in assembly := baseDirectory.value.getParentFile / "target" /
CrossVersion.binaryScalaVersion(scalaVersion.value)
- )
- ).dependsOn(core % "test->test; provided")
+ ) ++ include("examples/distributedshell")
+ ).dependsOn(core % "compile; test->test")
lazy val distributeservice = Project(
id = "gearpump-examples-distributeservice",
@@ -109,8 +108,8 @@ object BuildExamples extends sbt.Build {
"io.spray" %% "spray-can" % sprayVersion,
"io.spray" %% "spray-routing-shapeless2" % sprayVersion
)
- )
- ).dependsOn(core % "test->test; provided")
+ ) ++ include("examples/distributeservice")
+ ).dependsOn(core % "compile; test->test")
lazy val fsio = Project(
id = "gearpump-examples-fsio",
@@ -128,15 +127,16 @@ object BuildExamples extends sbt.Build {
exclude("asm", "asm")
exclude("org.ow2.asm", "asm")
)
- )
- ).dependsOn(core % "provided", streaming % "test->test; provided")
+ ) ++ include("examples/streaming/fsio")
+ ).dependsOn(core, streaming % "compile; test->test")
lazy val examples_kafka = Project(
id = "gearpump-examples-kafka",
base = file("examples/streaming/kafka"),
settings =
- exampleSettings("org.apache.gearpump.streaming.examples.kafka.wordcount.KafkaWordCount")
- ).dependsOn(core % "provided", streaming % "test->test; provided", external_kafka)
+ exampleSettings("org.apache.gearpump.streaming.examples.kafka.wordcount.KafkaWordCount") ++
+ include("examples/streaming/kafka", "external/kafka")
+ ).dependsOn(core, streaming % "compile; test->test", external_kafka)
lazy val examples_state = Project(
id = "gearpump-examples-state",
@@ -155,15 +155,18 @@ object BuildExamples extends sbt.Build {
exclude("org.ow2.asm", "asm"),
"org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion
)
- )
- ).dependsOn(core % "provided", streaming % "test->test; provided",
+ ) ++ include("examples/streaming/state",
+ "external/hadoopfs", "external/monoid", "external/serializer", "external/kafka")
+ ).dependsOn(core, streaming % "compile; test->test",
external_hadoopfs, external_monoid, external_serializer, external_kafka)
lazy val pagerank = Project(
id = "gearpump-examples-pagerank",
base = file("examples/pagerank"),
- settings = exampleSettings("org.apache.gearpump.experiments.pagerank.example.PageRankExample")
- ).dependsOn(core % "provided", streaming % "test->test; provided")
+ settings =
+ exampleSettings("org.apache.gearpump.experiments.pagerank.example.PageRankExample") ++
+ include("examples/pagerank")
+ ).dependsOn(core, streaming % "compile; test->test")
private def exampleSettings(className: String): Seq[Def.Setting[_]] =
commonSettings ++ noPublish ++ myAssemblySettings ++ Seq(
@@ -172,4 +175,12 @@ object BuildExamples extends sbt.Build {
target in assembly := baseDirectory.value.getParentFile.getParentFile / "target" /
CrossVersion.binaryScalaVersion(scalaVersion.value)
)
+
+ private def include(files: String*): Seq[Def.Setting[_]] = Seq(
+ assemblyExcludedJars in assembly := {
+ val cp = (fullClasspath in assembly).value
+ cp.filterNot(p =>
+ files.exists(p.data.getAbsolutePath.contains))
+ }
+ )
}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/project/BuildExperiments.scala
----------------------------------------------------------------------
diff --git a/project/BuildExperiments.scala b/project/BuildExperiments.scala
index 0d3adb0..4022e95 100644
--- a/project/BuildExperiments.scala
+++ b/project/BuildExperiments.scala
@@ -74,8 +74,9 @@ object BuildExperiments extends sbt.Build {
libraryDependencies ++= Seq(
"redis.clients" % "jedis" % jedisVersion
)
- )
- ).dependsOn(core % "provided", streaming % "test->test; provided")
+ ))
+ .dependsOn(core % "provided", streaming % "test->test; provided")
+ .disablePlugins(sbtassembly.AssemblyPlugin)
lazy val storm = Project(
id = "gearpump-experiments-storm",
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/ac8ac039/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 33a7e50..b8bae36 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -18,11 +18,13 @@
resolvers += Resolver.bintrayIvyRepo("fvunicorn", "sbt-plugins")
+resolvers += Resolver.bintrayIvyRepo("manuzhang", "sbt-plugins")
+
resolvers += Classpaths.sbtPluginReleases
addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.8")
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4"
+addSbtPlugin("io.gearpump.sbt" % "sbt-assembly" % "0.14.5"
exclude("org.apache.maven", "maven-plugin-api"))
addSbtPlugin("io.gearpump.sbt" % "sbt-pack" % "0.7.7")