You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by am...@apache.org on 2016/04/23 00:23:16 UTC
[4/5] incubator-beam git commit: Update README according to
dataflow->beam package rename
Update README according to dataflow->beam package rename
Annotate class with RunWith and some whitespace fix ups
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/7fd9e1e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/7fd9e1e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/7fd9e1e7
Branch: refs/heads/master
Commit: 7fd9e1e70ef607c47613022c4b7cf63c46353e37
Parents: f424b8d
Author: Sela <an...@paypal.com>
Authored: Thu Apr 21 22:48:14 2016 +0300
Committer: Sela <an...@paypal.com>
Committed: Sat Apr 23 01:11:25 2016 +0300
----------------------------------------------------------------------
runners/spark/README.md | 6 +++---
.../beam/runners/spark/SparkRunnerRegistrarTest.java | 11 +++++++----
2 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7fd9e1e7/runners/spark/README.md
----------------------------------------------------------------------
diff --git a/runners/spark/README.md b/runners/spark/README.md
index 1d75b35..5b2e732 100644
--- a/runners/spark/README.md
+++ b/runners/spark/README.md
@@ -93,7 +93,7 @@ Switch to the Spark runner directory:
Then run the [word count example][wc] from the SDK using a single threaded Spark instance
in local mode:
- mvn exec:exec -DmainClass=com.google.cloud.dataflow.examples.WordCount \
+ mvn exec:exec -DmainClass=org.apache.beam.examples.WordCount \
-Dinput=/tmp/kinglear.txt -Doutput=/tmp/out -Drunner=SparkPipelineRunner \
-DsparkMaster=local
@@ -104,7 +104,7 @@ Check the output by running:
__Note: running examples using `mvn exec:exec` only works for Spark local mode at the
moment. See the next section for how to run on a cluster.__
-[wc]: https://github.com/apache/incubator-beam/blob/master/examples/src/main/java/com/google/cloud/dataflow/examples/WordCount.java
+[wc]: https://github.com/apache/incubator-beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
## Running on a Cluster
Spark Beam pipelines can be run on a cluster using the `spark-submit` command.
@@ -117,7 +117,7 @@ Then run the word count example using Spark submit with the `yarn-client` master
(`yarn-cluster` works just as well):
spark-submit \
- --class com.google.cloud.dataflow.examples.WordCount \
+ --class org.apache.beam.examples.WordCount \
--master yarn-client \
target/spark-runner-*-spark-app.jar \
--inputFile=kinglear.txt --output=out --runner=SparkPipelineRunner --sparkMaster=yarn-client
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7fd9e1e7/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java
index d51403f..3643bac 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkRunnerRegistrarTest.java
@@ -23,6 +23,8 @@ import com.google.common.collect.Lists;
import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
import java.util.ServiceLoader;
@@ -32,12 +34,13 @@ import static org.junit.Assert.fail;
/**
* Test {@link SparkRunnerRegistrar}.
*/
+@RunWith(JUnit4.class)
public class SparkRunnerRegistrarTest {
@Test
public void testOptions() {
assertEquals(
- ImmutableList.of(SparkPipelineOptions.class, SparkStreamingPipelineOptions.class),
- new SparkRunnerRegistrar.Options().getPipelineOptions());
+ ImmutableList.of(SparkPipelineOptions.class, SparkStreamingPipelineOptions.class),
+ new SparkRunnerRegistrar.Options().getPipelineOptions());
}
@Test
@@ -51,7 +54,7 @@ public class SparkRunnerRegistrarTest {
for (PipelineOptionsRegistrar registrar :
Lists.newArrayList(ServiceLoader.load(PipelineOptionsRegistrar.class).iterator())) {
if (registrar instanceof SparkRunnerRegistrar.Options) {
- return;
+ return;
}
}
fail("Expected to find " + SparkRunnerRegistrar.Options.class);
@@ -62,7 +65,7 @@ public class SparkRunnerRegistrarTest {
for (PipelineRunnerRegistrar registrar :
Lists.newArrayList(ServiceLoader.load(PipelineRunnerRegistrar.class).iterator())) {
if (registrar instanceof SparkRunnerRegistrar.Runner) {
- return;
+ return;
}
}
fail("Expected to find " + SparkRunnerRegistrar.Runner.class);