You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/08/30 00:19:04 UTC
[1/2] incubator-beam git commit: [BEAM-572] Remove Spark Reference in
WordCount
Repository: incubator-beam
Updated Branches:
refs/heads/master e4e5e5ea2 -> 6d4c739c0
[BEAM-572] Remove Spark Reference in WordCount
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a25b3589
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a25b3589
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a25b3589
Branch: refs/heads/master
Commit: a25b3589db738ea507047b502092b0d177ea8e1f
Parents: e4e5e5e
Author: Mark Liu <ma...@markliu-macbookpro.roam.corp.google.com>
Authored: Fri Aug 26 16:13:17 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Aug 29 17:18:59 2016 -0700
----------------------------------------------------------------------
.../org/apache/beam/examples/WordCount.java | 24 +-------------------
.../org/apache/beam/examples/WordCountIT.java | 9 ++++++++
2 files changed, 10 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a25b3589/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
index 0275651..793ee4b 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
@@ -18,7 +18,6 @@
package org.apache.beam.examples;
import com.google.common.base.Strings;
-import com.google.common.io.Resources;
import java.io.IOException;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
@@ -165,7 +164,7 @@ public class WordCount {
*/
public static interface WordCountOptions extends PipelineOptions {
@Description("Path of the file to read from")
- @Default.InstanceFactory(InputFactory.class)
+ @Default.String("gs://apache-beam-samples/shakespeare/kinglear.txt")
String getInputFile();
void setInputFile(String value);
@@ -194,27 +193,6 @@ public class WordCount {
}
}
}
-
- /**
- * Return default input file path according to runner type.
- *
- * <p><ul>
- * <li>SparkRunner:
- * .../src/test/resources/LICENSE</li>
- * <li>other runners:
- * gs://apache-beam-samples/apache/LICENSE</li>
- * </ul>
- */
- public static class InputFactory implements DefaultValueFactory<String> {
- @Override
- public String create(PipelineOptions options) {
- if (options.getRunner().getName().contains("SparkRunner")) {
- return Resources.getResource("LICENSE").getPath();
- } else {
- return "gs://apache-beam-samples/apache/LICENSE";
- }
- }
- }
}
public static void main(String[] args) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a25b3589/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
----------------------------------------------------------------------
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
index ca0c9d6..b0e0fe0 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
@@ -18,6 +18,7 @@
package org.apache.beam.examples;
+import com.google.common.io.Resources;
import java.util.Date;
import org.apache.beam.examples.WordCount.WordCountOptions;
import org.apache.beam.sdk.options.Default;
@@ -61,6 +62,14 @@ public class WordCountIT {
options.setOnSuccessMatcher(
new FileChecksumMatcher(options.getOutputChecksum(), options.getOutput() + "*"));
+ String e2eTestInputPath = "gs://apache-beam-samples/apache/LICENSE";
+ // Spark runner currently doesn't support GCS I/O, change default input to:
+ // .../src/test/resources/LICENSE
+ if (options.getRunner().getName().contains("SparkRunner")) {
+ e2eTestInputPath = Resources.getResource("LICENSE").getPath();
+ }
+ options.setInputFile(e2eTestInputPath);
+
WordCount.main(TestPipeline.convertToArgs(options));
}
}
[2/2] incubator-beam git commit: Closes #891
Posted by dh...@apache.org.
Closes #891
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6d4c739c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6d4c739c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6d4c739c
Branch: refs/heads/master
Commit: 6d4c739c061ba1c8083c06000f8eb1eb9c7514ad
Parents: e4e5e5e a25b358
Author: Dan Halperin <dh...@google.com>
Authored: Mon Aug 29 17:19:00 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Aug 29 17:19:00 2016 -0700
----------------------------------------------------------------------
.../org/apache/beam/examples/WordCount.java | 24 +-------------------
.../org/apache/beam/examples/WordCountIT.java | 9 ++++++++
2 files changed, 10 insertions(+), 23 deletions(-)
----------------------------------------------------------------------