You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2016/08/25 18:55:15 UTC
[01/50] incubator-beam git commit: Port various Spark runner tests to
new DoFn
Repository: incubator-beam
Updated Branches:
refs/heads/gearpump-runner 02b2248a5 -> 091a15a07
Port various Spark runner tests to new DoFn
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f5df3583
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f5df3583
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f5df3583
Branch: refs/heads/gearpump-runner
Commit: f5df358320cfde6a1c4d012d4169af691f6a18e9
Parents: d6395e9
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Aug 5 12:31:07 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 11:35:17 2016 -0700
----------------------------------------------------------------------
.../apache/beam/runners/spark/TfIdfTest.java | 22 ++++++++++----------
.../spark/translation/CombinePerKeyTest.java | 6 +++---
.../translation/MultiOutputWordCountTest.java | 10 ++++-----
.../spark/translation/SerializationTest.java | 10 ++++-----
.../streaming/KafkaStreamingTest.java | 6 +++---
5 files changed, 27 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f5df3583/runners/spark/src/test/java/org/apache/beam/runners/spark/TfIdfTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/TfIdfTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/TfIdfTest.java
index 074e6aa..17bf6dd 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/TfIdfTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/TfIdfTest.java
@@ -24,8 +24,8 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Keys;
-import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.RemoveDuplicates;
@@ -101,8 +101,8 @@ public class TfIdfTest {
// of the words in the document associated with that that URI.
PCollection<KV<URI, String>> uriToWords = uriToContent
.apply("SplitWords", ParDo.of(
- new OldDoFn<KV<URI, String>, KV<URI, String>>() {
- @Override
+ new DoFn<KV<URI, String>, KV<URI, String>>() {
+ @ProcessElement
public void processElement(ProcessContext c) {
URI uri = c.element().getKey();
String line = c.element().getValue();
@@ -144,8 +144,8 @@ public class TfIdfTest {
// by the URI key.
PCollection<KV<URI, KV<String, Long>>> uriToWordAndCount = uriAndWordToCount
.apply("ShiftKeys", ParDo.of(
- new OldDoFn<KV<KV<URI, String>, Long>, KV<URI, KV<String, Long>>>() {
- @Override
+ new DoFn<KV<KV<URI, String>, Long>, KV<URI, KV<String, Long>>>() {
+ @ProcessElement
public void processElement(ProcessContext c) {
URI uri = c.element().getKey().getKey();
String word = c.element().getKey().getValue();
@@ -183,8 +183,8 @@ public class TfIdfTest {
// divided by the total number of words in the document.
PCollection<KV<String, KV<URI, Double>>> wordToUriAndTf = uriToWordAndCountAndTotal
.apply("ComputeTermFrequencies", ParDo.of(
- new OldDoFn<KV<URI, CoGbkResult>, KV<String, KV<URI, Double>>>() {
- @Override
+ new DoFn<KV<URI, CoGbkResult>, KV<String, KV<URI, Double>>>() {
+ @ProcessElement
public void processElement(ProcessContext c) {
URI uri = c.element().getKey();
Long wordTotal = c.element().getValue().getOnly(wordTotalsTag);
@@ -208,8 +208,8 @@ public class TfIdfTest {
PCollection<KV<String, Double>> wordToDf = wordToDocCount
.apply("ComputeDocFrequencies", ParDo
.withSideInputs(totalDocuments)
- .of(new OldDoFn<KV<String, Long>, KV<String, Double>>() {
- @Override
+ .of(new DoFn<KV<String, Long>, KV<String, Double>>() {
+ @ProcessElement
public void processElement(ProcessContext c) {
String word = c.element().getKey();
Long documentCount = c.element().getValue();
@@ -237,8 +237,8 @@ public class TfIdfTest {
// divided by the log of the document frequency.
return wordToUriAndTfAndDf
.apply("ComputeTfIdf", ParDo.of(
- new OldDoFn<KV<String, CoGbkResult>, KV<String, KV<URI, Double>>>() {
- @Override
+ new DoFn<KV<String, CoGbkResult>, KV<String, KV<URI, Double>>>() {
+ @ProcessElement
public void processElement(ProcessContext c) {
String word = c.element().getKey();
Double df = c.element().getValue().getOnly(dfTag);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f5df3583/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/CombinePerKeyTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/CombinePerKeyTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/CombinePerKeyTest.java
index dee9213..cdf2cfb 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/CombinePerKeyTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/CombinePerKeyTest.java
@@ -27,7 +27,7 @@ import org.apache.beam.sdk.coders.VarLongCoder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Sum;
@@ -70,8 +70,8 @@ public class CombinePerKeyTest {
private static class SumPerKey<T> extends PTransform<PCollection<T>, PCollection<KV<T, Long>>> {
@Override
public PCollection<KV<T, Long>> apply(PCollection<T> pcol) {
- PCollection<KV<T, Long>> withLongs = pcol.apply(ParDo.of(new OldDoFn<T, KV<T, Long>>() {
- @Override
+ PCollection<KV<T, Long>> withLongs = pcol.apply(ParDo.of(new DoFn<T, KV<T, Long>>() {
+ @ProcessElement
public void processElement(ProcessContext processContext) throws Exception {
processContext.output(KV.of(processContext.element(), 1L));
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f5df3583/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
index 066521b..291f7b2 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
@@ -30,9 +30,9 @@ import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.ApproximateUnique;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.Max;
-import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Sum;
@@ -105,7 +105,7 @@ public class MultiOutputWordCountTest {
/**
* A OldDoFn that tokenizes lines of text into individual words.
*/
- static class ExtractWordsFn extends OldDoFn<String, String> {
+ static class ExtractWordsFn extends DoFn<String, String> {
private final Aggregator<Integer, Integer> totalWords = createAggregator("totalWords",
new Sum.SumIntegerFn());
@@ -117,7 +117,7 @@ public class MultiOutputWordCountTest {
this.regex = regex;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
String[] words = c.element().split(c.sideInput(regex));
for (String word : words) {
@@ -170,8 +170,8 @@ public class MultiOutputWordCountTest {
}
}
- private static class FormatCountsFn extends OldDoFn<KV<String, Long>, String> {
- @Override
+ private static class FormatCountsFn extends DoFn<KV<String, Long>, String> {
+ @ProcessElement
public void processElement(ProcessContext c) {
c.output(c.element().getKey() + ": " + c.element().getValue());
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f5df3583/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java
index fb97b8b..019b107 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java
@@ -30,7 +30,7 @@ import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Sum;
@@ -147,12 +147,12 @@ public class SerializationTest {
/**
* A OldDoFn that tokenizes lines of text into individual words.
*/
- static class ExtractWordsFn extends OldDoFn<StringHolder, StringHolder> {
+ static class ExtractWordsFn extends DoFn<StringHolder, StringHolder> {
private static final Pattern WORD_BOUNDARY = Pattern.compile("[^a-zA-Z']+");
private final Aggregator<Long, Long> emptyLines =
createAggregator("emptyLines", new Sum.SumLongFn());
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
// Split the line into words.
String[] words = WORD_BOUNDARY.split(c.element().toString());
@@ -175,8 +175,8 @@ public class SerializationTest {
/**
* A OldDoFn that converts a Word and Count into a printable string.
*/
- private static class FormatCountsFn extends OldDoFn<KV<StringHolder, Long>, StringHolder> {
- @Override
+ private static class FormatCountsFn extends DoFn<KV<StringHolder, Long>, StringHolder> {
+ @ProcessElement
public void processElement(ProcessContext c) {
c.output(new StringHolder(c.element().getKey() + ": " + c.element().getValue()));
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f5df3583/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
index fa98ca3..17044aa 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
@@ -27,7 +27,7 @@ import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
@@ -121,8 +121,8 @@ public class KafkaStreamingTest {
EMBEDDED_ZOOKEEPER.shutdown();
}
- private static class FormatKVFn extends OldDoFn<KV<String, String>, String> {
- @Override
+ private static class FormatKVFn extends DoFn<KV<String, String>, String> {
+ @ProcessElement
public void processElement(ProcessContext c) {
c.output(c.element().getKey() + "," + c.element().getValue());
}
[20/50] incubator-beam git commit: Make
StreamingPCollectionViewWriterFn and its data public
Posted by ke...@apache.org.
Make StreamingPCollectionViewWriterFn and its data public
This enables a smooth migration path whereby the runner can
notice the class and extract these values, to run them
without OldDoFn.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d70a3a47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d70a3a47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d70a3a47
Branch: refs/heads/gearpump-runner
Commit: d70a3a47ee39fab8c4514a7bda4126827baf65a1
Parents: 7da1a3d
Author: Kenneth Knowles <kl...@google.com>
Authored: Tue Aug 9 14:36:28 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Aug 9 14:36:28 2016 -0700
----------------------------------------------------------------------
.../org/apache/beam/runners/dataflow/DataflowRunner.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d70a3a47/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index fadd9c7..6da33fa 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -2375,7 +2375,8 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
* A specialized {@link OldDoFn} for writing the contents of a {@link PCollection}
* to a streaming {@link PCollectionView} backend implementation.
*/
- private static class StreamingPCollectionViewWriterFn<T>
+ @Deprecated
+ public static class StreamingPCollectionViewWriterFn<T>
extends OldDoFn<Iterable<T>, T> implements OldDoFn.RequiresWindowAccess {
private final PCollectionView<?> view;
private final Coder<T> dataCoder;
@@ -2390,6 +2391,14 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
this.dataCoder = dataCoder;
}
+ public PCollectionView<?> getView() {
+ return view;
+ }
+
+ public Coder<T> getDataCoder() {
+ return dataCoder;
+ }
+
@Override
public void processElement(ProcessContext c) throws Exception {
List<WindowedValue<T>> output = new ArrayList<>();
[34/50] incubator-beam git commit: Use Datastore AncestorKey for
AutoComplete example
Posted by ke...@apache.org.
Use Datastore AncestorKey for AutoComplete example
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a41dab86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a41dab86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a41dab86
Branch: refs/heads/gearpump-runner
Commit: a41dab8685c086f60d0cd72b61ce7affd3559a34
Parents: 95e7f01
Author: Vikas Kedigehalli <vi...@google.com>
Authored: Thu Aug 4 11:29:25 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Wed Aug 10 10:39:15 2016 -0700
----------------------------------------------------------------------
.../beam/examples/complete/AutoComplete.java | 21 +++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a41dab86/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
index 1ab39c9..baae100 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
@@ -18,6 +18,7 @@
package org.apache.beam.examples.complete;
import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.datastore.v1beta3.client.DatastoreHelper.makeKey;
import static com.google.datastore.v1beta3.client.DatastoreHelper.makeValue;
import org.apache.beam.examples.common.ExampleBigQueryTableOptions;
@@ -61,7 +62,6 @@ import com.google.common.base.MoreObjects;
import com.google.datastore.v1beta3.Entity;
import com.google.datastore.v1beta3.Key;
import com.google.datastore.v1beta3.Value;
-import com.google.datastore.v1beta3.client.DatastoreHelper;
import org.joda.time.Duration;
@@ -384,18 +384,23 @@ public class AutoComplete {
/**
* Takes as input a the top candidates per prefix, and emits an entity
* suitable for writing to Datastore.
+ *
+ * <p>Note: We use ancestor keys for strong consistency. See the Cloud Datastore documentation on
+ * <a href="https://cloud.google.com/datastore/docs/concepts/structuring_for_strong_consistency">
+ * Structuring Data for Strong Consistency</a>
*/
static class FormatForDatastore extends DoFn<KV<String, List<CompletionCandidate>>, Entity> {
private String kind;
-
- public FormatForDatastore(String kind) {
+ private String ancestorKey;
+ public FormatForDatastore(String kind, String ancestorKey) {
this.kind = kind;
+ this.ancestorKey = ancestorKey;
}
@ProcessElement
public void processElement(ProcessContext c) {
Entity.Builder entityBuilder = Entity.newBuilder();
- Key key = DatastoreHelper.makeKey(kind, c.element().getKey()).build();
+ Key key = makeKey(makeKey(kind, ancestorKey).build(), kind, c.element().getKey()).build();
entityBuilder.setKey(key);
List<Value> candidates = new ArrayList<>();
@@ -444,6 +449,11 @@ public class AutoComplete {
Boolean getOutputToDatastore();
void setOutputToDatastore(Boolean value);
+ @Description("Datastore ancestor key")
+ @Default.String("root")
+ String getDatastoreAncestorKey();
+ void setDatastoreAncestorKey(String value);
+
@Description("Datastore output project ID, defaults to project ID")
String getOutputProject();
void setOutputProject(String value);
@@ -476,7 +486,8 @@ public class AutoComplete {
if (options.getOutputToDatastore()) {
toWrite
- .apply("FormatForDatastore", ParDo.of(new FormatForDatastore(options.getKind())))
+ .apply("FormatForDatastore", ParDo.of(new FormatForDatastore(options.getKind(),
+ options.getDatastoreAncestorKey())))
.apply(DatastoreIO.v1beta3().write().withProjectId(MoreObjects.firstNonNull(
options.getOutputProject(), options.getProject())));
}
[07/50] incubator-beam git commit: This closes #759
Posted by ke...@apache.org.
This closes #759
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/bb00810a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/bb00810a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/bb00810a
Branch: refs/heads/gearpump-runner
Commit: bb00810adeee27506a43487c6842bb0c9dcf58a9
Parents: 7629f97 4753f44
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Aug 8 13:37:03 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 13:37:03 2016 -0700
----------------------------------------------------------------------
.../beam/sdk/transforms/FlatMapElements.java | 2 +-
.../apache/beam/sdk/transforms/MapElements.java | 2 +-
.../beam/sdk/transforms/SimpleFunction.java | 14 +++++++++++++-
.../beam/sdk/transforms/FlatMapElementsTest.java | 19 +++++++++++++++++++
.../beam/sdk/transforms/MapElementsTest.java | 8 ++++++++
5 files changed, 42 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[27/50] incubator-beam git commit: This closes #792
Posted by ke...@apache.org.
This closes #792
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6a5310a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6a5310a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6a5310a5
Branch: refs/heads/gearpump-runner
Commit: 6a5310a560e67743873b7b046951f27ef3b67ed9
Parents: a035796 5e1185d
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Aug 10 09:20:52 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Aug 10 09:20:52 2016 -0700
----------------------------------------------------------------------
.travis.yml | 4 +-
.travis/README.md | 23 +++++++
.travis/settings.xml | 33 +++++++++
.travis/test_wordcount.sh | 125 ++++++++++++++++++++++++++++++++++
testing/travis/README.md | 23 -------
testing/travis/test_wordcount.sh | 125 ----------------------------------
6 files changed, 183 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
[17/50] incubator-beam git commit: Port easy parts of DataflowRunner
to new DoFn
Posted by ke...@apache.org.
Port easy parts of DataflowRunner to new DoFn
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/fcdd15b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/fcdd15b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/fcdd15b8
Branch: refs/heads/gearpump-runner
Commit: fcdd15b81b93f87de0aa02bfb3b09740bc259c4c
Parents: a1d601a
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Aug 8 20:35:59 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Aug 9 12:41:52 2016 -0700
----------------------------------------------------------------------
.../beam/runners/dataflow/DataflowRunner.java | 92 ++++++++++----------
1 file changed, 45 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fcdd15b8/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index fadd9c7..4d34ec4 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -761,31 +761,30 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
PTransform<PCollection<T>, PCollection<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>>> {
/**
- * A {@link OldDoFn} that for each element outputs a {@code KV} structure suitable for
+ * A {@link DoFn} that for each element outputs a {@code KV} structure suitable for
* grouping by the hash of the window's byte representation and sorting the grouped values
* using the window's byte representation.
*/
@SystemDoFnInternal
private static class UseWindowHashAsKeyAndWindowAsSortKeyDoFn<T, W extends BoundedWindow>
- extends OldDoFn<T, KV<Integer, KV<W, WindowedValue<T>>>> implements
- OldDoFn.RequiresWindowAccess {
+ extends DoFn<T, KV<Integer, KV<W, WindowedValue<T>>>> {
private final IsmRecordCoder<?> ismCoderForHash;
private UseWindowHashAsKeyAndWindowAsSortKeyDoFn(IsmRecordCoder<?> ismCoderForHash) {
this.ismCoderForHash = ismCoderForHash;
}
- @Override
- public void processElement(ProcessContext c) throws Exception {
+ @ProcessElement
+ public void processElement(ProcessContext c, BoundedWindow untypedWindow) throws Exception {
@SuppressWarnings("unchecked")
- W window = (W) c.window();
+ W window = (W) untypedWindow;
c.output(
KV.of(ismCoderForHash.hash(ImmutableList.of(window)),
KV.of(window,
WindowedValue.of(
c.element(),
c.timestamp(),
- c.window(),
+ window,
c.pane()))));
}
}
@@ -828,14 +827,14 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
extends PTransform<PCollection<T>, PCollectionView<T>> {
/**
- * A {@link OldDoFn} that outputs {@link IsmRecord}s. These records are structured as follows:
+ * A {@link DoFn} that outputs {@link IsmRecord}s. These records are structured as follows:
* <ul>
* <li>Key 1: Window
* <li>Value: Windowed value
* </ul>
*/
static class IsmRecordForSingularValuePerWindowDoFn<T, W extends BoundedWindow>
- extends OldDoFn<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>,
+ extends DoFn<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>,
IsmRecord<WindowedValue<T>>> {
private final Coder<W> windowCoder;
@@ -843,7 +842,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
this.windowCoder = windowCoder;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
Optional<Object> previousWindowStructuralValue = Optional.absent();
T previousValue = null;
@@ -902,7 +901,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
applyForSingleton(
DataflowRunner runner,
PCollection<T> input,
- OldDoFn<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>,
+ DoFn<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>,
IsmRecord<WindowedValue<FinalT>>> doFn,
boolean hasDefault,
FinalT defaultValue,
@@ -998,7 +997,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
static class BatchViewAsList<T>
extends PTransform<PCollection<T>, PCollectionView<List<T>>> {
/**
- * A {@link OldDoFn} which creates {@link IsmRecord}s assuming that each element is within the
+ * A {@link DoFn} which creates {@link IsmRecord}s assuming that each element is within the
* global window. Each {@link IsmRecord} has
* <ul>
* <li>Key 1: Global window</li>
@@ -1008,15 +1007,15 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
*/
@SystemDoFnInternal
static class ToIsmRecordForGlobalWindowDoFn<T>
- extends OldDoFn<T, IsmRecord<WindowedValue<T>>> {
+ extends DoFn<T, IsmRecord<WindowedValue<T>>> {
long indexInBundle;
- @Override
+ @StartBundle
public void startBundle(Context c) throws Exception {
indexInBundle = 0;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
c.output(IsmRecord.of(
ImmutableList.of(GlobalWindow.INSTANCE, indexInBundle),
@@ -1030,7 +1029,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
}
/**
- * A {@link OldDoFn} which creates {@link IsmRecord}s comparing successive elements windows
+ * A {@link DoFn} which creates {@link IsmRecord}s comparing successive elements windows
* to locate the window boundaries. The {@link IsmRecord} has:
* <ul>
* <li>Key 1: Window</li>
@@ -1040,7 +1039,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
*/
@SystemDoFnInternal
static class ToIsmRecordForNonGlobalWindowDoFn<T, W extends BoundedWindow>
- extends OldDoFn<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>,
+ extends DoFn<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>,
IsmRecord<WindowedValue<T>>> {
private final Coder<W> windowCoder;
@@ -1048,7 +1047,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
this.windowCoder = windowCoder;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
long elementsInWindow = 0;
Optional<Object> previousWindowStructuralValue = Optional.absent();
@@ -1174,7 +1173,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, V>>> {
/**
- * A {@link OldDoFn} which groups elements by window boundaries. For each group,
+ * A {@link DoFn} which groups elements by window boundaries. For each group,
* the group of elements is transformed into a {@link TransformedMap}.
* The transformed {@code Map<K, V>} is backed by a {@code Map<K, WindowedValue<V>>}
* and contains a function {@code WindowedValue<V> -> V}.
@@ -1188,7 +1187,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
* </ul>
*/
static class ToMapDoFn<K, V, W extends BoundedWindow>
- extends OldDoFn<KV<Integer, Iterable<KV<W, WindowedValue<KV<K, V>>>>>,
+ extends DoFn<KV<Integer, Iterable<KV<W, WindowedValue<KV<K, V>>>>>,
IsmRecord<WindowedValue<TransformedMap<K,
WindowedValue<V>,
V>>>> {
@@ -1198,7 +1197,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
this.windowCoder = windowCoder;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c)
throws Exception {
Optional<Object> previousWindowStructuralValue = Optional.absent();
@@ -1358,18 +1357,17 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
@SystemDoFnInternal
private static class GroupByKeyHashAndSortByKeyAndWindowDoFn<K, V, W>
- extends OldDoFn<KV<K, V>, KV<Integer, KV<KV<K, W>, WindowedValue<V>>>>
- implements OldDoFn.RequiresWindowAccess {
+ extends DoFn<KV<K, V>, KV<Integer, KV<KV<K, W>, WindowedValue<V>>>> {
private final IsmRecordCoder<?> coder;
private GroupByKeyHashAndSortByKeyAndWindowDoFn(IsmRecordCoder<?> coder) {
this.coder = coder;
}
- @Override
- public void processElement(ProcessContext c) throws Exception {
+ @ProcessElement
+ public void processElement(ProcessContext c, BoundedWindow untypedWindow) throws Exception {
@SuppressWarnings("unchecked")
- W window = (W) c.window();
+ W window = (W) untypedWindow;
c.output(
KV.of(coder.hash(ImmutableList.of(c.element().getKey())),
@@ -1377,7 +1375,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
WindowedValue.of(
c.element().getValue(),
c.timestamp(),
- (BoundedWindow) window,
+ untypedWindow,
c.pane()))));
}
}
@@ -1412,7 +1410,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
}
/**
- * A {@link OldDoFn} which creates {@link IsmRecord}s comparing successive elements windows
+ * A {@link DoFn} which creates {@link IsmRecord}s comparing successive elements windows
* and keys to locate window and key boundaries. The main output {@link IsmRecord}s have:
* <ul>
* <li>Key 1: Window</li>
@@ -1424,11 +1422,11 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
* <p>Additionally, we output all the unique keys per window seen to {@code outputForEntrySet}
* and the unique key count per window to {@code outputForSize}.
*
- * <p>Finally, if this OldDoFn has been requested to perform unique key checking, it will
+ * <p>Finally, if this {@link DoFn} has been requested to perform unique key checking, it will
* throw an {@link IllegalStateException} if more than one key per window is found.
*/
static class ToIsmRecordForMapLikeDoFn<K, V, W extends BoundedWindow>
- extends OldDoFn<KV<Integer, Iterable<KV<KV<K, W>, WindowedValue<V>>>>,
+ extends DoFn<KV<Integer, Iterable<KV<KV<K, W>, WindowedValue<V>>>>,
IsmRecord<WindowedValue<V>>> {
private final TupleTag<KV<Integer, KV<W, Long>>> outputForSize;
@@ -1452,7 +1450,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
this.uniqueKeysExpected = uniqueKeysExpected;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
long currentKeyIndex = 0;
// We use one based indexing while counting
@@ -1557,7 +1555,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
}
/**
- * A {@link OldDoFn} which outputs a metadata {@link IsmRecord} per window of:
+ * A {@link DoFn} which outputs a metadata {@link IsmRecord} per window of:
* <ul>
* <li>Key 1: META key</li>
* <li>Key 2: window</li>
@@ -1565,17 +1563,17 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
* <li>Value: sum of values for window</li>
* </ul>
*
- * <p>This {@link OldDoFn} is meant to be used to compute the number of unique keys
+ * <p>This {@link DoFn} is meant to be used to compute the number of unique keys
* per window for map and multimap side inputs.
*/
static class ToIsmMetadataRecordForSizeDoFn<K, V, W extends BoundedWindow>
- extends OldDoFn<KV<Integer, Iterable<KV<W, Long>>>, IsmRecord<WindowedValue<V>>> {
+ extends DoFn<KV<Integer, Iterable<KV<W, Long>>>, IsmRecord<WindowedValue<V>>> {
private final Coder<W> windowCoder;
ToIsmMetadataRecordForSizeDoFn(Coder<W> windowCoder) {
this.windowCoder = windowCoder;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
Iterator<KV<W, Long>> iterator = c.element().getValue().iterator();
KV<W, Long> currentValue = iterator.next();
@@ -1606,7 +1604,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
}
/**
- * A {@link OldDoFn} which outputs a metadata {@link IsmRecord} per window and key pair of:
+ * A {@link DoFn} which outputs a metadata {@link IsmRecord} per window and key pair of:
* <ul>
* <li>Key 1: META key</li>
* <li>Key 2: window</li>
@@ -1614,11 +1612,11 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
* <li>Value: key</li>
* </ul>
*
- * <p>This {@link OldDoFn} is meant to be used to output index to key records
+ * <p>This {@link DoFn} is meant to be used to output index to key records
* per window for map and multimap side inputs.
*/
static class ToIsmMetadataRecordForKeyDoFn<K, V, W extends BoundedWindow>
- extends OldDoFn<KV<Integer, Iterable<KV<W, K>>>, IsmRecord<WindowedValue<V>>> {
+ extends DoFn<KV<Integer, Iterable<KV<W, K>>>, IsmRecord<WindowedValue<V>>> {
private final Coder<K> keyCoder;
private final Coder<W> windowCoder;
@@ -1627,7 +1625,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
this.windowCoder = windowCoder;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
Iterator<KV<W, K>> iterator = c.element().getValue().iterator();
KV<W, K> currentValue = iterator.next();
@@ -1658,7 +1656,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
}
/**
- * A {@link OldDoFn} which partitions sets of elements by window boundaries. Within each
+ * A {@link DoFn} which partitions sets of elements by window boundaries. Within each
* partition, the set of elements is transformed into a {@link TransformedMap}.
* The transformed {@code Map<K, Iterable<V>>} is backed by a
* {@code Map<K, Iterable<WindowedValue<V>>>} and contains a function
@@ -1673,7 +1671,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
* </ul>
*/
static class ToMultimapDoFn<K, V, W extends BoundedWindow>
- extends OldDoFn<KV<Integer, Iterable<KV<W, WindowedValue<KV<K, V>>>>>,
+ extends DoFn<KV<Integer, Iterable<KV<W, WindowedValue<KV<K, V>>>>>,
IsmRecord<WindowedValue<TransformedMap<K,
Iterable<WindowedValue<V>>,
Iterable<V>>>>> {
@@ -1683,7 +1681,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
this.windowCoder = windowCoder;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c)
throws Exception {
Optional<Object> previousWindowStructuralValue = Optional.absent();
@@ -2335,8 +2333,8 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
// WindmillSink.
.apply(Reshuffle.<Integer, ValueWithRecordId<T>>of())
.apply("StripIds", ParDo.of(
- new OldDoFn<KV<Integer, ValueWithRecordId<T>>, T>() {
- @Override
+ new DoFn<KV<Integer, ValueWithRecordId<T>>, T>() {
+ @ProcessElement
public void processElement(ProcessContext c) {
c.output(c.element().getValue().getValue());
}
@@ -2372,7 +2370,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
}
/**
- * A specialized {@link OldDoFn} for writing the contents of a {@link PCollection}
+ * A specialized {@link DoFn} for writing the contents of a {@link PCollection}
* to a streaming {@link PCollectionView} backend implementation.
*/
private static class StreamingPCollectionViewWriterFn<T>
@@ -2553,8 +2551,8 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
}
}
- private static class WrapAsList<T> extends OldDoFn<T, List<T>> {
- @Override
+ private static class WrapAsList<T> extends DoFn<T, List<T>> {
+ @ProcessElement
public void processElement(ProcessContext c) {
c.output(Arrays.asList(c.element()));
}
[44/50] incubator-beam git commit: This closes #811
Posted by ke...@apache.org.
This closes #811
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2a1055dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2a1055dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2a1055dd
Branch: refs/heads/gearpump-runner
Commit: 2a1055dd8e669ee9181bda3d944e377f7e33b5ea
Parents: 6da92ad 65f9076
Author: bchambers <bc...@google.com>
Authored: Wed Aug 10 15:54:36 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Wed Aug 10 15:54:36 2016 -0700
----------------------------------------------------------------------
.../apache/beam/runners/direct/DirectRunnerTest.java | 15 +++++++++++++++
1 file changed, 15 insertions(+)
----------------------------------------------------------------------
[38/50] incubator-beam git commit: Move aggregator support classes
out of runners namespace, make private
Posted by ke...@apache.org.
Move aggregator support classes out of runners namespace, make private
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/adec254d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/adec254d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/adec254d
Branch: refs/heads/gearpump-runner
Commit: adec254d5fdb409e786a1fc2bcee38f8a7a04408
Parents: 9da4bbc
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Jul 1 14:56:20 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Aug 10 11:34:03 2016 -0700
----------------------------------------------------------------------
.../beam/runners/direct/DirectRunner.java | 7 +-
.../beam/runners/flink/FlinkRunnerResult.java | 4 +-
.../runners/dataflow/DataflowPipelineJob.java | 4 +-
.../beam/runners/dataflow/DataflowRunner.java | 4 +-
.../dataflow/DataflowPipelineJobTest.java | 4 +-
.../spark/translation/EvaluationContext.java | 4 +-
.../spark/translation/SparkRuntimeContext.java | 2 +-
.../translation/MultiOutputWordCountTest.java | 2 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 93 ++++++++
.../beam/sdk/AggregatorRetrievalException.java | 33 +++
.../org/apache/beam/sdk/AggregatorValues.java | 52 +++++
.../main/java/org/apache/beam/sdk/Pipeline.java | 10 +
.../org/apache/beam/sdk/PipelineResult.java | 2 -
.../runners/AggregatorPipelineExtractor.java | 93 --------
.../runners/AggregatorRetrievalException.java | 33 ---
.../beam/sdk/runners/AggregatorValues.java | 52 -----
.../sdk/AggregatorPipelineExtractorTest.java | 229 +++++++++++++++++++
.../AggregatorPipelineExtractorTest.java | 229 -------------------
.../apache/beam/sdk/transforms/DoFnTest.java | 1 +
.../apache/beam/sdk/transforms/OldDoFnTest.java | 3 +-
20 files changed, 434 insertions(+), 427 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index a9c8ecb..f2b781e 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -20,15 +20,14 @@ package org.apache.beam.runners.direct;
import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
import org.apache.beam.runners.direct.DirectRunner.DirectPipelineResult;
import org.apache.beam.runners.direct.ViewEvaluatorFactory.ViewOverrideFactory;
+import org.apache.beam.sdk.AggregatorRetrievalException;
+import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.io.Write;
import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.AggregatorPipelineExtractor;
-import org.apache.beam.sdk.runners.AggregatorRetrievalException;
-import org.apache.beam.sdk.runners.AggregatorValues;
import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.AppliedPTransform;
@@ -244,7 +243,7 @@ public class DirectRunner
executor.start(consumerTrackingVisitor.getRootTransforms());
Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
- new AggregatorPipelineExtractor(pipeline).getAggregatorSteps();
+ pipeline.getAggregatorSteps();
DirectPipelineResult result =
new DirectPipelineResult(executor, context, aggregatorSteps);
if (options.isBlockOnRun()) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
index cae0b2a..923d54c 100644
--- a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
+++ b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkRunnerResult.java
@@ -18,8 +18,8 @@
package org.apache.beam.runners.flink;
import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.runners.AggregatorRetrievalException;
-import org.apache.beam.sdk.runners.AggregatorValues;
+import org.apache.beam.sdk.AggregatorRetrievalException;
+import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.transforms.Aggregator;
import org.joda.time.Duration;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
index a6baa4f..e043e23 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
@@ -23,9 +23,9 @@ import org.apache.beam.runners.dataflow.internal.DataflowAggregatorTransforms;
import org.apache.beam.runners.dataflow.internal.DataflowMetricUpdateExtractor;
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.runners.dataflow.util.MonitoringUtil;
+import org.apache.beam.sdk.AggregatorRetrievalException;
+import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.runners.AggregatorRetrievalException;
-import org.apache.beam.sdk.runners.AggregatorValues;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.util.AttemptAndTimeBoundedExponentialBackOff;
import org.apache.beam.sdk.util.AttemptBoundedExponentialBackOff;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index fadd9c7..3b68e92 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -71,7 +71,6 @@ import org.apache.beam.sdk.io.Write;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
import org.apache.beam.sdk.options.StreamingOptions;
-import org.apache.beam.sdk.runners.AggregatorPipelineExtractor;
import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.runners.TransformTreeNode;
import org.apache.beam.sdk.transforms.Aggregator;
@@ -596,9 +595,8 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
// Obtain all of the extractors from the PTransforms used in the pipeline so the
// DataflowPipelineJob has access to them.
- AggregatorPipelineExtractor aggregatorExtractor = new AggregatorPipelineExtractor(pipeline);
Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
- aggregatorExtractor.getAggregatorSteps();
+ pipeline.getAggregatorSteps();
DataflowAggregatorTransforms aggregatorTransforms =
new DataflowAggregatorTransforms(aggregatorSteps, jobSpecification.getStepNames());
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
index 343d538..e6277d9 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineJobTest.java
@@ -35,10 +35,10 @@ import static org.mockito.Mockito.when;
import org.apache.beam.runners.dataflow.internal.DataflowAggregatorTransforms;
import org.apache.beam.runners.dataflow.testing.TestDataflowPipelineOptions;
import org.apache.beam.runners.dataflow.util.MonitoringUtil;
+import org.apache.beam.sdk.AggregatorRetrievalException;
+import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.PipelineResult.State;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.runners.AggregatorRetrievalException;
-import org.apache.beam.sdk.runners.AggregatorValues;
import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.AppliedPTransform;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 169c2af..4ccac0e 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -22,10 +22,10 @@ import static com.google.common.base.Preconditions.checkArgument;
import org.apache.beam.runners.spark.EvaluationResult;
import org.apache.beam.runners.spark.coders.CoderHelpers;
+import org.apache.beam.sdk.AggregatorRetrievalException;
+import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.runners.AggregatorRetrievalException;
-import org.apache.beam.sdk.runners.AggregatorValues;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
index 46f5b33..c2edd02 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
@@ -20,12 +20,12 @@ package org.apache.beam.runners.spark.translation;
import org.apache.beam.runners.spark.aggregators.AggAccumParam;
import org.apache.beam.runners.spark.aggregators.NamedAggregators;
+import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.AggregatorValues;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.Max;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
index 291f7b2..0d0c0b4 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
@@ -20,11 +20,11 @@ package org.apache.beam.runners.spark.translation;
import org.apache.beam.runners.spark.EvaluationResult;
import org.apache.beam.runners.spark.SparkRunner;
+import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.runners.AggregatorValues;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.ApproximateUnique;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorPipelineExtractor.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorPipelineExtractor.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorPipelineExtractor.java
new file mode 100644
index 0000000..ac215c9
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorPipelineExtractor.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk;
+
+import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.runners.TransformTreeNode;
+import org.apache.beam.sdk.transforms.Aggregator;
+import org.apache.beam.sdk.transforms.AggregatorRetriever;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PValue;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.SetMultimap;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Retrieves {@link Aggregator Aggregators} at each {@link ParDo} and returns a {@link Map} of
+ * {@link Aggregator} to the {@link PTransform PTransforms} in which it is present.
+ */
+class AggregatorPipelineExtractor {
+ private final Pipeline pipeline;
+
+ /**
+ * Creates an {@code AggregatorPipelineExtractor} for the given {@link Pipeline}.
+ */
+ public AggregatorPipelineExtractor(Pipeline pipeline) {
+ this.pipeline = pipeline;
+ }
+
+ /**
+ * Returns a {@link Map} between each {@link Aggregator} in the {@link Pipeline} to the {@link
+ * PTransform PTransforms} in which it is used.
+ */
+ public Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> getAggregatorSteps() {
+ HashMultimap<Aggregator<?, ?>, PTransform<?, ?>> aggregatorSteps = HashMultimap.create();
+ pipeline.traverseTopologically(new AggregatorVisitor(aggregatorSteps));
+ return aggregatorSteps.asMap();
+ }
+
+ private static class AggregatorVisitor extends PipelineVisitor.Defaults {
+ private final SetMultimap<Aggregator<?, ?>, PTransform<?, ?>> aggregatorSteps;
+
+ public AggregatorVisitor(SetMultimap<Aggregator<?, ?>, PTransform<?, ?>> aggregatorSteps) {
+ this.aggregatorSteps = aggregatorSteps;
+ }
+
+ @Override
+ public void visitPrimitiveTransform(TransformTreeNode node) {
+ PTransform<?, ?> transform = node.getTransform();
+ addStepToAggregators(transform, getAggregators(transform));
+ }
+
+ private Collection<Aggregator<?, ?>> getAggregators(PTransform<?, ?> transform) {
+ if (transform != null) {
+ if (transform instanceof ParDo.Bound) {
+ return AggregatorRetriever.getAggregators(((ParDo.Bound<?, ?>) transform).getFn());
+ } else if (transform instanceof ParDo.BoundMulti) {
+ return AggregatorRetriever.getAggregators(((ParDo.BoundMulti<?, ?>) transform).getFn());
+ }
+ }
+ return Collections.emptyList();
+ }
+
+ private void addStepToAggregators(
+ PTransform<?, ?> transform, Collection<Aggregator<?, ?>> aggregators) {
+ for (Aggregator<?, ?> aggregator : aggregators) {
+ aggregatorSteps.put(aggregator, transform);
+ }
+ }
+
+ @Override
+ public void visitValue(PValue value, TransformTreeNode producer) {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorRetrievalException.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorRetrievalException.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorRetrievalException.java
new file mode 100644
index 0000000..3040815
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorRetrievalException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk;
+
+import org.apache.beam.sdk.transforms.Aggregator;
+
+/**
+ * Signals that an exception has occurred while retrieving {@link Aggregator}s.
+ */
+public class AggregatorRetrievalException extends Exception {
+ /**
+ * Constructs a new {@code AggregatorRetrievalException} with the specified detail message and
+ * cause.
+ */
+ public AggregatorRetrievalException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorValues.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorValues.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorValues.java
new file mode 100644
index 0000000..efaad85
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorValues.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk;
+
+import org.apache.beam.sdk.transforms.Aggregator;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.OldDoFn;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * A collection of values associated with an {@link Aggregator}. Aggregators declared in a
+ * {@link OldDoFn} are emitted on a per-{@code OldDoFn}-application basis.
+ *
+ * @param <T> the output type of the aggregator
+ */
+public abstract class AggregatorValues<T> {
+ /**
+ * Get the values of the {@link Aggregator} at all steps it was used.
+ */
+ public Collection<T> getValues() {
+ return getValuesAtSteps().values();
+ }
+
+ /**
+ * Get the values of the {@link Aggregator} by the user name at each step it was used.
+ */
+ public abstract Map<String, T> getValuesAtSteps();
+
+ /**
+ * Get the total value of this {@link Aggregator} by applying the specified {@link CombineFn}.
+ */
+ public T getTotalValue(CombineFn<T, ?, T> combineFn) {
+ return combineFn.apply(getValues());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index e4f3e4a..1bbc56f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -26,6 +26,7 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.runners.TransformTreeNode;
+import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.PTransform;
@@ -47,6 +48,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -518,6 +520,14 @@ public class Pipeline {
}
/**
+ * Returns a {@link Map} from each {@link Aggregator} in the {@link Pipeline} to the {@link
+ * PTransform PTransforms} in which it is used.
+ */
+ public Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> getAggregatorSteps() {
+ return new AggregatorPipelineExtractor(this).getAggregatorSteps();
+ }
+
+ /**
* Builds a name from a "/"-delimited prefix and a name.
*/
private String buildName(String namePrefix, String name) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java
index 993962c..edfc924 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/PipelineResult.java
@@ -17,8 +17,6 @@
*/
package org.apache.beam.sdk;
-import org.apache.beam.sdk.runners.AggregatorRetrievalException;
-import org.apache.beam.sdk.runners.AggregatorValues;
import org.apache.beam.sdk.transforms.Aggregator;
import org.joda.time.Duration;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorPipelineExtractor.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorPipelineExtractor.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorPipelineExtractor.java
deleted file mode 100644
index 146ddfa..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorPipelineExtractor.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.runners;
-
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.Pipeline.PipelineVisitor;
-import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.beam.sdk.transforms.AggregatorRetriever;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.values.PValue;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.SetMultimap;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * Retrieves {@link Aggregator Aggregators} at each {@link ParDo} and returns a {@link Map} of
- * {@link Aggregator} to the {@link PTransform PTransforms} in which it is present.
- */
-public class AggregatorPipelineExtractor {
- private final Pipeline pipeline;
-
- /**
- * Creates an {@code AggregatorPipelineExtractor} for the given {@link Pipeline}.
- */
- public AggregatorPipelineExtractor(Pipeline pipeline) {
- this.pipeline = pipeline;
- }
-
- /**
- * Returns a {@link Map} between each {@link Aggregator} in the {@link Pipeline} to the {@link
- * PTransform PTransforms} in which it is used.
- */
- public Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> getAggregatorSteps() {
- HashMultimap<Aggregator<?, ?>, PTransform<?, ?>> aggregatorSteps = HashMultimap.create();
- pipeline.traverseTopologically(new AggregatorVisitor(aggregatorSteps));
- return aggregatorSteps.asMap();
- }
-
- private static class AggregatorVisitor extends PipelineVisitor.Defaults {
- private final SetMultimap<Aggregator<?, ?>, PTransform<?, ?>> aggregatorSteps;
-
- public AggregatorVisitor(SetMultimap<Aggregator<?, ?>, PTransform<?, ?>> aggregatorSteps) {
- this.aggregatorSteps = aggregatorSteps;
- }
-
- @Override
- public void visitPrimitiveTransform(TransformTreeNode node) {
- PTransform<?, ?> transform = node.getTransform();
- addStepToAggregators(transform, getAggregators(transform));
- }
-
- private Collection<Aggregator<?, ?>> getAggregators(PTransform<?, ?> transform) {
- if (transform != null) {
- if (transform instanceof ParDo.Bound) {
- return AggregatorRetriever.getAggregators(((ParDo.Bound<?, ?>) transform).getFn());
- } else if (transform instanceof ParDo.BoundMulti) {
- return AggregatorRetriever.getAggregators(((ParDo.BoundMulti<?, ?>) transform).getFn());
- }
- }
- return Collections.emptyList();
- }
-
- private void addStepToAggregators(
- PTransform<?, ?> transform, Collection<Aggregator<?, ?>> aggregators) {
- for (Aggregator<?, ?> aggregator : aggregators) {
- aggregatorSteps.put(aggregator, transform);
- }
- }
-
- @Override
- public void visitValue(PValue value, TransformTreeNode producer) {}
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorRetrievalException.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorRetrievalException.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorRetrievalException.java
deleted file mode 100644
index a0973c3..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorRetrievalException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.runners;
-
-import org.apache.beam.sdk.transforms.Aggregator;
-
-/**
- * Signals that an exception has occurred while retrieving {@link Aggregator}s.
- */
-public class AggregatorRetrievalException extends Exception {
- /**
- * Constructs a new {@code AggregatorRetrievalException} with the specified detail message and
- * cause.
- */
- public AggregatorRetrievalException(String message, Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorValues.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorValues.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorValues.java
deleted file mode 100644
index 6f6836e..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/AggregatorValues.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.runners;
-
-import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.beam.sdk.transforms.Combine.CombineFn;
-import org.apache.beam.sdk.transforms.OldDoFn;
-
-import java.util.Collection;
-import java.util.Map;
-
-/**
- * A collection of values associated with an {@link Aggregator}. Aggregators declared in a
- * {@link OldDoFn} are emitted on a per-{@code OldDoFn}-application basis.
- *
- * @param <T> the output type of the aggregator
- */
-public abstract class AggregatorValues<T> {
- /**
- * Get the values of the {@link Aggregator} at all steps it was used.
- */
- public Collection<T> getValues() {
- return getValuesAtSteps().values();
- }
-
- /**
- * Get the values of the {@link Aggregator} by the user name at each step it was used.
- */
- public abstract Map<String, T> getValuesAtSteps();
-
- /**
- * Get the total value of this {@link Aggregator} by applying the specified {@link CombineFn}.
- */
- public T getTotalValue(CombineFn<T, ?, T> combineFn) {
- return combineFn.apply(getValues());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/test/java/org/apache/beam/sdk/AggregatorPipelineExtractorTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/AggregatorPipelineExtractorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/AggregatorPipelineExtractorTest.java
new file mode 100644
index 0000000..930fbe7
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/AggregatorPipelineExtractorTest.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.sdk.Pipeline.PipelineVisitor;
+import org.apache.beam.sdk.runners.TransformTreeNode;
+import org.apache.beam.sdk.transforms.Aggregator;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.Max;
+import org.apache.beam.sdk.transforms.Min;
+import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Sum;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Tests for {@link AggregatorPipelineExtractor}.
+ */
+@RunWith(JUnit4.class)
+public class AggregatorPipelineExtractorTest {
+ @Mock
+ private Pipeline p;
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testGetAggregatorStepsWithParDoBoundExtractsSteps() {
+ @SuppressWarnings("rawtypes")
+ ParDo.Bound bound = mock(ParDo.Bound.class, "Bound");
+ AggregatorProvidingDoFn<ThreadGroup, StrictMath> fn = new AggregatorProvidingDoFn<>();
+ when(bound.getFn()).thenReturn(fn);
+
+ Aggregator<Long, Long> aggregatorOne = fn.addAggregator(new Sum.SumLongFn());
+ Aggregator<Integer, Integer> aggregatorTwo = fn.addAggregator(new Min.MinIntegerFn());
+
+ TransformTreeNode transformNode = mock(TransformTreeNode.class);
+ when(transformNode.getTransform()).thenReturn(bound);
+
+ doAnswer(new VisitNodesAnswer(ImmutableList.of(transformNode)))
+ .when(p)
+ .traverseTopologically(Mockito.any(PipelineVisitor.class));
+
+ AggregatorPipelineExtractor extractor = new AggregatorPipelineExtractor(p);
+
+ Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
+ extractor.getAggregatorSteps();
+
+ assertEquals(ImmutableSet.<PTransform<?, ?>>of(bound), aggregatorSteps.get(aggregatorOne));
+ assertEquals(ImmutableSet.<PTransform<?, ?>>of(bound), aggregatorSteps.get(aggregatorTwo));
+ assertEquals(aggregatorSteps.size(), 2);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testGetAggregatorStepsWithParDoBoundMultiExtractsSteps() {
+ @SuppressWarnings("rawtypes")
+ ParDo.BoundMulti bound = mock(ParDo.BoundMulti.class, "BoundMulti");
+ AggregatorProvidingDoFn<Object, Void> fn = new AggregatorProvidingDoFn<>();
+ when(bound.getFn()).thenReturn(fn);
+
+ Aggregator<Long, Long> aggregatorOne = fn.addAggregator(new Max.MaxLongFn());
+ Aggregator<Double, Double> aggregatorTwo = fn.addAggregator(new Min.MinDoubleFn());
+
+ TransformTreeNode transformNode = mock(TransformTreeNode.class);
+ when(transformNode.getTransform()).thenReturn(bound);
+
+ doAnswer(new VisitNodesAnswer(ImmutableList.of(transformNode)))
+ .when(p)
+ .traverseTopologically(Mockito.any(PipelineVisitor.class));
+
+ AggregatorPipelineExtractor extractor = new AggregatorPipelineExtractor(p);
+
+ Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
+ extractor.getAggregatorSteps();
+
+ assertEquals(ImmutableSet.<PTransform<?, ?>>of(bound), aggregatorSteps.get(aggregatorOne));
+ assertEquals(ImmutableSet.<PTransform<?, ?>>of(bound), aggregatorSteps.get(aggregatorTwo));
+ assertEquals(2, aggregatorSteps.size());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testGetAggregatorStepsWithOneAggregatorInMultipleStepsAddsSteps() {
+ @SuppressWarnings("rawtypes")
+ ParDo.Bound bound = mock(ParDo.Bound.class, "Bound");
+ @SuppressWarnings("rawtypes")
+ ParDo.BoundMulti otherBound = mock(ParDo.BoundMulti.class, "otherBound");
+ AggregatorProvidingDoFn<String, Math> fn = new AggregatorProvidingDoFn<>();
+ when(bound.getFn()).thenReturn(fn);
+ when(otherBound.getFn()).thenReturn(fn);
+
+ Aggregator<Long, Long> aggregatorOne = fn.addAggregator(new Sum.SumLongFn());
+ Aggregator<Double, Double> aggregatorTwo = fn.addAggregator(new Min.MinDoubleFn());
+
+ TransformTreeNode transformNode = mock(TransformTreeNode.class);
+ when(transformNode.getTransform()).thenReturn(bound);
+ TransformTreeNode otherTransformNode = mock(TransformTreeNode.class);
+ when(otherTransformNode.getTransform()).thenReturn(otherBound);
+
+ doAnswer(new VisitNodesAnswer(ImmutableList.of(transformNode, otherTransformNode)))
+ .when(p)
+ .traverseTopologically(Mockito.any(PipelineVisitor.class));
+
+ AggregatorPipelineExtractor extractor = new AggregatorPipelineExtractor(p);
+
+ Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
+ extractor.getAggregatorSteps();
+
+ assertEquals(
+ ImmutableSet.<PTransform<?, ?>>of(bound, otherBound), aggregatorSteps.get(aggregatorOne));
+ assertEquals(
+ ImmutableSet.<PTransform<?, ?>>of(bound, otherBound), aggregatorSteps.get(aggregatorTwo));
+ assertEquals(2, aggregatorSteps.size());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testGetAggregatorStepsWithDifferentStepsAddsSteps() {
+ @SuppressWarnings("rawtypes")
+ ParDo.Bound bound = mock(ParDo.Bound.class, "Bound");
+
+ AggregatorProvidingDoFn<ThreadGroup, Void> fn = new AggregatorProvidingDoFn<>();
+ Aggregator<Long, Long> aggregatorOne = fn.addAggregator(new Sum.SumLongFn());
+
+ when(bound.getFn()).thenReturn(fn);
+
+ @SuppressWarnings("rawtypes")
+ ParDo.BoundMulti otherBound = mock(ParDo.BoundMulti.class, "otherBound");
+
+ AggregatorProvidingDoFn<Long, Long> otherFn = new AggregatorProvidingDoFn<>();
+ Aggregator<Double, Double> aggregatorTwo = otherFn.addAggregator(new Sum.SumDoubleFn());
+
+ when(otherBound.getFn()).thenReturn(otherFn);
+
+ TransformTreeNode transformNode = mock(TransformTreeNode.class);
+ when(transformNode.getTransform()).thenReturn(bound);
+ TransformTreeNode otherTransformNode = mock(TransformTreeNode.class);
+ when(otherTransformNode.getTransform()).thenReturn(otherBound);
+
+ doAnswer(new VisitNodesAnswer(ImmutableList.of(transformNode, otherTransformNode)))
+ .when(p)
+ .traverseTopologically(Mockito.any(PipelineVisitor.class));
+
+ AggregatorPipelineExtractor extractor = new AggregatorPipelineExtractor(p);
+
+ Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
+ extractor.getAggregatorSteps();
+
+ assertEquals(ImmutableSet.<PTransform<?, ?>>of(bound), aggregatorSteps.get(aggregatorOne));
+ assertEquals(ImmutableSet.<PTransform<?, ?>>of(otherBound), aggregatorSteps.get(aggregatorTwo));
+ assertEquals(2, aggregatorSteps.size());
+ }
+
+ private static class VisitNodesAnswer implements Answer<Object> {
+ private final List<TransformTreeNode> nodes;
+
+ public VisitNodesAnswer(List<TransformTreeNode> nodes) {
+ this.nodes = nodes;
+ }
+
+ @Override
+ public Object answer(InvocationOnMock invocation) throws Throwable {
+ PipelineVisitor visitor = (PipelineVisitor) invocation.getArguments()[0];
+ for (TransformTreeNode node : nodes) {
+ visitor.visitPrimitiveTransform(node);
+ }
+ return null;
+ }
+ }
+
+ private static class AggregatorProvidingDoFn<InT, OuT> extends OldDoFn<InT, OuT> {
+ public <InputT, OutT> Aggregator<InputT, OutT> addAggregator(
+ CombineFn<InputT, ?, OutT> combiner) {
+ return createAggregator(randomName(), combiner);
+ }
+
+ private String randomName() {
+ return UUID.randomUUID().toString();
+ }
+
+ @Override
+ public void processElement(OldDoFn<InT, OuT>.ProcessContext c) throws Exception {
+ fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/AggregatorPipelineExtractorTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/AggregatorPipelineExtractorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/AggregatorPipelineExtractorTest.java
deleted file mode 100644
index 13476e2..0000000
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/AggregatorPipelineExtractorTest.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.runners;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.Pipeline.PipelineVisitor;
-import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.beam.sdk.transforms.Combine.CombineFn;
-import org.apache.beam.sdk.transforms.Max;
-import org.apache.beam.sdk.transforms.Min;
-import org.apache.beam.sdk.transforms.OldDoFn;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.Sum;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.MockitoAnnotations;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-/**
- * Tests for {@link AggregatorPipelineExtractor}.
- */
-@RunWith(JUnit4.class)
-public class AggregatorPipelineExtractorTest {
- @Mock
- private Pipeline p;
-
- @Before
- public void setup() {
- MockitoAnnotations.initMocks(this);
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testGetAggregatorStepsWithParDoBoundExtractsSteps() {
- @SuppressWarnings("rawtypes")
- ParDo.Bound bound = mock(ParDo.Bound.class, "Bound");
- AggregatorProvidingDoFn<ThreadGroup, StrictMath> fn = new AggregatorProvidingDoFn<>();
- when(bound.getFn()).thenReturn(fn);
-
- Aggregator<Long, Long> aggregatorOne = fn.addAggregator(new Sum.SumLongFn());
- Aggregator<Integer, Integer> aggregatorTwo = fn.addAggregator(new Min.MinIntegerFn());
-
- TransformTreeNode transformNode = mock(TransformTreeNode.class);
- when(transformNode.getTransform()).thenReturn(bound);
-
- doAnswer(new VisitNodesAnswer(ImmutableList.of(transformNode)))
- .when(p)
- .traverseTopologically(Mockito.any(PipelineVisitor.class));
-
- AggregatorPipelineExtractor extractor = new AggregatorPipelineExtractor(p);
-
- Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
- extractor.getAggregatorSteps();
-
- assertEquals(ImmutableSet.<PTransform<?, ?>>of(bound), aggregatorSteps.get(aggregatorOne));
- assertEquals(ImmutableSet.<PTransform<?, ?>>of(bound), aggregatorSteps.get(aggregatorTwo));
- assertEquals(aggregatorSteps.size(), 2);
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testGetAggregatorStepsWithParDoBoundMultiExtractsSteps() {
- @SuppressWarnings("rawtypes")
- ParDo.BoundMulti bound = mock(ParDo.BoundMulti.class, "BoundMulti");
- AggregatorProvidingDoFn<Object, Void> fn = new AggregatorProvidingDoFn<>();
- when(bound.getFn()).thenReturn(fn);
-
- Aggregator<Long, Long> aggregatorOne = fn.addAggregator(new Max.MaxLongFn());
- Aggregator<Double, Double> aggregatorTwo = fn.addAggregator(new Min.MinDoubleFn());
-
- TransformTreeNode transformNode = mock(TransformTreeNode.class);
- when(transformNode.getTransform()).thenReturn(bound);
-
- doAnswer(new VisitNodesAnswer(ImmutableList.of(transformNode)))
- .when(p)
- .traverseTopologically(Mockito.any(PipelineVisitor.class));
-
- AggregatorPipelineExtractor extractor = new AggregatorPipelineExtractor(p);
-
- Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
- extractor.getAggregatorSteps();
-
- assertEquals(ImmutableSet.<PTransform<?, ?>>of(bound), aggregatorSteps.get(aggregatorOne));
- assertEquals(ImmutableSet.<PTransform<?, ?>>of(bound), aggregatorSteps.get(aggregatorTwo));
- assertEquals(2, aggregatorSteps.size());
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testGetAggregatorStepsWithOneAggregatorInMultipleStepsAddsSteps() {
- @SuppressWarnings("rawtypes")
- ParDo.Bound bound = mock(ParDo.Bound.class, "Bound");
- @SuppressWarnings("rawtypes")
- ParDo.BoundMulti otherBound = mock(ParDo.BoundMulti.class, "otherBound");
- AggregatorProvidingDoFn<String, Math> fn = new AggregatorProvidingDoFn<>();
- when(bound.getFn()).thenReturn(fn);
- when(otherBound.getFn()).thenReturn(fn);
-
- Aggregator<Long, Long> aggregatorOne = fn.addAggregator(new Sum.SumLongFn());
- Aggregator<Double, Double> aggregatorTwo = fn.addAggregator(new Min.MinDoubleFn());
-
- TransformTreeNode transformNode = mock(TransformTreeNode.class);
- when(transformNode.getTransform()).thenReturn(bound);
- TransformTreeNode otherTransformNode = mock(TransformTreeNode.class);
- when(otherTransformNode.getTransform()).thenReturn(otherBound);
-
- doAnswer(new VisitNodesAnswer(ImmutableList.of(transformNode, otherTransformNode)))
- .when(p)
- .traverseTopologically(Mockito.any(PipelineVisitor.class));
-
- AggregatorPipelineExtractor extractor = new AggregatorPipelineExtractor(p);
-
- Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
- extractor.getAggregatorSteps();
-
- assertEquals(
- ImmutableSet.<PTransform<?, ?>>of(bound, otherBound), aggregatorSteps.get(aggregatorOne));
- assertEquals(
- ImmutableSet.<PTransform<?, ?>>of(bound, otherBound), aggregatorSteps.get(aggregatorTwo));
- assertEquals(2, aggregatorSteps.size());
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testGetAggregatorStepsWithDifferentStepsAddsSteps() {
- @SuppressWarnings("rawtypes")
- ParDo.Bound bound = mock(ParDo.Bound.class, "Bound");
-
- AggregatorProvidingDoFn<ThreadGroup, Void> fn = new AggregatorProvidingDoFn<>();
- Aggregator<Long, Long> aggregatorOne = fn.addAggregator(new Sum.SumLongFn());
-
- when(bound.getFn()).thenReturn(fn);
-
- @SuppressWarnings("rawtypes")
- ParDo.BoundMulti otherBound = mock(ParDo.BoundMulti.class, "otherBound");
-
- AggregatorProvidingDoFn<Long, Long> otherFn = new AggregatorProvidingDoFn<>();
- Aggregator<Double, Double> aggregatorTwo = otherFn.addAggregator(new Sum.SumDoubleFn());
-
- when(otherBound.getFn()).thenReturn(otherFn);
-
- TransformTreeNode transformNode = mock(TransformTreeNode.class);
- when(transformNode.getTransform()).thenReturn(bound);
- TransformTreeNode otherTransformNode = mock(TransformTreeNode.class);
- when(otherTransformNode.getTransform()).thenReturn(otherBound);
-
- doAnswer(new VisitNodesAnswer(ImmutableList.of(transformNode, otherTransformNode)))
- .when(p)
- .traverseTopologically(Mockito.any(PipelineVisitor.class));
-
- AggregatorPipelineExtractor extractor = new AggregatorPipelineExtractor(p);
-
- Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
- extractor.getAggregatorSteps();
-
- assertEquals(ImmutableSet.<PTransform<?, ?>>of(bound), aggregatorSteps.get(aggregatorOne));
- assertEquals(ImmutableSet.<PTransform<?, ?>>of(otherBound), aggregatorSteps.get(aggregatorTwo));
- assertEquals(2, aggregatorSteps.size());
- }
-
- private static class VisitNodesAnswer implements Answer<Object> {
- private final List<TransformTreeNode> nodes;
-
- public VisitNodesAnswer(List<TransformTreeNode> nodes) {
- this.nodes = nodes;
- }
-
- @Override
- public Object answer(InvocationOnMock invocation) throws Throwable {
- PipelineVisitor visitor = (PipelineVisitor) invocation.getArguments()[0];
- for (TransformTreeNode node : nodes) {
- visitor.visitPrimitiveTransform(node);
- }
- return null;
- }
- }
-
- private static class AggregatorProvidingDoFn<InT, OuT> extends OldDoFn<InT, OuT> {
- public <InputT, OutT> Aggregator<InputT, OutT> addAggregator(
- CombineFn<InputT, ?, OutT> combiner) {
- return createAggregator(randomName(), combiner);
- }
-
- private String randomName() {
- return UUID.randomUUID().toString();
- }
-
- @Override
- public void processElement(OldDoFn<InT, OuT>.ProcessContext c) throws Exception {
- fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTest.java
index 710e4ce..3fb3193 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTest.java
@@ -128,6 +128,7 @@ public class DoFnTest implements Serializable {
DoFn<Void, Void> doFn = new NoOpDoFn();
Aggregator<Double, Double> aggregatorOne =
+
doFn.createAggregator(nameOne, combiner);
Aggregator<Double, Double> aggregatorTwo =
doFn.createAggregator(nameTwo, combiner);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/adec254d/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/OldDoFnTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/OldDoFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/OldDoFnTest.java
index 9d144b3..5946d9a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/OldDoFnTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/OldDoFnTest.java
@@ -24,10 +24,10 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertThat;
+import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.runners.AggregatorValues;
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Combine.CombineFn;
@@ -36,6 +36,7 @@ import org.apache.beam.sdk.transforms.Sum.SumIntegerFn;
import org.apache.beam.sdk.transforms.display.DisplayData;
import com.google.common.collect.ImmutableMap;
+
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
[16/50] incubator-beam git commit: Lower maven central timeouts on
Travis-CI
Posted by ke...@apache.org.
Lower maven central timeouts on Travis-CI
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/71366165
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/71366165
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/71366165
Branch: refs/heads/gearpump-runner
Commit: 713661651befe72a844a181a33884d5f6c1e72e9
Parents: 7da1a3d
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Aug 4 15:18:13 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Aug 9 09:59:05 2016 -0700
----------------------------------------------------------------------
.travis.yml | 2 +-
.travis/settings.xml | 33 +++++++++++++++++++++++++++++++++
2 files changed, 34 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71366165/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 43ae7de..4674bf3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -52,7 +52,7 @@ install:
- rm -rf "$HOME/.m2/repository/org/apache/beam"
script:
- - travis_retry mvn --batch-mode --update-snapshots $MAVEN_OVERRIDE verify
+ - travis_retry mvn --settings .travis/settings.xml --batch-mode --update-snapshots $MAVEN_OVERRIDE verify
- travis_retry testing/travis/test_wordcount.sh
cache:
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71366165/.travis/settings.xml
----------------------------------------------------------------------
diff --git a/.travis/settings.xml b/.travis/settings.xml
new file mode 100644
index 0000000..e086aec
--- /dev/null
+++ b/.travis/settings.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<settings>
+ <servers>
+ <server>
+ <id>central</id>
+ <configuration>
+ <httpConfiguration>
+ <all>
+ <connectionTimeout>1000</connectionTimeout>
+ </all>
+ </httpConfiguration>
+ <timeout>5000</timeout>
+ </configuration>
+ </server>
+ </servers>
+</settings>
[36/50] incubator-beam git commit: SparkRunner batch interval as a
configuration instead of Beam Windows.
Posted by ke...@apache.org.
SparkRunner batch interval as a configuration instead of Beam Windows.
Add the batch interval to the pipeline options, default arbitrarily to 1000 msec.
Pick-up the batch interval from pipeline options and remove StreamingWindowPipelineDetector.
Use SDK API to get the window function.
Update the README
Update streaming tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/08dca30a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/08dca30a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/08dca30a
Branch: refs/heads/gearpump-runner
Commit: 08dca30a38e11c13e8a4b2db1529f1306cc489b0
Parents: 95e7f01
Author: Sela <an...@paypal.com>
Authored: Wed Aug 10 13:30:30 2016 +0300
Committer: Sela <an...@paypal.com>
Committed: Wed Aug 10 20:42:17 2016 +0300
----------------------------------------------------------------------
runners/spark/README.md | 3 +-
.../runners/spark/SparkPipelineOptions.java | 5 +
.../apache/beam/runners/spark/SparkRunner.java | 19 +---
.../streaming/StreamingTransformTranslator.java | 10 +-
.../StreamingWindowPipelineDetector.java | 102 -------------------
.../streaming/FlattenStreamingTest.java | 1 +
.../streaming/KafkaStreamingTest.java | 2 +
.../streaming/SimpleStreamingWordCountTest.java | 2 +
8 files changed, 18 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/08dca30a/runners/spark/README.md
----------------------------------------------------------------------
diff --git a/runners/spark/README.md b/runners/spark/README.md
index d2bfd3e..ef42fa7 100644
--- a/runners/spark/README.md
+++ b/runners/spark/README.md
@@ -63,8 +63,7 @@ The Spark runner provides support for batch processing of Beam bounded PCollecti
### Streaming
The Spark runner currently provides partial support for stream processing of Beam unbounded PCollections as Spark [DStream](http://spark.apache.org/docs/latest/streaming-programming-guide.html#discretized-streams-dstreams)s.
-Current implementation of *Windowing* takes the first window size in the pipeline and treats it as the Spark "batch interval", while following windows will be treated as *Processing Time* windows.
-Further work is required to provide full support for the Beam Model *event-time* and *out-of-order* stream processing.
+Currently, both *FixedWindows* and *SlidingWindows* are supported, but only with processing-time triggers and discarding pane.
### issue tracking
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/08dca30a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
index 6ef3741..080ff19 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
@@ -39,4 +39,9 @@ public interface SparkPipelineOptions extends PipelineOptions, StreamingOptions,
@Default.Long(-1)
Long getTimeout();
void setTimeout(Long batchInterval);
+
+ @Description("Batch interval for Spark streaming in milliseconds.")
+ @Default.Long(1000)
+ Long getBatchIntervalMillis();
+ void setBatchIntervalMillis(Long batchInterval);
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/08dca30a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index d994ec4..be50f70 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -26,7 +26,6 @@ import org.apache.beam.runners.spark.translation.SparkProcessContext;
import org.apache.beam.runners.spark.translation.TransformTranslator;
import org.apache.beam.runners.spark.translation.streaming.StreamingEvaluationContext;
import org.apache.beam.runners.spark.translation.streaming.StreamingTransformTranslator;
-import org.apache.beam.runners.spark.translation.streaming.StreamingWindowPipelineDetector;
import org.apache.beam.runners.spark.util.SinglePrimitiveOutputPTransform;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -145,24 +144,16 @@ public final class SparkRunner extends PipelineRunner<EvaluationResult> {
public EvaluationResult run(Pipeline pipeline) {
try {
LOG.info("Executing pipeline using the SparkRunner.");
- JavaSparkContext jsc = SparkContextFactory.getSparkContext(mOptions
- .getSparkMaster(), mOptions.getAppName());
+ JavaSparkContext jsc = SparkContextFactory.getSparkContext(mOptions.getSparkMaster(),
+ mOptions.getAppName());
if (mOptions.isStreaming()) {
SparkPipelineTranslator translator =
- new StreamingTransformTranslator.Translator(new TransformTranslator.Translator());
- // if streaming - fixed window should be defined on all UNBOUNDED inputs
- StreamingWindowPipelineDetector streamingWindowPipelineDetector =
- new StreamingWindowPipelineDetector(translator);
- pipeline.traverseTopologically(streamingWindowPipelineDetector);
- if (!streamingWindowPipelineDetector.isWindowing()) {
- throw new IllegalStateException("Spark streaming pipeline must be windowed!");
- }
-
- Duration batchInterval = streamingWindowPipelineDetector.getBatchDuration();
+ new StreamingTransformTranslator.Translator(new TransformTranslator.Translator());
+ Duration batchInterval = new Duration(mOptions.getBatchIntervalMillis());
LOG.info("Setting Spark streaming batchInterval to {} msec", batchInterval.milliseconds());
- EvaluationContext ctxt = createStreamingEvaluationContext(jsc, pipeline, batchInterval);
+ EvaluationContext ctxt = createStreamingEvaluationContext(jsc, pipeline, batchInterval);
pipeline.traverseTopologically(new SparkPipelineEvaluator(ctxt, translator));
ctxt.computeOutputs();
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/08dca30a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index b0fb931..2ce2c29 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -25,7 +25,6 @@ import org.apache.beam.runners.spark.translation.DoFnFunction;
import org.apache.beam.runners.spark.translation.EvaluationContext;
import org.apache.beam.runners.spark.translation.SparkPipelineTranslator;
import org.apache.beam.runners.spark.translation.TransformEvaluator;
-import org.apache.beam.runners.spark.translation.TransformTranslator;
import org.apache.beam.runners.spark.translation.WindowingHelpers;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.AvroIO;
@@ -35,7 +34,6 @@ import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
@@ -290,16 +288,12 @@ public final class StreamingTransformTranslator {
}
}
- private static final TransformTranslator.FieldGetter WINDOW_FG =
- new TransformTranslator.FieldGetter(Window.Bound.class);
-
- private static <T, W extends BoundedWindow> TransformEvaluator<Window.Bound<T>> window() {
+ private static <T> TransformEvaluator<Window.Bound<T>> window() {
return new TransformEvaluator<Window.Bound<T>>() {
@Override
public void evaluate(Window.Bound<T> transform, EvaluationContext context) {
StreamingEvaluationContext sec = (StreamingEvaluationContext) context;
- //--- first we apply windowing to the stream
- WindowFn<? super T, W> windowFn = WINDOW_FG.get("windowFn", transform);
+ WindowFn<? super T, ?> windowFn = transform.getWindowFn();
@SuppressWarnings("unchecked")
JavaDStream<WindowedValue<T>> dStream =
(JavaDStream<WindowedValue<T>>) sec.getStream(transform);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/08dca30a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingWindowPipelineDetector.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingWindowPipelineDetector.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingWindowPipelineDetector.java
deleted file mode 100644
index 394b2c5..0000000
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingWindowPipelineDetector.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.spark.translation.streaming;
-
-import org.apache.beam.runners.spark.SparkRunner;
-import org.apache.beam.runners.spark.translation.SparkPipelineTranslator;
-import org.apache.beam.runners.spark.translation.TransformTranslator;
-import org.apache.beam.sdk.runners.TransformTreeNode;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
-import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.POutput;
-import org.apache.spark.streaming.Duration;
-import org.apache.spark.streaming.Durations;
-
-
-/**
- * Pipeline {@link SparkRunner.Evaluator} to detect windowing.
- */
-public final class StreamingWindowPipelineDetector extends SparkRunner.Evaluator {
-
- // Currently, Spark streaming recommends batches no smaller then 500 msec
- private static final Duration SPARK_MIN_WINDOW = Durations.milliseconds(500);
-
- private boolean windowing;
- private Duration batchDuration;
-
- public StreamingWindowPipelineDetector(SparkPipelineTranslator translator) {
- super(translator);
- }
-
- private static final TransformTranslator.FieldGetter WINDOW_FG =
- new TransformTranslator.FieldGetter(Window.Bound.class);
-
- // Use the smallest window (fixed or sliding) as Spark streaming's batch duration
- @Override
- protected <TransformT extends PTransform<? super PInput, POutput>> void
- doVisitTransform(TransformTreeNode node) {
- @SuppressWarnings("unchecked")
- TransformT transform = (TransformT) node.getTransform();
- @SuppressWarnings("unchecked")
- Class<TransformT> transformClass = (Class<TransformT>) (Class<?>) transform.getClass();
- if (transformClass.isAssignableFrom(Window.Bound.class)) {
- WindowFn<?, ?> windowFn = WINDOW_FG.get("windowFn", transform);
- if (windowFn instanceof FixedWindows) {
- setBatchDuration(((FixedWindows) windowFn).getSize());
- } else if (windowFn instanceof SlidingWindows) {
- if (((SlidingWindows) windowFn).getOffset().getMillis() > 0) {
- throw new UnsupportedOperationException("Spark does not support window offsets");
- }
- // Sliding window size might as well set the batch duration. Applying the transformation
- // will add the "slide"
- setBatchDuration(((SlidingWindows) windowFn).getSize());
- } else if (!(windowFn instanceof GlobalWindows)) {
- throw new IllegalStateException("Windowing function not supported: " + windowFn);
- }
- }
- }
-
- private void setBatchDuration(org.joda.time.Duration duration) {
- Long durationMillis = duration.getMillis();
- // validate window size
- if (durationMillis < SPARK_MIN_WINDOW.milliseconds()) {
- throw new IllegalArgumentException("Windowing of size " + durationMillis
- + "msec is not supported!");
- }
- // choose the smallest duration to be Spark's batch duration, larger ones will be handled
- // as window functions over the batched-stream
- if (!windowing || this.batchDuration.milliseconds() > durationMillis) {
- this.batchDuration = Durations.milliseconds(durationMillis);
- }
- windowing = true;
- }
-
- public boolean isWindowing() {
- return windowing;
- }
-
- public Duration getBatchDuration() {
- return batchDuration;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/08dca30a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
index deb1b6a..6f4d8fb 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
@@ -61,6 +61,7 @@ public class FlattenStreamingTest {
PipelineOptionsFactory.as(SparkPipelineOptions.class);
options.setRunner(SparkRunner.class);
options.setStreaming(true);
+ // using the default 1000 msec interval
options.setTimeout(TEST_TIMEOUT_MSEC); // run for one interval
Pipeline p = Pipeline.create(options);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/08dca30a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
index 17044aa..2527152 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
@@ -40,6 +40,7 @@ import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.Serializer;
import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.spark.streaming.Durations;
import org.joda.time.Duration;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -92,6 +93,7 @@ public class KafkaStreamingTest {
PipelineOptionsFactory.as(SparkPipelineOptions.class);
options.setRunner(SparkRunner.class);
options.setStreaming(true);
+ options.setBatchIntervalMillis(Durations.seconds(1).milliseconds());
options.setTimeout(TEST_TIMEOUT_MSEC); // run for one interval
Pipeline p = Pipeline.create(options);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/08dca30a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
index 5627056..c761fae 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
@@ -32,6 +32,7 @@ import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.spark.streaming.Durations;
import org.joda.time.Duration;
import org.junit.Test;
@@ -58,6 +59,7 @@ public class SimpleStreamingWordCountTest implements Serializable {
PipelineOptionsFactory.as(SparkPipelineOptions.class);
options.setRunner(SparkRunner.class);
options.setStreaming(true);
+ options.setBatchIntervalMillis(Durations.seconds(1).milliseconds());
options.setTimeout(TEST_TIMEOUT_MSEC); // run for one interval
Pipeline p = Pipeline.create(options);
[05/50] incubator-beam git commit: Port BigQueryIO to new DoFn
Posted by ke...@apache.org.
Port BigQueryIO to new DoFn
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d6395e9d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d6395e9d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d6395e9d
Branch: refs/heads/gearpump-runner
Commit: d6395e9d45dcbeb9b3d3e2f8214a49866622b9cf
Parents: 87313f1
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Aug 5 12:26:53 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 11:35:17 2016 -0700
----------------------------------------------------------------------
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 21 ++++++++++----------
1 file changed, 10 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d6395e9d/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index ed2c32e..36e09f1 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -44,7 +44,6 @@ import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.SerializableFunction;
@@ -1785,7 +1784,7 @@ public class BigQueryIO {
return PDone.in(input.getPipeline());
}
- private class WriteBundles extends OldDoFn<TableRow, KV<String, Long>> {
+ private class WriteBundles extends DoFn<TableRow, KV<String, Long>> {
private TableRowWriter writer = null;
private final String tempFilePrefix;
@@ -1793,7 +1792,7 @@ public class BigQueryIO {
this.tempFilePrefix = tempFilePrefix;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
if (writer == null) {
writer = new TableRowWriter(tempFilePrefix);
@@ -1806,7 +1805,7 @@ public class BigQueryIO {
// Discard write result and close the write.
try {
writer.close();
- // The writer does not need to be reset, as this OldDoFn cannot be reused.
+ // The writer does not need to be reset, as this DoFn cannot be reused.
} catch (Exception closeException) {
// Do not mask the exception that caused the write to fail.
e.addSuppressed(closeException);
@@ -1815,7 +1814,7 @@ public class BigQueryIO {
}
}
- @Override
+ @FinishBundle
public void finishBundle(Context c) throws Exception {
if (writer != null) {
c.output(writer.close());
@@ -1959,7 +1958,7 @@ public class BigQueryIO {
/**
* Partitions temporary files based on number of files and file sizes.
*/
- static class WritePartition extends OldDoFn<String, KV<Long, List<String>>> {
+ static class WritePartition extends DoFn<String, KV<Long, List<String>>> {
private final PCollectionView<Iterable<KV<String, Long>>> resultsView;
private TupleTag<KV<Long, List<String>>> multiPartitionsTag;
private TupleTag<KV<Long, List<String>>> singlePartitionTag;
@@ -1973,7 +1972,7 @@ public class BigQueryIO {
this.singlePartitionTag = singlePartitionTag;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
List<KV<String, Long>> results = Lists.newArrayList(c.sideInput(resultsView));
if (results.isEmpty()) {
@@ -2015,7 +2014,7 @@ public class BigQueryIO {
/**
* Writes partitions to BigQuery tables.
*/
- static class WriteTables extends OldDoFn<KV<Long, Iterable<List<String>>>, String> {
+ static class WriteTables extends DoFn<KV<Long, Iterable<List<String>>>, String> {
private final boolean singlePartition;
private final BigQueryServices bqServices;
private final String jobIdToken;
@@ -2044,7 +2043,7 @@ public class BigQueryIO {
this.createDisposition = createDisposition;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
List<String> partition = Lists.newArrayList(c.element().getValue()).get(0);
String jobIdPrefix = String.format(jobIdToken + "_%05d", c.element().getKey());
@@ -2149,7 +2148,7 @@ public class BigQueryIO {
/**
* Copies temporary tables to destination table.
*/
- static class WriteRename extends OldDoFn<String, Void> {
+ static class WriteRename extends DoFn<String, Void> {
private final BigQueryServices bqServices;
private final String jobIdToken;
private final String jsonTableRef;
@@ -2172,7 +2171,7 @@ public class BigQueryIO {
this.tempTablesView = tempTablesView;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
List<String> tempTablesJson = Lists.newArrayList(c.sideInput(tempTablesView));
[09/50] incubator-beam git commit: This closes #768
Posted by ke...@apache.org.
This closes #768
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d60a0a0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d60a0a0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d60a0a0e
Branch: refs/heads/gearpump-runner
Commit: d60a0a0e412f51e98f6f024dfc7ec4f21d1355a4
Parents: 574c377 2b6c873
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Aug 8 13:43:07 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 13:43:07 2016 -0700
----------------------------------------------------------------------
.travis.yml | 5 +----
pom.xml | 21 +++++++--------------
2 files changed, 8 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d60a0a0e/pom.xml
----------------------------------------------------------------------
[14/50] incubator-beam git commit: This closes #791
Posted by ke...@apache.org.
This closes #791
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/7da1a3de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/7da1a3de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/7da1a3de
Branch: refs/heads/gearpump-runner
Commit: 7da1a3dee0089acb7c791ec6d06f6b498334970a
Parents: 5049011 630ae25
Author: Kenneth Knowles <kl...@google.com>
Authored: Tue Aug 9 09:50:54 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Aug 9 09:50:54 2016 -0700
----------------------------------------------------------------------
.travis.yml | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
[21/50] incubator-beam git commit: [BEAM-534] Fix dead links in
README.md
Posted by ke...@apache.org.
[BEAM-534] Fix dead links in README.md
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0812fe65
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0812fe65
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0812fe65
Branch: refs/heads/gearpump-runner
Commit: 0812fe6551a187e07627b1331f0aa5a8978aa939
Parents: 7da1a3d
Author: Jean-Baptiste Onofr� <jb...@apache.org>
Authored: Thu Aug 4 23:18:42 2016 +0200
Committer: Frances Perry <fj...@google.com>
Committed: Tue Aug 9 14:50:48 2016 -0700
----------------------------------------------------------------------
README.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0812fe65/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index e72aec2..3f03ea5 100644
--- a/README.md
+++ b/README.md
@@ -110,4 +110,5 @@ To get involved in Apache Beam:
## More Information
* [Apache Beam](http://beam.incubator.apache.org)
-* [Getting Started with Apache Beam](http://beam.incubator.apache.org/getting_started/)
+* [Overview](http://beam.incubator.apache.org/use/beam-overview/)
+* [Quickstart](http://beam.incubator.apache.org/use/quickstart/)
[43/50] incubator-beam git commit: Demonstrate that the DirectRunner
runs per-call
Posted by ke...@apache.org.
Demonstrate that the DirectRunner runs per-call
Add a field that is modified per output, which should occur twice.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/65f9076d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/65f9076d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/65f9076d
Branch: refs/heads/gearpump-runner
Commit: 65f9076d654be02cbdc07442d008f6c5245d1ab5
Parents: 6da92ad
Author: Thomas Groh <tg...@google.com>
Authored: Wed Aug 10 11:29:38 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Wed Aug 10 15:54:09 2016 -0700
----------------------------------------------------------------------
.../apache/beam/runners/direct/DirectRunnerTest.java | 15 +++++++++++++++
1 file changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/65f9076d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
index 1e73ec0..ddce458 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.runners.direct;
+import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
@@ -32,6 +33,7 @@ import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.OldDoFn;
@@ -58,6 +60,7 @@ import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
/**
* Tests for basic {@link DirectRunner} functionality.
@@ -109,10 +112,12 @@ public class DirectRunnerTest implements Serializable {
result.awaitCompletion();
}
+ private static AtomicInteger changed;
@Test
public void reusePipelineSucceeds() throws Throwable {
Pipeline p = getPipeline();
+ changed = new AtomicInteger(0);
PCollection<KV<String, Long>> counts =
p.apply(Create.of("foo", "bar", "foo", "baz", "bar", "foo"))
.apply(MapElements.via(new SimpleFunction<String, String>() {
@@ -131,6 +136,14 @@ public class DirectRunnerTest implements Serializable {
}
}));
+ counts.apply(ParDo.of(new DoFn<KV<String, Long>, Void>() {
+ @ProcessElement
+ public void updateChanged(ProcessContext c) {
+ changed.getAndIncrement();
+ }
+ }));
+
+
PAssert.that(countStrs).containsInAnyOrder("baz: 1", "bar: 2", "foo: 3");
DirectPipelineResult result = ((DirectPipelineResult) p.run());
@@ -138,6 +151,8 @@ public class DirectRunnerTest implements Serializable {
DirectPipelineResult otherResult = ((DirectPipelineResult) p.run());
otherResult.awaitCompletion();
+
+ assertThat("Each element should have been processed twice", changed.get(), equalTo(6));
}
@Test(timeout = 5000L)
[12/50] incubator-beam git commit: Fix MapAggregatorValues use of
toStringHelper
Posted by ke...@apache.org.
Fix MapAggregatorValues use of toStringHelper
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c867790a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c867790a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c867790a
Branch: refs/heads/gearpump-runner
Commit: c867790abad182bcbffa9a7a88de76a608b6d039
Parents: d60a0a0
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Jul 21 20:18:05 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 13:55:24 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/beam/sdk/util/MapAggregatorValues.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c867790a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MapAggregatorValues.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MapAggregatorValues.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MapAggregatorValues.java
index fe02666..3d949ec 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MapAggregatorValues.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MapAggregatorValues.java
@@ -43,7 +43,7 @@ public class MapAggregatorValues<T> extends AggregatorValues<T> {
@Override
public String toString() {
- return MoreObjects.toStringHelper(MapAggregatorValues.class)
+ return MoreObjects.toStringHelper(this)
.add("stepValues", stepValues)
.toString();
}
[46/50] incubator-beam git commit: move integration tests to profile
Posted by ke...@apache.org.
move integration tests to profile
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1672b548
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1672b548
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1672b548
Branch: refs/heads/gearpump-runner
Commit: 1672b5483e029292816397248dc6fe63bf51f4af
Parents: 2a0ba61
Author: manuzhang <ow...@gmail.com>
Authored: Sat Jul 23 14:10:15 2016 +0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu Aug 25 11:37:17 2016 -0700
----------------------------------------------------------------------
runners/gearpump/pom.xml | 107 ++++++++++++++++++++++++------------------
1 file changed, 61 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1672b548/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index c725dae..85994ad 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -57,6 +57,67 @@
</repository>
</repositories>
+ <profiles>
+ <profile>
+ <id>local-runnable-on-service-tests</id>
+ <activation><activeByDefault>false</activeByDefault></activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.19.1</version>
+ <executions>
+ <execution>
+ <id>runnable-on-service-tests</id>
+ <configuration>
+ <groups>org.apache.beam.sdk.testing.RunnableOnService</groups>
+ <parallel>none</parallel>
+ <failIfNoTests>true</failIfNoTests>
+ <dependenciesToScan>
+ <dependency>org.apache.beam:beam-sdks-java-core</dependency>
+ <dependency>org.apache.beam:beam-runners-java-core</dependency>
+ </dependenciesToScan>
+ <excludes>
+ <!-- side input is not supported in Gearpump -->
+ <exclude>
+ org.apache.beam.sdk.io.BigQueryIOTest,
+ org.apache.beam.sdk.io.CountingInputTest,
+ org.apache.beam.sdk.io.CountingSourceTest,
+ org.apache.beam.sdk.testing.PAssertTest,
+ org.apache.beam.sdk.transforms.ApproximateUniqueTest,
+ org.apache.beam.sdk.transforms.CombineTest,
+ org.apache.beam.sdk.transforms.CombineFnsTest,
+ org.apache.beam.sdk.transforms.CountTest,
+ org.apache.beam.sdk.transforms.FlattenTest,
+ org.apache.beam.sdk.transforms.ParDoTest,
+ org.apache.beam.sdk.transforms.SampleTest,
+ org.apache.beam.sdk.transforms.ViewTest,
+ org.apache.beam.sdk.transforms.join.CoGroupByKeyTest
+ </exclude>
+ <!-- merging windows is not supported in Gearpump -->
+ <exclude>
+ org.apache.beam.sdk.transforms.windowing.WindowingTest,
+ org.apache.beam.sdk.util.ReshuffleTest
+ </exclude>
+ </excludes>
+ <systemPropertyVariables>
+ <beamTestPipelineOptions>
+ [
+ "--runner=TestGearpumpRunner",
+ "--streaming=true"
+ ]
+ </beamTestPipelineOptions>
+ </systemPropertyVariables>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
<dependencies>
<dependency>
<groupId>org.apache.gearpump</groupId>
@@ -228,52 +289,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
- <version>2.19.1</version>
- <executions>
- <execution>
- <id>runnable-on-service-tests</id>
- <configuration>
- <groups>org.apache.beam.sdk.testing.RunnableOnService</groups>
- <parallel>none</parallel>
- <failIfNoTests>true</failIfNoTests>
- <dependenciesToScan>
- <dependency>org.apache.beam:beam-sdks-java-core</dependency>
- <dependency>org.apache.beam:beam-runners-java-core</dependency>
- </dependenciesToScan>
- <excludes>
- <!-- side input is not supported in Gearpump -->
- <exclude>
- org.apache.beam.sdk.io.BigQueryIOTest,
- org.apache.beam.sdk.io.CountingInputTest,
- org.apache.beam.sdk.io.CountingSourceTest,
- org.apache.beam.sdk.testing.PAssertTest,
- org.apache.beam.sdk.transforms.ApproximateUniqueTest,
- org.apache.beam.sdk.transforms.CombineTest,
- org.apache.beam.sdk.transforms.CombineFnsTest,
- org.apache.beam.sdk.transforms.CountTest,
- org.apache.beam.sdk.transforms.FlattenTest,
- org.apache.beam.sdk.transforms.ParDoTest,
- org.apache.beam.sdk.transforms.SampleTest,
- org.apache.beam.sdk.transforms.ViewTest,
- org.apache.beam.sdk.transforms.join.CoGroupByKeyTest
- </exclude>
- <!-- merging windows is not supported in Gearpump -->
- <exclude>
- org.apache.beam.sdk.transforms.windowing.WindowingTest,
- org.apache.beam.sdk.util.ReshuffleTest
- </exclude>
- </excludes>
- <systemPropertyVariables>
- <beamTestPipelineOptions>
- [
- "--runner=TestGearpumpRunner",
- "--streaming=true"
- ]
- </beamTestPipelineOptions>
- </systemPropertyVariables>
- </configuration>
- </execution>
- </executions>
</plugin>
<!-- uber jar -->
[26/50] incubator-beam git commit: [BEAM-124] Spark Running
WordCountIT Example
Posted by ke...@apache.org.
[BEAM-124] Spark Running WordCountIT Example
This closes #769
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a035796c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a035796c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a035796c
Branch: refs/heads/gearpump-runner
Commit: a035796c2b189ad06a39260590b17057e1a1da2c
Parents: 063ff2f e979c82
Author: Luke Cwik <lc...@google.com>
Authored: Wed Aug 10 08:25:40 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Wed Aug 10 08:25:40 2016 -0700
----------------------------------------------------------------------
examples/java/pom.xml | 24 +++
.../org/apache/beam/examples/WordCount.java | 24 ++-
.../org/apache/beam/examples/WordCountIT.java | 2 +-
examples/java/src/test/resources/LICENSE | 202 +++++++++++++++++++
4 files changed, 250 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[35/50] incubator-beam git commit: Closes #788
Posted by ke...@apache.org.
Closes #788
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0ab9495f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0ab9495f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0ab9495f
Branch: refs/heads/gearpump-runner
Commit: 0ab9495f88aa8719789213c01808fe852ad89a91
Parents: 95e7f01 a41dab8
Author: Dan Halperin <dh...@google.com>
Authored: Wed Aug 10 10:39:15 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Wed Aug 10 10:39:15 2016 -0700
----------------------------------------------------------------------
.../beam/examples/complete/AutoComplete.java | 21 +++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[02/50] incubator-beam git commit: Port Flink fork of examples to new
DoFn
Posted by ke...@apache.org.
Port Flink fork of examples to new DoFn
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/87313f1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/87313f1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/87313f1c
Branch: refs/heads/gearpump-runner
Commit: 87313f1c3d8cf874e04aaf528161478afa030f38
Parents: ae1f6d1
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Aug 5 12:24:24 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 11:35:17 2016 -0700
----------------------------------------------------------------------
.../beam/runners/flink/examples/TFIDF.java | 28 +++++++--------
.../beam/runners/flink/examples/WordCount.java | 5 +--
.../flink/examples/streaming/AutoComplete.java | 37 ++++++++++----------
.../flink/examples/streaming/JoinExamples.java | 14 ++++----
.../examples/streaming/KafkaIOExamples.java | 7 ++--
.../KafkaWindowedWordCountExample.java | 10 +++---
.../examples/streaming/WindowedWordCount.java | 10 +++---
7 files changed, 57 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/87313f1c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/TFIDF.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/TFIDF.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/TFIDF.java
index 716c8ad..4deca12 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/TFIDF.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/TFIDF.java
@@ -32,7 +32,7 @@ import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.Validation;
import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.Keys;
import org.apache.beam.sdk.transforms.PTransform;
@@ -230,10 +230,10 @@ public class TFIDF {
// Create a collection of pairs mapping a URI to each
// of the words in the document associated with that that URI.
PCollection<KV<URI, String>> uriToWords = uriToContent
- .apply("SplitWords", ParDo.of(new OldDoFn<KV<URI, String>, KV<URI, String>>() {
+ .apply("SplitWords", ParDo.of(new DoFn<KV<URI, String>, KV<URI, String>>() {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
URI uri = c.element().getKey();
String line = c.element().getValue();
@@ -275,10 +275,10 @@ public class TFIDF {
// by the URI key.
PCollection<KV<URI, KV<String, Long>>> uriToWordAndCount = uriAndWordToCount
.apply("ShiftKeys", ParDo.of(
- new OldDoFn<KV<KV<URI, String>, Long>, KV<URI, KV<String, Long>>>() {
+ new DoFn<KV<KV<URI, String>, Long>, KV<URI, KV<String, Long>>>() {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
URI uri = c.element().getKey().getKey();
String word = c.element().getKey().getValue();
@@ -316,10 +316,10 @@ public class TFIDF {
// divided by the total number of words in the document.
PCollection<KV<String, KV<URI, Double>>> wordToUriAndTf = uriToWordAndCountAndTotal
.apply("ComputeTermFrequencies", ParDo.of(
- new OldDoFn<KV<URI, CoGbkResult>, KV<String, KV<URI, Double>>>() {
+ new DoFn<KV<URI, CoGbkResult>, KV<String, KV<URI, Double>>>() {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
URI uri = c.element().getKey();
Long wordTotal = c.element().getValue().getOnly(wordTotalsTag);
@@ -339,14 +339,14 @@ public class TFIDF {
// documents in which the word appears divided by the total
// number of documents in the corpus. Note how the total number of
// documents is passed as a side input; the same value is
- // presented to each invocation of the OldDoFn.
+ // presented to each invocation of the DoFn.
PCollection<KV<String, Double>> wordToDf = wordToDocCount
.apply("ComputeDocFrequencies", ParDo
.withSideInputs(totalDocuments)
- .of(new OldDoFn<KV<String, Long>, KV<String, Double>>() {
+ .of(new DoFn<KV<String, Long>, KV<String, Double>>() {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
String word = c.element().getKey();
Long documentCount = c.element().getValue();
@@ -375,10 +375,10 @@ public class TFIDF {
return wordToUriAndTfAndDf
.apply("ComputeTfIdf", ParDo.of(
- new OldDoFn<KV<String, CoGbkResult>, KV<String, KV<URI, Double>>>() {
+ new DoFn<KV<String, CoGbkResult>, KV<String, KV<URI, Double>>>() {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
String word = c.element().getKey();
Double df = c.element().getValue().getOnly(dfTag);
@@ -416,10 +416,10 @@ public class TFIDF {
@Override
public PDone apply(PCollection<KV<String, KV<URI, Double>>> wordToUriAndTfIdf) {
return wordToUriAndTfIdf
- .apply("Format", ParDo.of(new OldDoFn<KV<String, KV<URI, Double>>, String>() {
+ .apply("Format", ParDo.of(new DoFn<KV<String, KV<URI, Double>>, String>() {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
c.output(String.format("%s,\t%s,\t%f",
c.element().getKey(),
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/87313f1c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/WordCount.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/WordCount.java
index 080cdc9..fdffd39 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/WordCount.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/WordCount.java
@@ -27,6 +27,7 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.Validation;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
@@ -38,11 +39,11 @@ import org.apache.beam.sdk.values.PCollection;
public class WordCount {
- public static class ExtractWordsFn extends OldDoFn<String, String> {
+ public static class ExtractWordsFn extends DoFn<String, String> {
private final Aggregator<Long, Long> emptyLines =
createAggregator("emptyLines", new Sum.SumLongFn());
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
if (c.element().trim().isEmpty()) {
emptyLines.addValue(1L);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/87313f1c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/AutoComplete.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/AutoComplete.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/AutoComplete.java
index 068404a..aff1a35 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/AutoComplete.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/AutoComplete.java
@@ -29,7 +29,7 @@ import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Filter;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.PTransform;
@@ -40,6 +40,7 @@ import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.Top;
import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
@@ -92,10 +93,10 @@ public class AutoComplete {
// Map the KV outputs of Count into our own CompletionCandiate class.
.apply("CreateCompletionCandidates", ParDo.of(
- new OldDoFn<KV<String, Long>, CompletionCandidate>() {
+ new DoFn<KV<String, Long>, CompletionCandidate>() {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
CompletionCandidate cand = new CompletionCandidate(c.element().getKey(), c.element().getValue());
c.output(cand);
@@ -182,10 +183,10 @@ public class AutoComplete {
}
private static class FlattenTops
- extends OldDoFn<KV<String, List<CompletionCandidate>>, CompletionCandidate> {
+ extends DoFn<KV<String, List<CompletionCandidate>>, CompletionCandidate> {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
for (CompletionCandidate cc : c.element().getValue()) {
c.output(cc);
@@ -236,10 +237,10 @@ public class AutoComplete {
}
/**
- * A OldDoFn that keys each candidate by all its prefixes.
+ * A DoFn that keys each candidate by all its prefixes.
*/
private static class AllPrefixes
- extends OldDoFn<CompletionCandidate, KV<String, CompletionCandidate>> {
+ extends DoFn<CompletionCandidate, KV<String, CompletionCandidate>> {
private static final long serialVersionUID = 0;
private final int minPrefix;
@@ -251,7 +252,7 @@ public class AutoComplete {
this.minPrefix = minPrefix;
this.maxPrefix = maxPrefix;
}
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
String word = c.element().value;
for (int i = minPrefix; i <= Math.min(word.length(), maxPrefix); i++) {
@@ -314,11 +315,11 @@ public class AutoComplete {
}
}
- static class ExtractWordsFn extends OldDoFn<String, String> {
+ static class ExtractWordsFn extends DoFn<String, String> {
private final Aggregator<Long, Long> emptyLines =
createAggregator("emptyLines", new Sum.SumLongFn());
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
if (c.element().trim().isEmpty()) {
emptyLines.addValue(1L);
@@ -337,21 +338,21 @@ public class AutoComplete {
}
/**
- * Takes as input a the top candidates per prefix, and emits an entity
- * suitable for writing to Datastore.
+ * Takes as input a the top candidates per prefix, and emits an entity suitable for writing to
+ * Datastore.
*/
- static class FormatForPerTaskLocalFile extends OldDoFn<KV<String, List<CompletionCandidate>>, String>
- implements OldDoFn.RequiresWindowAccess{
+ static class FormatForPerTaskLocalFile
+ extends DoFn<KV<String, List<CompletionCandidate>>, String> {
private static final long serialVersionUID = 0;
- @Override
- public void processElement(ProcessContext c) {
+ @ProcessElement
+ public void processElement(ProcessContext c, BoundedWindow window) {
StringBuilder str = new StringBuilder();
KV<String, List<CompletionCandidate>> elem = c.element();
- str.append(elem.getKey() +" @ "+ c.window() +" -> ");
- for(CompletionCandidate cand: elem.getValue()) {
+ str.append(elem.getKey() +" @ "+ window +" -> ");
+ for (CompletionCandidate cand: elem.getValue()) {
str.append(cand.toString() + " ");
}
System.out.println(str.toString());
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/87313f1c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/JoinExamples.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/JoinExamples.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/JoinExamples.java
index 7d7c0c7..458a263 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/JoinExamples.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/JoinExamples.java
@@ -23,7 +23,7 @@ import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.join.CoGbkResult;
import org.apache.beam.sdk.transforms.join.CoGroupByKey;
@@ -76,10 +76,10 @@ public class JoinExamples {
// country code 'key' -> string of <event info>, <country name>
PCollection<KV<String, String>> finalResultCollection =
kvpCollection.apply("Process", ParDo.of(
- new OldDoFn<KV<String, CoGbkResult>, KV<String, String>>() {
+ new DoFn<KV<String, CoGbkResult>, KV<String, String>>() {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
KV<String, CoGbkResult> e = c.element();
String key = e.getKey();
@@ -98,10 +98,10 @@ public class JoinExamples {
}));
return finalResultCollection
- .apply("Format", ParDo.of(new OldDoFn<KV<String, String>, String>() {
+ .apply("Format", ParDo.of(new DoFn<KV<String, String>, String>() {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
String result = c.element().getKey() + " -> " + c.element().getValue();
System.out.println(result);
@@ -110,10 +110,10 @@ public class JoinExamples {
}));
}
- static class ExtractEventDataFn extends OldDoFn<String, KV<String, String>> {
+ static class ExtractEventDataFn extends DoFn<String, KV<String, String>> {
private static final long serialVersionUID = 0;
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
String line = c.element().toLowerCase();
String key = line.split("\\s")[0];
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/87313f1c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaIOExamples.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaIOExamples.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaIOExamples.java
index 395b409..68a9edc 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaIOExamples.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaIOExamples.java
@@ -30,9 +30,10 @@ import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.PCollection;
+
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08;
@@ -326,9 +327,9 @@ public class KafkaIOExamples {
* Print contents to stdout
* @param <T> type of the input
*/
- private static class PrintFn<T> extends OldDoFn<T, T> {
+ private static class PrintFn<T> extends DoFn<T, T> {
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
System.out.println(c.element().toString());
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/87313f1c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaWindowedWordCountExample.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaWindowedWordCountExample.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaWindowedWordCountExample.java
index 8c31783..39ce225 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaWindowedWordCountExample.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaWindowedWordCountExample.java
@@ -27,7 +27,7 @@ import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
@@ -49,11 +49,11 @@ public class KafkaWindowedWordCountExample {
static final String GROUP_ID = "myGroup"; // Default groupId
static final String ZOOKEEPER = "localhost:2181"; // Default zookeeper to connect to for Kafka
- public static class ExtractWordsFn extends OldDoFn<String, String> {
+ public static class ExtractWordsFn extends DoFn<String, String> {
private final Aggregator<Long, Long> emptyLines =
createAggregator("emptyLines", new Sum.SumLongFn());
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
if (c.element().trim().isEmpty()) {
emptyLines.addValue(1L);
@@ -71,8 +71,8 @@ public class KafkaWindowedWordCountExample {
}
}
- public static class FormatAsStringFn extends OldDoFn<KV<String, Long>, String> {
- @Override
+ public static class FormatAsStringFn extends DoFn<KV<String, Long>, String> {
+ @ProcessElement
public void processElement(ProcessContext c) {
String row = c.element().getKey() + " - " + c.element().getValue() + " @ " + c.timestamp().toString();
System.out.println(row);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/87313f1c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/WindowedWordCount.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/WindowedWordCount.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/WindowedWordCount.java
index d149e4e..fe8e627 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/WindowedWordCount.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/WindowedWordCount.java
@@ -27,7 +27,7 @@ import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
@@ -59,19 +59,19 @@ public class WindowedWordCount {
static final long WINDOW_SIZE = 10; // Default window duration in seconds
static final long SLIDE_SIZE = 5; // Default window slide in seconds
- static class FormatAsStringFn extends OldDoFn<KV<String, Long>, String> {
- @Override
+ static class FormatAsStringFn extends DoFn<KV<String, Long>, String> {
+ @ProcessElement
public void processElement(ProcessContext c) {
String row = c.element().getKey() + " - " + c.element().getValue() + " @ " + c.timestamp().toString();
c.output(row);
}
}
- static class ExtractWordsFn extends OldDoFn<String, String> {
+ static class ExtractWordsFn extends DoFn<String, String> {
private final Aggregator<Long, Long> emptyLines =
createAggregator("emptyLines", new Sum.SumLongFn());
- @Override
+ @ProcessElement
public void processElement(ProcessContext c) {
if (c.element().trim().isEmpty()) {
emptyLines.addValue(1L);
[25/50] incubator-beam git commit: [BEAM-124] Spark Running
WordCountIT Example
Posted by ke...@apache.org.
[BEAM-124] Spark Running WordCountIT Example
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9983ff9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9983ff9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9983ff9c
Branch: refs/heads/gearpump-runner
Commit: 9983ff9c57a477bb2bce2010768dfaf16aed73d6
Parents: 063ff2f
Author: Mark Liu <ma...@markliu0.mtv.corp.google.com>
Authored: Tue Aug 2 14:56:28 2016 -0700
Committer: Mark Liu <ma...@markliu-macbookpro.roam.corp.google.com>
Committed: Tue Aug 9 15:28:24 2016 -0700
----------------------------------------------------------------------
examples/java/pom.xml | 24 +++
.../org/apache/beam/examples/WordCount.java | 23 ++-
.../org/apache/beam/examples/WordCountIT.java | 2 +-
examples/java/src/test/resources/LICENSE | 202 +++++++++++++++++++
4 files changed, 249 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9983ff9c/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 10e7503..cdf80cb 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -34,6 +34,10 @@
<packaging>jar</packaging>
+ <properties>
+ <spark.version>1.6.2</spark.version>
+ </properties>
+
<build>
<plugins>
<plugin>
@@ -292,6 +296,26 @@
</dependency>
<dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-spark</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<scope>runtime</scope>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9983ff9c/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
index d3768a8..d136b88 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.examples;
+import org.apache.beam.runners.spark.SparkRunner;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.options.Default;
@@ -166,7 +167,7 @@ public class WordCount {
*/
public static interface WordCountOptions extends PipelineOptions {
@Description("Path of the file to read from")
- @Default.String("gs://dataflow-samples/shakespeare/kinglear.txt")
+ @Default.InstanceFactory(InputFactory.class)
String getInputFile();
void setInputFile(String value);
@@ -196,6 +197,26 @@ public class WordCount {
}
}
+ /**
+ * Return default input file path according to runner type.
+ *
+ * <p><ul>
+ * <li>SparkRunner:
+ * .../src/test/resources/LICENSE</li>
+ * <li>other runners:
+ * gs://dataflow-samples/apache/LICENSE</li>
+ * </ul>
+ */
+ public static class InputFactory implements DefaultValueFactory<String> {
+ @Override
+ public String create(PipelineOptions options) {
+ if (options.getRunner().isAssignableFrom(SparkRunner.class)) {
+ return WordCount.class.getClass().getResource("/LICENSE").getPath();
+ } else {
+ return "gs://dataflow-samples/apache/LICENSE";
+ }
+ }
+ }
}
public static void main(String[] args) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9983ff9c/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
----------------------------------------------------------------------
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
index 6ba46d7..c087c67 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
@@ -86,7 +86,7 @@ public class WordCountIT {
private static final Logger LOG = LoggerFactory.getLogger(WordCountOnSuccessMatcher.class);
- private static final String EXPECTED_CHECKSUM = "8ae94f799f97cfd1cb5e8125951b32dfb52e1f12";
+ private static final String EXPECTED_CHECKSUM = "c04722202dee29c442b55ead54c6000693e85e77";
private String actualChecksum;
private final String outputPath;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9983ff9c/examples/java/src/test/resources/LICENSE
----------------------------------------------------------------------
diff --git a/examples/java/src/test/resources/LICENSE b/examples/java/src/test/resources/LICENSE
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/examples/java/src/test/resources/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
[11/50] incubator-beam git commit: Inline MapAggregatorValues to
remove dependencies
Posted by ke...@apache.org.
Inline MapAggregatorValues to remove dependencies
This class is trivial. Adding it to the public API of the SDK is
not desirable, since it is just for runners. Adding it to runners-core
would be OK but is really overkill for a glorified Map.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9da4bbcd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9da4bbcd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9da4bbcd
Branch: refs/heads/gearpump-runner
Commit: 9da4bbcdaf3c19ee5f78836b7cffaab947861a58
Parents: c867790
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Jul 21 20:24:17 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 13:55:24 2016 -0700
----------------------------------------------------------------------
.../beam/runners/direct/DirectRunner.java | 18 +++++--
.../runners/dataflow/DataflowPipelineJob.java | 17 ++++++-
.../beam/sdk/util/MapAggregatorValues.java | 50 --------------------
3 files changed, 30 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9da4bbcd/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index 743c565..a9c8ecb 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -36,7 +36,6 @@ import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
-import org.apache.beam.sdk.util.MapAggregatorValues;
import org.apache.beam.sdk.util.TimerInternals.TimerData;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.util.WindowedValue;
@@ -47,6 +46,7 @@ import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
import org.apache.beam.sdk.values.PValue;
+import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -321,7 +321,7 @@ public class DirectRunner
throws AggregatorRetrievalException {
AggregatorContainer aggregators = evaluationContext.getAggregatorContainer();
Collection<PTransform<?, ?>> steps = aggregatorSteps.get(aggregator);
- Map<String, T> stepValues = new HashMap<>();
+ final Map<String, T> stepValues = new HashMap<>();
for (AppliedPTransform<?, ?, ?> transform : evaluationContext.getSteps()) {
if (steps.contains(transform.getTransform())) {
T aggregate = aggregators.getAggregate(
@@ -331,7 +331,19 @@ public class DirectRunner
}
}
}
- return new MapAggregatorValues<>(stepValues);
+ return new AggregatorValues<T>() {
+ @Override
+ public Map<String, T> getValuesAtSteps() {
+ return stepValues;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("stepValues", stepValues)
+ .toString();
+ }
+ };
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9da4bbcd/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
index 3194f7c..a6baa4f 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
@@ -29,7 +29,6 @@ import org.apache.beam.sdk.runners.AggregatorValues;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.util.AttemptAndTimeBoundedExponentialBackOff;
import org.apache.beam.sdk.util.AttemptBoundedExponentialBackOff;
-import org.apache.beam.sdk.util.MapAggregatorValues;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.util.BackOff;
@@ -41,6 +40,7 @@ import com.google.api.services.dataflow.model.JobMessage;
import com.google.api.services.dataflow.model.JobMetrics;
import com.google.api.services.dataflow.model.MetricUpdate;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
import org.joda.time.Duration;
import org.slf4j.Logger;
@@ -369,7 +369,20 @@ public class DataflowPipelineJob implements PipelineResult {
public <OutputT> AggregatorValues<OutputT> getAggregatorValues(Aggregator<?, OutputT> aggregator)
throws AggregatorRetrievalException {
try {
- return new MapAggregatorValues<>(fromMetricUpdates(aggregator));
+ final Map<String, OutputT> stepValues = fromMetricUpdates(aggregator);
+ return new AggregatorValues<OutputT>() {
+ @Override
+ public Map<String, OutputT> getValuesAtSteps() {
+ return stepValues;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("stepValues", stepValues)
+ .toString();
+ }
+ };
} catch (IOException e) {
throw new AggregatorRetrievalException(
"IOException when retrieving Aggregator values for Aggregator " + aggregator, e);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9da4bbcd/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MapAggregatorValues.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MapAggregatorValues.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MapAggregatorValues.java
deleted file mode 100644
index 3d949ec..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MapAggregatorValues.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import org.apache.beam.sdk.runners.AggregatorValues;
-import org.apache.beam.sdk.transforms.Aggregator;
-
-import com.google.common.base.MoreObjects;
-
-import java.util.Map;
-
-/**
- * An {@link AggregatorValues} implementation that is backed by an in-memory map.
- *
- * @param <T> the output type of the {@link Aggregator}
- */
-public class MapAggregatorValues<T> extends AggregatorValues<T> {
- private final Map<String, T> stepValues;
-
- public MapAggregatorValues(Map<String, T> stepValues) {
- this.stepValues = stepValues;
- }
-
- @Override
- public Map<String, T> getValuesAtSteps() {
- return stepValues;
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("stepValues", stepValues)
- .toString();
- }
-}
[06/50] incubator-beam git commit: Add ViewFn and port SDK to use it
Posted by ke...@apache.org.
Add ViewFn and port SDK to use it
This is a preliminary step towards the architecture at
https://s.apache.org/beam-side-input-1-pager
This separates the ViewFn part of each PCollectionView
class/transform, toward eliminating extraneous public
subclasses of PCollectionView and PTransform.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c376b45c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c376b45c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c376b45c
Branch: refs/heads/gearpump-runner
Commit: c376b45cac8568d7242d29725f4a9a701673df75
Parents: 2b5c6bc
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Jun 22 08:39:33 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 12:31:11 2016 -0700
----------------------------------------------------------------------
.../beam/runners/direct/SideInputContainer.java | 2 +-
.../runners/direct/ViewEvaluatorFactory.java | 5 +-
.../functions/FlinkProcessContext.java | 2 +-
.../functions/SideInputInitializer.java | 2 +-
.../spark/translation/SparkProcessContext.java | 2 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 2 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 45 ++++
.../beam/sdk/util/DirectSideInputReader.java | 4 +-
.../apache/beam/sdk/util/PCollectionViews.java | 228 +++++++++++++------
.../apache/beam/sdk/values/PCollectionView.java | 29 ++-
.../sdk/testing/PCollectionViewTesting.java | 35 +--
11 files changed, 262 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
index 7a19ed9..6458215 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
@@ -247,7 +247,7 @@ class SideInputContainer {
@SuppressWarnings("unchecked") Iterable<WindowedValue<?>> values =
(Iterable<WindowedValue<?>>) viewContents.getUnchecked(PCollectionViewWindow.of(view,
window)).get();
- return view.fromIterableInternal(values);
+ return view.getViewFn().apply(values);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
index 7a0b0f7..362e903 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
@@ -133,8 +133,9 @@ class ViewEvaluatorFactory implements TransformEvaluatorFactory {
/**
* An in-process implementation of the {@link CreatePCollectionView} primitive.
*
- * This implementation requires the input {@link PCollection} to be an iterable, which is provided
- * to {@link PCollectionView#fromIterableInternal(Iterable)}.
+ * This implementation requires the input {@link PCollection} to be an iterable
+ * of {@code WindowedValue<ElemT>}, which is provided
+ * to {@link PCollectionView#getViewFn()} for conversion to {@link ViewT}.
*/
public static final class WriteView<ElemT, ViewT>
extends PTransform<PCollection<Iterable<ElemT>>, PCollectionView<ViewT>> {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkProcessContext.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkProcessContext.java b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkProcessContext.java
index 3954d1f..64b93c8 100644
--- a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkProcessContext.java
+++ b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkProcessContext.java
@@ -249,7 +249,7 @@ class FlinkProcessContext<InputT, OutputT>
view.getTagInternal().getId(), new SideInputInitializer<>(view));
ViewT result = sideInputs.get(sideInputWindow);
if (result == null) {
- result = view.fromIterableInternal(Collections.<WindowedValue<?>>emptyList());
+ result = view.getViewFn().apply(Collections.<WindowedValue<?>>emptyList());
}
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/functions/SideInputInitializer.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/functions/SideInputInitializer.java b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/functions/SideInputInitializer.java
index 451b31b..a577b68 100644
--- a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/functions/SideInputInitializer.java
+++ b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/functions/SideInputInitializer.java
@@ -67,7 +67,7 @@ public class SideInputInitializer<ElemT, ViewT, W extends BoundedWindow>
Iterable<WindowedValue<?>> elementsIterable =
(List<WindowedValue<?>>) (List<?>) elements.getValue();
- resultMap.put(elements.getKey(), view.fromIterableInternal(elementsIterable));
+ resultMap.put(elements.getKey(), view.getViewFn().apply(elementsIterable));
}
return resultMap;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
index 58ac03c..2f06a1c 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkProcessContext.java
@@ -84,7 +84,7 @@ public abstract class SparkProcessContext<InputT, OutputT, ValueT>
BroadcastHelper<Iterable<WindowedValue<?>>> broadcastHelper =
(BroadcastHelper<Iterable<WindowedValue<?>>>) mSideInputs.get(view.getTagInternal());
Iterable<WindowedValue<?>> contents = broadcastHelper.getValue();
- return view.fromIterableInternal(contents);
+ return view.getViewFn().apply(contents);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index 8de1066..e2764eb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -543,7 +543,7 @@ public class DoFnTester<InputT, OutputT> {
return windowValue;
}
}
- return view.fromIterableInternal(Collections.<WindowedValue<?>>emptyList());
+ return view.getViewFn().apply(Collections.<WindowedValue<?>>emptyList());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ViewFn.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ViewFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ViewFn.java
new file mode 100644
index 0000000..aa3cb0d
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ViewFn.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms;
+
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+
+import java.io.Serializable;
+
+/**
+ * A function to adapt a primitive "view" of a {@link PCollection} - some materialization
+ * specified in the Beam model and implemented by the runner - to a user-facing view type
+ * for side input.
+ *
+ * <p>Both the underlying primitive view and the user-facing view are immutable.
+ *
+ * <p>The most common case is using the {@link View} transforms to prepare a {@link PCollection}
+ * for use as a side input to {@link ParDo}. See {@link View#asSingleton()},
+ * {@link View#asIterable()}, and {@link View#asMap()} for more detail on specific views
+ * available in the SDK.
+ *
+ * @param <PrimitiveViewT> the type of the underlying primitive view, provided by the runner
+ * <ViewT> the type of the value(s) accessible via this {@link PCollectionView}
+ */
+public abstract class ViewFn<PrimitiveViewT, ViewT> implements Serializable {
+ /**
+ * A function to adapt a primitive view type to a desired view type.
+ */
+ public abstract ViewT apply(PrimitiveViewT contents);
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DirectSideInputReader.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DirectSideInputReader.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DirectSideInputReader.java
index f44c06e..c8d360c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DirectSideInputReader.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/DirectSideInputReader.java
@@ -60,9 +60,9 @@ public class DirectSideInputReader implements SideInputReader {
}
if (view.getWindowingStrategyInternal().getWindowFn() instanceof GlobalWindows) {
- return view.fromIterableInternal(sideInputValues.get(tag));
+ return view.getViewFn().apply(sideInputValues.get(tag));
} else {
- return view.fromIterableInternal(
+ return view.getViewFn().apply(
Iterables.filter(sideInputValues.get(tag),
new Predicate<WindowedValue<?>>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java
index d63fb96..581a98a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PCollectionViews.java
@@ -20,6 +20,7 @@ package org.apache.beam.sdk.util;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.transforms.ViewFn;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.InvalidWindows;
import org.apache.beam.sdk.values.KV;
@@ -63,8 +64,14 @@ public class PCollectionViews {
Pipeline pipeline,
WindowingStrategy<?, W> windowingStrategy,
boolean hasDefault,
- T defaultValue,
+ @Nullable T defaultValue,
Coder<T> valueCoder) {
+ // TODO: as soon as runners are ported off the indicator classes,
+ // return new SimplePCollectionView<>(
+ // pipeline,
+ // new SingletonViewFn<K, V>(hasDefault, defaultValue, valueCoder),
+ // windowingStrategy,
+ // valueCoder);
return new SingletonPCollectionView<>(
pipeline, windowingStrategy, hasDefault, defaultValue, valueCoder);
}
@@ -77,6 +84,9 @@ public class PCollectionViews {
Pipeline pipeline,
WindowingStrategy<?, W> windowingStrategy,
Coder<T> valueCoder) {
+ // TODO: as soon as runners are ported off the indicator classes,
+ // return new SimplePCollectionView<>(
+ // pipeline, new IterableViewFn<T>(), windowingStrategy, valueCoder);
return new IterablePCollectionView<>(pipeline, windowingStrategy, valueCoder);
}
@@ -88,6 +98,9 @@ public class PCollectionViews {
Pipeline pipeline,
WindowingStrategy<?, W> windowingStrategy,
Coder<T> valueCoder) {
+ // TODO: as soon as runners are ported off the indicator classes,
+ // return new SimplePCollectionView<>(
+ // pipeline, new ListViewFn<T>(), windowingStrategy, valueCoder);
return new ListPCollectionView<>(pipeline, windowingStrategy, valueCoder);
}
@@ -99,8 +112,10 @@ public class PCollectionViews {
Pipeline pipeline,
WindowingStrategy<?, W> windowingStrategy,
Coder<KV<K, V>> valueCoder) {
-
- return new MapPCollectionView<K, V, W>(pipeline, windowingStrategy, valueCoder);
+ // TODO: as soon as runners are ported off the indicator classes,
+ // return new SimplePCollectionView<>(
+ // pipeline, new MapViewFn<K, V>(), windowingStrategy, valueCoder);
+ return new MapPCollectionView<>(pipeline, windowingStrategy, valueCoder);
}
/**
@@ -111,27 +126,119 @@ public class PCollectionViews {
Pipeline pipeline,
WindowingStrategy<?, W> windowingStrategy,
Coder<KV<K, V>> valueCoder) {
- return new MultimapPCollectionView<K, V, W>(pipeline, windowingStrategy, valueCoder);
+ // TODO: as soon as runners are ported off the indicator classes,
+ // return new SimplePCollectionView<>(
+ // pipeline, new MultimapViewFn<K, V>(), windowingStrategy, valueCoder);
+ return new MultimapPCollectionView<>(pipeline, windowingStrategy, valueCoder);
+ }
+
+ /**
+ * A public indicator class that this view is a singleton view.
+ *
+ * @deprecated Runners should not inspect the {@link PCollectionView} subclass, as it is an
+ * implementation detail. To specialize a side input, a runner should inspect the
+ * language-independent metadata of the {@link ViewFn}.
+ */
+ @Deprecated
+ public static class SingletonPCollectionView<T, W extends BoundedWindow>
+ extends SimplePCollectionView<T, T, W> {
+ public SingletonPCollectionView(
+ Pipeline pipeline,
+ WindowingStrategy<?, W> windowingStrategy,
+ boolean hasDefault,
+ T defaultValue,
+ Coder<T> valueCoder) {
+ super(
+ pipeline,
+ new SingletonViewFn<>(hasDefault, defaultValue, valueCoder),
+ windowingStrategy,
+ valueCoder);
+ }
+
+ public T getDefaultValue() {
+ return ((SingletonViewFn<T>) viewFn).getDefaultValue();
+ }
+ }
+
+ /**
+ * A public indicator class that this view is an iterable view.
+ *
+ * @deprecated Runners should not inspect the {@link PCollectionView} subclass, as it is an
+ * implementation detail. To specialize a side input, a runner should inspect the
+ * language-independent metadata of the {@link ViewFn}.
+ */
+ @Deprecated
+ public static class IterablePCollectionView<ElemT, W extends BoundedWindow>
+ extends SimplePCollectionView<ElemT, Iterable<ElemT>, W> {
+ public IterablePCollectionView(
+ Pipeline pipeline, WindowingStrategy<?, W> windowingStrategy, Coder<ElemT> valueCoder) {
+ super(pipeline, new IterableViewFn<ElemT>(), windowingStrategy, valueCoder);
+ }
+ }
+
+ /**
+ * A public indicator class that this view is a list view.
+ *
+ * @deprecated Runners should not inspect the {@link PCollectionView} subclass, as it is an
+ * implementation detail. To specialize a side input, a runner should inspect the
+ * language-independent metadata of the {@link ViewFn}.
+ */
+ @Deprecated
+ public static class ListPCollectionView<ElemT, W extends BoundedWindow>
+ extends SimplePCollectionView<ElemT, List<ElemT>, W> {
+ public ListPCollectionView(
+ Pipeline pipeline, WindowingStrategy<?, W> windowingStrategy, Coder<ElemT> valueCoder) {
+ super(pipeline, new ListViewFn<ElemT>(), windowingStrategy, valueCoder);
+ }
+ }
+
+ /**
+ * A public indicator class that this view is a map view.
+ *
+ * @deprecated Runners should not inspect the {@link PCollectionView} subclass, as it is an
+ * implementation detail. To specialize a side input, a runner should inspect the
+ * language-independent metadata of the {@link ViewFn}.
+ */
+ @Deprecated
+ public static class MapPCollectionView<K, V, W extends BoundedWindow>
+ extends SimplePCollectionView<KV<K, V>, Map<K, V>, W> {
+ public MapPCollectionView(
+ Pipeline pipeline, WindowingStrategy<?, W> windowingStrategy, Coder<KV<K, V>> valueCoder) {
+ super(pipeline, new MapViewFn<K, V>(), windowingStrategy, valueCoder);
+ }
}
/**
+ * A public indicator class that this view is a multimap view.
+ *
+ * @deprecated Runners should not inspect the {@link PCollectionView} subclass, as it is an
+ * implementation detail. To specialize a side input, a runner should inspect the
+ * language-independent metadata of the {@link ViewFn}.
+ */
+ @Deprecated
+ public static class MultimapPCollectionView<K, V, W extends BoundedWindow>
+ extends SimplePCollectionView<KV<K, V>, Map<K, Iterable<V>>, W> {
+ public MultimapPCollectionView(
+ Pipeline pipeline, WindowingStrategy<?, W> windowingStrategy, Coder<KV<K, V>> valueCoder) {
+ super(pipeline, new MultimapViewFn<K, V>(), windowingStrategy, valueCoder);
+ }
+ }
+
+
+ /**
* Implementation of conversion of singleton {@code Iterable<WindowedValue<T>>} to {@code T}.
*
* <p>For internal use only.
*
* <p>Instantiate via {@link PCollectionViews#singletonView}.
*/
- public static class SingletonPCollectionView<T, W extends BoundedWindow>
- extends PCollectionViewBase<T, T, W> {
+ private static class SingletonViewFn<T> extends ViewFn<Iterable<WindowedValue<T>>, T> {
@Nullable private byte[] encodedDefaultValue;
@Nullable private transient T defaultValue;
@Nullable private Coder<T> valueCoder;
private boolean hasDefault;
- private SingletonPCollectionView(
- Pipeline pipeline, WindowingStrategy<?, W> windowingStrategy,
- boolean hasDefault, T defaultValue, Coder<T> valueCoder) {
- super(pipeline, windowingStrategy, valueCoder);
+ private SingletonViewFn(boolean hasDefault, T defaultValue, Coder<T> valueCoder) {
this.hasDefault = hasDefault;
this.defaultValue = defaultValue;
this.valueCoder = valueCoder;
@@ -170,7 +277,7 @@ public class PCollectionViews {
}
@Override
- protected T fromElements(Iterable<WindowedValue<T>> contents) {
+ public T apply(Iterable<WindowedValue<T>> contents) {
try {
return Iterables.getOnlyElement(contents).getValue();
} catch (NoSuchElementException exc) {
@@ -178,7 +285,7 @@ public class PCollectionViews {
} catch (IllegalArgumentException exc) {
throw new IllegalArgumentException(
"PCollection with more than one element "
- + "accessed as a singleton view.");
+ + "accessed as a singleton view.");
}
}
}
@@ -190,15 +297,11 @@ public class PCollectionViews {
*
* <p>Instantiate via {@link PCollectionViews#iterableView}.
*/
- public static class IterablePCollectionView<T, W extends BoundedWindow>
- extends PCollectionViewBase<T, Iterable<T>, W> {
- private IterablePCollectionView(
- Pipeline pipeline, WindowingStrategy<?, W> windowingStrategy, Coder<T> valueCoder) {
- super(pipeline, windowingStrategy, valueCoder);
- }
+ private static class IterableViewFn<T>
+ extends ViewFn<Iterable<WindowedValue<T>>, Iterable<T>> {
@Override
- protected Iterable<T> fromElements(Iterable<WindowedValue<T>> contents) {
+ public Iterable<T> apply(Iterable<WindowedValue<T>> contents) {
return Iterables.unmodifiableIterable(
Iterables.transform(contents, new Function<WindowedValue<T>, T>() {
@SuppressWarnings("unchecked")
@@ -217,15 +320,9 @@ public class PCollectionViews {
*
* <p>Instantiate via {@link PCollectionViews#listView}.
*/
- public static class ListPCollectionView<T, W extends BoundedWindow>
- extends PCollectionViewBase<T, List<T>, W> {
- private ListPCollectionView(
- Pipeline pipeline, WindowingStrategy<?, W> windowingStrategy, Coder<T> valueCoder) {
- super(pipeline, windowingStrategy, valueCoder);
- }
-
+ private static class ListViewFn<T> extends ViewFn<Iterable<WindowedValue<T>>, List<T>> {
@Override
- protected List<T> fromElements(Iterable<WindowedValue<T>> contents) {
+ public List<T> apply(Iterable<WindowedValue<T>> contents) {
return ImmutableList.copyOf(
Iterables.transform(contents, new Function<WindowedValue<T>, T>() {
@SuppressWarnings("unchecked")
@@ -240,20 +337,12 @@ public class PCollectionViews {
/**
* Implementation of conversion {@code Iterable<WindowedValue<KV<K, V>>>}
* to {@code Map<K, Iterable<V>>}.
- *
- * <p>For internal use only.
*/
- public static class MultimapPCollectionView<K, V, W extends BoundedWindow>
- extends PCollectionViewBase<KV<K, V>, Map<K, Iterable<V>>, W> {
- private MultimapPCollectionView(
- Pipeline pipeline,
- WindowingStrategy<?, W> windowingStrategy,
- Coder<KV<K, V>> valueCoder) {
- super(pipeline, windowingStrategy, valueCoder);
- }
+ private static class MultimapViewFn<K, V>
+ extends ViewFn<Iterable<WindowedValue<KV<K, V>>>, Map<K, Iterable<V>>> {
@Override
- protected Map<K, Iterable<V>> fromElements(Iterable<WindowedValue<KV<K, V>>> elements) {
+ public Map<K, Iterable<V>> apply(Iterable<WindowedValue<KV<K, V>>> elements) {
Multimap<K, V> multimap = HashMultimap.create();
for (WindowedValue<KV<K, V>> elem : elements) {
KV<K, V> kv = elem.getValue();
@@ -267,25 +356,16 @@ public class PCollectionViews {
}
/**
- * Implementation of conversion {@code Iterable<WindowedValue<KV<K, V>>} with
- * one value per key to {@code Map<K, V>}.
- *
- * <p>For internal use only.
+ * Implementation of conversion {@code Iterable<WindowedValue<KV<K, V>>} with one value per key
+ * to {@code Map<K, V>}.
*/
- public static class MapPCollectionView<K, V, W extends BoundedWindow>
- extends PCollectionViewBase<KV<K, V>, Map<K, V>, W> {
- private MapPCollectionView(
- Pipeline pipeline,
- WindowingStrategy<?, W> windowingStrategy,
- Coder<KV<K, V>> valueCoder) {
- super(pipeline, windowingStrategy, valueCoder);
- }
-
+ private static class MapViewFn<K, V>
+ extends ViewFn<Iterable<WindowedValue<KV<K, V>>>, Map<K, V>> {
/**
* Input iterable must actually be {@code Iterable<WindowedValue<KV<K, V>>>}.
*/
@Override
- protected Map<K, V> fromElements(Iterable<WindowedValue<KV<K, V>>> elements) {
+ public Map<K, V> apply(Iterable<WindowedValue<KV<K, V>>> elements) {
Map<K, V> map = new HashMap<>();
for (WindowedValue<KV<K, V>> elem : elements) {
KV<K, V> kv = elem.getValue();
@@ -302,7 +382,7 @@ public class PCollectionViews {
* A base class for {@link PCollectionView} implementations, with additional type parameters
* that are not visible at pipeline assembly time when the view is used as a side input.
*/
- private abstract static class PCollectionViewBase<ElemT, ViewT, W extends BoundedWindow>
+ private static class SimplePCollectionView<ElemT, ViewT, W extends BoundedWindow>
extends PValueBase
implements PCollectionView<ViewT> {
/** A unique tag for the view, typed according to the elements underlying the view. */
@@ -315,18 +395,23 @@ public class PCollectionViews {
private Coder<Iterable<WindowedValue<ElemT>>> coder;
/**
- * Implement this to complete the implementation. It is a conversion function from
- * all of the elements of the underlying {@link PCollection} to the value of the view.
+ * The typed {@link ViewFn} for this view.
+ *
+ * @deprecated Access to this variable from subclasses is temporary, for migrating away
+ * from language-specific inspections.
*/
- protected abstract ViewT fromElements(Iterable<WindowedValue<ElemT>> elements);
+ @Deprecated
+ protected ViewFn<Iterable<WindowedValue<ElemT>>, ViewT> viewFn;
/**
* Call this constructor to initialize the fields for which this base class provides
* boilerplate accessors.
*/
- protected PCollectionViewBase(
+ // TODO: make private as soon as runners are ported off indicator subclasses
+ protected SimplePCollectionView(
Pipeline pipeline,
TupleTag<Iterable<WindowedValue<ElemT>>> tag,
+ ViewFn<Iterable<WindowedValue<ElemT>>, ViewT> viewFn,
WindowingStrategy<?, W> windowingStrategy,
Coder<ElemT> valueCoder) {
super(pipeline);
@@ -335,6 +420,7 @@ public class PCollectionViews {
}
this.tag = tag;
this.windowingStrategy = windowingStrategy;
+ this.viewFn = viewFn;
this.coder =
IterableCoder.of(WindowedValue.getFullCoder(
valueCoder, windowingStrategy.getWindowFn().windowCoder()));
@@ -344,30 +430,42 @@ public class PCollectionViews {
* Call this constructor to initialize the fields for which this base class provides
* boilerplate accessors, with an auto-generated tag.
*/
- protected PCollectionViewBase(
+ // TODO: make private as soon as runners are ported off indicator subclasses
+ protected SimplePCollectionView(
Pipeline pipeline,
+ ViewFn<Iterable<WindowedValue<ElemT>>, ViewT> viewFn,
WindowingStrategy<?, W> windowingStrategy,
Coder<ElemT> valueCoder) {
- this(pipeline, new TupleTag<Iterable<WindowedValue<ElemT>>>(), windowingStrategy, valueCoder);
+ this(
+ pipeline,
+ new TupleTag<Iterable<WindowedValue<ElemT>>>(),
+ viewFn,
+ windowingStrategy,
+ valueCoder);
}
/**
- * For serialization only. Do not use directly. Subclasses should call from their own
- * protected no-argument constructor.
+ * For serialization only. Do not use directly.
*/
@SuppressWarnings("unused") // used for serialization
- protected PCollectionViewBase() {
+ protected SimplePCollectionView() {
super();
}
@Override
- public ViewT fromIterableInternal(Iterable<WindowedValue<?>> elements) {
+ public ViewFn<Iterable<WindowedValue<?>>, ViewT> getViewFn() {
// Safe cast: it is required that the rest of the SDK maintain the invariant
// that a PCollectionView is only provided an iterable for the elements of an
// appropriately typed PCollection.
@SuppressWarnings({"rawtypes", "unchecked"})
- Iterable<WindowedValue<ElemT>> typedElements = (Iterable) elements;
- return fromElements(typedElements);
+ ViewFn<Iterable<WindowedValue<?>>, ViewT> untypedViewFn = (ViewFn) viewFn;
+ return untypedViewFn;
+ }
+
+ @Override
+ @Deprecated
+ public ViewT fromIterableInternal(Iterable<WindowedValue<?>> elements) {
+ return getViewFn().apply(elements);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java
index fb3bfab..20f1071 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionView.java
@@ -20,6 +20,7 @@ package org.apache.beam.sdk.values;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.ViewFn;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingStrategy;
@@ -44,22 +45,42 @@ import java.io.Serializable;
*/
public interface PCollectionView<T> extends PValue, Serializable {
/**
- * A unique identifier, for internal use.
+ * @deprecated this method will be removed entirely. The {@link PCollection} underlying a side
+ * input, is part of the side input's specification with a {@link ParDo} transform, which will
+ * obtain that information via a package-private channel.
*/
+ @Deprecated
public TupleTag<Iterable<WindowedValue<?>>> getTagInternal();
/**
- * For internal use only.
+ * @deprecated use {@link #getViewFn()} for now, but eventually get the needed information via the
+ * side input specification on the {@link ParDo} transform.
*/
+ @Deprecated
public T fromIterableInternal(Iterable<WindowedValue<?>> contents);
/**
- * For internal use only.
+ * @deprecated this method will be removed entirely. The {@link ViewFn} for a side input is an
+ * attribute of the side input's specification with a {@link ParDo} transform, which will
+ * obtain this specification via a package-private channel.
*/
+ @Deprecated
+ public ViewFn<Iterable<WindowedValue<?>>, T> getViewFn();
+
+ /**
+ * @deprecated this method will be removed entirely. The {@link PCollection} underlying a side
+ * input, including its {@link WindowingStrategy}, is part of the side input's specification
+ * with a {@link ParDo} transform, which will obtain that information via a package-private
+ * channel.
+ */
+ @Deprecated
public WindowingStrategy<?, ?> getWindowingStrategyInternal();
/**
- * For internal use only.
+ * @deprecated this method will be removed entirely. The {@link PCollection} underlying a side
+ * input, including its {@link Coder}, is part of the side input's specification with a {@link
+ * ParDo} transform, which will obtain that information via a package-private channel.
*/
+ @Deprecated
public Coder<Iterable<WindowedValue<?>>> getCoderInternal();
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c376b45c/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
index 427f2da..517ed68 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
@@ -19,7 +19,7 @@ package org.apache.beam.sdk.testing;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
-import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.ViewFn;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
@@ -83,16 +83,9 @@ public final class PCollectionViewTesting {
DEFAULT_NONEMPTY_WINDOW.maxTimestamp().plus(DEFAULT_WINDOW_MSECS));
/**
- * A specialization of {@link SerializableFunction} just for putting together
- * {@link PCollectionView} instances.
- */
- public static interface ViewFn<ElemT, ViewT>
- extends SerializableFunction<Iterable<WindowedValue<ElemT>>, ViewT> { }
-
- /**
* A {@link ViewFn} that returns the provided contents as a fully lazy iterable.
*/
- public static class IdentityViewFn<T> implements ViewFn<T, Iterable<T>> {
+ public static class IdentityViewFn<T> extends ViewFn<Iterable<WindowedValue<T>>, Iterable<T>> {
@Override
public Iterable<T> apply(Iterable<WindowedValue<T>> contents) {
return Iterables.transform(contents, new Function<WindowedValue<T>, T>() {
@@ -110,7 +103,7 @@ public final class PCollectionViewTesting {
* <p>Only for use in testing scenarios with small collections. If there are more elements
* provided than {@code Integer.MAX_VALUE} then behavior is unpredictable.
*/
- public static class LengthViewFn<T> implements ViewFn<T, Long> {
+ public static class LengthViewFn<T> extends ViewFn<Iterable<WindowedValue<T>>, Long> {
@Override
public Long apply(Iterable<WindowedValue<T>> contents) {
return (long) Iterables.size(contents);
@@ -120,7 +113,8 @@ public final class PCollectionViewTesting {
/**
* A {@link ViewFn} that always returns the value with which it is instantiated.
*/
- public static class ConstantViewFn<ElemT, ViewT> implements ViewFn<ElemT, ViewT> {
+ public static class ConstantViewFn<ElemT, ViewT>
+ extends ViewFn<Iterable<WindowedValue<ElemT>>, ViewT> {
private ViewT value;
public ConstantViewFn(ViewT value) {
@@ -148,7 +142,7 @@ public final class PCollectionViewTesting {
*/
public static <ElemT, ViewT> PCollectionView<ViewT> testingView(
TupleTag<Iterable<WindowedValue<ElemT>>> tag,
- ViewFn<ElemT, ViewT> viewFn,
+ ViewFn<Iterable<WindowedValue<ElemT>>, ViewT> viewFn,
Coder<ElemT> elemCoder) {
return testingView(
tag,
@@ -178,7 +172,7 @@ public final class PCollectionViewTesting {
*/
public static <ElemT, ViewT> PCollectionView<ViewT> testingView(
TupleTag<Iterable<WindowedValue<ElemT>>> tag,
- ViewFn<ElemT, ViewT> viewFn,
+ ViewFn<Iterable<WindowedValue<ElemT>>, ViewT> viewFn,
Coder<ElemT> elemCoder,
WindowingStrategy<?, ?> windowingStrategy) {
return new PCollectionViewFromParts<>(
@@ -233,13 +227,13 @@ public final class PCollectionViewTesting {
extends PValueBase
implements PCollectionView<ViewT> {
private TupleTag<Iterable<WindowedValue<ElemT>>> tag;
- private ViewFn<ElemT, ViewT> viewFn;
+ private ViewFn<Iterable<WindowedValue<ElemT>>, ViewT> viewFn;
private WindowingStrategy<?, ?> windowingStrategy;
private Coder<Iterable<WindowedValue<ElemT>>> coder;
public PCollectionViewFromParts(
TupleTag<Iterable<WindowedValue<ElemT>>> tag,
- ViewFn<ElemT, ViewT> viewFn,
+ ViewFn<Iterable<WindowedValue<ElemT>>, ViewT> viewFn,
WindowingStrategy<?, ?> windowingStrategy,
Coder<Iterable<WindowedValue<ElemT>>> coder) {
this.tag = tag;
@@ -256,8 +250,17 @@ public final class PCollectionViewTesting {
@SuppressWarnings({"unchecked", "rawtypes"})
@Override
+ @Deprecated
public ViewT fromIterableInternal(Iterable<WindowedValue<?>> contents) {
- return (ViewT) viewFn.apply((Iterable) contents);
+ return getViewFn().apply(contents);
+ }
+
+ @Override
+ public ViewFn<Iterable<WindowedValue<?>>, ViewT> getViewFn() {
+ // Safe cast; runners must maintain type safety
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ ViewFn<Iterable<WindowedValue<?>>, ViewT> untypedViewFn = (ViewFn) viewFn;
+ return untypedViewFn;
}
@Override
[42/50] incubator-beam git commit: This closes #813
Posted by ke...@apache.org.
This closes #813
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6da92ad9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6da92ad9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6da92ad9
Branch: refs/heads/gearpump-runner
Commit: 6da92ad94701efce2643969ba9158371f6527a80
Parents: b4c0bb9 9fe6ce2
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Aug 10 14:28:12 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Aug 10 14:28:12 2016 -0700
----------------------------------------------------------------------
.../apache/beam/sdk/util/TimerInternals.java | 15 ++++++-
.../beam/sdk/util/TimerInternalsTest.java | 47 ++++++++++++++++++++
2 files changed, 61 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[50/50] incubator-beam git commit: This closess #750
Posted by ke...@apache.org.
This closess #750
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/091a15a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/091a15a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/091a15a0
Branch: refs/heads/gearpump-runner
Commit: 091a15a07c7625ae3009cefaecece3a29a34c109
Parents: 02b2248 bc1b354
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Aug 25 11:40:03 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu Aug 25 11:40:03 2016 -0700
----------------------------------------------------------------------
.travis.yml | 17 +-
.travis/README.md | 23 +
.travis/settings.xml | 33 +
.travis/test_wordcount.sh | 125 +
KEYS | 141 -
NOTICE | 1 -
README.md | 25 +-
examples/java/pom.xml | 48 +-
.../beam/examples/DebuggingWordCount.java | 2 +-
.../apache/beam/examples/MinimalWordCount.java | 2 +-
.../apache/beam/examples/WindowedWordCount.java | 6 +-
.../org/apache/beam/examples/WordCount.java | 26 +-
.../beam/examples/common/ExampleUtils.java | 4 +-
.../examples/common/PubsubFileInjector.java | 10 +-
.../beam/examples/complete/AutoComplete.java | 37 +-
.../examples/complete/StreamingWordExtract.java | 12 +-
.../apache/beam/examples/complete/TfIdf.java | 12 +-
.../examples/complete/TopWikipediaSessions.java | 29 +-
.../examples/complete/TrafficMaxLaneFlow.java | 8 +-
.../beam/examples/complete/TrafficRoutes.java | 10 +-
.../examples/cookbook/BigQueryTornadoes.java | 6 +-
.../cookbook/CombinePerKeyExamples.java | 6 +-
.../examples/cookbook/DatastoreWordCount.java | 9 +-
.../beam/examples/cookbook/FilterExamples.java | 10 +-
.../beam/examples/cookbook/JoinExamples.java | 10 +-
.../examples/cookbook/MaxPerKeyExamples.java | 6 +-
.../beam/examples/cookbook/TriggerExample.java | 21 +-
.../org/apache/beam/examples/WordCountIT.java | 2 +-
.../org/apache/beam/examples/WordCountTest.java | 3 +-
.../examples/complete/AutoCompleteTest.java | 2 +-
.../examples/cookbook/TriggerExampleTest.java | 2 +-
examples/java/src/test/resources/LICENSE | 202 ++
examples/java8/pom.xml | 7 +-
.../beam/examples/complete/game/GameStats.java | 16 +-
.../beam/examples/complete/game/UserScore.java | 2 +-
.../complete/game/utils/WriteToBigQuery.java | 18 +-
.../game/utils/WriteWindowedToBigQuery.java | 14 +-
.../examples/complete/game/UserScoreTest.java | 2 +-
examples/pom.xml | 2 +-
pom.xml | 72 +-
runners/core-java/pom.xml | 2 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 28 +-
.../core/UnboundedReadFromBoundedSource.java | 6 +-
.../apache/beam/runners/core/package-info.java | 22 +
.../apache/beam/sdk/util/AssignWindowsDoFn.java | 10 +-
.../org/apache/beam/sdk/util/DoFnRunner.java | 21 +-
.../apache/beam/sdk/util/DoFnRunnerBase.java | 82 +-
.../org/apache/beam/sdk/util/DoFnRunners.java | 104 +-
.../beam/sdk/util/GroupAlsoByWindowsDoFn.java | 16 +-
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 14 +-
.../sdk/util/GroupByKeyViaGroupByKeyOnly.java | 85 +-
.../sdk/util/LateDataDroppingDoFnRunner.java | 4 +-
.../apache/beam/sdk/util/PaneInfoTracker.java | 1 -
.../apache/beam/sdk/util/ReduceFnRunner.java | 4 +-
.../apache/beam/sdk/util/SimpleDoFnRunner.java | 19 +-
.../org/apache/beam/sdk/util/WatermarkHold.java | 1 -
.../beam/sdk/util/common/package-info.java | 20 +
.../org/apache/beam/sdk/util/package-info.java | 20 +
.../UnboundedReadFromBoundedSourceTest.java | 2 +-
.../sdk/util/GroupAlsoByWindowsProperties.java | 50 +-
...oupAlsoByWindowsViaOutputBufferDoFnTest.java | 8 +-
.../beam/sdk/util/ReduceFnRunnerTest.java | 1 +
.../apache/beam/sdk/util/ReduceFnTester.java | 1 +
.../beam/sdk/util/SimpleDoFnRunnerTest.java | 6 +-
runners/direct-java/pom.xml | 2 +-
.../runners/direct/AggregatorContainer.java | 183 ++
.../direct/BoundedReadEvaluatorFactory.java | 26 +-
.../beam/runners/direct/CloningThreadLocal.java | 43 +
.../beam/runners/direct/CommittedResult.java | 23 +-
.../beam/runners/direct/CompletionCallback.java | 8 +
.../beam/runners/direct/DirectRegistrar.java | 4 +-
.../beam/runners/direct/DirectRunner.java | 57 +-
.../beam/runners/direct/EvaluationContext.java | 57 +-
.../beam/runners/direct/EvaluatorKey.java | 55 -
.../direct/ExecutorServiceParallelExecutor.java | 186 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 33 +-
.../ImmutabilityCheckingBundleFactory.java | 4 +-
.../beam/runners/direct/ParDoEvaluator.java | 24 +-
.../direct/ParDoMultiEvaluatorFactory.java | 53 +-
.../direct/ParDoSingleEvaluatorFactory.java | 59 +-
...rializableCloningThreadLocalCacheLoader.java | 54 -
.../beam/runners/direct/SideInputContainer.java | 2 +-
.../runners/direct/StepTransformResult.java | 95 +-
.../direct/TransformEvaluatorFactory.java | 10 +-
.../beam/runners/direct/TransformExecutor.java | 3 +-
.../beam/runners/direct/TransformResult.java | 16 +-
.../direct/UnboundedReadEvaluatorFactory.java | 32 +-
.../runners/direct/ViewEvaluatorFactory.java | 14 +-
.../direct/WriteWithShardingFactory.java | 11 +-
.../runners/direct/AggregatorContainerTest.java | 134 +
.../runners/direct/CloningThreadLocalTest.java | 92 +
.../runners/direct/CommittedResultTest.java | 26 +-
.../ConsumerTrackingPipelineVisitorTest.java | 22 +-
.../runners/direct/DirectRegistrarTest.java | 17 +-
.../beam/runners/direct/DirectRunnerTest.java | 72 +-
.../runners/direct/EvaluationContextTest.java | 32 +-
.../ImmutabilityCheckingBundleFactoryTest.java | 6 +-
.../ImmutabilityEnforcementFactoryTest.java | 6 +-
.../direct/KeyedPValueTrackingVisitorTest.java | 6 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 18 +-
.../direct/ParDoMultiEvaluatorFactoryTest.java | 41 +-
.../direct/ParDoSingleEvaluatorFactoryTest.java | 41 +-
...izableCloningThreadLocalCacheLoaderTest.java | 99 -
.../runners/direct/StepTransformResultTest.java | 91 +
.../runners/direct/TransformExecutorTest.java | 33 +-
.../runners/direct/WatermarkManagerTest.java | 18 +-
runners/flink/README.md | 25 +-
runners/flink/examples/pom.xml | 13 +-
.../beam/runners/flink/examples/TFIDF.java | 12 +-
.../beam/runners/flink/examples/WordCount.java | 7 +-
.../flink/examples/streaming/AutoComplete.java | 25 +-
.../flink/examples/streaming/JoinExamples.java | 6 +-
.../examples/streaming/KafkaIOExamples.java | 3 +-
.../KafkaWindowedWordCountExample.java | 4 +-
.../examples/streaming/WindowedWordCount.java | 4 +-
runners/flink/pom.xml | 2 +-
runners/flink/runner/pom.xml | 116 +-
.../beam/runners/flink/FlinkRunnerResult.java | 26 +-
.../FlinkBatchTransformTranslators.java | 12 +-
.../FlinkStreamingTransformTranslators.java | 9 +-
.../functions/FlinkDoFnFunction.java | 10 +-
.../FlinkMergingNonShuffleReduceFunction.java | 8 +-
.../functions/FlinkMultiOutputDoFnFunction.java | 10 +-
.../FlinkMultiOutputProcessContext.java | 6 +-
.../functions/FlinkNoElementAssignContext.java | 8 +-
.../functions/FlinkPartialReduceFunction.java | 8 +-
.../functions/FlinkProcessContext.java | 18 +-
.../functions/FlinkReduceFunction.java | 8 +-
.../functions/SideInputInitializer.java | 2 +-
.../streaming/FlinkAbstractParDoWrapper.java | 18 +-
.../FlinkGroupAlsoByWindowWrapper.java | 41 +-
.../streaming/FlinkParDoBoundMultiWrapper.java | 4 +-
.../streaming/FlinkParDoBoundWrapper.java | 4 +-
.../state/AbstractFlinkTimerInternals.java | 4 +-
.../beam/runners/flink/PipelineOptionsTest.java | 6 +-
.../beam/runners/flink/ReadSourceITCase.java | 2 +-
.../flink/ReadSourceStreamingITCase.java | 4 +-
.../flink/streaming/GroupByNullKeyTest.java | 8 +-
.../streaming/TopWikipediaSessionsITCase.java | 6 +-
runners/gearpump/pom.xml | 111 +-
.../gearpump/GearpumpPipelineResult.java | 23 +-
.../gearpump/GearpumpPipelineRunner.java | 6 +-
.../gearpump/examples/StreamingWordCount.java | 6 +-
.../runners/gearpump/examples/package-info.java | 22 +
.../beam/runners/gearpump/package-info.java | 22 +
.../translators/ParDoBoundMultiTranslator.java | 3 +-
.../translators/ParDoBoundTranslator.java | 3 +-
.../translators/functions/DoFnFunction.java | 3 +-
.../translators/functions/package-info.java | 22 +
.../gearpump/translators/io/package-info.java | 22 +
.../gearpump/translators/package-info.java | 22 +
.../translators/utils/GearpumpDoFnRunner.java | 23 +-
.../translators/utils/package-info.java | 22 +
runners/google-cloud-dataflow-java/pom.xml | 35 +-
.../dataflow/BlockingDataflowRunner.java | 5 +-
.../runners/dataflow/DataflowPipelineJob.java | 106 +-
.../dataflow/DataflowPipelineTranslator.java | 6 +-
.../beam/runners/dataflow/DataflowRunner.java | 198 +-
.../dataflow/internal/AssignWindows.java | 6 +-
.../DataflowUnboundedReadFromBoundedSource.java | 4 +-
.../options/DataflowPipelineDebugOptions.java | 43 -
.../options/DataflowPipelineOptions.java | 6 +-
.../runners/dataflow/options/package-info.java | 22 +
.../beam/runners/dataflow/package-info.java | 22 +
.../dataflow/testing/TestDataflowRunner.java | 7 +-
.../runners/dataflow/testing/package-info.java | 24 +
.../dataflow/util/DataflowPathValidator.java | 98 -
.../beam/runners/dataflow/util/DoFnInfo.java | 16 +-
.../runners/dataflow/util/package-info.java | 20 +
.../dataflow/BlockingDataflowRunnerTest.java | 9 +-
.../dataflow/DataflowPipelineJobTest.java | 47 +-
.../DataflowPipelineTranslatorTest.java | 12 +-
.../runners/dataflow/DataflowRunnerTest.java | 39 +-
.../options/DataflowPipelineOptionsTest.java | 3 +
.../testing/TestDataflowRunnerTest.java | 34 +-
.../util/DataflowPathValidatorTest.java | 94 -
runners/pom.xml | 72 +-
runners/spark/README.md | 3 +-
runners/spark/pom.xml | 27 +-
.../runners/spark/SparkPipelineOptions.java | 16 +-
.../apache/beam/runners/spark/SparkRunner.java | 33 +-
.../runners/spark/SparkRunnerRegistrar.java | 6 +-
.../spark/SparkStreamingPipelineOptions.java | 41 -
.../beam/runners/spark/TestSparkRunner.java | 2 -
.../runners/spark/aggregators/package-info.java | 20 +
.../beam/runners/spark/coders/package-info.java | 22 +
.../beam/runners/spark/examples/WordCount.java | 137 +
.../runners/spark/io/hadoop/package-info.java | 22 +
.../beam/runners/spark/io/package-info.java | 22 +
.../apache/beam/runners/spark/package-info.java | 22 +
.../runners/spark/translation/DoFnFunction.java | 8 +-
.../spark/translation/EvaluationContext.java | 25 +-
.../spark/translation/MultiDoFnFunction.java | 8 +-
.../spark/translation/SparkContextFactory.java | 8 +-
.../spark/translation/SparkProcessContext.java | 20 +-
.../spark/translation/SparkRuntimeContext.java | 2 +-
.../spark/translation/TransformTranslator.java | 73 +-
.../runners/spark/translation/package-info.java | 22 +
.../streaming/StreamingTransformTranslator.java | 14 +-
.../StreamingWindowPipelineDetector.java | 102 -
.../translation/streaming/package-info.java | 22 +
.../beam/runners/spark/util/package-info.java | 22 +
.../apache/beam/runners/spark/DeDupTest.java | 2 +-
.../beam/runners/spark/EmptyInputTest.java | 2 +-
.../beam/runners/spark/SimpleWordCountTest.java | 80 +-
.../runners/spark/SparkRunnerRegistrarTest.java | 2 +-
.../apache/beam/runners/spark/TfIdfTest.java | 201 +-
.../beam/runners/spark/io/AvroPipelineTest.java | 7 +-
.../beam/runners/spark/io/NumShardsTest.java | 4 +-
.../io/hadoop/HadoopFileFormatPipelineTest.java | 7 +-
.../spark/translation/CombineGloballyTest.java | 2 +-
.../spark/translation/CombinePerKeyTest.java | 9 +-
.../spark/translation/DoFnOutputTest.java | 10 +-
.../translation/MultiOutputWordCountTest.java | 15 +-
.../spark/translation/SerializationTest.java | 10 +-
.../spark/translation/SideEffectsTest.java | 10 +-
.../translation/TransformTranslatorTest.java | 13 +-
.../translation/WindowedWordCountTest.java | 12 +-
.../streaming/FlattenStreamingTest.java | 11 +-
.../streaming/KafkaStreamingTest.java | 19 +-
.../streaming/SimpleStreamingWordCountTest.java | 18 +-
.../streaming/utils/EmbeddedKafkaCluster.java | 4 +-
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/checkstyle.xml | 14 +
.../src/main/resources/beam/findbugs-filter.xml | 13 +-
.../src/main/resources/beam/suppressions.xml | 24 +
sdks/java/core/pom.xml | 27 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 93 +
.../beam/sdk/AggregatorRetrievalException.java | 33 +
.../org/apache/beam/sdk/AggregatorValues.java | 52 +
.../main/java/org/apache/beam/sdk/Pipeline.java | 12 +-
.../org/apache/beam/sdk/PipelineResult.java | 40 +-
.../org/apache/beam/sdk/coders/AvroCoder.java | 1 -
.../java/org/apache/beam/sdk/coders/Coder.java | 2 +-
.../apache/beam/sdk/coders/CoderRegistry.java | 38 +-
.../apache/beam/sdk/coders/DurationCoder.java | 1 -
.../apache/beam/sdk/coders/InstantCoder.java | 1 -
.../org/apache/beam/sdk/coders/JAXBCoder.java | 2 +-
.../apache/beam/sdk/coders/StandardCoder.java | 14 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 3 +-
.../java/org/apache/beam/sdk/io/BigQueryIO.java | 2447 ----------------
.../org/apache/beam/sdk/io/FileBasedSink.java | 2 +-
.../java/org/apache/beam/sdk/io/PubsubIO.java | 14 +-
.../apache/beam/sdk/io/PubsubUnboundedSink.java | 9 +-
.../beam/sdk/io/PubsubUnboundedSource.java | 3 +-
.../java/org/apache/beam/sdk/io/Source.java | 2 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 17 +-
.../org/apache/beam/sdk/io/package-info.java | 3 +-
.../org/apache/beam/sdk/options/GcpOptions.java | 11 +-
.../org/apache/beam/sdk/options/GcsOptions.java | 44 +
.../beam/sdk/options/PipelineOptions.java | 3 +-
.../sdk/options/PipelineOptionsFactory.java | 3 +-
.../sdk/options/PipelineOptionsReflector.java | 1 +
.../runners/AggregatorPipelineExtractor.java | 93 -
.../runners/AggregatorRetrievalException.java | 33 -
.../beam/sdk/runners/AggregatorValues.java | 52 -
.../beam/sdk/testing/CoderProperties.java | 82 +-
.../org/apache/beam/sdk/testing/PAssert.java | 19 +-
.../beam/sdk/testing/SerializableMatchers.java | 23 +-
.../apache/beam/sdk/testing/TestPipeline.java | 1 -
.../beam/sdk/testing/TestPipelineOptions.java | 1 +
.../apache/beam/sdk/transforms/Aggregator.java | 32 +-
.../sdk/transforms/AggregatorRetriever.java | 6 +-
.../org/apache/beam/sdk/transforms/Combine.java | 16 +-
.../apache/beam/sdk/transforms/CombineFns.java | 10 +-
.../org/apache/beam/sdk/transforms/Count.java | 2 +-
.../org/apache/beam/sdk/transforms/Create.java | 2 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 450 ++-
.../beam/sdk/transforms/DoFnReflector.java | 234 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 153 +-
.../beam/sdk/transforms/DoFnWithContext.java | 429 ---
.../org/apache/beam/sdk/transforms/Filter.java | 7 +-
.../beam/sdk/transforms/FlatMapElements.java | 126 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 2 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 2 +-
.../transforms/IntraBundleParallelization.java | 40 +-
.../org/apache/beam/sdk/transforms/Keys.java | 2 +-
.../org/apache/beam/sdk/transforms/KvSwap.java | 2 +-
.../apache/beam/sdk/transforms/MapElements.java | 60 +-
.../org/apache/beam/sdk/transforms/OldDoFn.java | 567 ++++
.../apache/beam/sdk/transforms/PTransform.java | 2 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 227 +-
.../apache/beam/sdk/transforms/Partition.java | 2 +-
.../beam/sdk/transforms/RemoveDuplicates.java | 2 +-
.../org/apache/beam/sdk/transforms/Sample.java | 2 +-
.../beam/sdk/transforms/SimpleFunction.java | 56 +-
.../org/apache/beam/sdk/transforms/Values.java | 2 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 45 +
.../apache/beam/sdk/transforms/WithKeys.java | 2 +-
.../beam/sdk/transforms/WithTimestamps.java | 2 +-
.../sdk/transforms/display/DisplayData.java | 1 -
.../beam/sdk/transforms/join/CoGbkResult.java | 1 -
.../beam/sdk/transforms/join/CoGroupByKey.java | 6 +-
.../sdk/transforms/windowing/AfterEach.java | 1 +
.../windowing/AfterProcessingTime.java | 6 +-
.../transforms/windowing/IntervalWindow.java | 1 -
.../beam/sdk/transforms/windowing/Never.java | 1 +
.../beam/sdk/transforms/windowing/PaneInfo.java | 12 +-
.../beam/sdk/transforms/windowing/Window.java | 25 +-
.../org/apache/beam/sdk/util/AvroUtils.java | 207 --
.../beam/sdk/util/BaseExecutionContext.java | 4 +-
.../apache/beam/sdk/util/BigQueryServices.java | 177 --
.../beam/sdk/util/BigQueryServicesImpl.java | 515 ----
.../beam/sdk/util/BigQueryTableInserter.java | 469 ---
.../beam/sdk/util/BigQueryTableRowIterator.java | 472 ---
.../apache/beam/sdk/util/BucketingFunction.java | 1 +
.../org/apache/beam/sdk/util/CoderUtils.java | 4 +-
.../beam/sdk/util/CombineContextFactory.java | 6 +-
.../apache/beam/sdk/util/CounterAggregator.java | 35 +-
.../beam/sdk/util/DirectSideInputReader.java | 4 +-
.../apache/beam/sdk/util/ExecutionContext.java | 8 +-
.../beam/sdk/util/FileIOChannelFactory.java | 31 +-
.../apache/beam/sdk/util/GcsPathValidator.java | 97 +
.../apache/beam/sdk/util/IOChannelUtils.java | 1 +
.../beam/sdk/util/MapAggregatorValues.java | 50 -
.../apache/beam/sdk/util/MovingFunction.java | 1 +
.../apache/beam/sdk/util/PCollectionViews.java | 228 +-
.../beam/sdk/util/PerKeyCombineFnRunner.java | 44 +-
.../beam/sdk/util/PerKeyCombineFnRunners.java | 30 +-
.../org/apache/beam/sdk/util/PubsubClient.java | 1 +
.../apache/beam/sdk/util/PubsubTestClient.java | 1 +
.../sdk/util/ReifyTimestampAndWindowsDoFn.java | 6 +-
.../org/apache/beam/sdk/util/Reshuffle.java | 2 +-
.../apache/beam/sdk/util/SerializableUtils.java | 2 +-
.../org/apache/beam/sdk/util/StringUtils.java | 2 +-
.../beam/sdk/util/SystemDoFnInternal.java | 6 +-
.../org/apache/beam/sdk/util/TimeDomain.java | 2 +-
.../apache/beam/sdk/util/TimerInternals.java | 16 +-
.../apache/beam/sdk/util/ValueWithRecordId.java | 6 +-
.../org/apache/beam/sdk/util/WindowedValue.java | 1 -
.../beam/sdk/util/WindowingInternals.java | 4 +-
.../apache/beam/sdk/util/WindowingStrategy.java | 6 +-
.../util/common/ElementByteSizeObserver.java | 18 +-
.../beam/sdk/util/common/ReflectHelpers.java | 1 +
.../beam/sdk/util/common/package-info.java | 3 +-
.../beam/sdk/util/gcsfs/package-info.java | 3 +-
.../org/apache/beam/sdk/util/package-info.java | 3 +-
.../CopyOnAccessInMemoryStateInternals.java | 4 +-
.../sdk/util/state/StateInternalsFactory.java | 36 +
.../beam/sdk/util/state/StateNamespaces.java | 2 +-
.../apache/beam/sdk/util/state/StateTags.java | 6 +-
.../beam/sdk/util/state/package-info.java | 22 +
.../org/apache/beam/sdk/values/PCollection.java | 34 +-
.../apache/beam/sdk/values/PCollectionList.java | 2 +-
.../apache/beam/sdk/values/PCollectionView.java | 29 +-
.../org/apache/beam/sdk/values/POutput.java | 2 +-
.../beam/sdk/values/TimestampedValue.java | 1 -
.../org/apache/beam/sdk/values/TupleTag.java | 2 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 13 +
.../dataflow/util/GcsPathValidatorTest.java | 104 +
.../sdk/AggregatorPipelineExtractorTest.java | 229 ++
.../java/org/apache/beam/sdk/PipelineTest.java | 2 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 2 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 15 +-
.../beam/sdk/coders/BigIntegerCoderTest.java | 14 +-
.../beam/sdk/coders/ByteArrayCoderTest.java | 5 +-
.../beam/sdk/coders/CoderRegistryTest.java | 22 +-
.../beam/sdk/coders/SerializableCoderTest.java | 4 +-
.../beam/sdk/coders/StandardCoderTest.java | 25 +
.../org/apache/beam/sdk/io/AvroSourceTest.java | 5 +-
.../org/apache/beam/sdk/io/BigQueryIOTest.java | 1231 --------
.../io/BoundedReadFromUnboundedSourceTest.java | 1 +
.../beam/sdk/io/CompressedSourceTest.java | 1 +
.../apache/beam/sdk/io/CountingInputTest.java | 3 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 2 +-
.../beam/sdk/io/OffsetBasedSourceTest.java | 1 +
.../beam/sdk/io/PubsubUnboundedSinkTest.java | 2 +-
.../java/org/apache/beam/sdk/io/ReadTest.java | 1 +
.../java/org/apache/beam/sdk/io/TextIOTest.java | 1 +
.../java/org/apache/beam/sdk/io/WriteTest.java | 17 +-
.../org/apache/beam/sdk/io/XmlSinkTest.java | 1 +
.../apache/beam/sdk/options/GcpOptionsTest.java | 3 +
.../sdk/options/GoogleApiDebugOptionsTest.java | 1 -
.../sdk/options/PipelineOptionsFactoryTest.java | 1 -
.../beam/sdk/options/PipelineOptionsTest.java | 1 -
.../sdk/options/ProxyInvocationHandlerTest.java | 32 +-
.../AggregatorPipelineExtractorTest.java | 229 --
.../beam/sdk/testing/CoderPropertiesTest.java | 26 +
.../sdk/testing/DataflowJUnitTestRunner.java | 130 -
.../apache/beam/sdk/testing/PAssertTest.java | 1 -
.../sdk/testing/PCollectionViewTesting.java | 35 +-
.../beam/sdk/testing/TestPipelineTest.java | 1 -
.../transforms/ApproximateQuantilesTest.java | 1 +
.../sdk/transforms/ApproximateUniqueTest.java | 3 +-
.../beam/sdk/transforms/CombineFnsTest.java | 4 +-
.../apache/beam/sdk/transforms/CombineTest.java | 10 +-
.../apache/beam/sdk/transforms/CreateTest.java | 2 +-
.../beam/sdk/transforms/DoFnContextTest.java | 69 -
.../DoFnDelegatingAggregatorTest.java | 16 +-
.../beam/sdk/transforms/DoFnReflectorTest.java | 294 +-
.../apache/beam/sdk/transforms/DoFnTest.java | 68 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 10 +-
.../sdk/transforms/DoFnWithContextTest.java | 237 --
.../sdk/transforms/FlatMapElementsTest.java | 67 +
.../apache/beam/sdk/transforms/FlattenTest.java | 4 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 6 +-
.../IntraBundleParallelizationTest.java | 23 +-
.../beam/sdk/transforms/MapElementsTest.java | 93 +
.../org/apache/beam/sdk/transforms/MaxTest.java | 1 +
.../org/apache/beam/sdk/transforms/MinTest.java | 2 +
.../apache/beam/sdk/transforms/NoOpDoFn.java | 144 -
.../apache/beam/sdk/transforms/NoOpOldDoFn.java | 144 +
.../beam/sdk/transforms/OldDoFnContextTest.java | 69 +
.../apache/beam/sdk/transforms/OldDoFnTest.java | 243 ++
.../apache/beam/sdk/transforms/ParDoTest.java | 108 +-
.../beam/sdk/transforms/PartitionTest.java | 1 +
.../apache/beam/sdk/transforms/SampleTest.java | 1 +
.../org/apache/beam/sdk/transforms/TopTest.java | 1 +
.../apache/beam/sdk/transforms/ViewTest.java | 396 +--
.../beam/sdk/transforms/WithTimestampsTest.java | 4 +-
.../display/DisplayDataEvaluatorTest.java | 4 +-
.../display/DisplayDataMatchersTest.java | 1 +
.../sdk/transforms/display/DisplayDataTest.java | 4 +-
.../dofnreflector/DoFnReflectorTestHelper.java | 116 +
.../sdk/transforms/join/CoGroupByKeyTest.java | 26 +-
.../sdk/transforms/windowing/NeverTest.java | 1 +
.../sdk/transforms/windowing/WindowTest.java | 4 +-
.../sdk/transforms/windowing/WindowingTest.java | 17 +-
.../org/apache/beam/sdk/util/AvroUtilsTest.java | 78 -
.../beam/sdk/util/BigQueryServicesImplTest.java | 303 --
.../sdk/util/BigQueryTableInserterTest.java | 311 --
.../sdk/util/BigQueryTableRowIteratorTest.java | 256 --
.../apache/beam/sdk/util/BigQueryUtilTest.java | 485 ---
.../beam/sdk/util/BucketingFunctionTest.java | 4 +-
.../beam/sdk/util/FileIOChannelFactoryTest.java | 26 +-
.../beam/sdk/util/IOChannelUtilsTest.java | 8 +
.../beam/sdk/util/MovingFunctionTest.java | 4 +-
.../util/RetryHttpRequestInitializerTest.java | 20 +-
.../beam/sdk/util/SerializableUtilsTest.java | 1 -
.../apache/beam/sdk/util/SerializerTest.java | 1 -
.../apache/beam/sdk/util/StringUtilsTest.java | 16 +-
.../beam/sdk/util/TimerInternalsTest.java | 47 +
.../org/apache/beam/sdk/util/TriggerTester.java | 1 +
.../beam/sdk/util/common/CounterTest.java | 1 +
.../beam/sdk/util/common/CounterTestUtils.java | 57 -
.../beam/sdk/values/PCollectionTupleTest.java | 2 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 20 +-
sdks/java/extensions/join-library/pom.xml | 2 +-
.../beam/sdk/extensions/joinlibrary/Join.java | 6 +-
.../extensions/joinlibrary/package-info.java | 23 +
sdks/java/extensions/pom.xml | 2 +-
sdks/java/io/google-cloud-platform/pom.xml | 56 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 236 ++
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 2749 ++++++++++++++++++
.../sdk/io/gcp/bigquery/BigQueryServices.java | 191 ++
.../io/gcp/bigquery/BigQueryServicesImpl.java | 722 +++++
.../io/gcp/bigquery/BigQueryTableInserter.java | 221 ++
.../gcp/bigquery/BigQueryTableRowIterator.java | 474 +++
.../beam/sdk/io/gcp/bigquery/package-info.java | 24 +
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 21 +-
.../beam/sdk/io/gcp/datastore/V1Beta3.java | 817 +++---
.../io/gcp/bigquery/BigQueryAvroUtilsTest.java | 143 +
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 1438 +++++++++
.../gcp/bigquery/BigQueryServicesImplTest.java | 359 +++
.../gcp/bigquery/BigQueryTableInserterTest.java | 249 ++
.../bigquery/BigQueryTableRowIteratorTest.java | 256 ++
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 454 +++
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 25 +
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1Beta3Test.java | 466 ++-
.../sdk/io/gcp/datastore/V1Beta3TestUtil.java | 3 +-
sdks/java/io/hdfs/pom.xml | 2 +-
.../apache/beam/sdk/io/hdfs/package-info.java | 22 +
.../sdk/io/hdfs/simpleauth/package-info.java | 22 +
sdks/java/io/jms/pom.xml | 134 +
.../beam/sdk/io/jms/JmsCheckpointMark.java | 82 +
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 516 ++++
.../org/apache/beam/sdk/io/jms/JmsRecord.java | 153 +
.../apache/beam/sdk/io/jms/package-info.java | 22 +
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 145 +
sdks/java/io/kafka/pom.xml | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 13 +-
.../apache/beam/sdk/io/kafka/package-info.java | 22 +
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 5 +-
sdks/java/io/pom.xml | 3 +-
sdks/java/java8tests/pom.xml | 2 +-
.../transforms/RemoveDuplicatesJava8Test.java | 10 +-
.../beam/sdk/transforms/WithKeysJava8Test.java | 11 +-
.../sdk/transforms/WithTimestampsJava8Test.java | 4 +-
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 7 +
.../src/main/java/DebuggingWordCount.java | 4 +-
.../src/main/java/MinimalWordCount.java | 6 +-
.../src/main/java/WindowedWordCount.java | 8 +-
.../src/main/java/WordCount.java | 6 +-
.../main/java/common/DataflowExampleUtils.java | 5 +-
.../main/java/common/PubsubFileInjector.java | 4 +-
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../src/main/java/StarterPipeline.java | 6 +-
.../src/main/java/it/pkg/StarterPipeline.java | 6 +-
sdks/java/microbenchmarks/pom.xml | 2 +-
.../microbenchmarks/coders/package-info.java | 22 +
.../transforms/DoFnReflectorBenchmark.java | 62 +-
.../transforms/package-info.java | 22 +
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 2 +-
testing/travis/README.md | 23 -
testing/travis/test_wordcount.sh | 125 -
499 files changed, 17733 insertions(+), 13202 deletions(-)
----------------------------------------------------------------------
[13/50] incubator-beam git commit: Cache .m2 directory on Travis-CI
Posted by ke...@apache.org.
Cache .m2 directory on Travis-CI
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/630ae250
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/630ae250
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/630ae250
Branch: refs/heads/gearpump-runner
Commit: 630ae25017764a83cfd6dd6c1ac3d8eae2abfbb4
Parents: 5049011
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Aug 4 15:12:03 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 14:41:19 2016 -0700
----------------------------------------------------------------------
.travis.yml | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/630ae250/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index bef61ff..43ae7de 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -47,6 +47,20 @@ before_install:
- if [ "$TRAVIS_OS_NAME" == "linux" ]; then jdk_switcher use "$CUSTOM_JDK"; fi
- export BEAM_SUREFIRE_ARGLINE="-Xmx512m"
+install:
+ # Removing this here protects from inadvertent caching
+ - rm -rf "$HOME/.m2/repository/org/apache/beam"
+
script:
- travis_retry mvn --batch-mode --update-snapshots $MAVEN_OVERRIDE verify
- travis_retry testing/travis/test_wordcount.sh
+
+cache:
+ directories:
+ - $HOME/.m2/repository
+
+before_cache:
+ # Removing here increases cache hits (makes the above
+ # rm in `install` redundant unless our config has a bug,
+ # but it will be idempotent)
+ - rm -rf "$HOME/.m2/repository/org/apache/beam"
[45/50] incubator-beam git commit: Merge branch master into
gearpump-runner
Posted by ke...@apache.org.
Merge branch master into gearpump-runner
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2a0ba61e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2a0ba61e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2a0ba61e
Branch: refs/heads/gearpump-runner
Commit: 2a0ba61e8507e1539115b583749a78f14d577bd8
Parents: 02b2248 2a1055d
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Aug 25 11:36:45 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu Aug 25 11:36:45 2016 -0700
----------------------------------------------------------------------
.travis.yml | 17 +-
.travis/README.md | 23 +
.travis/settings.xml | 33 +
.travis/test_wordcount.sh | 125 +
KEYS | 141 -
NOTICE | 1 -
README.md | 25 +-
examples/java/pom.xml | 48 +-
.../beam/examples/DebuggingWordCount.java | 2 +-
.../apache/beam/examples/MinimalWordCount.java | 2 +-
.../apache/beam/examples/WindowedWordCount.java | 6 +-
.../org/apache/beam/examples/WordCount.java | 26 +-
.../beam/examples/common/ExampleUtils.java | 4 +-
.../examples/common/PubsubFileInjector.java | 10 +-
.../beam/examples/complete/AutoComplete.java | 37 +-
.../examples/complete/StreamingWordExtract.java | 12 +-
.../apache/beam/examples/complete/TfIdf.java | 12 +-
.../examples/complete/TopWikipediaSessions.java | 29 +-
.../examples/complete/TrafficMaxLaneFlow.java | 8 +-
.../beam/examples/complete/TrafficRoutes.java | 10 +-
.../examples/cookbook/BigQueryTornadoes.java | 6 +-
.../cookbook/CombinePerKeyExamples.java | 6 +-
.../examples/cookbook/DatastoreWordCount.java | 9 +-
.../beam/examples/cookbook/FilterExamples.java | 10 +-
.../beam/examples/cookbook/JoinExamples.java | 10 +-
.../examples/cookbook/MaxPerKeyExamples.java | 6 +-
.../beam/examples/cookbook/TriggerExample.java | 21 +-
.../org/apache/beam/examples/WordCountIT.java | 2 +-
.../org/apache/beam/examples/WordCountTest.java | 3 +-
.../examples/complete/AutoCompleteTest.java | 2 +-
.../examples/cookbook/TriggerExampleTest.java | 2 +-
examples/java/src/test/resources/LICENSE | 202 ++
examples/java8/pom.xml | 7 +-
.../beam/examples/complete/game/GameStats.java | 16 +-
.../beam/examples/complete/game/UserScore.java | 2 +-
.../complete/game/utils/WriteToBigQuery.java | 18 +-
.../game/utils/WriteWindowedToBigQuery.java | 14 +-
.../examples/complete/game/UserScoreTest.java | 2 +-
examples/pom.xml | 2 +-
pom.xml | 72 +-
runners/core-java/pom.xml | 2 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 28 +-
.../core/UnboundedReadFromBoundedSource.java | 6 +-
.../apache/beam/runners/core/package-info.java | 22 +
.../apache/beam/sdk/util/AssignWindowsDoFn.java | 10 +-
.../org/apache/beam/sdk/util/DoFnRunner.java | 21 +-
.../apache/beam/sdk/util/DoFnRunnerBase.java | 82 +-
.../org/apache/beam/sdk/util/DoFnRunners.java | 104 +-
.../beam/sdk/util/GroupAlsoByWindowsDoFn.java | 16 +-
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 14 +-
.../sdk/util/GroupByKeyViaGroupByKeyOnly.java | 85 +-
.../sdk/util/LateDataDroppingDoFnRunner.java | 4 +-
.../apache/beam/sdk/util/PaneInfoTracker.java | 1 -
.../apache/beam/sdk/util/ReduceFnRunner.java | 4 +-
.../apache/beam/sdk/util/SimpleDoFnRunner.java | 19 +-
.../org/apache/beam/sdk/util/WatermarkHold.java | 1 -
.../beam/sdk/util/common/package-info.java | 20 +
.../org/apache/beam/sdk/util/package-info.java | 20 +
.../UnboundedReadFromBoundedSourceTest.java | 2 +-
.../sdk/util/GroupAlsoByWindowsProperties.java | 50 +-
...oupAlsoByWindowsViaOutputBufferDoFnTest.java | 8 +-
.../beam/sdk/util/ReduceFnRunnerTest.java | 1 +
.../apache/beam/sdk/util/ReduceFnTester.java | 1 +
.../beam/sdk/util/SimpleDoFnRunnerTest.java | 6 +-
runners/direct-java/pom.xml | 2 +-
.../runners/direct/AggregatorContainer.java | 183 ++
.../direct/BoundedReadEvaluatorFactory.java | 26 +-
.../beam/runners/direct/CloningThreadLocal.java | 43 +
.../beam/runners/direct/CommittedResult.java | 23 +-
.../beam/runners/direct/CompletionCallback.java | 8 +
.../beam/runners/direct/DirectRegistrar.java | 4 +-
.../beam/runners/direct/DirectRunner.java | 57 +-
.../beam/runners/direct/EvaluationContext.java | 57 +-
.../beam/runners/direct/EvaluatorKey.java | 55 -
.../direct/ExecutorServiceParallelExecutor.java | 186 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 33 +-
.../ImmutabilityCheckingBundleFactory.java | 4 +-
.../beam/runners/direct/ParDoEvaluator.java | 24 +-
.../direct/ParDoMultiEvaluatorFactory.java | 53 +-
.../direct/ParDoSingleEvaluatorFactory.java | 59 +-
...rializableCloningThreadLocalCacheLoader.java | 54 -
.../beam/runners/direct/SideInputContainer.java | 2 +-
.../runners/direct/StepTransformResult.java | 95 +-
.../direct/TransformEvaluatorFactory.java | 10 +-
.../beam/runners/direct/TransformExecutor.java | 3 +-
.../beam/runners/direct/TransformResult.java | 16 +-
.../direct/UnboundedReadEvaluatorFactory.java | 32 +-
.../runners/direct/ViewEvaluatorFactory.java | 14 +-
.../direct/WriteWithShardingFactory.java | 11 +-
.../runners/direct/AggregatorContainerTest.java | 134 +
.../runners/direct/CloningThreadLocalTest.java | 92 +
.../runners/direct/CommittedResultTest.java | 26 +-
.../ConsumerTrackingPipelineVisitorTest.java | 22 +-
.../runners/direct/DirectRegistrarTest.java | 17 +-
.../beam/runners/direct/DirectRunnerTest.java | 72 +-
.../runners/direct/EvaluationContextTest.java | 32 +-
.../ImmutabilityCheckingBundleFactoryTest.java | 6 +-
.../ImmutabilityEnforcementFactoryTest.java | 6 +-
.../direct/KeyedPValueTrackingVisitorTest.java | 6 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 18 +-
.../direct/ParDoMultiEvaluatorFactoryTest.java | 41 +-
.../direct/ParDoSingleEvaluatorFactoryTest.java | 41 +-
...izableCloningThreadLocalCacheLoaderTest.java | 99 -
.../runners/direct/StepTransformResultTest.java | 91 +
.../runners/direct/TransformExecutorTest.java | 33 +-
.../runners/direct/WatermarkManagerTest.java | 18 +-
runners/flink/README.md | 25 +-
runners/flink/examples/pom.xml | 13 +-
.../beam/runners/flink/examples/TFIDF.java | 12 +-
.../beam/runners/flink/examples/WordCount.java | 7 +-
.../flink/examples/streaming/AutoComplete.java | 25 +-
.../flink/examples/streaming/JoinExamples.java | 6 +-
.../examples/streaming/KafkaIOExamples.java | 3 +-
.../KafkaWindowedWordCountExample.java | 4 +-
.../examples/streaming/WindowedWordCount.java | 4 +-
runners/flink/pom.xml | 2 +-
runners/flink/runner/pom.xml | 116 +-
.../beam/runners/flink/FlinkRunnerResult.java | 26 +-
.../FlinkBatchTransformTranslators.java | 12 +-
.../FlinkStreamingTransformTranslators.java | 9 +-
.../functions/FlinkDoFnFunction.java | 10 +-
.../FlinkMergingNonShuffleReduceFunction.java | 8 +-
.../functions/FlinkMultiOutputDoFnFunction.java | 10 +-
.../FlinkMultiOutputProcessContext.java | 6 +-
.../functions/FlinkNoElementAssignContext.java | 8 +-
.../functions/FlinkPartialReduceFunction.java | 8 +-
.../functions/FlinkProcessContext.java | 18 +-
.../functions/FlinkReduceFunction.java | 8 +-
.../functions/SideInputInitializer.java | 2 +-
.../streaming/FlinkAbstractParDoWrapper.java | 18 +-
.../FlinkGroupAlsoByWindowWrapper.java | 41 +-
.../streaming/FlinkParDoBoundMultiWrapper.java | 4 +-
.../streaming/FlinkParDoBoundWrapper.java | 4 +-
.../state/AbstractFlinkTimerInternals.java | 4 +-
.../beam/runners/flink/PipelineOptionsTest.java | 6 +-
.../beam/runners/flink/ReadSourceITCase.java | 2 +-
.../flink/ReadSourceStreamingITCase.java | 4 +-
.../flink/streaming/GroupByNullKeyTest.java | 8 +-
.../streaming/TopWikipediaSessionsITCase.java | 6 +-
runners/google-cloud-dataflow-java/pom.xml | 35 +-
.../dataflow/BlockingDataflowRunner.java | 5 +-
.../runners/dataflow/DataflowPipelineJob.java | 106 +-
.../dataflow/DataflowPipelineTranslator.java | 6 +-
.../beam/runners/dataflow/DataflowRunner.java | 198 +-
.../dataflow/internal/AssignWindows.java | 6 +-
.../DataflowUnboundedReadFromBoundedSource.java | 4 +-
.../options/DataflowPipelineDebugOptions.java | 43 -
.../options/DataflowPipelineOptions.java | 6 +-
.../runners/dataflow/options/package-info.java | 22 +
.../beam/runners/dataflow/package-info.java | 22 +
.../dataflow/testing/TestDataflowRunner.java | 7 +-
.../runners/dataflow/testing/package-info.java | 24 +
.../dataflow/util/DataflowPathValidator.java | 98 -
.../beam/runners/dataflow/util/DoFnInfo.java | 16 +-
.../runners/dataflow/util/package-info.java | 20 +
.../dataflow/BlockingDataflowRunnerTest.java | 9 +-
.../dataflow/DataflowPipelineJobTest.java | 47 +-
.../DataflowPipelineTranslatorTest.java | 12 +-
.../runners/dataflow/DataflowRunnerTest.java | 39 +-
.../options/DataflowPipelineOptionsTest.java | 3 +
.../testing/TestDataflowRunnerTest.java | 34 +-
.../util/DataflowPathValidatorTest.java | 94 -
runners/pom.xml | 72 +-
runners/spark/README.md | 3 +-
runners/spark/pom.xml | 27 +-
.../runners/spark/SparkPipelineOptions.java | 16 +-
.../apache/beam/runners/spark/SparkRunner.java | 33 +-
.../runners/spark/SparkRunnerRegistrar.java | 6 +-
.../spark/SparkStreamingPipelineOptions.java | 41 -
.../beam/runners/spark/TestSparkRunner.java | 2 -
.../runners/spark/aggregators/package-info.java | 20 +
.../beam/runners/spark/coders/package-info.java | 22 +
.../beam/runners/spark/examples/WordCount.java | 137 +
.../runners/spark/io/hadoop/package-info.java | 22 +
.../beam/runners/spark/io/package-info.java | 22 +
.../apache/beam/runners/spark/package-info.java | 22 +
.../runners/spark/translation/DoFnFunction.java | 8 +-
.../spark/translation/EvaluationContext.java | 25 +-
.../spark/translation/MultiDoFnFunction.java | 8 +-
.../spark/translation/SparkContextFactory.java | 8 +-
.../spark/translation/SparkProcessContext.java | 20 +-
.../spark/translation/SparkRuntimeContext.java | 2 +-
.../spark/translation/TransformTranslator.java | 73 +-
.../runners/spark/translation/package-info.java | 22 +
.../streaming/StreamingTransformTranslator.java | 14 +-
.../StreamingWindowPipelineDetector.java | 102 -
.../translation/streaming/package-info.java | 22 +
.../beam/runners/spark/util/package-info.java | 22 +
.../apache/beam/runners/spark/DeDupTest.java | 2 +-
.../beam/runners/spark/EmptyInputTest.java | 2 +-
.../beam/runners/spark/SimpleWordCountTest.java | 80 +-
.../runners/spark/SparkRunnerRegistrarTest.java | 2 +-
.../apache/beam/runners/spark/TfIdfTest.java | 201 +-
.../beam/runners/spark/io/AvroPipelineTest.java | 7 +-
.../beam/runners/spark/io/NumShardsTest.java | 4 +-
.../io/hadoop/HadoopFileFormatPipelineTest.java | 7 +-
.../spark/translation/CombineGloballyTest.java | 2 +-
.../spark/translation/CombinePerKeyTest.java | 9 +-
.../spark/translation/DoFnOutputTest.java | 10 +-
.../translation/MultiOutputWordCountTest.java | 15 +-
.../spark/translation/SerializationTest.java | 10 +-
.../spark/translation/SideEffectsTest.java | 10 +-
.../translation/TransformTranslatorTest.java | 13 +-
.../translation/WindowedWordCountTest.java | 12 +-
.../streaming/FlattenStreamingTest.java | 11 +-
.../streaming/KafkaStreamingTest.java | 19 +-
.../streaming/SimpleStreamingWordCountTest.java | 18 +-
.../streaming/utils/EmbeddedKafkaCluster.java | 4 +-
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/checkstyle.xml | 14 +
.../src/main/resources/beam/findbugs-filter.xml | 13 +-
.../src/main/resources/beam/suppressions.xml | 24 +
sdks/java/core/pom.xml | 27 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 93 +
.../beam/sdk/AggregatorRetrievalException.java | 33 +
.../org/apache/beam/sdk/AggregatorValues.java | 52 +
.../main/java/org/apache/beam/sdk/Pipeline.java | 12 +-
.../org/apache/beam/sdk/PipelineResult.java | 40 +-
.../org/apache/beam/sdk/coders/AvroCoder.java | 1 -
.../java/org/apache/beam/sdk/coders/Coder.java | 2 +-
.../apache/beam/sdk/coders/CoderRegistry.java | 38 +-
.../apache/beam/sdk/coders/DurationCoder.java | 1 -
.../apache/beam/sdk/coders/InstantCoder.java | 1 -
.../org/apache/beam/sdk/coders/JAXBCoder.java | 2 +-
.../apache/beam/sdk/coders/StandardCoder.java | 14 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 3 +-
.../java/org/apache/beam/sdk/io/BigQueryIO.java | 2447 ----------------
.../org/apache/beam/sdk/io/FileBasedSink.java | 2 +-
.../java/org/apache/beam/sdk/io/PubsubIO.java | 14 +-
.../apache/beam/sdk/io/PubsubUnboundedSink.java | 9 +-
.../beam/sdk/io/PubsubUnboundedSource.java | 3 +-
.../java/org/apache/beam/sdk/io/Source.java | 2 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 17 +-
.../org/apache/beam/sdk/io/package-info.java | 3 +-
.../org/apache/beam/sdk/options/GcpOptions.java | 11 +-
.../org/apache/beam/sdk/options/GcsOptions.java | 44 +
.../beam/sdk/options/PipelineOptions.java | 3 +-
.../sdk/options/PipelineOptionsFactory.java | 3 +-
.../sdk/options/PipelineOptionsReflector.java | 1 +
.../runners/AggregatorPipelineExtractor.java | 93 -
.../runners/AggregatorRetrievalException.java | 33 -
.../beam/sdk/runners/AggregatorValues.java | 52 -
.../beam/sdk/testing/CoderProperties.java | 82 +-
.../org/apache/beam/sdk/testing/PAssert.java | 19 +-
.../beam/sdk/testing/SerializableMatchers.java | 23 +-
.../apache/beam/sdk/testing/TestPipeline.java | 1 -
.../beam/sdk/testing/TestPipelineOptions.java | 1 +
.../apache/beam/sdk/transforms/Aggregator.java | 32 +-
.../sdk/transforms/AggregatorRetriever.java | 6 +-
.../org/apache/beam/sdk/transforms/Combine.java | 16 +-
.../apache/beam/sdk/transforms/CombineFns.java | 10 +-
.../org/apache/beam/sdk/transforms/Count.java | 2 +-
.../org/apache/beam/sdk/transforms/Create.java | 2 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 450 ++-
.../beam/sdk/transforms/DoFnReflector.java | 234 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 153 +-
.../beam/sdk/transforms/DoFnWithContext.java | 429 ---
.../org/apache/beam/sdk/transforms/Filter.java | 7 +-
.../beam/sdk/transforms/FlatMapElements.java | 126 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 2 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 2 +-
.../transforms/IntraBundleParallelization.java | 40 +-
.../org/apache/beam/sdk/transforms/Keys.java | 2 +-
.../org/apache/beam/sdk/transforms/KvSwap.java | 2 +-
.../apache/beam/sdk/transforms/MapElements.java | 60 +-
.../org/apache/beam/sdk/transforms/OldDoFn.java | 567 ++++
.../apache/beam/sdk/transforms/PTransform.java | 2 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 227 +-
.../apache/beam/sdk/transforms/Partition.java | 2 +-
.../beam/sdk/transforms/RemoveDuplicates.java | 2 +-
.../org/apache/beam/sdk/transforms/Sample.java | 2 +-
.../beam/sdk/transforms/SimpleFunction.java | 56 +-
.../org/apache/beam/sdk/transforms/Values.java | 2 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 45 +
.../apache/beam/sdk/transforms/WithKeys.java | 2 +-
.../beam/sdk/transforms/WithTimestamps.java | 2 +-
.../sdk/transforms/display/DisplayData.java | 1 -
.../beam/sdk/transforms/join/CoGbkResult.java | 1 -
.../beam/sdk/transforms/join/CoGroupByKey.java | 6 +-
.../sdk/transforms/windowing/AfterEach.java | 1 +
.../windowing/AfterProcessingTime.java | 6 +-
.../transforms/windowing/IntervalWindow.java | 1 -
.../beam/sdk/transforms/windowing/Never.java | 1 +
.../beam/sdk/transforms/windowing/PaneInfo.java | 12 +-
.../beam/sdk/transforms/windowing/Window.java | 25 +-
.../org/apache/beam/sdk/util/AvroUtils.java | 207 --
.../beam/sdk/util/BaseExecutionContext.java | 4 +-
.../apache/beam/sdk/util/BigQueryServices.java | 177 --
.../beam/sdk/util/BigQueryServicesImpl.java | 515 ----
.../beam/sdk/util/BigQueryTableInserter.java | 469 ---
.../beam/sdk/util/BigQueryTableRowIterator.java | 472 ---
.../apache/beam/sdk/util/BucketingFunction.java | 1 +
.../org/apache/beam/sdk/util/CoderUtils.java | 4 +-
.../beam/sdk/util/CombineContextFactory.java | 6 +-
.../apache/beam/sdk/util/CounterAggregator.java | 35 +-
.../beam/sdk/util/DirectSideInputReader.java | 4 +-
.../apache/beam/sdk/util/ExecutionContext.java | 8 +-
.../beam/sdk/util/FileIOChannelFactory.java | 31 +-
.../apache/beam/sdk/util/GcsPathValidator.java | 97 +
.../apache/beam/sdk/util/IOChannelUtils.java | 1 +
.../beam/sdk/util/MapAggregatorValues.java | 50 -
.../apache/beam/sdk/util/MovingFunction.java | 1 +
.../apache/beam/sdk/util/PCollectionViews.java | 228 +-
.../beam/sdk/util/PerKeyCombineFnRunner.java | 44 +-
.../beam/sdk/util/PerKeyCombineFnRunners.java | 30 +-
.../org/apache/beam/sdk/util/PubsubClient.java | 1 +
.../apache/beam/sdk/util/PubsubTestClient.java | 1 +
.../sdk/util/ReifyTimestampAndWindowsDoFn.java | 6 +-
.../org/apache/beam/sdk/util/Reshuffle.java | 2 +-
.../apache/beam/sdk/util/SerializableUtils.java | 2 +-
.../org/apache/beam/sdk/util/StringUtils.java | 2 +-
.../beam/sdk/util/SystemDoFnInternal.java | 6 +-
.../org/apache/beam/sdk/util/TimeDomain.java | 2 +-
.../apache/beam/sdk/util/TimerInternals.java | 16 +-
.../apache/beam/sdk/util/ValueWithRecordId.java | 6 +-
.../org/apache/beam/sdk/util/WindowedValue.java | 1 -
.../beam/sdk/util/WindowingInternals.java | 4 +-
.../apache/beam/sdk/util/WindowingStrategy.java | 6 +-
.../util/common/ElementByteSizeObserver.java | 18 +-
.../beam/sdk/util/common/ReflectHelpers.java | 1 +
.../beam/sdk/util/common/package-info.java | 3 +-
.../beam/sdk/util/gcsfs/package-info.java | 3 +-
.../org/apache/beam/sdk/util/package-info.java | 3 +-
.../CopyOnAccessInMemoryStateInternals.java | 4 +-
.../sdk/util/state/StateInternalsFactory.java | 36 +
.../beam/sdk/util/state/StateNamespaces.java | 2 +-
.../apache/beam/sdk/util/state/StateTags.java | 6 +-
.../beam/sdk/util/state/package-info.java | 22 +
.../org/apache/beam/sdk/values/PCollection.java | 34 +-
.../apache/beam/sdk/values/PCollectionList.java | 2 +-
.../apache/beam/sdk/values/PCollectionView.java | 29 +-
.../org/apache/beam/sdk/values/POutput.java | 2 +-
.../beam/sdk/values/TimestampedValue.java | 1 -
.../org/apache/beam/sdk/values/TupleTag.java | 2 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 13 +
.../dataflow/util/GcsPathValidatorTest.java | 104 +
.../sdk/AggregatorPipelineExtractorTest.java | 229 ++
.../java/org/apache/beam/sdk/PipelineTest.java | 2 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 2 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 15 +-
.../beam/sdk/coders/BigIntegerCoderTest.java | 14 +-
.../beam/sdk/coders/ByteArrayCoderTest.java | 5 +-
.../beam/sdk/coders/CoderRegistryTest.java | 22 +-
.../beam/sdk/coders/SerializableCoderTest.java | 4 +-
.../beam/sdk/coders/StandardCoderTest.java | 25 +
.../org/apache/beam/sdk/io/AvroSourceTest.java | 5 +-
.../org/apache/beam/sdk/io/BigQueryIOTest.java | 1231 --------
.../io/BoundedReadFromUnboundedSourceTest.java | 1 +
.../beam/sdk/io/CompressedSourceTest.java | 1 +
.../apache/beam/sdk/io/CountingInputTest.java | 3 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 2 +-
.../beam/sdk/io/OffsetBasedSourceTest.java | 1 +
.../beam/sdk/io/PubsubUnboundedSinkTest.java | 2 +-
.../java/org/apache/beam/sdk/io/ReadTest.java | 1 +
.../java/org/apache/beam/sdk/io/TextIOTest.java | 1 +
.../java/org/apache/beam/sdk/io/WriteTest.java | 17 +-
.../org/apache/beam/sdk/io/XmlSinkTest.java | 1 +
.../apache/beam/sdk/options/GcpOptionsTest.java | 3 +
.../sdk/options/GoogleApiDebugOptionsTest.java | 1 -
.../sdk/options/PipelineOptionsFactoryTest.java | 1 -
.../beam/sdk/options/PipelineOptionsTest.java | 1 -
.../sdk/options/ProxyInvocationHandlerTest.java | 32 +-
.../AggregatorPipelineExtractorTest.java | 229 --
.../beam/sdk/testing/CoderPropertiesTest.java | 26 +
.../sdk/testing/DataflowJUnitTestRunner.java | 130 -
.../apache/beam/sdk/testing/PAssertTest.java | 1 -
.../sdk/testing/PCollectionViewTesting.java | 35 +-
.../beam/sdk/testing/TestPipelineTest.java | 1 -
.../transforms/ApproximateQuantilesTest.java | 1 +
.../sdk/transforms/ApproximateUniqueTest.java | 3 +-
.../beam/sdk/transforms/CombineFnsTest.java | 4 +-
.../apache/beam/sdk/transforms/CombineTest.java | 10 +-
.../apache/beam/sdk/transforms/CreateTest.java | 2 +-
.../beam/sdk/transforms/DoFnContextTest.java | 69 -
.../DoFnDelegatingAggregatorTest.java | 16 +-
.../beam/sdk/transforms/DoFnReflectorTest.java | 294 +-
.../apache/beam/sdk/transforms/DoFnTest.java | 68 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 10 +-
.../sdk/transforms/DoFnWithContextTest.java | 237 --
.../sdk/transforms/FlatMapElementsTest.java | 67 +
.../apache/beam/sdk/transforms/FlattenTest.java | 4 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 6 +-
.../IntraBundleParallelizationTest.java | 23 +-
.../beam/sdk/transforms/MapElementsTest.java | 93 +
.../org/apache/beam/sdk/transforms/MaxTest.java | 1 +
.../org/apache/beam/sdk/transforms/MinTest.java | 2 +
.../apache/beam/sdk/transforms/NoOpDoFn.java | 144 -
.../apache/beam/sdk/transforms/NoOpOldDoFn.java | 144 +
.../beam/sdk/transforms/OldDoFnContextTest.java | 69 +
.../apache/beam/sdk/transforms/OldDoFnTest.java | 243 ++
.../apache/beam/sdk/transforms/ParDoTest.java | 108 +-
.../beam/sdk/transforms/PartitionTest.java | 1 +
.../apache/beam/sdk/transforms/SampleTest.java | 1 +
.../org/apache/beam/sdk/transforms/TopTest.java | 1 +
.../apache/beam/sdk/transforms/ViewTest.java | 396 +--
.../beam/sdk/transforms/WithTimestampsTest.java | 4 +-
.../display/DisplayDataEvaluatorTest.java | 4 +-
.../display/DisplayDataMatchersTest.java | 1 +
.../sdk/transforms/display/DisplayDataTest.java | 4 +-
.../dofnreflector/DoFnReflectorTestHelper.java | 116 +
.../sdk/transforms/join/CoGroupByKeyTest.java | 26 +-
.../sdk/transforms/windowing/NeverTest.java | 1 +
.../sdk/transforms/windowing/WindowTest.java | 4 +-
.../sdk/transforms/windowing/WindowingTest.java | 17 +-
.../org/apache/beam/sdk/util/AvroUtilsTest.java | 78 -
.../beam/sdk/util/BigQueryServicesImplTest.java | 303 --
.../sdk/util/BigQueryTableInserterTest.java | 311 --
.../sdk/util/BigQueryTableRowIteratorTest.java | 256 --
.../apache/beam/sdk/util/BigQueryUtilTest.java | 485 ---
.../beam/sdk/util/BucketingFunctionTest.java | 4 +-
.../beam/sdk/util/FileIOChannelFactoryTest.java | 26 +-
.../beam/sdk/util/IOChannelUtilsTest.java | 8 +
.../beam/sdk/util/MovingFunctionTest.java | 4 +-
.../util/RetryHttpRequestInitializerTest.java | 20 +-
.../beam/sdk/util/SerializableUtilsTest.java | 1 -
.../apache/beam/sdk/util/SerializerTest.java | 1 -
.../apache/beam/sdk/util/StringUtilsTest.java | 16 +-
.../beam/sdk/util/TimerInternalsTest.java | 47 +
.../org/apache/beam/sdk/util/TriggerTester.java | 1 +
.../beam/sdk/util/common/CounterTest.java | 1 +
.../beam/sdk/util/common/CounterTestUtils.java | 57 -
.../beam/sdk/values/PCollectionTupleTest.java | 2 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 20 +-
sdks/java/extensions/join-library/pom.xml | 2 +-
.../beam/sdk/extensions/joinlibrary/Join.java | 6 +-
.../extensions/joinlibrary/package-info.java | 23 +
sdks/java/extensions/pom.xml | 2 +-
sdks/java/io/google-cloud-platform/pom.xml | 56 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 236 ++
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 2749 ++++++++++++++++++
.../sdk/io/gcp/bigquery/BigQueryServices.java | 191 ++
.../io/gcp/bigquery/BigQueryServicesImpl.java | 722 +++++
.../io/gcp/bigquery/BigQueryTableInserter.java | 221 ++
.../gcp/bigquery/BigQueryTableRowIterator.java | 474 +++
.../beam/sdk/io/gcp/bigquery/package-info.java | 24 +
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 21 +-
.../beam/sdk/io/gcp/datastore/V1Beta3.java | 817 +++---
.../io/gcp/bigquery/BigQueryAvroUtilsTest.java | 143 +
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 1438 +++++++++
.../gcp/bigquery/BigQueryServicesImplTest.java | 359 +++
.../gcp/bigquery/BigQueryTableInserterTest.java | 249 ++
.../bigquery/BigQueryTableRowIteratorTest.java | 256 ++
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 454 +++
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 25 +
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1Beta3Test.java | 466 ++-
.../sdk/io/gcp/datastore/V1Beta3TestUtil.java | 3 +-
sdks/java/io/hdfs/pom.xml | 2 +-
.../apache/beam/sdk/io/hdfs/package-info.java | 22 +
.../sdk/io/hdfs/simpleauth/package-info.java | 22 +
sdks/java/io/jms/pom.xml | 134 +
.../beam/sdk/io/jms/JmsCheckpointMark.java | 82 +
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 516 ++++
.../org/apache/beam/sdk/io/jms/JmsRecord.java | 153 +
.../apache/beam/sdk/io/jms/package-info.java | 22 +
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 145 +
sdks/java/io/kafka/pom.xml | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 13 +-
.../apache/beam/sdk/io/kafka/package-info.java | 22 +
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 5 +-
sdks/java/io/pom.xml | 3 +-
sdks/java/java8tests/pom.xml | 2 +-
.../transforms/RemoveDuplicatesJava8Test.java | 10 +-
.../beam/sdk/transforms/WithKeysJava8Test.java | 11 +-
.../sdk/transforms/WithTimestampsJava8Test.java | 4 +-
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 7 +
.../src/main/java/DebuggingWordCount.java | 4 +-
.../src/main/java/MinimalWordCount.java | 6 +-
.../src/main/java/WindowedWordCount.java | 8 +-
.../src/main/java/WordCount.java | 6 +-
.../main/java/common/DataflowExampleUtils.java | 5 +-
.../main/java/common/PubsubFileInjector.java | 4 +-
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../src/main/java/StarterPipeline.java | 6 +-
.../src/main/java/it/pkg/StarterPipeline.java | 6 +-
sdks/java/microbenchmarks/pom.xml | 2 +-
.../microbenchmarks/coders/package-info.java | 22 +
.../transforms/DoFnReflectorBenchmark.java | 62 +-
.../transforms/package-info.java | 22 +
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 2 +-
testing/travis/README.md | 23 -
testing/travis/test_wordcount.sh | 125 -
485 files changed, 17493 insertions(+), 13132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2a0ba61e/runners/pom.xml
----------------------------------------------------------------------
diff --cc runners/pom.xml
index 3f985c5,605c3b2..6b8434a
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@@ -40,44 -40,45 +40,54 @@@
</modules>
<profiles>
+ <profile>
+ <id>java8</id>
+ <activation>
+ <jdk>[1.8,)</jdk>
+ </activation>
+ <modules>
+ <module>gearpump</module>
+ </modules>
+ </profile>
+
+ <!-- A profile that adds an integration test phase if and only if
+ the runnableOnServicePipelineOptions maven property has been set.
+ It should be set to a valid PipelineOptions JSON string. -->
+ <profile>
+ <id>runnable-on-service-tests</id>
+ <activation>
+ <property><name>runnableOnServicePipelineOptions</name></property>
+ </activation>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>runnable-on-service-tests</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <groups>org.apache.beam.sdk.testing.RunnableOnService</groups>
+ <parallel>all</parallel>
+ <threadCount>4</threadCount>
+ <dependenciesToScan>
+ <dependency>org.apache.beam:beam-sdks-java-core</dependency>
+ </dependenciesToScan>
+ <systemPropertyVariables>
+ <beamTestPipelineOptions>${runnableOnServicePipelineOptions}</beamTestPipelineOptions>
+ </systemPropertyVariables>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
</profiles>
- <build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <executions>
- <execution>
- <id>runnable-on-service-tests</id>
- <phase>integration-test</phase>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <groups>org.apache.beam.sdk.testing.RunnableOnService</groups>
- <parallel>all</parallel>
- <threadCount>4</threadCount>
- <dependenciesToScan>
- <dependency>org.apache.beam:beam-sdks-java-core</dependency>
- </dependenciesToScan>
- <systemPropertyVariables>
- <beamTestPipelineOptions>${runnableOnServicePipelineOptions}</beamTestPipelineOptions>
- </systemPropertyVariables>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2a0ba61e/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java
----------------------------------------------------------------------
[08/50] incubator-beam git commit: This closes #797
Posted by ke...@apache.org.
This closes #797
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/574c3777
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/574c3777
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/574c3777
Branch: refs/heads/gearpump-runner
Commit: 574c3777ddb6d9e8f5afb39e7ec14913ea52a8e8
Parents: bb00810 f5df358
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Aug 8 13:40:21 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 13:40:21 2016 -0700
----------------------------------------------------------------------
.../beam/runners/flink/examples/TFIDF.java | 28 +--
.../beam/runners/flink/examples/WordCount.java | 5 +-
.../flink/examples/streaming/AutoComplete.java | 37 ++--
.../flink/examples/streaming/JoinExamples.java | 14 +-
.../examples/streaming/KafkaIOExamples.java | 7 +-
.../KafkaWindowedWordCountExample.java | 10 +-
.../examples/streaming/WindowedWordCount.java | 10 +-
.../beam/runners/flink/ReadSourceITCase.java | 6 +-
.../flink/ReadSourceStreamingITCase.java | 8 +-
.../apache/beam/runners/spark/TfIdfTest.java | 22 +--
.../spark/translation/CombinePerKeyTest.java | 6 +-
.../translation/MultiOutputWordCountTest.java | 10 +-
.../spark/translation/SerializationTest.java | 10 +-
.../streaming/KafkaStreamingTest.java | 6 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 26 +--
.../org/apache/beam/sdk/transforms/Combine.java | 20 +-
.../org/apache/beam/sdk/transforms/Filter.java | 4 +-
.../beam/sdk/transforms/windowing/Window.java | 7 +-
.../org/apache/beam/sdk/util/Reshuffle.java | 6 +-
.../java/org/apache/beam/sdk/io/WriteTest.java | 22 ++-
.../apache/beam/sdk/transforms/DoFnTest.java | 29 +--
.../apache/beam/sdk/transforms/ViewTest.java | 192 ++++++++++---------
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 21 +-
.../sdk/transforms/WithTimestampsJava8Test.java | 6 +-
24 files changed, 248 insertions(+), 264 deletions(-)
----------------------------------------------------------------------
[15/50] incubator-beam git commit: Relocate Travis utilities to
.travis
Posted by ke...@apache.org.
Relocate Travis utilities to .travis
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5e1185d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5e1185d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5e1185d6
Branch: refs/heads/gearpump-runner
Commit: 5e1185d67cea4112bb08251de72f513dc9b4b871
Parents: 7136616
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Aug 4 19:35:39 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Aug 9 09:59:05 2016 -0700
----------------------------------------------------------------------
.travis.yml | 2 +-
.travis/README.md | 23 +++++++
.travis/test_wordcount.sh | 125 ++++++++++++++++++++++++++++++++++
testing/travis/README.md | 23 -------
testing/travis/test_wordcount.sh | 125 ----------------------------------
5 files changed, 149 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5e1185d6/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 4674bf3..656aba0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -53,7 +53,7 @@ install:
script:
- travis_retry mvn --settings .travis/settings.xml --batch-mode --update-snapshots $MAVEN_OVERRIDE verify
- - travis_retry testing/travis/test_wordcount.sh
+ - travis_retry .travis/test_wordcount.sh
cache:
directories:
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5e1185d6/.travis/README.md
----------------------------------------------------------------------
diff --git a/.travis/README.md b/.travis/README.md
new file mode 100644
index 0000000..e0c13f2
--- /dev/null
+++ b/.travis/README.md
@@ -0,0 +1,23 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Travis Scripts
+
+This directory contains scripts used for [Travis CI](https://travis-ci.org/GoogleCloudPlatform/DataflowJavaSDK)
+testing.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5e1185d6/.travis/test_wordcount.sh
----------------------------------------------------------------------
diff --git a/.travis/test_wordcount.sh b/.travis/test_wordcount.sh
new file mode 100755
index 0000000..e059a35
--- /dev/null
+++ b/.travis/test_wordcount.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script runs WordCount example locally in a few different ways.
+# Specifically, all combinations of:
+# a) using mvn exec, or java -cp with a bundled jar file;
+# b) input filename with no directory component, with a relative directory, or
+# with an absolute directory; AND
+# c) input filename containing wildcards or not.
+#
+# The one optional parameter is a path from the directory containing the script
+# to the directory containing the top-level (parent) pom.xml. If no parameter
+# is provided, the script assumes that directory is equal to the directory
+# containing the script itself.
+#
+# The exit-code of the script indicates success or a failure.
+
+set -e
+set -o pipefail
+
+PASS=1
+VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[')
+JAR_FILE=examples/java/target/beam-examples-java-bundled-${VERSION}.jar
+
+function check_result_hash {
+ local name=$1
+ local outfile_prefix=$2
+ local expected=$3
+
+ local actual=$(LC_ALL=C sort $outfile_prefix-* | md5sum | awk '{print $1}' \
+ || LC_ALL=C sort $outfile_prefix-* | md5 -q) || exit 2 # OSX
+ if [[ "$actual" != "$expected" ]]
+ then
+ echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected."
+ PASS=""
+ echo "head hexdump of actual:"
+ head $outfile_prefix-* | hexdump -c
+ else
+ echo "pass $name"
+ # Output files are left behind in /tmp
+ fi
+}
+
+function get_outfile_prefix {
+ local name=$1
+ # NOTE: mktemp on OSX doesn't support --tmpdir
+ mktemp -u "/tmp/$name.out.XXXXXXXXXX"
+}
+
+function run_via_mvn {
+ local name=$1
+ local input=$2
+ local expected_hash=$3
+
+ local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
+ local cmd='mvn exec:java -f pom.xml -pl examples/java \
+ -Dexec.mainClass=org.apache.beam.examples.WordCount \
+ -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"'
+ echo "$name: Running $cmd" >&2
+ sh -c "$cmd"
+ check_result_hash "$name" "$outfile_prefix" "$expected_hash"
+}
+
+function run_bundled {
+ local name=$1
+ local input=$2
+ local expected_hash=$3
+
+ local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
+ local cmd='java -cp '"$JAR_FILE"' \
+ org.apache.beam.examples.WordCount \
+ --runner=DirectRunner \
+ --inputFile='"'$input'"' \
+ --output='"$outfile_prefix"
+ echo "$name: Running $cmd" >&2
+ sh -c "$cmd"
+ check_result_hash "$name" "$outfile_prefix" "$expected_hash"
+}
+
+function run_all_ways {
+ local name=$1
+ local input=$2
+ local expected_hash=$3
+
+ run_via_mvn ${name}a "$input" $expected_hash
+ check_for_jar_file
+ run_bundled ${name}b "$input" $expected_hash
+}
+
+function check_for_jar_file {
+ if [[ ! -f $JAR_FILE ]]
+ then
+ echo "Jar file $JAR_FILE not created" >&2
+ exit 2
+ fi
+}
+
+run_all_ways wordcount1 "LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount2 "./LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount3 "$PWD/LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount4 "L*N?E*" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount5 "./LICE*N?E" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount6 "$PWD/*LIC?NSE" c5350a5ad4bb51e3e018612b4b044097
+
+if [[ ! "$PASS" ]]
+then
+ echo "One or more tests FAILED."
+ exit 1
+fi
+echo "All tests PASS"
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5e1185d6/testing/travis/README.md
----------------------------------------------------------------------
diff --git a/testing/travis/README.md b/testing/travis/README.md
deleted file mode 100644
index e0c13f2..0000000
--- a/testing/travis/README.md
+++ /dev/null
@@ -1,23 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Travis Scripts
-
-This directory contains scripts used for [Travis CI](https://travis-ci.org/GoogleCloudPlatform/DataflowJavaSDK)
-testing.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5e1185d6/testing/travis/test_wordcount.sh
----------------------------------------------------------------------
diff --git a/testing/travis/test_wordcount.sh b/testing/travis/test_wordcount.sh
deleted file mode 100755
index e059a35..0000000
--- a/testing/travis/test_wordcount.sh
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script runs WordCount example locally in a few different ways.
-# Specifically, all combinations of:
-# a) using mvn exec, or java -cp with a bundled jar file;
-# b) input filename with no directory component, with a relative directory, or
-# with an absolute directory; AND
-# c) input filename containing wildcards or not.
-#
-# The one optional parameter is a path from the directory containing the script
-# to the directory containing the top-level (parent) pom.xml. If no parameter
-# is provided, the script assumes that directory is equal to the directory
-# containing the script itself.
-#
-# The exit-code of the script indicates success or a failure.
-
-set -e
-set -o pipefail
-
-PASS=1
-VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[')
-JAR_FILE=examples/java/target/beam-examples-java-bundled-${VERSION}.jar
-
-function check_result_hash {
- local name=$1
- local outfile_prefix=$2
- local expected=$3
-
- local actual=$(LC_ALL=C sort $outfile_prefix-* | md5sum | awk '{print $1}' \
- || LC_ALL=C sort $outfile_prefix-* | md5 -q) || exit 2 # OSX
- if [[ "$actual" != "$expected" ]]
- then
- echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected."
- PASS=""
- echo "head hexdump of actual:"
- head $outfile_prefix-* | hexdump -c
- else
- echo "pass $name"
- # Output files are left behind in /tmp
- fi
-}
-
-function get_outfile_prefix {
- local name=$1
- # NOTE: mktemp on OSX doesn't support --tmpdir
- mktemp -u "/tmp/$name.out.XXXXXXXXXX"
-}
-
-function run_via_mvn {
- local name=$1
- local input=$2
- local expected_hash=$3
-
- local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
- local cmd='mvn exec:java -f pom.xml -pl examples/java \
- -Dexec.mainClass=org.apache.beam.examples.WordCount \
- -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"'
- echo "$name: Running $cmd" >&2
- sh -c "$cmd"
- check_result_hash "$name" "$outfile_prefix" "$expected_hash"
-}
-
-function run_bundled {
- local name=$1
- local input=$2
- local expected_hash=$3
-
- local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
- local cmd='java -cp '"$JAR_FILE"' \
- org.apache.beam.examples.WordCount \
- --runner=DirectRunner \
- --inputFile='"'$input'"' \
- --output='"$outfile_prefix"
- echo "$name: Running $cmd" >&2
- sh -c "$cmd"
- check_result_hash "$name" "$outfile_prefix" "$expected_hash"
-}
-
-function run_all_ways {
- local name=$1
- local input=$2
- local expected_hash=$3
-
- run_via_mvn ${name}a "$input" $expected_hash
- check_for_jar_file
- run_bundled ${name}b "$input" $expected_hash
-}
-
-function check_for_jar_file {
- if [[ ! -f $JAR_FILE ]]
- then
- echo "Jar file $JAR_FILE not created" >&2
- exit 2
- fi
-}
-
-run_all_ways wordcount1 "LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount2 "./LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount3 "$PWD/LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount4 "L*N?E*" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount5 "./LICE*N?E" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount6 "$PWD/*LIC?NSE" c5350a5ad4bb51e3e018612b4b044097
-
-if [[ ! "$PASS" ]]
-then
- echo "One or more tests FAILED."
- exit 1
-fi
-echo "All tests PASS"
[31/50] incubator-beam git commit: This closes #789
Posted by ke...@apache.org.
This closes #789
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/eef26f42
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/eef26f42
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/eef26f42
Branch: refs/heads/gearpump-runner
Commit: eef26f4238fb72e1a852297d3dc6b558dc567530
Parents: 038eeca 743923b
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Aug 10 09:44:42 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Aug 10 09:44:42 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/beam/findbugs-filter.xml | 6 -----
.../apache/beam/sdk/coders/StandardCoder.java | 14 ++++++-----
.../beam/sdk/coders/StandardCoderTest.java | 25 ++++++++++++++++++++
3 files changed, 33 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/eef26f42/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
----------------------------------------------------------------------
[22/50] incubator-beam git commit: [BEAM-534] Fix dead links in
README.md
Posted by ke...@apache.org.
[BEAM-534] Fix dead links in README.md
This closes #790
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ac5242e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ac5242e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ac5242e4
Branch: refs/heads/gearpump-runner
Commit: ac5242e46e73871b49f30f84efbbfebc56792acd
Parents: 7da1a3d 0812fe6
Author: Frances Perry <fj...@google.com>
Authored: Tue Aug 9 14:52:39 2016 -0700
Committer: Frances Perry <fj...@google.com>
Committed: Tue Aug 9 14:52:39 2016 -0700
----------------------------------------------------------------------
README.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[40/50] incubator-beam git commit: This closes #803
Posted by ke...@apache.org.
This closes #803
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b4c0bb9f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b4c0bb9f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b4c0bb9f
Branch: refs/heads/gearpump-runner
Commit: b4c0bb9fa9951e6cfbc6aa56edcb8235a81216b1
Parents: fa0e395 fcdd15b
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Aug 10 13:41:29 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Aug 10 13:41:29 2016 -0700
----------------------------------------------------------------------
.../beam/runners/dataflow/DataflowRunner.java | 92 ++++++++++----------
.../apache/beam/sdk/transforms/DoFnTester.java | 29 +++++-
.../org/apache/beam/sdk/transforms/ParDo.java | 6 ++
3 files changed, 77 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/b4c0bb9f/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
----------------------------------------------------------------------
diff --cc runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 54d1923,4d34ec4..bea6264
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@@ -2370,11 -2370,10 +2368,11 @@@ public class DataflowRunner extends Pip
}
/**
- * A specialized {@link OldDoFn} for writing the contents of a {@link PCollection}
+ * A specialized {@link DoFn} for writing the contents of a {@link PCollection}
* to a streaming {@link PCollectionView} backend implementation.
*/
- private static class StreamingPCollectionViewWriterFn<T>
+ @Deprecated
+ public static class StreamingPCollectionViewWriterFn<T>
extends OldDoFn<Iterable<T>, T> implements OldDoFn.RequiresWindowAccess {
private final PCollectionView<?> view;
private final Coder<T> dataCoder;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/b4c0bb9f/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
----------------------------------------------------------------------
[28/50] incubator-beam git commit: Fixed javadoc for processBundle
Posted by ke...@apache.org.
Fixed javadoc for processBundle
The 2nd `li` element for `processBundle(InputT... inputElements)` was missing a proper closing tag.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a9a76b01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a9a76b01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a9a76b01
Branch: refs/heads/gearpump-runner
Commit: a9a76b01204cde989d69ec4a79b96d430f821f6f
Parents: 6a5310a
Author: Stas Levin <st...@users.noreply.github.com>
Authored: Wed Aug 10 15:20:50 2016 +0300
Committer: Dan Halperin <dh...@google.com>
Committed: Wed Aug 10 09:21:21 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a9a76b01/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index e2764eb..93d7884 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -170,7 +170,7 @@ public class DoFnTester<InputT, OutputT> {
*
* <ol>
* <li>Calls {@link #startBundle}.</li>
- * <li>Calls {@link #processElement} on each of the arguments.<li>
+ * <li>Calls {@link #processElement} on each of the arguments.</li>
* <li>Calls {@link #finishBundle}.</li>
* <li>Returns the result of {@link #takeOutputElements}.</li>
* </ol>
[41/50] incubator-beam git commit: Make TimerData#compareTo
consistent with equals
Posted by ke...@apache.org.
Make TimerData#compareTo consistent with equals
Timers are equal if the domain, timestamp, and namespace are equal.
Compare these values in compareTo. The ordering of TimerData that are
not in the same namespace or domain is arbitrary.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9fe6ce22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9fe6ce22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9fe6ce22
Branch: refs/heads/gearpump-runner
Commit: 9fe6ce22ca12b48704deb0e7cf3c583dff9b1870
Parents: fc87a0c
Author: Thomas Groh <tg...@google.com>
Authored: Wed Aug 10 13:52:14 2016 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Wed Aug 10 13:55:54 2016 -0700
----------------------------------------------------------------------
.../apache/beam/sdk/util/TimerInternals.java | 15 ++++++-
.../beam/sdk/util/TimerInternalsTest.java | 47 ++++++++++++++++++++
2 files changed, 61 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9fe6ce22/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerInternals.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerInternals.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerInternals.java
index 3212d64..eb49b9d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerInternals.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerInternals.java
@@ -30,6 +30,7 @@ import org.apache.beam.sdk.util.state.StateNamespace;
import org.apache.beam.sdk.util.state.StateNamespaces;
import com.google.common.base.MoreObjects;
+import com.google.common.collect.ComparisonChain;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -205,9 +206,21 @@ public interface TimerInternals {
.toString();
}
+ /**
+ * {@inheritDoc}.
+ *
+ * <p>The ordering of {@link TimerData} that are not in the same namespace or domain is
+ * arbitrary.
+ */
@Override
public int compareTo(TimerData o) {
- return Long.compare(timestamp.getMillis(), o.getTimestamp().getMillis());
+ ComparisonChain chain =
+ ComparisonChain.start().compare(timestamp, o.getTimestamp()).compare(domain, o.domain);
+ if (chain.result() == 0) {
+ // Obtaining the stringKey may be expensive; only do so if required
+ chain = chain.compare(namespace.stringKey(), o.namespace.stringKey());
+ }
+ return chain.result();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9fe6ce22/sdks/java/core/src/test/java/org/apache/beam/sdk/util/TimerInternalsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/TimerInternalsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/TimerInternalsTest.java
index baf911a..bc2930c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/TimerInternalsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/TimerInternalsTest.java
@@ -17,12 +17,18 @@
*/
package org.apache.beam.sdk.util;
+import static org.hamcrest.Matchers.comparesEqualTo;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.testing.CoderProperties;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.util.TimerInternals.TimerData;
import org.apache.beam.sdk.util.TimerInternals.TimerDataCoder;
+import org.apache.beam.sdk.util.state.StateNamespace;
import org.apache.beam.sdk.util.state.StateNamespaces;
import org.joda.time.Instant;
@@ -50,4 +56,45 @@ public class TimerInternalsTest {
windowCoder, new IntervalWindow(new Instant(0), new Instant(100))),
new Instant(99), TimeDomain.PROCESSING_TIME));
}
+
+ @Test
+ public void testCompareTo() {
+ Instant firstTimestamp = new Instant(100);
+ Instant secondTimestamp = new Instant(200);
+ IntervalWindow firstWindow = new IntervalWindow(new Instant(0), firstTimestamp);
+ IntervalWindow secondWindow = new IntervalWindow(firstTimestamp, secondTimestamp);
+ Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
+
+ StateNamespace firstWindowNs = StateNamespaces.window(windowCoder, firstWindow);
+ StateNamespace secondWindowNs = StateNamespaces.window(windowCoder, secondWindow);
+
+ TimerData firstEventTime = TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.EVENT_TIME);
+ TimerData secondEventTime = TimerData.of(firstWindowNs, secondTimestamp, TimeDomain.EVENT_TIME);
+ TimerData thirdEventTime = TimerData.of(secondWindowNs, secondTimestamp, TimeDomain.EVENT_TIME);
+
+ TimerData firstProcTime =
+ TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.PROCESSING_TIME);
+ TimerData secondProcTime =
+ TimerData.of(firstWindowNs, secondTimestamp, TimeDomain.PROCESSING_TIME);
+ TimerData thirdProcTime =
+ TimerData.of(secondWindowNs, secondTimestamp, TimeDomain.PROCESSING_TIME);
+
+ assertThat(firstEventTime,
+ comparesEqualTo(TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.EVENT_TIME)));
+ assertThat(firstEventTime, lessThan(secondEventTime));
+ assertThat(secondEventTime, lessThan(thirdEventTime));
+ assertThat(firstEventTime, lessThan(thirdEventTime));
+
+ assertThat(secondProcTime,
+ comparesEqualTo(TimerData.of(firstWindowNs, secondTimestamp, TimeDomain.PROCESSING_TIME)));
+ assertThat(firstProcTime, lessThan(secondProcTime));
+ assertThat(secondProcTime, lessThan(thirdProcTime));
+ assertThat(firstProcTime, lessThan(thirdProcTime));
+
+ assertThat(firstEventTime, not(comparesEqualTo(firstProcTime)));
+ assertThat(firstProcTime,
+ not(comparesEqualTo(TimerData.of(firstWindowNs,
+ firstTimestamp,
+ TimeDomain.SYNCHRONIZED_PROCESSING_TIME))));
+ }
}
[33/50] incubator-beam git commit: This closes #807
Posted by ke...@apache.org.
This closes #807
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/95e7f017
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/95e7f017
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/95e7f017
Branch: refs/heads/gearpump-runner
Commit: 95e7f017d16878d185a806a327eb24cd22900393
Parents: eef26f4 20208d6
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Aug 10 10:00:56 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Aug 10 10:00:56 2016 -0700
----------------------------------------------------------------------
.../org/apache/beam/sdk/transforms/DoFn.java | 44 +++++-
.../beam/sdk/transforms/DoFnReflector.java | 92 +++++++----
.../beam/sdk/transforms/DoFnReflectorTest.java | 157 ++++++++++++++-----
.../transforms/DoFnReflectorBenchmark.java | 13 +-
4 files changed, 214 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
[23/50] incubator-beam git commit: This closes #805
Posted by ke...@apache.org.
This closes #805
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/063ff2f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/063ff2f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/063ff2f4
Branch: refs/heads/gearpump-runner
Commit: 063ff2f42290654cefe6c8bc4ea066d94f9aeff6
Parents: ac5242e d70a3a4
Author: Kenneth Knowles <kl...@google.com>
Authored: Tue Aug 9 15:05:40 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Aug 9 15:05:40 2016 -0700
----------------------------------------------------------------------
.../org/apache/beam/runners/dataflow/DataflowRunner.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[37/50] incubator-beam git commit: [BEAM-542] Spark batch interval
should be a configuration instead of an interpretation of the Pipeline's
windows
Posted by ke...@apache.org.
[BEAM-542] Spark batch interval should be a configuration instead of an interpretation of the Pipeline's windows
This closes #808
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/fc87a0ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/fc87a0ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/fc87a0ca
Branch: refs/heads/gearpump-runner
Commit: fc87a0ca73598bf2e9021697425bcff52d962f10
Parents: 0ab9495 08dca30
Author: Sela <an...@paypal.com>
Authored: Wed Aug 10 20:50:15 2016 +0300
Committer: Sela <an...@paypal.com>
Committed: Wed Aug 10 20:50:15 2016 +0300
----------------------------------------------------------------------
runners/spark/README.md | 3 +-
.../runners/spark/SparkPipelineOptions.java | 5 +
.../apache/beam/runners/spark/SparkRunner.java | 19 +---
.../streaming/StreamingTransformTranslator.java | 10 +-
.../StreamingWindowPipelineDetector.java | 102 -------------------
.../streaming/FlattenStreamingTest.java | 1 +
.../streaming/KafkaStreamingTest.java | 2 +
.../streaming/SimpleStreamingWordCountTest.java | 2 +
8 files changed, 18 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
[10/50] incubator-beam git commit: This closes #520
Posted by ke...@apache.org.
This closes #520
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5049011a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5049011a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5049011a
Branch: refs/heads/gearpump-runner
Commit: 5049011a2602acc1ce0e1997b467ddee38c66c10
Parents: d60a0a0 c376b45
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Aug 8 13:51:21 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 13:51:21 2016 -0700
----------------------------------------------------------------------
.../beam/runners/direct/SideInputContainer.java | 2 +-
.../runners/direct/ViewEvaluatorFactory.java | 5 +-
.../functions/FlinkProcessContext.java | 2 +-
.../functions/SideInputInitializer.java | 2 +-
.../spark/translation/SparkProcessContext.java | 2 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 2 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 45 ++++
.../beam/sdk/util/DirectSideInputReader.java | 4 +-
.../apache/beam/sdk/util/PCollectionViews.java | 228 +++++++++++++------
.../apache/beam/sdk/values/PCollectionView.java | 29 ++-
.../sdk/testing/PCollectionViewTesting.java | 35 +--
11 files changed, 262 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
[29/50] incubator-beam git commit: Closes #809
Posted by ke...@apache.org.
Closes #809
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/038eeca2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/038eeca2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/038eeca2
Branch: refs/heads/gearpump-runner
Commit: 038eeca2b9e083c4fc0b8b89ecb2b25ecac547cc
Parents: 6a5310a a9a76b0
Author: Dan Halperin <dh...@google.com>
Authored: Wed Aug 10 09:21:22 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Wed Aug 10 09:21:22 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[49/50] incubator-beam git commit: Rename DoFn to OldDoFn in Gearpump
runner
Posted by ke...@apache.org.
Rename DoFn to OldDoFn in Gearpump runner
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/bc1b3549
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/bc1b3549
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/bc1b3549
Branch: refs/heads/gearpump-runner
Commit: bc1b354949416db3b52c4f37c66968bdb86f0813
Parents: 40be715
Author: manuzhang <ow...@gmail.com>
Authored: Fri Aug 12 07:22:00 2016 +0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu Aug 25 11:38:08 2016 -0700
----------------------------------------------------------------------
.../gearpump/GearpumpPipelineResult.java | 23 ++++++++++++++++++--
.../gearpump/GearpumpPipelineRunner.java | 6 ++---
.../gearpump/examples/StreamingWordCount.java | 6 ++---
.../translators/ParDoBoundMultiTranslator.java | 3 ++-
.../translators/ParDoBoundTranslator.java | 3 ++-
.../translators/functions/DoFnFunction.java | 3 ++-
.../translators/utils/GearpumpDoFnRunner.java | 23 ++++++++++----------
7 files changed, 45 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bc1b3549/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
index bc27147..6184bc3 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
@@ -17,11 +17,14 @@
*/
package org.apache.beam.runners.gearpump;
+import org.apache.beam.sdk.AggregatorRetrievalException;
+import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.runners.AggregatorRetrievalException;
-import org.apache.beam.sdk.runners.AggregatorValues;
import org.apache.beam.sdk.transforms.Aggregator;
+import org.joda.time.Duration;
+
+import java.io.IOException;
/**
* Result of executing a {@link Pipeline} with Gearpump.
@@ -33,10 +36,26 @@ public class GearpumpPipelineResult implements PipelineResult {
}
@Override
+ public State cancel() throws IOException {
+ return null;
+ }
+
+ @Override
+ public State waitUntilFinish(Duration duration) throws IOException, InterruptedException {
+ return null;
+ }
+
+ @Override
+ public State waitUntilFinish() throws IOException, InterruptedException {
+ return null;
+ }
+
+ @Override
public <T> AggregatorValues<T> getAggregatorValues(Aggregator<?, T> aggregator)
throws AggregatorRetrievalException {
throw new AggregatorRetrievalException(
"PipelineResult getAggregatorValues not supported in Gearpump pipeline",
new UnsupportedOperationException());
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bc1b3549/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineRunner.java
index 660d703..4182ee4 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineRunner.java
@@ -23,8 +23,8 @@ import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -151,7 +151,7 @@ public class GearpumpPipelineRunner extends PipelineRunner<GearpumpPipelineResul
private final Window.Bound<T> wrapped;
- public AssignWindowsAndSetStrategy(Window.Bound<T> wrapped) {
+ AssignWindowsAndSetStrategy(Window.Bound<T> wrapped) {
this.wrapped = wrapped;
}
@@ -184,7 +184,7 @@ public class GearpumpPipelineRunner extends PipelineRunner<GearpumpPipelineResul
}
}
- private static class IdentityFn<T> extends DoFn<T, T> {
+ private static class IdentityFn<T> extends OldDoFn<T, T> {
@Override
public void processElement(ProcessContext c) {
c.output(c.element());
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bc1b3549/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/StreamingWordCount.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/StreamingWordCount.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/StreamingWordCount.java
index c51289d..5f35c6b 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/StreamingWordCount.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/StreamingWordCount.java
@@ -25,7 +25,7 @@ import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
@@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory;
*/
public class StreamingWordCount {
- static class ExtractWordsFn extends DoFn<String, String> {
+ static class ExtractWordsFn extends OldDoFn<String, String> {
private final Aggregator<Long, Long> emptyLines =
createAggregator("emptyLines", new Sum.SumLongFn());
@@ -66,7 +66,7 @@ public class StreamingWordCount {
}
}
- static class FormatAsStringFn extends DoFn<KV<String, Long>, String> {
+ static class FormatAsStringFn extends OldDoFn<KV<String, Long>, String> {
private static final Logger LOG = LoggerFactory.getLogger(FormatAsStringFn.class);
@Override
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bc1b3549/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
index af5bcbc..d5ed0d2 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
@@ -23,6 +23,7 @@ import org.apache.beam.runners.gearpump.translators.utils.GearpumpDoFnRunner;
import org.apache.beam.runners.gearpump.translators.utils.NoOpSideInputReader;
import org.apache.beam.runners.gearpump.translators.utils.NoOpStepContext;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.util.DoFnRunner;
import org.apache.beam.sdk.util.DoFnRunners;
@@ -92,7 +93,7 @@ public class ParDoBoundMultiTranslator<InputT, OutputT> implements
public DoFnMultiFunction(
GearpumpPipelineOptions pipelineOptions,
- DoFn<InputT, OutputT> doFn,
+ OldDoFn<InputT, OutputT> doFn,
TupleTag<OutputT> mainOutputTag,
TupleTagList sideOutputTags,
WindowingStrategy<?, ?> windowingStrategy,
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bc1b3549/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
index 689bc08..b97cbb4 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
@@ -21,6 +21,7 @@ package org.apache.beam.runners.gearpump.translators;
import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
import org.apache.beam.runners.gearpump.translators.utils.NoOpSideInputReader;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingStrategy;
@@ -38,7 +39,7 @@ public class ParDoBoundTranslator<InputT, OutputT> implements
@Override
public void translate(ParDo.Bound<InputT, OutputT> transform, TranslationContext context) {
- DoFn<InputT, OutputT> doFn = transform.getFn();
+ OldDoFn<InputT, OutputT> doFn = transform.getFn();
PCollection<OutputT> output = context.getOutput(transform);
WindowingStrategy<?, ?> windowingStrategy = output.getWindowingStrategy();
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bc1b3549/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index 088fc14..b1ebd2a 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -22,6 +22,7 @@ import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
import org.apache.beam.runners.gearpump.translators.utils.GearpumpDoFnRunner;
import org.apache.beam.runners.gearpump.translators.utils.NoOpStepContext;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.util.DoFnRunner;
import org.apache.beam.sdk.util.DoFnRunners;
import org.apache.beam.sdk.util.SideInputReader;
@@ -50,7 +51,7 @@ public class DoFnFunction<InputT, OutputT> implements
public DoFnFunction(
GearpumpPipelineOptions pipelineOptions,
- DoFn<InputT, OutputT> doFn,
+ OldDoFn<InputT, OutputT> doFn,
WindowingStrategy<?, ?> windowingStrategy,
SideInputReader sideInputReader) {
this.doFnRunner = new GearpumpDoFnRunner<>(
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bc1b3549/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/GearpumpDoFnRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/GearpumpDoFnRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/GearpumpDoFnRunner.java
index 608ad7c..be0d025 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/GearpumpDoFnRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/GearpumpDoFnRunner.java
@@ -25,6 +25,7 @@ import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
@@ -64,7 +65,7 @@ import java.util.Set;
public class GearpumpDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, OutputT>,
Serializable {
- private final DoFn<InputT, OutputT> fn;
+ private final OldDoFn<InputT, OutputT> fn;
private final transient PipelineOptions options;
private final SideInputReader sideInputReader;
private final DoFnRunners.OutputManager outputManager;
@@ -76,7 +77,7 @@ public class GearpumpDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, O
public GearpumpDoFnRunner(
GearpumpPipelineOptions pipelineOptions,
- DoFn<InputT, OutputT> doFn,
+ OldDoFn<InputT, OutputT> doFn,
SideInputReader sideInputReader,
DoFnRunners.OutputManager outputManager,
TupleTag<OutputT> mainOutputTag,
@@ -119,7 +120,7 @@ public class GearpumpDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, O
@Override
public void processElement(WindowedValue<InputT> elem) {
if (elem.getWindows().size() <= 1
- || (!DoFn.RequiresWindowAccess.class.isAssignableFrom(fn.getClass())
+ || (!OldDoFn.RequiresWindowAccess.class.isAssignableFrom(fn.getClass())
&& context.sideInputReader.isEmpty())) {
invokeProcessElement(elem);
} else {
@@ -144,7 +145,7 @@ public class GearpumpDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, O
}
private void invokeProcessElement(WindowedValue<InputT> elem) {
- final DoFn<InputT, OutputT>.ProcessContext processContext =
+ final OldDoFn<InputT, OutputT>.ProcessContext processContext =
new DoFnProcessContext<>(fn, context, elem);
// This can contain user code. Wrap it in case it throws an exception.
try {
@@ -169,11 +170,11 @@ public class GearpumpDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, O
* @param <OutputT> the type of the DoFn's (main) output elements
*/
private static class DoFnContext<InputT, OutputT>
- extends DoFn<InputT, OutputT>.Context {
+ extends OldDoFn<InputT, OutputT>.Context {
private static final int MAX_SIDE_OUTPUTS = 1000;
final transient PipelineOptions options;
- final DoFn<InputT, OutputT> fn;
+ final OldDoFn<InputT, OutputT> fn;
final SideInputReader sideInputReader;
final DoFnRunners.OutputManager outputManager;
final TupleTag<OutputT> mainOutputTag;
@@ -187,7 +188,7 @@ public class GearpumpDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, O
private final Set<TupleTag<?>> outputTags;
public DoFnContext(PipelineOptions options,
- DoFn<InputT, OutputT> fn,
+ OldDoFn<InputT, OutputT> fn,
SideInputReader sideInputReader,
DoFnRunners.OutputManager outputManager,
TupleTag<OutputT> mainOutputTag,
@@ -357,14 +358,14 @@ public class GearpumpDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, O
* @param <OutputT> the type of the DoFn's (main) output elements
*/
private static class DoFnProcessContext<InputT, OutputT>
- extends DoFn<InputT, OutputT>.ProcessContext {
+ extends OldDoFn<InputT, OutputT>.ProcessContext {
- final DoFn<InputT, OutputT> fn;
+ final OldDoFn<InputT, OutputT> fn;
final DoFnContext<InputT, OutputT> context;
final WindowedValue<InputT> windowedValue;
- public DoFnProcessContext(DoFn<InputT, OutputT> fn,
+ public DoFnProcessContext(OldDoFn<InputT, OutputT> fn,
DoFnContext<InputT, OutputT> context,
WindowedValue<InputT> windowedValue) {
fn.super();
@@ -409,7 +410,7 @@ public class GearpumpDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, O
@Override
public BoundedWindow window() {
- if (!(fn instanceof DoFn.RequiresWindowAccess)) {
+ if (!(fn instanceof OldDoFn.RequiresWindowAccess)) {
throw new UnsupportedOperationException(
"window() is only available in the context of a DoFn marked as RequiresWindow.");
}
[47/50] incubator-beam git commit: add package-info.java
Posted by ke...@apache.org.
add package-info.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/276a2e10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/276a2e10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/276a2e10
Branch: refs/heads/gearpump-runner
Commit: 276a2e106aa1a5736666fc2eb2426b640f63cf68
Parents: 1672b54
Author: manuzhang <ow...@gmail.com>
Authored: Thu Jul 28 16:30:13 2016 +0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu Aug 25 11:37:27 2016 -0700
----------------------------------------------------------------------
.../runners/gearpump/examples/package-info.java | 22 ++++++++++++++++++++
.../beam/runners/gearpump/package-info.java | 22 ++++++++++++++++++++
.../translators/functions/package-info.java | 22 ++++++++++++++++++++
.../gearpump/translators/io/package-info.java | 22 ++++++++++++++++++++
.../gearpump/translators/package-info.java | 22 ++++++++++++++++++++
.../translators/utils/package-info.java | 22 ++++++++++++++++++++
6 files changed, 132 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/276a2e10/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/package-info.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/package-info.java
new file mode 100644
index 0000000..a62a6c0
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Examples showcase Beam application over Gearpump runner.
+ */
+package org.apache.beam.runners.gearpump.examples;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/276a2e10/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/package-info.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/package-info.java
new file mode 100644
index 0000000..5013616
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Internal implementation of the Beam runner for Apache Gearpump.
+ */
+package org.apache.beam.runners.gearpump;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/276a2e10/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/package-info.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/package-info.java
new file mode 100644
index 0000000..cba2363
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Gearpump specific wrappers for Beam DoFn.
+ */
+package org.apache.beam.runners.gearpump.translators.functions;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/276a2e10/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/package-info.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/package-info.java
new file mode 100644
index 0000000..dfdf51a
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Gearpump specific wrappers for Beam I/O.
+ */
+package org.apache.beam.runners.gearpump.translators.io;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/276a2e10/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/package-info.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/package-info.java
new file mode 100644
index 0000000..612096a
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Gearpump specific translators.
+ */
+package org.apache.beam.runners.gearpump.translators;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/276a2e10/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/package-info.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/package-info.java
new file mode 100644
index 0000000..ab2a6ea
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities for translators.
+ */
+package org.apache.beam.runners.gearpump.translators.utils;
[39/50] incubator-beam git commit: This closes #699
Posted by ke...@apache.org.
This closes #699
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/fa0e395f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/fa0e395f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/fa0e395f
Branch: refs/heads/gearpump-runner
Commit: fa0e395f0ad3e874206066881a273869e75d96a3
Parents: fc87a0c adec254
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Aug 10 11:34:17 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Aug 10 11:34:17 2016 -0700
----------------------------------------------------------------------
.../beam/runners/direct/DirectRunner.java | 25 +-
.../beam/runners/flink/FlinkRunnerResult.java | 4 +-
.../runners/dataflow/DataflowPipelineJob.java | 21 +-
.../beam/runners/dataflow/DataflowRunner.java | 4 +-
.../dataflow/DataflowPipelineJobTest.java | 4 +-
.../spark/translation/EvaluationContext.java | 4 +-
.../spark/translation/SparkRuntimeContext.java | 2 +-
.../translation/MultiOutputWordCountTest.java | 2 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 93 ++++++++
.../beam/sdk/AggregatorRetrievalException.java | 33 +++
.../org/apache/beam/sdk/AggregatorValues.java | 52 +++++
.../main/java/org/apache/beam/sdk/Pipeline.java | 10 +
.../org/apache/beam/sdk/PipelineResult.java | 2 -
.../runners/AggregatorPipelineExtractor.java | 93 --------
.../runners/AggregatorRetrievalException.java | 33 ---
.../beam/sdk/runners/AggregatorValues.java | 52 -----
.../beam/sdk/util/MapAggregatorValues.java | 50 ----
.../sdk/AggregatorPipelineExtractorTest.java | 229 +++++++++++++++++++
.../AggregatorPipelineExtractorTest.java | 229 -------------------
.../apache/beam/sdk/transforms/DoFnTest.java | 1 +
.../apache/beam/sdk/transforms/OldDoFnTest.java | 3 +-
21 files changed, 464 insertions(+), 482 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fa0e395f/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
----------------------------------------------------------------------
[18/50] incubator-beam git commit: Unwrap UserCodeException in
DoFnTester
Posted by ke...@apache.org.
Unwrap UserCodeException in DoFnTester
The execution of user code and system code is intertwined in the OldDoFn
wrapper of DoFn. So DoFnTester will sometimes encounter a wrapped
UserCodeException where previously the exception would not have
been wrapped.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a1d601af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a1d601af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a1d601af
Branch: refs/heads/gearpump-runner
Commit: a1d601afd0b98bf6183b14a8bbd5e6b8bee0233c
Parents: 2c8eb42
Author: Kenneth Knowles <kl...@google.com>
Authored: Tue Aug 9 12:39:41 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Aug 9 12:41:52 2016 -0700
----------------------------------------------------------------------
.../apache/beam/sdk/transforms/DoFnTester.java | 29 ++++++++++++++++++--
1 file changed, 26 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a1d601af/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index e2764eb..a2ce6c9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -26,6 +26,7 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.util.TimerInternals;
+import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingInternals;
import org.apache.beam.sdk.util.state.InMemoryStateInternals;
@@ -190,10 +191,24 @@ public class DoFnTester<InputT, OutputT> {
initializeState();
TestContext<InputT, OutputT> context = createContext(fn);
context.setupDelegateAggregators();
- fn.startBundle(context);
+ try {
+ fn.startBundle(context);
+ } catch (UserCodeException e) {
+ unwrapUserCodeException(e);
+ }
state = State.STARTED;
}
+ private static void unwrapUserCodeException(UserCodeException e) throws Exception {
+ if (e.getCause() instanceof Exception) {
+ throw (Exception) e.getCause();
+ } else if (e.getCause() instanceof Error) {
+ throw (Error) e.getCause();
+ } else {
+ throw e;
+ }
+ }
+
/**
* Calls {@link OldDoFn#processElement} on the {@code OldDoFn} under test, in a
* context where {@link OldDoFn.ProcessContext#element} returns the
@@ -212,7 +227,11 @@ public class DoFnTester<InputT, OutputT> {
if (state == State.UNSTARTED) {
startBundle();
}
- fn.processElement(createProcessContext(fn, element));
+ try {
+ fn.processElement(createProcessContext(fn, element));
+ } catch (UserCodeException e) {
+ unwrapUserCodeException(e);
+ }
}
/**
@@ -231,7 +250,11 @@ public class DoFnTester<InputT, OutputT> {
if (state == State.UNSTARTED) {
startBundle();
}
- fn.finishBundle(createContext(fn));
+ try {
+ fn.finishBundle(createContext(fn));
+ } catch (UserCodeException e) {
+ unwrapUserCodeException(e);
+ }
state = State.FINISHED;
}
[04/50] incubator-beam git commit: Port Java 8 tests to new DoFn
Posted by ke...@apache.org.
Port Java 8 tests to new DoFn
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/879f18fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/879f18fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/879f18fd
Branch: refs/heads/gearpump-runner
Commit: 879f18fd1694b0540e3e695416566f24220ecb1e
Parents: 331f523
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Aug 5 12:12:12 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 11:35:17 2016 -0700
----------------------------------------------------------------------
.../apache/beam/sdk/transforms/WithTimestampsJava8Test.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/879f18fd/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/WithTimestampsJava8Test.java
----------------------------------------------------------------------
diff --git a/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/WithTimestampsJava8Test.java b/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/WithTimestampsJava8Test.java
index 1141e88..03aa647 100644
--- a/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/WithTimestampsJava8Test.java
+++ b/sdks/java/java8tests/src/test/java/org/apache/beam/sdk/transforms/WithTimestampsJava8Test.java
@@ -47,9 +47,9 @@ public class WithTimestampsJava8Test implements Serializable {
.apply(WithTimestamps.of((String input) -> new Instant(Long.valueOf(yearTwoThousand))));
PCollection<KV<String, Instant>> timestampedVals =
- timestamped.apply(ParDo.of(new OldDoFn<String, KV<String, Instant>>() {
- @Override
- public void processElement(OldDoFn<String, KV<String, Instant>>.ProcessContext c)
+ timestamped.apply(ParDo.of(new DoFn<String, KV<String, Instant>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c)
throws Exception {
c.output(KV.of(c.element(), c.timestamp()));
}
[03/50] incubator-beam git commit: Port Flink integration tests to
new DoFn
Posted by ke...@apache.org.
Port Flink integration tests to new DoFn
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ae1f6d18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ae1f6d18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ae1f6d18
Branch: refs/heads/gearpump-runner
Commit: ae1f6d181ebe3c0bdffc35c833a6fdc858937d6c
Parents: 879f18f
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Aug 5 12:17:20 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 8 11:35:17 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/runners/flink/ReadSourceITCase.java | 6 +++---
.../apache/beam/runners/flink/ReadSourceStreamingITCase.java | 8 +++++---
2 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ae1f6d18/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/ReadSourceITCase.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/ReadSourceITCase.java b/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/ReadSourceITCase.java
index ca70096..516c7ba 100644
--- a/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/ReadSourceITCase.java
+++ b/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/ReadSourceITCase.java
@@ -20,7 +20,7 @@ package org.apache.beam.runners.flink;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.CountingInput;
import org.apache.beam.sdk.io.TextIO;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.PCollection;
@@ -72,8 +72,8 @@ public class ReadSourceITCase extends JavaProgramTestBase {
PCollection<String> result = p
.apply(CountingInput.upTo(10))
- .apply(ParDo.of(new OldDoFn<Long, String>() {
- @Override
+ .apply(ParDo.of(new DoFn<Long, String>() {
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
c.output(c.element().toString());
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ae1f6d18/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/ReadSourceStreamingITCase.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/ReadSourceStreamingITCase.java b/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/ReadSourceStreamingITCase.java
index bc69f34..ea58d0d 100644
--- a/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/ReadSourceStreamingITCase.java
+++ b/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/ReadSourceStreamingITCase.java
@@ -20,9 +20,11 @@ package org.apache.beam.runners.flink;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.CountingInput;
import org.apache.beam.sdk.io.TextIO;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
+
import com.google.common.base.Joiner;
+
import org.apache.flink.streaming.util.StreamingProgramTestBase;
/**
@@ -59,8 +61,8 @@ public class ReadSourceStreamingITCase extends StreamingProgramTestBase {
p
.apply(CountingInput.upTo(10))
- .apply(ParDo.of(new OldDoFn<Long, String>() {
- @Override
+ .apply(ParDo.of(new DoFn<Long, String>() {
+ @ProcessElement
public void processElement(ProcessContext c) throws Exception {
c.output(c.element().toString());
}
[19/50] incubator-beam git commit: Deprecate more uses of OldDoFn
Posted by ke...@apache.org.
Deprecate more uses of OldDoFn
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2c8eb429
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2c8eb429
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2c8eb429
Branch: refs/heads/gearpump-runner
Commit: 2c8eb429b6710c3a637dda3a707fcc887e16858a
Parents: 7da1a3d
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Aug 8 13:27:00 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Aug 9 12:41:52 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/beam/sdk/transforms/ParDo.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2c8eb429/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index 91f6203..ca6d9b2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -516,7 +516,10 @@ public class ParDo {
* {@code PCollection<OutputT>}, inferred from the types of the argument
* {@code OldDoFn<InputT, OutputT>}. It is ready to be applied, or further
* properties can be set on it first.
+ *
+ * @deprecated please port your {@link OldDoFn} to a {@link DoFn}
*/
+ @Deprecated
public static <InputT, OutputT> Bound<InputT, OutputT> of(OldDoFn<InputT, OutputT> fn) {
return of(fn, fn.getClass());
}
@@ -620,7 +623,10 @@ public class ParDo {
* not modify this transform. The resulting {@link PTransform} is
* sufficiently specified to be applied, but more properties can
* still be specified.
+ *
+ * @deprecated please port your {@link OldDoFn} to a {@link DoFn}
*/
+ @Deprecated
public <InputT, OutputT> Bound<InputT, OutputT> of(OldDoFn<InputT, OutputT> fn) {
return of(fn, fn.getClass());
}
[32/50] incubator-beam git commit: Remove WindowingInternals support
from DoFnReflector
Posted by ke...@apache.org.
Remove WindowingInternals support from DoFnReflector
The test themselves are replaced by mostly-hidden placeholders, to
ensure that our code for handling generic parameters remains in place
until new context parameters that use generics are added back.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/20208d68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/20208d68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/20208d68
Branch: refs/heads/gearpump-runner
Commit: 20208d68142e756800507048d9b8339041f2db70
Parents: 063ff2f
Author: Kenneth Knowles <kl...@google.com>
Authored: Tue Aug 9 20:42:04 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Aug 10 10:00:40 2016 -0700
----------------------------------------------------------------------
.../org/apache/beam/sdk/transforms/DoFn.java | 44 +++++-
.../beam/sdk/transforms/DoFnReflector.java | 92 +++++++----
.../beam/sdk/transforms/DoFnReflectorTest.java | 157 ++++++++++++++-----
.../transforms/DoFnReflectorBenchmark.java | 13 +-
4 files changed, 214 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/20208d68/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index 6f9a6b6..a06467e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -28,7 +28,6 @@ import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.HasDisplayData;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.WindowingInternals;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TypeDescriptor;
@@ -302,11 +301,43 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
BoundedWindow window();
/**
- * Construct the {@link WindowingInternals} to use within a {@link DoFn} that
- * needs it. This is called if the {@link ProcessElement} method has a parameter of type
- * {@link WindowingInternals}.
+ * A placeholder for testing purposes. The return type itself is package-private and not
+ * implemented.
*/
- WindowingInternals<InputT, OutputT> windowingInternals();
+ InputProvider<InputT> inputProvider();
+
+ /**
+ * A placeholder for testing purposes. The return type itself is package-private and not
+ * implemented.
+ */
+ OutputReceiver<OutputT> outputReceiver();
+ }
+
+ static interface OutputReceiver<T> {
+ void output(T output);
+ }
+
+ static interface InputProvider<T> {
+ T get();
+ }
+
+ /** For testing only, this {@link ExtraContextFactory} returns {@code null} for all parameters. */
+ public static class FakeExtraContextFactory<InputT, OutputT>
+ implements ExtraContextFactory<InputT, OutputT> {
+ @Override
+ public BoundedWindow window() {
+ return null;
+ }
+
+ @Override
+ public InputProvider<InputT> inputProvider() {
+ return null;
+ }
+
+ @Override
+ public OutputReceiver<OutputT> outputReceiver() {
+ return null;
+ }
}
/////////////////////////////////////////////////////////////////////////////
@@ -331,8 +362,7 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
* <ul>
* <li>It must have at least one argument.
* <li>Its first argument must be a {@link DoFn.ProcessContext}.
- * <li>Its remaining arguments must be {@link BoundedWindow}, or
- * {@link WindowingInternals WindowingInternals<InputT, OutputT>}.
+ * <li>Its remaining argument, if any, must be {@link BoundedWindow}.
* </ul>
*/
@Documented
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/20208d68/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnReflector.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnReflector.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnReflector.java
index c6168b3..3dfda55 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnReflector.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnReflector.java
@@ -26,7 +26,6 @@ import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.UserCodeException;
-import org.apache.beam.sdk.util.WindowingInternals;
import org.apache.beam.sdk.util.common.ReflectHelpers;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
@@ -35,6 +34,7 @@ import org.apache.beam.sdk.values.TypeDescriptor;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -91,6 +91,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -121,20 +122,35 @@ public abstract class DoFnReflector {
/** Any {@link BoundedWindow} parameter is populated by the window of the current element. */
WINDOW_OF_ELEMENT(Availability.PROCESS_ELEMENT_ONLY, BoundedWindow.class, "window") {
@Override
- public <InputT, OutputT> TypeToken<?>
- tokenFor(TypeToken<InputT> in, TypeToken<OutputT> out) {
+ public <InputT, OutputT> TypeToken<?> tokenFor(TypeToken<InputT> in, TypeToken<OutputT> out) {
return TypeToken.of(BoundedWindow.class);
}
},
- WINDOWING_INTERNALS(Availability.PROCESS_ELEMENT_ONLY,
- WindowingInternals.class, "windowingInternals") {
+ INPUT_PROVIDER(Availability.PROCESS_ELEMENT_ONLY, DoFn.InputProvider.class, "inputProvider") {
@Override
- public <InputT, OutputT> TypeToken<?> tokenFor(
- TypeToken<InputT> in, TypeToken<OutputT> out) {
- return new TypeToken<WindowingInternals<InputT, OutputT>>() {}
- .where(new TypeParameter<InputT>() {}, in)
- .where(new TypeParameter<OutputT>() {}, out);
+ public <InputT, OutputT> TypeToken<?> tokenFor(TypeToken<InputT> in, TypeToken<OutputT> out) {
+ return new TypeToken<DoFn.InputProvider<InputT>>() {}.where(
+ new TypeParameter<InputT>() {}, in);
+ }
+
+ @Override
+ public boolean isHidden() {
+ return true;
+ }
+ },
+
+ OUTPUT_RECEIVER(
+ Availability.PROCESS_ELEMENT_ONLY, DoFn.OutputReceiver.class, "outputReceiver") {
+ @Override
+ public <InputT, OutputT> TypeToken<?> tokenFor(TypeToken<InputT> in, TypeToken<OutputT> out) {
+ return new TypeToken<DoFn.OutputReceiver<OutputT>>() {}.where(
+ new TypeParameter<OutputT>() {}, out);
+ }
+
+ @Override
+ public boolean isHidden() {
+ return true;
}
};
@@ -146,6 +162,14 @@ public abstract class DoFnReflector {
abstract <InputT, OutputT> TypeToken<?> tokenFor(
TypeToken<InputT> in, TypeToken<OutputT> out);
+ /**
+ * Indicates whether this enum is for testing only, hence should not appear in error messages,
+ * etc. Defaults to {@code false}.
+ */
+ boolean isHidden() {
+ return false;
+ }
+
private final Class<?> rawType;
private final Availability availability;
private final transient MethodDescription method;
@@ -241,16 +265,17 @@ public abstract class DoFnReflector {
final TypeToken<?> in, final TypeToken<?> out) {
return FluentIterable
.from(extraProcessContexts.values())
+ .filter(new Predicate<AdditionalParameter>() {
+ @Override
+ public boolean apply(@Nonnull AdditionalParameter additionalParameter) {
+ return !additionalParameter.isHidden();
+ }
+ })
.transform(new Function<AdditionalParameter, String>() {
-
@Override
- @Nullable
- public String apply(@Nullable AdditionalParameter input) {
- if (input == null) {
- return null;
- } else {
- return formatType(input.tokenFor(in, out));
- }
+ @Nonnull
+ public String apply(@Nonnull AdditionalParameter input) {
+ return formatType(input.tokenFor(in, out));
}
})
.toSortedSet(String.CASE_INSENSITIVE_ORDER);
@@ -285,10 +310,9 @@ public abstract class DoFnReflector {
* <li>The method has at least one argument.
* <li>The first argument is of type firstContextArg.
* <li>The remaining arguments have raw types that appear in {@code contexts}
- * <li>Any generics on the extra context arguments match what is expected. Eg.,
- * {@code WindowingInternals<InputT, OutputT>} either matches the
- * {@code InputT} and {@code OutputT} parameters of the
- * {@code OldDoFn<InputT, OutputT>.ProcessContext}, or it uses a wildcard, etc.
+ * <li>Any generics on the extra context arguments match what is expected. Currently, this
+ * is exercised only by placeholders. For example, {@code InputReceiver<InputT> must either match
+ * the {@code InputT} {@code OldDoFn<InputT, OutputT>.ProcessContext} or use a wildcard, etc.
* </ol>
*
* @param m the method to verify
@@ -298,7 +322,8 @@ public abstract class DoFnReflector {
* @param iParam TypeParameter representing the input type
* @param oParam TypeParameter representing the output type
*/
- @VisibleForTesting static <InputT, OutputT> List<AdditionalParameter> verifyMethodArguments(
+ @VisibleForTesting
+ static <InputT, OutputT> List<AdditionalParameter> verifyMethodArguments(
Method m,
Map<Class<?>, AdditionalParameter> contexts,
TypeToken<?> firstContextArg,
@@ -607,11 +632,13 @@ public abstract class DoFnReflector {
}
@Override
- public WindowingInternals<InputT, OutputT> windowingInternals() {
- // The DoFn doesn't allow us to ask for these outside ProcessElements, so this
- // should be unreachable.
- throw new UnsupportedOperationException(
- "Can only get the windowingInternals in ProcessElements");
+ public DoFn.InputProvider<InputT> inputProvider() {
+ throw new UnsupportedOperationException("inputProvider() exists only for testing");
+ }
+
+ @Override
+ public DoFn.OutputReceiver<OutputT> outputReceiver() {
+ throw new UnsupportedOperationException("outputReceiver() exists only for testing");
}
}
@@ -679,8 +706,13 @@ public abstract class DoFnReflector {
}
@Override
- public WindowingInternals<InputT, OutputT> windowingInternals() {
- return context.windowingInternals();
+ public DoFn.InputProvider<InputT> inputProvider() {
+ throw new UnsupportedOperationException("inputProvider() exists only for testing");
+ }
+
+ @Override
+ public DoFn.OutputReceiver<OutputT> outputReceiver() {
+ throw new UnsupportedOperationException("outputReceiver() exists only for testing");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/20208d68/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnReflectorTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnReflectorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnReflectorTest.java
index df9e441..c47e0cf 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnReflectorTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnReflectorTest.java
@@ -28,7 +28,6 @@ import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
import org.apache.beam.sdk.transforms.dofnreflector.DoFnReflectorTestHelper;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.UserCodeException;
-import org.apache.beam.sdk.util.WindowingInternals;
import org.junit.Before;
import org.junit.Rule;
@@ -71,7 +70,9 @@ public class DoFnReflectorTest {
@Mock
private BoundedWindow mockWindow;
@Mock
- private WindowingInternals<String, String> mockWindowingInternals;
+ private DoFn.InputProvider<String> mockInputProvider;
+ @Mock
+ private DoFn.OutputReceiver<String> mockOutputReceiver;
private ExtraContextFactory<String, String> extraContextFactory;
@@ -85,8 +86,13 @@ public class DoFnReflectorTest {
}
@Override
- public WindowingInternals<String, String> windowingInternals() {
- return mockWindowingInternals;
+ public DoFn.InputProvider<String> inputProvider() {
+ return mockInputProvider;
+ }
+
+ @Override
+ public DoFn.OutputReceiver<String> outputReceiver() {
+ return mockOutputReceiver;
}
};
}
@@ -257,16 +263,35 @@ public class DoFnReflectorTest {
}
@Test
- public void testDoFnWithWindowingInternals() throws Exception {
+ public void testDoFnWithOutputReceiver() throws Exception {
+ final Invocations invocations = new Invocations("AnonymousClass");
+ DoFnReflector reflector = underTest(new DoFn<String, String>() {
+
+ @ProcessElement
+ public void processElement(ProcessContext c, DoFn.OutputReceiver<String> o)
+ throws Exception {
+ invocations.wasProcessElementInvoked = true;
+ assertSame(c, mockContext);
+ assertSame(o, mockOutputReceiver);
+ }
+ });
+
+ assertFalse(reflector.usesSingleWindow());
+
+ checkInvokeProcessElementWorks(reflector, invocations);
+ }
+
+ @Test
+ public void testDoFnWithInputProvider() throws Exception {
final Invocations invocations = new Invocations("AnonymousClass");
DoFnReflector reflector = underTest(new DoFn<String, String>() {
@ProcessElement
- public void processElement(ProcessContext c, WindowingInternals<String, String> w)
+ public void processElement(ProcessContext c, DoFn.InputProvider<String> i)
throws Exception {
invocations.wasProcessElementInvoked = true;
assertSame(c, mockContext);
- assertSame(w, mockWindowingInternals);
+ assertSame(i, mockInputProvider);
}
});
@@ -513,7 +538,7 @@ public class DoFnReflectorTest {
thrown.expectMessage(
"Integer is not a valid context parameter for method "
+ getClass().getName() + "#badExtraProcessContext(ProcessContext, Integer)"
- + ". Should be one of [BoundedWindow, WindowingInternals<Integer, String>]");
+ + ". Should be one of [BoundedWindow]");
DoFnReflector.verifyProcessMethodArguments(
getClass().getDeclaredMethod("badExtraProcessContext",
@@ -534,102 +559,148 @@ public class DoFnReflectorTest {
}
@SuppressWarnings("unused")
- private void goodGenerics(DoFn<Integer, String>.ProcessContext c,
- WindowingInternals<Integer, String> i1) {}
+ private void goodGenerics(
+ DoFn<Integer, String>.ProcessContext c,
+ DoFn.InputProvider<Integer> input,
+ DoFn.OutputReceiver<String> output) {}
@Test
public void testValidGenerics() throws Exception {
- Method method = getClass().getDeclaredMethod("goodGenerics",
- DoFn.ProcessContext.class, WindowingInternals.class);
+ Method method =
+ getClass()
+ .getDeclaredMethod(
+ "goodGenerics",
+ DoFn.ProcessContext.class,
+ DoFn.InputProvider.class,
+ DoFn.OutputReceiver.class);
DoFnReflector.verifyProcessMethodArguments(method);
}
@SuppressWarnings("unused")
- private void goodWildcards(DoFn<Integer, String>.ProcessContext c,
- WindowingInternals<?, ?> i1) {}
+ private void goodWildcards(
+ DoFn<Integer, String>.ProcessContext c,
+ DoFn.InputProvider<?> input,
+ DoFn.OutputReceiver<?> output) {}
@Test
public void testGoodWildcards() throws Exception {
- Method method = getClass().getDeclaredMethod("goodWildcards",
- DoFn.ProcessContext.class, WindowingInternals.class);
+ Method method =
+ getClass()
+ .getDeclaredMethod(
+ "goodWildcards",
+ DoFn.ProcessContext.class,
+ DoFn.InputProvider.class,
+ DoFn.OutputReceiver.class);
DoFnReflector.verifyProcessMethodArguments(method);
}
@SuppressWarnings("unused")
- private void goodBoundedWildcards(DoFn<Integer, String>.ProcessContext c,
- WindowingInternals<? super Integer, ? super String> i1) {}
+ private void goodBoundedWildcards(
+ DoFn<Integer, String>.ProcessContext c,
+ DoFn.InputProvider<? super Integer> input,
+ DoFn.OutputReceiver<? super String> output) {}
@Test
public void testGoodBoundedWildcards() throws Exception {
- Method method = getClass().getDeclaredMethod("goodBoundedWildcards",
- DoFn.ProcessContext.class, WindowingInternals.class);
+ Method method =
+ getClass()
+ .getDeclaredMethod(
+ "goodBoundedWildcards",
+ DoFn.ProcessContext.class,
+ DoFn.InputProvider.class,
+ DoFn.OutputReceiver.class);
DoFnReflector.verifyProcessMethodArguments(method);
}
@SuppressWarnings("unused")
private <InputT, OutputT> void goodTypeVariables(
DoFn<InputT, OutputT>.ProcessContext c,
- WindowingInternals<InputT, OutputT> i1) {}
+ DoFn.InputProvider<InputT> input,
+ DoFn.OutputReceiver<OutputT> output) {}
@Test
public void testGoodTypeVariables() throws Exception {
- Method method = getClass().getDeclaredMethod("goodTypeVariables",
- DoFn.ProcessContext.class, WindowingInternals.class);
+ Method method =
+ getClass()
+ .getDeclaredMethod(
+ "goodTypeVariables",
+ DoFn.ProcessContext.class,
+ DoFn.InputProvider.class,
+ DoFn.OutputReceiver.class);
DoFnReflector.verifyProcessMethodArguments(method);
}
@SuppressWarnings("unused")
- private void badGenericTwoArgs(DoFn<Integer, String>.ProcessContext c,
- WindowingInternals<Integer, Integer> i1) {}
+ private void badGenericTwoArgs(
+ DoFn<Integer, String>.ProcessContext c,
+ DoFn.InputProvider<Integer> input,
+ DoFn.OutputReceiver<Integer> output) {}
@Test
public void testBadGenericsTwoArgs() throws Exception {
- Method method = getClass().getDeclaredMethod("badGenericTwoArgs",
- DoFn.ProcessContext.class, WindowingInternals.class);
+ Method method =
+ getClass()
+ .getDeclaredMethod(
+ "badGenericTwoArgs",
+ DoFn.ProcessContext.class,
+ DoFn.InputProvider.class,
+ DoFn.OutputReceiver.class);
thrown.expect(IllegalStateException.class);
thrown.expectMessage("Incompatible generics in context parameter "
- + "WindowingInternals<Integer, Integer> "
+ + "OutputReceiver<Integer> "
+ "for method " + getClass().getName()
- + "#badGenericTwoArgs(ProcessContext, WindowingInternals). Should be "
- + "WindowingInternals<Integer, String>");
+ + "#badGenericTwoArgs(ProcessContext, InputProvider, OutputReceiver). Should be "
+ + "OutputReceiver<String>");
DoFnReflector.verifyProcessMethodArguments(method);
}
@SuppressWarnings("unused")
- private void badGenericWildCards(DoFn<Integer, String>.ProcessContext c,
- WindowingInternals<Integer, ? super Integer> i1) {}
+ private void badGenericWildCards(
+ DoFn<Integer, String>.ProcessContext c,
+ DoFn.InputProvider<Integer> input,
+ DoFn.OutputReceiver<? super Integer> output) {}
@Test
public void testBadGenericWildCards() throws Exception {
- Method method = getClass().getDeclaredMethod("badGenericWildCards",
- DoFn.ProcessContext.class, WindowingInternals.class);
+ Method method =
+ getClass()
+ .getDeclaredMethod(
+ "badGenericWildCards",
+ DoFn.ProcessContext.class,
+ DoFn.InputProvider.class,
+ DoFn.OutputReceiver.class);
thrown.expect(IllegalStateException.class);
thrown.expectMessage("Incompatible generics in context parameter "
- + "WindowingInternals<Integer, ? super Integer> for method "
+ + "OutputReceiver<? super Integer> for method "
+ getClass().getName()
- + "#badGenericWildCards(ProcessContext, WindowingInternals). Should be "
- + "WindowingInternals<Integer, String>");
+ + "#badGenericWildCards(ProcessContext, InputProvider, OutputReceiver). Should be "
+ + "OutputReceiver<String>");
DoFnReflector.verifyProcessMethodArguments(method);
}
@SuppressWarnings("unused")
private <InputT, OutputT> void badTypeVariables(DoFn<InputT, OutputT>.ProcessContext c,
- WindowingInternals<InputT, InputT> i1) {}
+ DoFn.InputProvider<InputT> input, DoFn.OutputReceiver<InputT> output) {}
@Test
public void testBadTypeVariables() throws Exception {
- Method method = getClass().getDeclaredMethod("badTypeVariables",
- DoFn.ProcessContext.class, WindowingInternals.class);
+ Method method =
+ getClass()
+ .getDeclaredMethod(
+ "badTypeVariables",
+ DoFn.ProcessContext.class,
+ DoFn.InputProvider.class,
+ DoFn.OutputReceiver.class);
thrown.expect(IllegalStateException.class);
thrown.expectMessage("Incompatible generics in context parameter "
- + "WindowingInternals<InputT, InputT> for method " + getClass().getName()
- + "#badTypeVariables(ProcessContext, WindowingInternals). Should be "
- + "WindowingInternals<InputT, OutputT>");
+ + "OutputReceiver<InputT> for method " + getClass().getName()
+ + "#badTypeVariables(ProcessContext, InputProvider, OutputReceiver). Should be "
+ + "OutputReceiver<OutputT>");
DoFnReflector.verifyProcessMethodArguments(method);
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/20208d68/sdks/java/microbenchmarks/src/main/java/org/apache/beam/sdk/microbenchmarks/transforms/DoFnReflectorBenchmark.java
----------------------------------------------------------------------
diff --git a/sdks/java/microbenchmarks/src/main/java/org/apache/beam/sdk/microbenchmarks/transforms/DoFnReflectorBenchmark.java b/sdks/java/microbenchmarks/src/main/java/org/apache/beam/sdk/microbenchmarks/transforms/DoFnReflectorBenchmark.java
index 233b8be..91ecd16 100644
--- a/sdks/java/microbenchmarks/src/main/java/org/apache/beam/sdk/microbenchmarks/transforms/DoFnReflectorBenchmark.java
+++ b/sdks/java/microbenchmarks/src/main/java/org/apache/beam/sdk/microbenchmarks/transforms/DoFnReflectorBenchmark.java
@@ -58,18 +58,7 @@ public class DoFnReflectorBenchmark {
private StubDoFnProcessContext stubDoFnContext =
new StubDoFnProcessContext(doFn, ELEMENT);
private ExtraContextFactory<String, String> extraContextFactory =
- new ExtraContextFactory<String, String>() {
-
- @Override
- public BoundedWindow window() {
- return null;
- }
-
- @Override
- public WindowingInternals<String, String> windowingInternals() {
- return null;
- }
- };
+ new DoFn.FakeExtraContextFactory<>();
private DoFnReflector doFnReflector;
private OldDoFn<String, String> adaptedDoFnWithContext;
[30/50] incubator-beam git commit: [BEAM-401] Use StringBuilder in
StandardCoder.toString()
Posted by ke...@apache.org.
[BEAM-401] Use StringBuilder in StandardCoder.toString()
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/743923b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/743923b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/743923b1
Branch: refs/heads/gearpump-runner
Commit: 743923b1788eea0672db750362daf38a71ed7657
Parents: 388816a
Author: Chandni Singh <ch...@capitalone.com>
Authored: Thu Aug 4 12:00:41 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Aug 10 09:43:40 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/beam/findbugs-filter.xml | 6 -----
.../apache/beam/sdk/coders/StandardCoder.java | 14 ++++++-----
.../beam/sdk/coders/StandardCoderTest.java | 25 ++++++++++++++++++++
3 files changed, 33 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/743923b1/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
----------------------------------------------------------------------
diff --git a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
index 28fd0df..7d8f110 100644
--- a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
@@ -42,12 +42,6 @@
<!--[BEAM-398] Inconsistent synchronization-->
</Match>
<Match>
- <Class name="org.apache.beam.sdk.coders.StandardCoder"/>
- <Method name="toString"/>
- <Bug pattern="SBSC_USE_STRINGBUFFER_CONCATENATION"/>
- <!--[BEAM-401] Method invokes inefficient new String(String) constructor-->
- </Match>
- <Match>
<Class name="org.apache.beam.sdk.io.range.OffsetRangeTracker"/>
<Field name="done"/>
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/743923b1/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StandardCoder.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StandardCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StandardCoder.java
index 423d482..8b7b788 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StandardCoder.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/StandardCoder.java
@@ -102,23 +102,25 @@ public abstract class StandardCoder<T> implements Coder<T> {
@Override
public String toString() {
+ StringBuilder builder = new StringBuilder();
String s = getClass().getName();
- s = s.substring(s.lastIndexOf('.') + 1);
+ builder.append(s.substring(s.lastIndexOf('.') + 1));
+
List<? extends Coder<?>> componentCoders = getComponents();
if (!componentCoders.isEmpty()) {
- s += "(";
+ builder.append('(');
boolean first = true;
for (Coder<?> componentCoder : componentCoders) {
if (first) {
first = false;
} else {
- s += ", ";
+ builder.append(',');
}
- s += componentCoder.toString();
+ builder.append(componentCoder.toString());
}
- s += ")";
+ builder.append(')');
}
- return s;
+ return builder.toString();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/743923b1/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StandardCoderTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StandardCoderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StandardCoderTest.java
index a3a0f45..1dc2210 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StandardCoderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/coders/StandardCoderTest.java
@@ -19,6 +19,10 @@ package org.apache.beam.sdk.coders;
import org.apache.beam.sdk.testing.CoderProperties;
+import com.google.common.collect.ImmutableList;
+
+import org.hamcrest.CoreMatchers;
+import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -174,4 +178,25 @@ public class StandardCoderTest {
}
}
}
+
+ /**
+ * Test for verifying {@link StandardCoder#toString()}.
+ */
+ @Test
+ public void testToString() {
+ Assert.assertThat(new ObjectIdentityBooleanCoder().toString(),
+ CoreMatchers.equalTo("StandardCoderTest$ObjectIdentityBooleanCoder"));
+
+ ObjectIdentityBooleanCoder coderWithArgs = new ObjectIdentityBooleanCoder() {
+ @Override
+ public List<? extends Coder<?>> getCoderArguments() {
+ return ImmutableList.<Coder<?>>builder()
+ .add(BigDecimalCoder.of(), BigIntegerCoder.of())
+ .build();
+ }
+ };
+
+ Assert.assertThat(coderWithArgs.toString(),
+ CoreMatchers.equalTo("StandardCoderTest$1(BigDecimalCoder,BigIntegerCoder)"));
+ }
}
[48/50] incubator-beam git commit: Update Gearpump runner version to
0.3.0-incubating
Posted by ke...@apache.org.
Update Gearpump runner version to 0.3.0-incubating
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/40be715a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/40be715a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/40be715a
Branch: refs/heads/gearpump-runner
Commit: 40be715a696bb1218b209f7ad9a979b7e5d088d3
Parents: 276a2e1
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Aug 10 10:26:57 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu Aug 25 11:37:36 2016 -0700
----------------------------------------------------------------------
runners/gearpump/pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/40be715a/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index 85994ad..8fc4635 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>0.2.0-incubating-SNAPSHOT</version>
+ <version>0.3.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -308,4 +308,4 @@
</plugin>
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
[24/50] incubator-beam git commit: Change WordCount default input
Posted by ke...@apache.org.
Change WordCount default input
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e979c826
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e979c826
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e979c826
Branch: refs/heads/gearpump-runner
Commit: e979c826873b61f55c296cc07262829d63610a99
Parents: 9983ff9
Author: Mark Liu <ma...@markliu-macbookpro.roam.corp.google.com>
Authored: Tue Aug 9 15:27:20 2016 -0700
Committer: Mark Liu <ma...@markliu-macbookpro.roam.corp.google.com>
Committed: Tue Aug 9 15:28:24 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/beam/examples/WordCount.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e979c826/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
index d136b88..72ad35f 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
@@ -39,6 +39,7 @@ import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import com.google.common.base.Strings;
+import com.google.common.io.Resources;
import java.io.IOException;
@@ -204,16 +205,16 @@ public class WordCount {
* <li>SparkRunner:
* .../src/test/resources/LICENSE</li>
* <li>other runners:
- * gs://dataflow-samples/apache/LICENSE</li>
+ * gs://apache-beam-samples/apache/LICENSE</li>
* </ul>
*/
public static class InputFactory implements DefaultValueFactory<String> {
@Override
public String create(PipelineOptions options) {
if (options.getRunner().isAssignableFrom(SparkRunner.class)) {
- return WordCount.class.getClass().getResource("/LICENSE").getPath();
+ return Resources.getResource("LICENSE").getPath();
} else {
- return "gs://dataflow-samples/apache/LICENSE";
+ return "gs://apache-beam-samples/apache/LICENSE";
}
}
}