You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/08/07 23:53:26 UTC
[01/50] [abbrv] beam git commit: [BEAM-79] Support merging windows in
GearpumpRunner
Repository: beam
Updated Branches:
refs/heads/master c9abd15e5 -> 89236e3b5
[BEAM-79] Support merging windows in GearpumpRunner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7af64720
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7af64720
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7af64720
Branch: refs/heads/master
Commit: 7af6472082cbc7f3853e87831ed4bdc72978a3a3
Parents: 4001aeb
Author: manuzhang <ow...@gmail.com>
Authored: Tue Feb 7 22:14:18 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Wed Feb 15 14:59:42 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 5 -
.../gearpump/GearpumpPipelineResult.java | 8 +-
.../beam/runners/gearpump/GearpumpRunner.java | 24 +---
.../translators/GroupByKeyTranslator.java | 133 +++++++++++--------
.../translators/WindowBoundTranslator.java | 53 +-------
.../gearpump/translators/io/GearpumpSource.java | 6 +-
.../translators/utils/DoFnRunnerFactory.java | 1 +
.../translators/utils/TranslatorUtils.java | 20 +++
.../translators/utils/TranslatorUtilsTest.java | 75 +++++++++++
9 files changed, 186 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/7af64720/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index 7c6fa76..6f91c50 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -93,11 +93,6 @@
org.apache.beam.sdk.transforms.ViewTest,
org.apache.beam.sdk.transforms.join.CoGroupByKeyTest
</exclude>
- <!-- merging windows is not supported in Gearpump -->
- <exclude>
- org.apache.beam.sdk.transforms.windowing.WindowingTest,
- org.apache.beam.sdk.util.ReshuffleTest
- </exclude>
</excludes>
<systemPropertyVariables>
<beamTestPipelineOptions>
http://git-wip-us.apache.org/repos/asf/beam/blob/7af64720/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
index 9e53517..a3740b7 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
@@ -27,7 +27,7 @@ import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.metrics.MetricResults;
import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.gearpump.cluster.MasterToAppMaster;
+import org.apache.gearpump.cluster.ApplicationStatus;
import org.apache.gearpump.cluster.MasterToAppMaster.AppMasterData;
import org.apache.gearpump.cluster.client.ClientContext;
import org.joda.time.Duration;
@@ -105,7 +105,7 @@ public class GearpumpPipelineResult implements PipelineResult {
}
private State getGearpumpState() {
- String status = null;
+ ApplicationStatus status = null;
List<AppMasterData> apps =
JavaConverters.<AppMasterData>seqAsJavaListConverter(
(Seq<AppMasterData>) client.listApps().appMasters()).asJava();
@@ -114,9 +114,9 @@ public class GearpumpPipelineResult implements PipelineResult {
status = app.status();
}
}
- if (null == status || status.equals(MasterToAppMaster.AppMasterNonExist())) {
+ if (null == status || status instanceof ApplicationStatus.NONEXIST$) {
return State.UNKNOWN;
- } else if (status.equals(MasterToAppMaster.AppMasterActive())) {
+ } else if (status instanceof ApplicationStatus.ACTIVE$) {
return State.RUNNING;
} else {
return State.STOPPED;
http://git-wip-us.apache.org/repos/asf/beam/blob/7af64720/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
index 01fdb3b..9ca1eb2 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
@@ -29,13 +29,8 @@ import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.IdentityWindowFn;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionList;
@@ -74,11 +69,7 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
public <OutputT extends POutput, InputT extends PInput> OutputT apply(
PTransform<InputT, OutputT> transform, InputT input) {
- if (Window.Bound.class.equals(transform.getClass())
- && isNullOrIdentityWindowFn(((Window.Bound) transform).getWindowFn())) {
- return (OutputT) super.apply(
- ParDo.of(new IdentityFn()), input);
- } else if (Flatten.FlattenPCollectionList.class.equals(transform.getClass())
+ if (Flatten.FlattenPCollectionList.class.equals(transform.getClass())
&& ((PCollectionList<?>) input).size() == 0) {
return (OutputT) Pipeline.applyTransform(input.getPipeline().begin(), Create.of());
} else if (Create.Values.class.equals(transform.getClass())) {
@@ -108,7 +99,7 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
TranslationContext translationContext = new TranslationContext(streamApp, options);
GearpumpPipelineTranslator translator = new GearpumpPipelineTranslator(translationContext);
translator.translate(pipeline);
- int appId = streamApp.submit();
+ int appId = streamApp.submit().appId();
return new GearpumpPipelineResult(clientContext, appId);
}
@@ -140,15 +131,4 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
return config.withValue(GEARPUMP_SERIALIZERS, ConfigValueFactory.fromMap(serializers));
}
- private static class IdentityFn<T> extends DoFn<T, T> {
-
- @ProcessElement
- public void process(ProcessContext c) {
- c.output(c.element());
- }
- }
-
- private boolean isNullOrIdentityWindowFn(WindowFn windowFn) {
- return windowFn == null || windowFn.getClass().equals(IdentityWindowFn.class);
- }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/7af64720/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
index ac8e218..69a1d11 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
@@ -23,9 +23,8 @@ import com.google.common.collect.Lists;
import java.io.Serializable;
import java.nio.ByteBuffer;
-import java.time.Instant;
+import java.util.ArrayList;
import java.util.Collection;
-import java.util.LinkedList;
import java.util.List;
import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
@@ -34,9 +33,8 @@ import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.OutputTimeFn;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
@@ -48,15 +46,14 @@ import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
import org.apache.gearpump.streaming.dsl.javaapi.functions.GroupByFunction;
import org.apache.gearpump.streaming.dsl.window.api.Discarding$;
import org.apache.gearpump.streaming.dsl.window.api.EventTimeTrigger$;
-import org.apache.gearpump.streaming.dsl.window.api.Window;
-import org.apache.gearpump.streaming.dsl.window.api.WindowFn;
-import org.apache.gearpump.streaming.dsl.window.impl.Bucket;
-import scala.collection.JavaConversions;
-
+import org.apache.gearpump.streaming.dsl.window.api.WindowFunction;
+import org.apache.gearpump.streaming.dsl.window.api.Windows;
+import org.apache.gearpump.streaming.dsl.window.impl.Window;
/**
* {@link GroupByKey} is translated to Gearpump groupBy function.
*/
+@SuppressWarnings({"rawtypes", "unchecked"})
public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKey<K, V>> {
@Override
public void translate(GroupByKey<K, V> transform, TranslationContext context) {
@@ -67,69 +64,51 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
int parallelism = context.getPipelineOptions().getParallelism();
OutputTimeFn<? super BoundedWindow> outputTimeFn = (OutputTimeFn<? super BoundedWindow>)
input.getWindowingStrategy().getOutputTimeFn();
+ WindowFn<KV<K, V>, BoundedWindow> windowFn = (WindowFn<KV<K, V>, BoundedWindow>)
+ input.getWindowingStrategy().getWindowFn();
JavaStream<WindowedValue<KV<K, Iterable<V>>>> outputStream = inputStream
- .window(Window.apply(new GearpumpWindowFn(input.getWindowingStrategy().getWindowFn()),
+ .window(Windows.apply(
+ new GearpumpWindowFn(windowFn.isNonMerging()),
EventTimeTrigger$.MODULE$, Discarding$.MODULE$), "assign_window")
.groupBy(new GroupByFn<K, V>(inputKeyCoder), parallelism, "group_by_Key_and_Window")
.map(new ValueToIterable<K, V>(), "map_value_to_iterable")
- .map(new KeyedByTimestamp<K, V>(), "keyed_by_timestamp")
- .reduce(new Merge<K, V>(outputTimeFn), "merge")
+ .map(new KeyedByTimestamp<K, V>((OutputTimeFn<? super BoundedWindow>)
+ input.getWindowingStrategy().getOutputTimeFn()), "keyed_by_timestamp")
+ .reduce(new Merge<>(windowFn, outputTimeFn), "merge")
.map(new Values<K, V>(), "values");
context.setOutputStream(context.getOutput(transform), outputStream);
}
- private static class GearpumpWindowFn<T, W extends BoundedWindow> implements WindowFn,
- Serializable {
+ private static class GearpumpWindowFn<T, W extends BoundedWindow>
+ implements WindowFunction<WindowedValue<T>>, Serializable {
- private org.apache.beam.sdk.transforms.windowing.WindowFn<T, W> windowFn;
+ private final boolean isNonMerging;
- GearpumpWindowFn(org.apache.beam.sdk.transforms.windowing.WindowFn<T, W> windowFn) {
- this.windowFn = windowFn;
+ public GearpumpWindowFn(boolean isNonMerging) {
+ this.isNonMerging = isNonMerging;
}
@Override
- public scala.collection.immutable.List<Bucket> apply(final Instant timestamp) {
+ public Window[] apply(Context<WindowedValue<T>> context) {
try {
- Collection<W> windows = windowFn.assignWindows(windowFn.new AssignContext() {
- @Override
- public T element() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public org.joda.time.Instant timestamp() {
- return TranslatorUtils.java8TimeToJodaTime(timestamp);
- }
-
- @Override
- public W window() {
- throw new UnsupportedOperationException();
- }
- });
-
- List<Bucket> buckets = new LinkedList<>();
- for (BoundedWindow window : windows) {
- buckets.add(getBucket(window));
- }
- return JavaConversions.asScalaBuffer(buckets).toList();
+ return toGearpumpWindows(context.element().getWindows().toArray(new BoundedWindow[0]));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
- private Bucket getBucket(BoundedWindow window) {
- if (window instanceof IntervalWindow) {
- IntervalWindow intervalWindow = (IntervalWindow) window;
- Instant start = TranslatorUtils.jodaTimeToJava8Time(intervalWindow.start());
- Instant end = TranslatorUtils.jodaTimeToJava8Time(intervalWindow.end());
- return new Bucket(start, end);
- } else if (window instanceof GlobalWindow) {
- Instant end = TranslatorUtils.jodaTimeToJava8Time(window.maxTimestamp());
- return new Bucket(Instant.MIN, end);
- } else {
- throw new RuntimeException("unknown window " + window.getClass().getName());
+ @Override
+ public boolean isNonMerging() {
+ return isNonMerging;
+ }
+
+ private Window[] toGearpumpWindows(BoundedWindow[] windows) {
+ Window[] gwins = new Window[windows.length];
+ for (int i = 0; i < windows.length; i++) {
+ gwins[i] = TranslatorUtils.boundedWindowToGearpumpWindow(windows[i]);
}
+ return gwins;
}
}
@@ -166,19 +145,30 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
extends MapFunction<WindowedValue<KV<K, Iterable<V>>>,
KV<org.joda.time.Instant, WindowedValue<KV<K, Iterable<V>>>>> {
+ private final OutputTimeFn<? super BoundedWindow> outputTimeFn;
+
+ public KeyedByTimestamp(OutputTimeFn<? super BoundedWindow> outputTimeFn) {
+ this.outputTimeFn = outputTimeFn;
+ }
+
@Override
public KV<org.joda.time.Instant, WindowedValue<KV<K, Iterable<V>>>> apply(
WindowedValue<KV<K, Iterable<V>>> wv) {
- return KV.of(wv.getTimestamp(), wv);
+ org.joda.time.Instant timestamp = outputTimeFn.assignOutputTime(wv.getTimestamp(),
+ Iterables.getOnlyElement(wv.getWindows()));
+ return KV.of(timestamp, wv);
}
}
private static class Merge<K, V> extends
ReduceFunction<KV<org.joda.time.Instant, WindowedValue<KV<K, Iterable<V>>>>> {
+ private final WindowFn<KV<K, V>, BoundedWindow> windowFn;
private final OutputTimeFn<? super BoundedWindow> outputTimeFn;
- Merge(OutputTimeFn<? super BoundedWindow> outputTimeFn) {
+ Merge(WindowFn<KV<K, V>, BoundedWindow> windowFn,
+ OutputTimeFn<? super BoundedWindow> outputTimeFn) {
+ this.windowFn = windowFn;
this.outputTimeFn = outputTimeFn;
}
@@ -189,13 +179,40 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
org.joda.time.Instant t1 = kv1.getKey();
org.joda.time.Instant t2 = kv2.getKey();
- WindowedValue<KV<K, Iterable<V>>> wv1 = kv1.getValue();
- WindowedValue<KV<K, Iterable<V>>> wv2 = kv2.getValue();
+ final WindowedValue<KV<K, Iterable<V>>> wv1 = kv1.getValue();
+ final WindowedValue<KV<K, Iterable<V>>> wv2 = kv2.getValue();
+
+ final List<BoundedWindow> mergedWindows = new ArrayList<>();
+ if (!windowFn.isNonMerging()) {
+ try {
+ windowFn.mergeWindows(windowFn.new MergeContext() {
+
+ @Override
+ public Collection<BoundedWindow> windows() {
+ ArrayList<BoundedWindow> windows = new ArrayList<>();
+ windows.addAll(wv1.getWindows());
+ windows.addAll(wv2.getWindows());
+ return windows;
+ }
+
+ @Override
+ public void merge(Collection<BoundedWindow> toBeMerged,
+ BoundedWindow mergeResult) throws Exception {
+ mergedWindows.add(mergeResult);
+ }
+ });
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ mergedWindows.addAll(wv1.getWindows());
+ }
- return KV.of(outputTimeFn.combine(t1, t2),
+ org.joda.time.Instant timestamp = outputTimeFn.combine(t1, t2);
+ return KV.of(timestamp,
WindowedValue.of(KV.of(wv1.getValue().getKey(),
- Iterables.concat(wv1.getValue().getValue(), wv2.getValue().getValue())),
- wv1.getTimestamp(), wv1.getWindows(), wv1.getPane()));
+ Iterables.concat(wv1.getValue().getValue(), wv2.getValue().getValue())), timestamp,
+ mergedWindows, wv1.getPane()));
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/7af64720/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
index 9bf1936..c0de2df 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
@@ -21,23 +21,15 @@ package org.apache.beam.runners.gearpump.translators;
import com.google.common.collect.Iterables;
import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.OutputTimeFn;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.gearpump.Message;
-import org.apache.gearpump.cluster.UserConfig;
+import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
-import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
-import org.apache.gearpump.streaming.javaapi.Task;
-import org.apache.gearpump.streaming.task.TaskContext;
import org.joda.time.Instant;
/**
@@ -52,34 +44,25 @@ public class WindowBoundTranslator<T> implements TransformTranslator<Window.Bou
JavaStream<WindowedValue<T>> inputStream = context.getInputStream(input);
WindowingStrategy<?, ?> outputStrategy =
transform.getOutputStrategyInternal(input.getWindowingStrategy());
- WindowFn<T, BoundedWindow> windowFn =
- (WindowFn<T, BoundedWindow>) outputStrategy.getWindowFn();
- OutputTimeFn<? super BoundedWindow> outputTimeFn = (OutputTimeFn<? super BoundedWindow>)
- outputStrategy.getOutputTimeFn();
+ WindowFn<T, BoundedWindow> windowFn = (WindowFn<T, BoundedWindow>) outputStrategy.getWindowFn();
JavaStream<WindowedValue<T>> outputStream =
inputStream
- .flatMap(new AssignWindows(windowFn, outputTimeFn), "assign_windows")
- .process(AssignTimestampTask.class, 1, UserConfig.empty(), "assign_timestamp");
+ .map(new AssignWindows(windowFn), "assign_windows");
context.setOutputStream(context.getOutput(transform), outputStream);
}
private static class AssignWindows<T> extends
- FlatMapFunction<WindowedValue<T>, WindowedValue<T>> {
+ MapFunction<WindowedValue<T>, WindowedValue<T>> {
private final WindowFn<T, BoundedWindow> windowFn;
- private final OutputTimeFn<? super BoundedWindow> outputTimeFn;
- AssignWindows(
- WindowFn<T, BoundedWindow> windowFn,
- OutputTimeFn<? super BoundedWindow> outputTimeFn) {
+ AssignWindows(WindowFn<T, BoundedWindow> windowFn) {
this.windowFn = windowFn;
- this.outputTimeFn = outputTimeFn;
}
@Override
- public Iterator<WindowedValue<T>> apply(final WindowedValue<T> value) {
- List<WindowedValue<T>> ret = new LinkedList<>();
+ public WindowedValue<T> apply(final WindowedValue<T> value) {
try {
Collection<BoundedWindow> windows = windowFn.assignWindows(windowFn.new AssignContext() {
@Override
@@ -97,32 +80,10 @@ public class WindowBoundTranslator<T> implements TransformTranslator<Window.Bou
return Iterables.getOnlyElement(value.getWindows());
}
});
- for (BoundedWindow window: windows) {
- Instant timestamp = outputTimeFn.assignOutputTime(value.getTimestamp(), window);
- ret.add(WindowedValue.of(
- value.getValue(), timestamp, window, value.getPane()));
- }
+ return WindowedValue.of(value.getValue(), value.getTimestamp(), windows, value.getPane());
} catch (Exception e) {
throw new RuntimeException(e);
}
- return ret.iterator();
- }
- }
-
- /**
- * Assign WindowedValue timestamp to Gearpump message.
- * @param <T> element type of WindowedValue
- */
- public static class AssignTimestampTask<T> extends Task {
-
- public AssignTimestampTask(TaskContext taskContext, UserConfig userConfig) {
- super(taskContext, userConfig);
- }
-
- @Override
- public void onNext(Message message) {
- final WindowedValue<T> value = (WindowedValue<T>) message.msg();
- context.output(Message.apply(value, value.getTimestamp().getMillis()));
}
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/7af64720/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
index 6e5b2de..3d0d7c8 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
@@ -62,9 +62,8 @@ public abstract class GearpumpSource<T> implements DataSource {
this.reader = createReader(options);
this.available = reader.start();
} catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
close();
+ throw new RuntimeException(e);
}
}
@@ -81,9 +80,8 @@ public abstract class GearpumpSource<T> implements DataSource {
timestamp.getMillis());
}
} catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
close();
+ throw new RuntimeException(e);
}
return message;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/7af64720/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
index 7e1402f..aaefb88 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
@@ -38,6 +38,7 @@ import org.apache.beam.sdk.values.TupleTag;
*/
public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
+ private static final long serialVersionUID = 1083167395296383469L;
private final DoFn<InputT, OutputT> fn;
private final transient PipelineOptions options;
private final SideInputReader sideInputReader;
http://git-wip-us.apache.org/repos/asf/beam/blob/7af64720/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
index 9b72275..656fc6a 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
@@ -20,6 +20,12 @@ package org.apache.beam.runners.gearpump.translators.utils;
import java.time.Instant;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.gearpump.streaming.dsl.window.impl.Window;
+
+
/**
* Utility methods for translators.
*/
@@ -32,4 +38,18 @@ public class TranslatorUtils {
public static org.joda.time.Instant java8TimeToJodaTime(Instant time) {
return new org.joda.time.Instant(time.toEpochMilli());
}
+
+ public static Window boundedWindowToGearpumpWindow(BoundedWindow window) {
+ Instant end = TranslatorUtils.jodaTimeToJava8Time(window.maxTimestamp().plus(1L));
+ if (window instanceof IntervalWindow) {
+ IntervalWindow intervalWindow = (IntervalWindow) window;
+ Instant start = TranslatorUtils.jodaTimeToJava8Time(intervalWindow.start());
+ return new Window(start, end);
+ } else if (window instanceof GlobalWindow) {
+ return new Window(TranslatorUtils.jodaTimeToJava8Time(BoundedWindow.TIMESTAMP_MIN_VALUE),
+ end);
+ } else {
+ throw new RuntimeException("unknown window " + window.getClass().getName());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/7af64720/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
new file mode 100644
index 0000000..10976e8
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.utils;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.Lists;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.gearpump.streaming.dsl.window.impl.Window;
+import org.junit.Test;
+
+
+/**
+ * Tests for {@link TranslatorUtils}.
+ */
+public class TranslatorUtilsTest {
+
+ private static final List<KV<org.joda.time.Instant, Instant>> TEST_VALUES = Lists.newArrayList(
+ KV.of(new org.joda.time.Instant(0), Instant.EPOCH),
+ KV.of(new org.joda.time.Instant(42), Instant.ofEpochMilli(42)),
+ KV.of(new org.joda.time.Instant(Long.MIN_VALUE), Instant.ofEpochMilli(Long.MIN_VALUE)),
+ KV.of(new org.joda.time.Instant(Long.MAX_VALUE), Instant.ofEpochMilli(Long.MAX_VALUE)));
+
+ @Test
+ public void testJodaTimeAndJava8TimeConversion() {
+ for (KV<org.joda.time.Instant, Instant> kv: TEST_VALUES) {
+ assertThat(TranslatorUtils.jodaTimeToJava8Time(kv.getKey()),
+ equalTo(kv.getValue()));
+ assertThat(TranslatorUtils.java8TimeToJodaTime(kv.getValue()),
+ equalTo(kv.getKey()));
+ }
+ }
+
+ @Test
+ public void testBoundedWindowToGearpumpWindow() {
+ assertThat(TranslatorUtils.boundedWindowToGearpumpWindow(
+ new IntervalWindow(new org.joda.time.Instant(0),
+ new org.joda.time.Instant(Long.MAX_VALUE))),
+ equalTo(Window.apply(Instant.EPOCH, Instant.ofEpochMilli(Long.MAX_VALUE))));
+ assertThat(TranslatorUtils.boundedWindowToGearpumpWindow(
+ new IntervalWindow(new org.joda.time.Instant(Long.MIN_VALUE),
+ new org.joda.time.Instant(Long.MAX_VALUE))),
+ equalTo(Window.apply(Instant.ofEpochMilli(Long.MIN_VALUE),
+ Instant.ofEpochMilli(Long.MAX_VALUE))));
+ BoundedWindow globalWindow = GlobalWindow.INSTANCE;
+ assertThat(TranslatorUtils.boundedWindowToGearpumpWindow(globalWindow),
+ equalTo(Window.apply(Instant.ofEpochMilli(Long.MIN_VALUE / 1000),
+ Instant.ofEpochMilli(Long.MAX_VALUE / 1000).minus(Duration.ofDays(1)).plusMillis(1))));
+ }
+}
[24/50] [abbrv] beam git commit: Merge branch 'master' of
https://github.com/apache/beam into sync-master
Posted by ke...@apache.org.
Merge branch 'master' of https://github.com/apache/beam into sync-master
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9e6c9061
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9e6c9061
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9e6c9061
Branch: refs/heads/master
Commit: 9e6c9061c1b78bfd840f2505badac108ce77f4ae
Parents: 99221e7 462335c
Author: manuzhang <ow...@gmail.com>
Authored: Mon Jun 5 10:15:27 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Jun 5 10:15:27 2017 +0800
----------------------------------------------------------------------
.gitignore | 1 +
.../jenkins/common_job_properties.groovy | 6 +-
..._beam_PostCommit_Java_JDKVersionsTest.groovy | 60 +
.../cassandra/LargeITCluster/setup.sh | 21 +
.../cassandra/LargeITCluster/start-up.sh | 22 -
.../cassandra/LargeITCluster/teardown.sh | 1 -
.../cassandra/SmallITCluster/setup.sh | 22 +
.../cassandra/SmallITCluster/start-up.sh | 23 -
.../cassandra/SmallITCluster/teardown.sh | 1 -
.../LargeProductionCluster/setup.sh | 21 +
.../LargeProductionCluster/start-up.sh | 22 -
.../LargeProductionCluster/teardown.sh | 1 -
.../elasticsearch/SmallITCluster/setup.sh | 22 +
.../elasticsearch/SmallITCluster/start-up.sh | 23 -
.../elasticsearch/SmallITCluster/teardown.sh | 1 -
pom.xml | 102 +-
runners/apex/pom.xml | 5 +
.../apache/beam/runners/apex/ApexRunner.java | 126 +-
.../translation/ApexPipelineTranslator.java | 15 +-
.../apex/translation/ParDoTranslator.java | 11 +-
.../apex/translation/TranslationContext.java | 5 +-
.../operators/ApexGroupByKeyOperator.java | 4 +-
.../operators/ApexParDoOperator.java | 10 +-
.../apex/translation/utils/NoOpStepContext.java | 37 +-
runners/core-construction-java/pom.xml | 12 +-
.../core/construction/CoderTranslation.java | 193 ++
.../beam/runners/core/construction/Coders.java | 193 --
.../core/construction/CombineTranslation.java | 125 +
.../construction/ElementAndRestriction.java | 42 +
.../ElementAndRestrictionCoder.java | 88 +
.../core/construction/FlattenTranslator.java | 63 +
.../construction/GroupByKeyTranslation.java | 62 +
.../construction/PCollectionTranslation.java | 98 +
.../runners/core/construction/PCollections.java | 97 -
.../construction/PTransformTranslation.java | 193 ++
.../runners/core/construction/PTransforms.java | 110 -
.../core/construction/ParDoTranslation.java | 540 ++++
.../core/construction/ReadTranslation.java | 242 ++
.../construction/RunnerPCollectionView.java | 88 +
.../core/construction/SdkComponents.java | 26 +-
.../core/construction/SplittableParDo.java | 301 ++
.../TransformPayloadTranslatorRegistrar.java | 29 +
.../core/construction/TriggerTranslation.java | 336 ++
.../runners/core/construction/Triggers.java | 336 --
.../construction/WindowIntoTranslation.java | 109 +
.../core/construction/WindowingStrategies.java | 275 --
.../WindowingStrategyTranslation.java | 371 +++
.../construction/WriteFilesTranslation.java | 152 +
.../core/construction/CoderTranslationTest.java | 165 +
.../runners/core/construction/CodersTest.java | 164 -
.../construction/CombineTranslationTest.java | 130 +
.../ElementAndRestrictionCoderTest.java | 126 +
.../construction/GroupByKeyTranslationTest.java | 44 +
.../PCollectionTranslationTest.java | 203 ++
.../core/construction/PCollectionsTest.java | 189 --
.../construction/PTransformTranslationTest.java | 189 ++
.../core/construction/PTransformsTest.java | 188 --
.../core/construction/ParDoTranslationTest.java | 296 ++
.../core/construction/ReadTranslationTest.java | 179 ++
.../core/construction/SdkComponentsTest.java | 7 +-
.../core/construction/SplittableParDoTest.java | 157 +
.../construction/TriggerTranslationTest.java | 112 +
.../runners/core/construction/TriggersTest.java | 111 -
.../construction/WindowIntoTranslationTest.java | 127 +
.../construction/WindowingStrategiesTest.java | 110 -
.../WindowingStrategyTranslationTest.java | 120 +
.../construction/WriteFilesTranslationTest.java | 186 ++
.../beam/runners/core/BaseExecutionContext.java | 164 -
.../apache/beam/runners/core/DoFnRunners.java | 3 +-
.../runners/core/ElementAndRestriction.java | 42 -
.../core/ElementAndRestrictionCoder.java | 88 -
.../beam/runners/core/ExecutionContext.java | 99 -
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 4 +-
.../beam/runners/core/ProcessFnRunner.java | 7 +-
.../beam/runners/core/SimpleDoFnRunner.java | 394 +--
.../beam/runners/core/SplittableParDo.java | 602 ----
.../core/SplittableParDoViaKeyedWorkItems.java | 418 +++
.../apache/beam/runners/core/StepContext.java | 33 +
.../core/ElementAndRestrictionCoderTest.java | 127 -
.../beam/runners/core/ReduceFnTester.java | 10 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 4 +-
.../core/SplittableParDoProcessFnTest.java | 515 ++++
.../beam/runners/core/SplittableParDoTest.java | 606 ----
.../runners/core/StatefulDoFnRunnerTest.java | 4 +-
runners/direct-java/pom.xml | 184 +-
.../direct/BoundedReadEvaluatorFactory.java | 14 +-
.../runners/direct/DirectExecutionContext.java | 35 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 9 +-
.../apache/beam/runners/direct/DirectGraph.java | 26 +-
.../beam/runners/direct/DirectGraphVisitor.java | 28 +-
.../beam/runners/direct/DirectGroupByKey.java | 21 +-
.../beam/runners/direct/DirectRunner.java | 14 +-
.../beam/runners/direct/EmptyInputProvider.java | 8 +-
.../beam/runners/direct/EvaluationContext.java | 46 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 26 +-
.../direct/KeyedPValueTrackingVisitor.java | 9 +-
.../beam/runners/direct/ParDoEvaluator.java | 18 +-
.../runners/direct/ParDoEvaluatorFactory.java | 2 +-
.../direct/ParDoMultiOverrideFactory.java | 15 +-
.../runners/direct/ReadEvaluatorFactory.java | 97 +
.../beam/runners/direct/RootInputProvider.java | 7 +-
.../runners/direct/RootProviderRegistry.java | 28 +-
.../apache/beam/runners/direct/SourceShard.java | 33 +
...littableProcessElementsEvaluatorFactory.java | 13 +-
.../direct/StatefulParDoEvaluatorFactory.java | 2 +-
.../direct/TestStreamEvaluatorFactory.java | 28 +-
.../direct/TransformEvaluatorRegistry.java | 130 +-
.../direct/UnboundedReadEvaluatorFactory.java | 31 +-
.../runners/direct/ViewOverrideFactory.java | 12 +-
.../beam/runners/direct/WatermarkManager.java | 19 +-
.../runners/direct/DirectGraphVisitorTest.java | 3 +
.../beam/runners/direct/DirectGraphs.java | 10 +-
.../runners/direct/EvaluationContextTest.java | 22 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 4 +-
.../StatefulParDoEvaluatorFactoryTest.java | 4 +-
.../direct/WindowEvaluatorFactoryTest.java | 10 +
runners/flink/pom.xml | 4 +-
.../flink/FlinkBatchPipelineTranslator.java | 2 +-
.../FlinkPipelineExecutionEnvironment.java | 4 +-
.../runners/flink/FlinkPipelineOptions.java | 11 +
.../apache/beam/runners/flink/FlinkRunner.java | 10 +-
.../flink/FlinkStreamingPipelineTranslator.java | 11 +-
.../FlinkStreamingTransformTranslators.java | 29 +-
.../functions/FlinkDoFnFunction.java | 14 +-
.../functions/FlinkNoOpStepContext.java | 36 +-
.../functions/FlinkStatefulDoFnFunction.java | 10 +-
.../translation/types/CoderTypeSerializer.java | 55 +
.../types/EncodedValueSerializer.java | 18 +-
.../wrappers/streaming/DoFnOperator.java | 111 +-
.../streaming/SplittableDoFnOperator.java | 12 +-
.../state/FlinkBroadcastStateInternals.java | 29 +-
.../streaming/state/FlinkStateInternals.java | 6 +-
.../streaming/UnboundedSourceWrapperTest.java | 37 +
runners/google-cloud-dataflow-java/pom.xml | 124 +-
.../beam/runners/dataflow/DataflowMetrics.java | 2 +-
.../dataflow/DataflowPipelineTranslator.java | 10 +-
.../beam/runners/dataflow/DataflowRunner.java | 2 +-
.../beam/runners/dataflow/ReadTranslator.java | 5 +-
.../beam/runners/dataflow/util/DoFnInfo.java | 25 -
.../runners/dataflow/util/PropertyNames.java | 46 -
.../runners/dataflow/util/RandomAccessData.java | 2 +-
runners/spark/pom.xml | 6 +-
.../apache/beam/runners/spark/SparkRunner.java | 2 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 4 +-
.../spark/translation/MultiDoFnFunction.java | 110 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 4 +-
.../spark/translation/SparkProcessContext.java | 50 +-
.../spark/translation/TransformTranslator.java | 83 +-
.../streaming/StreamingTransformTranslator.java | 4 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 2 +-
.../streaming/TrackStreamingSourcesTest.java | 14 +-
sdks/common/fn-api/pom.xml | 5 +
.../fn-api/src/main/proto/beam_fn_api.proto | 25 +-
.../src/main/proto/beam_runner_api.proto | 43 +-
.../src/main/proto/standard_window_fns.proto | 53 +
.../src/main/resources/beam/findbugs-filter.xml | 7 +
sdks/java/core/pom.xml | 173 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 50 +-
.../apache/beam/sdk/coders/InstantCoder.java | 81 +-
.../org/apache/beam/sdk/io/CountingSource.java | 42 +
.../beam/sdk/io/DefaultFilenamePolicy.java | 151 +-
.../org/apache/beam/sdk/io/FileSystems.java | 6 -
.../java/org/apache/beam/sdk/io/TextIO.java | 8 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 9 +-
.../beam/sdk/runners/TransformHierarchy.java | 6 +-
.../org/apache/beam/sdk/state/StateSpec.java | 53 +
.../org/apache/beam/sdk/state/StateSpecs.java | 53 +-
.../org/apache/beam/sdk/testing/LargeKeys.java | 48 +
.../apache/beam/sdk/testing/StaticWindows.java | 12 +
.../sdk/transforms/ApproximateQuantiles.java | 4 +-
.../org/apache/beam/sdk/transforms/Count.java | 10 +
.../apache/beam/sdk/transforms/Distinct.java | 80 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 12 +-
.../org/apache/beam/sdk/transforms/Max.java | 4 +-
.../org/apache/beam/sdk/transforms/Min.java | 4 +-
.../apache/beam/sdk/transforms/PTransform.java | 8 +-
.../org/apache/beam/sdk/transforms/Sum.java | 30 +
.../org/apache/beam/sdk/transforms/Top.java | 36 +-
.../org/apache/beam/sdk/transforms/View.java | 5 +-
.../sdk/transforms/windowing/BoundedWindow.java | 20 +-
.../transforms/windowing/CalendarWindows.java | 36 +
.../sdk/transforms/windowing/FixedWindows.java | 11 +
.../sdk/transforms/windowing/GlobalWindows.java | 17 +-
.../windowing/IncompatibleWindowException.java | 38 +
.../transforms/windowing/InvalidWindows.java | 11 +
.../beam/sdk/transforms/windowing/Sessions.java | 11 +
.../transforms/windowing/SlidingWindows.java | 11 +
.../beam/sdk/transforms/windowing/Window.java | 6 +-
.../beam/sdk/transforms/windowing/WindowFn.java | 21 +
.../org/apache/beam/sdk/util/ClassPath.java | 8 +-
.../apache/beam/sdk/util/IdentityWindowFn.java | 11 +
.../apache/beam/sdk/coders/AvroCoderTest.java | 4 +-
.../beam/sdk/io/DefaultFilenamePolicyTest.java | 57 +
.../java/org/apache/beam/sdk/io/TextIOTest.java | 10 -
.../sdk/runners/TransformHierarchyTest.java | 2 +-
.../beam/sdk/testing/StaticWindowsTest.java | 12 +
.../transforms/ApproximateQuantilesTest.java | 2 +-
.../beam/sdk/transforms/DistinctTest.java | 130 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 74 +
.../sdk/transforms/GroupIntoBatchesTest.java | 4 +-
.../org/apache/beam/sdk/transforms/MaxTest.java | 2 +-
.../org/apache/beam/sdk/transforms/MinTest.java | 2 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 76 +-
.../org/apache/beam/sdk/transforms/TopTest.java | 10 +-
.../windowing/CalendarWindowsTest.java | 24 +
.../transforms/windowing/FixedWindowsTest.java | 7 +
.../sdk/transforms/windowing/SessionsTest.java | 14 +
.../windowing/SlidingWindowsTest.java | 11 +
.../sdk/transforms/windowing/WindowTest.java | 44 +
.../sdk/util/IdentitySideInputWindowFn.java | 4 +
.../java/org/apache/beam/sdk/util/GcsUtil.java | 8 +-
.../sdk/util/RetryHttpRequestInitializer.java | 12 +-
.../org/apache/beam/sdk/util/GcsUtilTest.java | 16 +-
.../beam/sdk/extensions/joinlibrary/Join.java | 65 +-
.../joinlibrary/OuterFullJoinTest.java | 179 ++
.../beam/fn/harness/fake/FakeStepContext.java | 33 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 36 +-
.../control/ProcessBundleHandlerTest.java | 2 +-
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 4 +-
sdks/java/io/google-cloud-platform/pom.xml | 78 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 8 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 22 +-
.../sdk/io/gcp/bigquery/BigQueryServices.java | 8 +-
.../io/gcp/bigquery/BigQueryServicesImpl.java | 39 +-
.../sdk/io/gcp/bigquery/InsertRetryPolicy.java | 86 +
.../sdk/io/gcp/bigquery/StreamingInserts.java | 35 +-
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 50 +-
.../io/gcp/bigquery/StreamingWriteTables.java | 26 +-
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 27 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 26 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 17 +
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 5 +-
.../io/gcp/spanner/MutationSizeEstimator.java | 98 +
.../beam/sdk/io/gcp/spanner/SpannerIO.java | 350 +++
.../beam/sdk/io/gcp/spanner/package-info.java | 23 +
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 6 +-
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 69 +
.../gcp/bigquery/BigQueryServicesImplTest.java | 88 +-
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 11 +-
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 75 +-
.../io/gcp/bigquery/InsertRetryPolicyTest.java | 79 +
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 36 +
.../beam/sdk/io/gcp/datastore/V1ReadIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 15 +-
.../beam/sdk/io/gcp/datastore/V1WriteIT.java | 36 +-
.../gcp/spanner/MutationSizeEstimatorTest.java | 138 +
.../beam/sdk/io/gcp/spanner/SpannerIOTest.java | 244 ++
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 159 +
.../io/hadoop/SerializableConfiguration.java | 18 +-
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 7 +
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 33 +
.../hadoop/inputformat/HadoopInputFormatIO.java | 53 +-
.../inputformat/HadoopInputFormatIOTest.java | 80 +-
.../inputformat/HIFIOWithElasticTest.java | 10 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 5 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 21 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 5 +-
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 78 +
sdks/java/io/kafka/pom.xml | 2 +-
.../apache/beam/sdk/io/kafka/ConsumerSpEL.java | 56 +
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 44 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 122 +-
.../examples-java8/generate-sources.sh | 2 +-
.../main/resources/archetype-resources/pom.xml | 52 +-
.../examples/generate-sources.sh | 2 +-
.../main/resources/archetype-resources/pom.xml | 49 +-
.../main/resources/archetype-resources/pom.xml | 12 +-
.../resources/projects/basic/reference/pom.xml | 12 +-
sdks/python/MANIFEST.in | 1 +
sdks/python/apache_beam/coders/coders.py | 9 +-
.../examples/complete/autocomplete.py | 19 +-
.../examples/complete/autocomplete_test.py | 31 +-
.../examples/complete/estimate_pi.py | 11 +-
.../examples/complete/estimate_pi_test.py | 12 +-
.../examples/complete/game/hourly_team_score.py | 19 +-
.../examples/complete/game/user_score.py | 15 +-
.../complete/juliaset/juliaset/juliaset.py | 44 +-
.../apache_beam/examples/complete/tfidf.py | 21 +-
.../apache_beam/examples/complete/tfidf_test.py | 28 +-
.../examples/complete/top_wikipedia_sessions.py | 12 +-
.../complete/top_wikipedia_sessions_test.py | 9 +-
.../examples/cookbook/bigquery_schema.py | 159 +-
.../examples/cookbook/bigquery_side_input.py | 51 +-
.../cookbook/bigquery_side_input_test.py | 39 +-
.../examples/cookbook/bigquery_tornadoes.py | 33 +-
.../cookbook/bigquery_tornadoes_test.py | 19 +-
.../apache_beam/examples/cookbook/coders.py | 16 +-
.../examples/cookbook/coders_test.py | 14 +-
.../examples/cookbook/custom_ptransform.py | 27 +-
.../examples/cookbook/custom_ptransform_test.py | 11 +-
.../examples/cookbook/datastore_wordcount.py | 20 +-
.../apache_beam/examples/cookbook/filters.py | 21 +-
.../examples/cookbook/group_with_coder.py | 43 +-
.../examples/cookbook/group_with_coder_test.py | 4 +-
.../examples/cookbook/mergecontacts.py | 115 +-
.../examples/cookbook/mergecontacts_test.py | 3 +-
.../examples/cookbook/multiple_output_pardo.py | 72 +-
.../cookbook/multiple_output_pardo_test.py | 2 +-
.../apache_beam/examples/snippets/snippets.py | 576 ++--
.../examples/snippets/snippets_test.py | 402 +--
.../apache_beam/examples/streaming_wordcap.py | 22 +-
.../apache_beam/examples/streaming_wordcount.py | 42 +-
sdks/python/apache_beam/examples/wordcount.py | 1 -
.../apache_beam/examples/wordcount_debugging.py | 55 +-
.../apache_beam/examples/wordcount_minimal.py | 33 +-
.../python/apache_beam/io/filebasedsink_test.py | 16 +-
.../io/gcp/datastore/v1/datastoreio.py | 15 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 16 +
sdks/python/apache_beam/io/gcp/gcsio.py | 2 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 22 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 137 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 57 +-
.../apache_beam/options/value_provider.py | 1 -
sdks/python/apache_beam/pipeline.py | 19 +-
sdks/python/apache_beam/runners/api/__init__.py | 13 +-
.../apache_beam/runners/api/beam_fn_api_pb2.py | 2742 -----------------
.../runners/api/beam_fn_api_pb2_grpc.py | 205 --
.../runners/api/beam_runner_api_pb2.py | 2872 ------------------
.../runners/dataflow/dataflow_runner.py | 49 +-
.../runners/dataflow/dataflow_runner_test.py | 11 +
.../runners/dataflow/internal/dependency.py | 3 +-
.../runners/portability/fn_api_runner.py | 2 +-
.../runners/worker/data_plane_test.py | 2 +-
.../runners/worker/log_handler_test.py | 2 +-
.../runners/worker/operation_specs.py | 11 +-
.../apache_beam/runners/worker/operations.py | 2 +-
.../runners/worker/sdk_worker_test.py | 2 +-
.../apache_beam/transforms/combiners_test.py | 58 +-
sdks/python/apache_beam/transforms/window.py | 57 +-
.../apache_beam/transforms/window_test.py | 147 +-
.../transforms/write_ptransform_test.py | 7 +-
.../typehints/typed_pipeline_test.py | 22 +-
sdks/python/apache_beam/utils/proto_utils.py | 6 +
sdks/python/apache_beam/utils/urns.py | 10 +-
sdks/python/gen_protos.py | 125 +
sdks/python/run_pylint.sh | 4 +-
sdks/python/setup.py | 47 +-
sdks/python/tox.ini | 9 +
338 files changed, 13182 insertions(+), 12472 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/9e6c9061/pom.xml
----------------------------------------------------------------------
[06/50] [abbrv] beam git commit: This closes #2150: Add SideInput
support for GearpumpRunner
Posted by ke...@apache.org.
This closes #2150: Add SideInput support for GearpumpRunner
enable ParDoTest
[BEAM-79] Add SideInput support for GearpumpRunner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/15a8ad62
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/15a8ad62
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/15a8ad62
Branch: refs/heads/master
Commit: 15a8ad62afdfaef20d87a9384dd8417979bee750
Parents: 2d0aed9 3dc8fc8
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Mar 10 13:56:37 2017 -0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Mar 10 13:56:37 2017 -0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 19 -
.../gearpump/GearpumpPipelineResult.java | 47 +--
.../gearpump/GearpumpPipelineTranslator.java | 8 +
.../beam/runners/gearpump/GearpumpRunner.java | 373 ++++++++++++++++++-
.../runners/gearpump/TestGearpumpRunner.java | 39 +-
...CreateGearpumpPCollectionViewTranslator.java | 44 +++
.../CreatePCollectionViewTranslator.java | 43 +++
.../translators/CreateValuesTranslator.java | 2 +
.../FlattenPCollectionTranslator.java | 38 ++
.../translators/GroupByKeyTranslator.java | 82 ++--
.../translators/ParDoBoundMultiTranslator.java | 165 +++-----
.../translators/ParDoBoundTranslator.java | 32 +-
.../translators/WindowBoundTranslator.java | 21 +-
.../translators/functions/DoFnFunction.java | 158 ++++++--
.../translators/io/BoundedSourceWrapper.java | 1 +
.../gearpump/translators/io/GearpumpSource.java | 23 +-
.../gearpump/translators/io/ValuesSource.java | 14 +-
.../translators/utils/DoFnRunnerFactory.java | 20 +-
.../translators/utils/NoOpSideInputReader.java | 48 ---
.../translators/utils/TranslatorUtils.java | 147 ++++++++
20 files changed, 1001 insertions(+), 323 deletions(-)
----------------------------------------------------------------------
[33/50] [abbrv] beam git commit: Fix kryo exception
Posted by ke...@apache.org.
Fix kryo exception
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/11caa978
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/11caa978
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/11caa978
Branch: refs/heads/master
Commit: 11caa9789fa843991231cdba73fbd738cb6d8fe2
Parents: a7b5d98
Author: manuzhang <ow...@gmail.com>
Authored: Mon Jun 19 19:12:45 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Jun 19 21:29:53 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java | 2 ++
.../beam/runners/gearpump/translators/functions/DoFnFunction.java | 1 -
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/11caa978/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
index ae59121..5febf3c 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
@@ -106,6 +106,8 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
serializers.put("org.apache.beam.sdk.values.KV", "");
serializers.put("org.apache.beam.sdk.transforms.windowing.IntervalWindow", "");
serializers.put("org.apache.beam.sdk.values.TimestampedValue", "");
+ serializers.put(
+ "org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils$RawUnionValue", "");
if (userSerializers != null && !userSerializers.isEmpty()) {
serializers.putAll(userSerializers);
http://git-wip-us.apache.org/repos/asf/beam/blob/11caa978/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index b20896a..fde265a 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -144,7 +144,6 @@ public class DoFnFunction<InputT, OutputT> extends
Object emptyValue = WindowedValue.of(
Lists.newArrayList(), value.getTimestamp(), sideInputWindow, value.getPane());
sideInputReader.addSideInputValue(sideInput, (WindowedValue<Iterable<?>>) emptyValue);
- System.out.println(sideInput + " in " + sideInputWindow.toString() + " not ready");
}
}
}
[50/50] [abbrv] beam git commit: This closes #3636: Merge
GearpumpRunner to master branch
Posted by ke...@apache.org.
This closes #3636: Merge GearpumpRunner to master branch
Remove gearpump-runner in precommit tests
Override maven-enforcer-plugin for precommit tests
Require java 8 for precommit tests and add gearpump-runner
Deactivate integration-tests for gearpump-runner by default
Add beam-runners-gearpump dependency to javadoc
Upgrade BEAM version to 2.2.0-SNAPSHOT in gearpump-runner
Revert accidental changes to sdks/java/pom.xml
Fix ParDoTest#testPipelineOptionsParameter
Upgrade to gearpump 0.8.4
Remove unused codes
Fix kryo exception
Fix PCollectionView translation
upgrade to gearpump 0.8.4-SNAPSHOT
Fix side input handling in DoFnFunction
Respect WindowFn#getOutputTime in gearpump-runner
Activate Gearpump local-validates-runner-tests in precommit
Update against master changes
Update gearpump-runner against master changes
Update gearpump-runner against master changes
Update gearpump-runner against master changes.
[BEAM-972] Add more unit test to Gearpump runner
[BEAM-972] Add unit tests to Gearpump runner
[BEAM-79] Fix gearpump-runner merge conflicts and test failure
enable ParDoTest
[BEAM-79] Add SideInput support for GearpumpRunner
[BEAM-79] Support merging windows in GearpumpRunner
[BEAM-79] Fix PostCommit test confs for Gearpump runner
note thread is interrupted on InterruptedException
Remove cache for Gearpump on travis
reduce timeout to wait for result
fix ParDo.BoundMulti translation
return encoded key for GroupByKey translation
support OutputTimeFn
update to latest gearpump dsl function interface
fix group by window
activate ROS on Gearpump by default
update ROS configurations
[BEAM-1180] Implement GearpumpPipelineResult
[BEAM-79] Upgrade to beam-0.5.0-incubating-SNAPSHOT
[BEAM-79] Update to latest Gearpump API
Fix NoOpAggregatorFactory
Remove print to stdout
Skip window assignment when windows don't change
Add Window.Bound translator
Upgrade Gearpump version
[BEAM-79] fix gearpump runner build failure
[BEAM-79] update GearpumpPipelineResult
[BEAM-79] Port Gearpump runner from OldDoFn to new DoFn
upgrade gearpump-runner to 0.4.0-incubating-SNAPSHOT
remove "pipeline" in runner name
post-merge fix
[BEAM-79] fix integration-test failure
fix import order
!fixup Minor javadoc clean-up
Added even more javadoc to TextIO#withHeader and TextIO#withFooter (2).
Added even more javadoc to TextIO#withHeader and TextIO#withFooter.
Added javadoc to TextIO#withHeader and TextIO#withFooter.
Reverted header and footer to be of type String.
Revised according to comments following a code review.
Add header/footer support to TextIO.Write
[BEAM-242] Enable and fix checkstyle in Flink runner examples
Remove timeout in JAXBCoderTest
Be more accepting in UnboundedReadDeduplicatorTest
BigQuery: limit max job polling time to 1 minute
[BEAM-242] Enable checkstyle and fix checkstyle errors in Flink runner
[BEAM-456] Add MongoDbIO
FluentBackoff: a replacement for a variety of custom backoff implementations
Remove the DataflowRunner instructions from examples
Put classes in runners-core package into runners.core namespace
Delegate populateDipslayData to wrapped combineFn's
Fixed Combine display data
Cloud Datastore naming clean-up
DatastoreIO SplitQueryFn integration test
Add Latest CombineFn and PTransforms
Remove empty unused method in TestStreamEvaluatorFactory
Test that multiple instances of TestStream are supported
Correct some accidental renames
Fix condition in FlinkStreamingPipelineTranslator
Address comments of Flink Side-Input PR
[BEAM-569] Define maxNumRecords default value to Long.MAX_VALUE in JmsIO
Add LeaderBoardTest
take advantage of setup/teardown for KafkaWriter
Returned KafkaIO getWatermark log line in debug mode
[BEAM-572] Remove Spark Reference in WordCount
Update Dataflow Container Version
[BEAM-313] Provide a context for SparkRunner
DataflowRunner: get PBegin from PInput
[BEAM-592] Fix SparkRunner Dependency Problem in WordCount
Fix javadoc in Kinesis
Organize imports in Kinesis
kinesis: a connector for Amazon Kinesis
[BEAM-589] Fixing IO.Read transformation
Query latest timestamp
travis.yml: disable updating snapshots
Added support for reporting aggregator values to Spark sinks
[BEAM-294] Rename dataflow references to beam
Modified BigtableIO to use DoFn setup/tearDown methods instead of startBundle/finishBundle
checkstyle: prohibit API client repackaged Guava
Make WriteTest more resilient to Randomness
Update DoFn javadocs to remove references to OldDoFn and Dataflow
...
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/89236e3b
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/89236e3b
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/89236e3b
Branch: refs/heads/master
Commit: 89236e3b588cd8ab5092667b1fe155738c69dbcf
Parents: c9abd15 c35febf
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Aug 7 16:51:42 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Aug 7 16:51:42 2017 -0700
----------------------------------------------------------------------
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 2 +-
examples/java8/pom.xml | 12 +
pom.xml | 6 +
runners/gearpump/README.md | 61 ++++
runners/gearpump/pom.xml | 280 +++++++++++++++++++
.../gearpump/GearpumpPipelineOptions.java | 67 +++++
.../gearpump/GearpumpPipelineResult.java | 109 ++++++++
.../beam/runners/gearpump/GearpumpRunner.java | 119 ++++++++
.../gearpump/GearpumpRunnerRegistrar.java | 64 +++++
.../runners/gearpump/TestGearpumpRunner.java | 63 +++++
.../beam/runners/gearpump/package-info.java | 22 ++
...CreateGearpumpPCollectionViewTranslator.java | 45 +++
.../CreateStreamingGearpumpView.java | 157 +++++++++++
.../FlattenPCollectionsTranslator.java | 83 ++++++
.../translators/GearpumpPipelineTranslator.java | 143 ++++++++++
.../translators/GroupByKeyTranslator.java | 258 +++++++++++++++++
.../translators/ParDoMultiOutputTranslator.java | 103 +++++++
.../translators/ReadBoundedTranslator.java | 46 +++
.../translators/ReadUnboundedTranslator.java | 48 ++++
.../translators/TransformTranslator.java | 30 ++
.../translators/TranslationContext.java | 105 +++++++
.../translators/WindowAssignTranslator.java | 103 +++++++
.../translators/functions/DoFnFunction.java | 193 +++++++++++++
.../translators/functions/package-info.java | 22 ++
.../translators/io/BoundedSourceWrapper.java | 45 +++
.../gearpump/translators/io/GearpumpSource.java | 113 ++++++++
.../translators/io/UnboundedSourceWrapper.java | 46 +++
.../gearpump/translators/io/ValuesSource.java | 173 ++++++++++++
.../gearpump/translators/io/package-info.java | 22 ++
.../gearpump/translators/package-info.java | 22 ++
.../translators/utils/DoFnRunnerFactory.java | 82 ++++++
.../translators/utils/NoOpStepContext.java | 41 +++
.../translators/utils/TranslatorUtils.java | 218 +++++++++++++++
.../translators/utils/package-info.java | 22 ++
.../gearpump/GearpumpRunnerRegistrarTest.java | 55 ++++
.../runners/gearpump/PipelineOptionsTest.java | 73 +++++
...teGearpumpPCollectionViewTranslatorTest.java | 56 ++++
.../FlattenPCollectionsTranslatorTest.java | 149 ++++++++++
.../translators/GroupByKeyTranslatorTest.java | 152 ++++++++++
.../translators/ReadBoundedTranslatorTest.java | 70 +++++
.../ReadUnboundedTranslatorTest.java | 70 +++++
.../translators/WindowAssignTranslatorTest.java | 110 ++++++++
.../translators/io/GearpumpSourceTest.java | 99 +++++++
.../gearpump/translators/io/ValueSoureTest.java | 79 ++++++
.../translators/utils/TranslatorUtilsTest.java | 73 +++++
runners/pom.xml | 9 +
sdks/java/javadoc/pom.xml | 5 +
47 files changed, 3924 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/89236e3b/examples/java8/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/89236e3b/pom.xml
----------------------------------------------------------------------
[14/50] [abbrv] beam git commit: This closes #2521: Add more unit
test to Gearpump runner
Posted by ke...@apache.org.
This closes #2521: Add more unit test to Gearpump runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ebbb6139
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ebbb6139
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ebbb6139
Branch: refs/heads/master
Commit: ebbb6139057deda05691fc357799506e5f9f3bf2
Parents: f4f2333 f3138dd
Author: Kenneth Knowles <kl...@google.com>
Authored: Tue Apr 18 13:25:20 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Apr 18 13:25:20 2017 -0700
----------------------------------------------------------------------
.../FlattenPCollectionsTranslator.java | 2 +-
.../translators/GroupByKeyTranslator.java | 20 ++-
.../translators/WindowAssignTranslator.java | 5 +-
...teGearpumpPCollectionViewTranslatorTest.java | 57 +++++++
.../CreatePCollectionViewTranslatorTest.java | 55 +++++++
.../FlattenPCollectionsTranslatorTest.java | 137 +++++++++++++++++
.../translators/GroupByKeyTranslatorTest.java | 151 +++++++++++++++++++
.../translators/ReadBoundedTranslatorTest.java | 70 +++++++++
.../ReadUnboundedTranslatorTest.java | 70 +++++++++
.../translators/WindowAssignTranslatorTest.java | 110 ++++++++++++++
.../translators/io/GearpumpSourceTest.java | 36 ++---
.../gearpump/translators/io/ValueSoureTest.java | 15 +-
12 files changed, 695 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
[13/50] [abbrv] beam git commit: [BEAM-972] Add more unit test to
Gearpump runner
Posted by ke...@apache.org.
[BEAM-972] Add more unit test to Gearpump runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f3138dde
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f3138dde
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f3138dde
Branch: refs/heads/master
Commit: f3138dde12b0c6a5cfdb8fefab916a6060b7f5ea
Parents: f4f2333
Author: huafengw <fv...@gmail.com>
Authored: Wed Apr 12 19:11:09 2017 +0800
Committer: huafengw <fv...@gmail.com>
Committed: Thu Apr 13 10:06:55 2017 +0800
----------------------------------------------------------------------
.../FlattenPCollectionsTranslator.java | 2 +-
.../translators/GroupByKeyTranslator.java | 20 ++-
.../translators/WindowAssignTranslator.java | 5 +-
...teGearpumpPCollectionViewTranslatorTest.java | 57 +++++++
.../CreatePCollectionViewTranslatorTest.java | 55 +++++++
.../FlattenPCollectionsTranslatorTest.java | 137 +++++++++++++++++
.../translators/GroupByKeyTranslatorTest.java | 151 +++++++++++++++++++
.../translators/ReadBoundedTranslatorTest.java | 70 +++++++++
.../ReadUnboundedTranslatorTest.java | 70 +++++++++
.../translators/WindowAssignTranslatorTest.java | 110 ++++++++++++++
.../translators/io/GearpumpSourceTest.java | 36 ++---
.../gearpump/translators/io/ValueSoureTest.java | 15 +-
12 files changed, 695 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
index 3a465cb..56f7d1a 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
@@ -47,7 +47,6 @@ public class FlattenPCollectionsTranslator<T> implements
Set<PCollection<T>> unique = new HashSet<>();
for (TaggedPValue input: context.getInputs()) {
PCollection<T> collection = (PCollection<T>) input.getValue();
- unique.add(collection);
JavaStream<T> inputStream = context.getInputStream(collection);
if (null == merged) {
merged = inputStream;
@@ -60,6 +59,7 @@ public class FlattenPCollectionsTranslator<T> implements
merged = merged.merge(inputStream, transform.getName());
}
+ unique.add(collection);
}
if (null == merged) {
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
index 5dfd3e9..54c8737 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
@@ -83,7 +83,10 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
context.setOutputStream(context.getOutput(), outputStream);
}
- private static class GearpumpWindowFn<T, W extends BoundedWindow>
+ /**
+ * A transform used internally to translate Beam's Window to Gearpump's Window.
+ */
+ protected static class GearpumpWindowFn<T, W extends BoundedWindow>
implements WindowFunction<WindowedValue<T>>, Serializable {
private final boolean isNonMerging;
@@ -115,7 +118,10 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
}
}
- private static class GroupByFn<K, V> extends
+ /**
+ * A transform used internally to group KV message by its key.
+ */
+ protected static class GroupByFn<K, V> extends
GroupByFunction<WindowedValue<KV<K, V>>, ByteBuffer> {
private static final long serialVersionUID = -807905402490735530L;
@@ -135,7 +141,10 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
}
}
- private static class KeyedByTimestamp<K, V>
+ /**
+ * A transform used internally to transform WindowedValue to KV.
+ */
+ protected static class KeyedByTimestamp<K, V>
extends MapFunction<WindowedValue<KV<K, V>>,
KV<Instant, WindowedValue<KV<K, V>>>> {
@@ -154,7 +163,10 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
}
}
- private static class Merge<K, V> extends
+ /**
+ * A transform used internally by Gearpump which encapsulates the merge logic.
+ */
+ protected static class Merge<K, V> extends
FoldFunction<KV<Instant, WindowedValue<KV<K, V>>>,
KV<Instant, WindowedValue<KV<K, List<V>>>>> {
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
index 29d8f02..2d70b63 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
@@ -57,7 +57,10 @@ public class WindowAssignTranslator<T> implements TransformTranslator<Window.Ass
context.setOutputStream(output, outputStream);
}
- private static class AssignWindows<T> extends
+ /**
+ * A Function used internally by Gearpump to wrap the actual Beam's WindowFn.
+ */
+ protected static class AssignWindows<T> extends
FlatMapFunction<WindowedValue<T>, WindowedValue<T>> {
private static final long serialVersionUID = 7284565861938681360L;
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java
new file mode 100644
index 0000000..b23b0c6
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.runners.gearpump.GearpumpPipelineTranslator;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.junit.Test;
+
+/** Tests for {@link CreateGearpumpPCollectionViewTranslator}. */
+public class CreateGearpumpPCollectionViewTranslatorTest {
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslate() {
+ CreateGearpumpPCollectionViewTranslator translator =
+ new CreateGearpumpPCollectionViewTranslator();
+
+ GearpumpPipelineTranslator.CreateGearpumpPCollectionView pCollectionView =
+ mock(GearpumpPipelineTranslator.CreateGearpumpPCollectionView.class);
+
+ JavaStream javaStream = mock(JavaStream.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ PValue mockInput = mock(PValue.class);
+ when(translationContext.getInput()).thenReturn(mockInput);
+ when(translationContext.getInputStream(mockInput)).thenReturn(javaStream);
+
+ PCollectionView view = mock(PCollectionView.class);
+ when(translationContext.getOutput()).thenReturn(view);
+
+ translator.translate(pCollectionView, translationContext);
+ verify(translationContext, times(1)).setOutputStream(view, javaStream);
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslatorTest.java
new file mode 100644
index 0000000..42ff14e
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslatorTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.junit.Test;
+
+/** Tests for {@link CreatePCollectionViewTranslator}. */
+public class CreatePCollectionViewTranslatorTest {
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslate() {
+ CreatePCollectionViewTranslator translator = new CreatePCollectionViewTranslator();
+ View.CreatePCollectionView<String, Iterable<String>> createView =
+ mock(View.CreatePCollectionView.class);
+
+ JavaStream javaStream = mock(JavaStream.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ PValue mockInput = mock(PValue.class);
+ when(translationContext.getInput()).thenReturn(mockInput);
+ when(translationContext.getInputStream(mockInput)).thenReturn(javaStream);
+
+ PCollectionView view = mock(PCollectionView.class);
+ when(translationContext.getOutput()).thenReturn(view);
+
+ translator.translate(createView, translationContext);
+ verify(translationContext, times(1)).setOutputStream(view, javaStream);
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
new file mode 100644
index 0000000..fa89d4a
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.Lists;
+import java.util.Collections;
+import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TaggedPValue;
+import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.source.DataSource;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+
+/** Tests for {@link FlattenPCollectionsTranslator}. */
+public class FlattenPCollectionsTranslatorTest {
+
+ private FlattenPCollectionsTranslator translator = new FlattenPCollectionsTranslator();
+ private Flatten.PCollections transform = mock(Flatten.PCollections.class);
+
+ class UnboundedSourceWrapperMatcher extends ArgumentMatcher<DataSource> {
+ @Override
+ public boolean matches(Object o) {
+ return o instanceof UnboundedSourceWrapper;
+ }
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslateWithEmptyCollection() {
+ PValue mockOutput = mock(PValue.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ when(translationContext.getInputs()).thenReturn(Collections.EMPTY_LIST);
+ when(translationContext.getOutput()).thenReturn(mockOutput);
+
+ translator.translate(transform, translationContext);
+ verify(translationContext).getSourceStream(argThat(new UnboundedSourceWrapperMatcher()));
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslateWithOneCollection() {
+ JavaStream javaStream = mock(JavaStream.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ TaggedPValue mockInput = mock(TaggedPValue.class);
+ PCollection mockCollection = mock(PCollection.class);
+ when(mockInput.getValue()).thenReturn(mockCollection);
+
+ when(translationContext.getInputs()).thenReturn(Lists.newArrayList(mockInput));
+ when(translationContext.getInputStream(mockCollection)).thenReturn(javaStream);
+
+ PValue mockOutput = mock(PValue.class);
+ when(translationContext.getOutput()).thenReturn(mockOutput);
+
+ translator.translate(transform, translationContext);
+ verify(translationContext, times(1)).setOutputStream(mockOutput, javaStream);
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testWithMoreThanOneCollections() {
+ String transformName = "transform";
+ when(transform.getName()).thenReturn(transformName);
+
+ JavaStream javaStream1 = mock(JavaStream.class);
+ JavaStream javaStream2 = mock(JavaStream.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ TaggedPValue mockInput1 = mock(TaggedPValue.class);
+ PCollection mockCollection1 = mock(PCollection.class);
+ when(mockInput1.getValue()).thenReturn(mockCollection1);
+
+ TaggedPValue mockInput2 = mock(TaggedPValue.class);
+ PCollection mockCollection2 = mock(PCollection.class);
+ when(mockInput2.getValue()).thenReturn(mockCollection2);
+
+ when(translationContext.getInputs()).thenReturn(Lists.newArrayList(mockInput1, mockInput2));
+ when(translationContext.getInputStream(mockCollection1)).thenReturn(javaStream1);
+ when(translationContext.getInputStream(mockCollection2)).thenReturn(javaStream2);
+
+ translator.translate(transform, translationContext);
+ verify(javaStream1).merge(javaStream2, transformName);
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testWithDuplicatedCollections() {
+ String transformName = "transform";
+ when(transform.getName()).thenReturn(transformName);
+
+ JavaStream javaStream1 = mock(JavaStream.class);
+ TranslationContext translationContext = mock(TranslationContext.class);
+
+ PCollection mockCollection1 = mock(PCollection.class);
+ TaggedPValue mockInput1 = mock(TaggedPValue.class);
+ when(mockInput1.getValue()).thenReturn(mockCollection1);
+
+ TaggedPValue mockInput2 = mock(TaggedPValue.class);
+ when(mockInput2.getValue()).thenReturn(mockCollection1);
+
+ when(translationContext.getInputs()).thenReturn(Lists.newArrayList(mockInput1, mockInput2));
+ when(translationContext.getInputStream(mockCollection1)).thenReturn(javaStream1);
+
+ translator.translate(transform, translationContext);
+ verify(javaStream1).map(any(MapFunction.class), eq("dummy"));
+ verify(javaStream1).merge(any(JavaStream.class), eq(transformName));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
new file mode 100644
index 0000000..9135022
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.beam.runners.gearpump.translators.GroupByKeyTranslator.GearpumpWindowFn;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.OutputTimeFn;
+import org.apache.beam.sdk.transforms.windowing.OutputTimeFns;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.apache.gearpump.streaming.dsl.window.api.WindowFunction;
+import org.apache.gearpump.streaming.dsl.window.impl.Window;
+import org.joda.time.Duration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/** Tests for {@link GroupByKeyTranslator}. */
+@RunWith(Parameterized.class)
+public class GroupByKeyTranslatorTest {
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testGearpumpWindowFn() {
+ GearpumpWindowFn windowFn = new GearpumpWindowFn(true);
+ List<BoundedWindow> windows =
+ Lists.newArrayList(
+ new IntervalWindow(new org.joda.time.Instant(0), new org.joda.time.Instant(10)),
+ new IntervalWindow(new org.joda.time.Instant(5), new org.joda.time.Instant(15)));
+
+ WindowFunction.Context<WindowedValue<String>> context =
+ new WindowFunction.Context<WindowedValue<String>>() {
+ @Override
+ public Instant timestamp() {
+ return Instant.EPOCH;
+ }
+
+ @Override
+ public WindowedValue<String> element() {
+ return WindowedValue.of(
+ "v1", new org.joda.time.Instant(6), windows, PaneInfo.NO_FIRING);
+ }
+ };
+
+ Window[] result = windowFn.apply(context);
+ List<Window> expected = Lists.newArrayList();
+ for (BoundedWindow w : windows) {
+ expected.add(TranslatorUtils.boundedWindowToGearpumpWindow(w));
+ }
+ assertThat(result, equalTo(expected.toArray()));
+ }
+
+ @Parameterized.Parameters(name = "{index}: {0}")
+ public static Iterable<OutputTimeFn<BoundedWindow>> data() {
+ return ImmutableList.of(
+ OutputTimeFns.outputAtEarliestInputTimestamp(),
+ OutputTimeFns.outputAtLatestInputTimestamp(),
+ OutputTimeFns.outputAtEndOfWindow());
+ }
+
+ @Parameterized.Parameter(0)
+ public OutputTimeFn<? super BoundedWindow> outputTimeFn;
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testKeyedByTimestamp() {
+ BoundedWindow window =
+ new IntervalWindow(new org.joda.time.Instant(0), new org.joda.time.Instant(10));
+ GroupByKeyTranslator.KeyedByTimestamp keyedByTimestamp =
+ new GroupByKeyTranslator.KeyedByTimestamp(outputTimeFn);
+ WindowedValue<KV<String, String>> value =
+ WindowedValue.of(
+ KV.of("key", "val"), org.joda.time.Instant.now(), window, PaneInfo.NO_FIRING);
+ KV<org.joda.time.Instant, WindowedValue<KV<String, String>>> result =
+ keyedByTimestamp.map(value);
+ org.joda.time.Instant time =
+ outputTimeFn.assignOutputTime(
+ value.getTimestamp(), Iterables.getOnlyElement(value.getWindows()));
+ assertThat(result, equalTo(KV.of(time, value)));
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testMerge() {
+ WindowFn slidingWindows = Sessions.withGapDuration(Duration.millis(10));
+ GroupByKeyTranslator.Merge merge = new GroupByKeyTranslator.Merge(slidingWindows, outputTimeFn);
+ org.joda.time.Instant key1 = new org.joda.time.Instant(5);
+ WindowedValue<KV<String, String>> value1 =
+ WindowedValue.of(
+ KV.of("key1", "value1"),
+ key1,
+ new IntervalWindow(new org.joda.time.Instant(5), new org.joda.time.Instant(10)),
+ PaneInfo.NO_FIRING);
+
+ org.joda.time.Instant key2 = new org.joda.time.Instant(10);
+ WindowedValue<KV<String, String>> value2 =
+ WindowedValue.of(
+ KV.of("key2", "value2"),
+ key2,
+ new IntervalWindow(new org.joda.time.Instant(9), new org.joda.time.Instant(14)),
+ PaneInfo.NO_FIRING);
+
+ KV<org.joda.time.Instant, WindowedValue<KV<String, List<String>>>> result1 =
+ merge.fold(KV.of(null, null), KV.of(key1, value1));
+ assertThat(result1.getKey(), equalTo(key1));
+ assertThat(result1.getValue().getValue().getValue(), equalTo(Lists.newArrayList("value1")));
+
+ KV<org.joda.time.Instant, WindowedValue<KV<String, List<String>>>> result2 =
+ merge.fold(result1, KV.of(key2, value2));
+ assertThat(result2.getKey(), equalTo(outputTimeFn.combine(key1, key2)));
+ Collection<? extends BoundedWindow> resultWindows = result2.getValue().getWindows();
+ assertThat(resultWindows.size(), equalTo(1));
+ IntervalWindow expectedWindow =
+ new IntervalWindow(new org.joda.time.Instant(5), new org.joda.time.Instant(14));
+ assertThat(resultWindows.toArray()[0], equalTo(expectedWindow));
+ assertThat(
+ result2.getValue().getValue().getValue(), equalTo(Lists.newArrayList("value1", "value2")));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslatorTest.java
new file mode 100644
index 0000000..20ee1a2
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslatorTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.runners.gearpump.translators.io.BoundedSourceWrapper;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.source.DataSource;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+
+/** Tests for {@link ReadBoundedTranslator}. */
+public class ReadBoundedTranslatorTest {
+
+ class BoundedSourceWrapperMatcher extends ArgumentMatcher<DataSource> {
+ @Override
+ public boolean matches(Object o) {
+ return o instanceof BoundedSourceWrapper;
+ }
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslate() {
+ ReadBoundedTranslator translator = new ReadBoundedTranslator();
+ GearpumpPipelineOptions options =
+ PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
+ Read.Bounded transform = mock(Read.Bounded.class);
+ BoundedSource source = mock(BoundedSource.class);
+ when(transform.getSource()).thenReturn(source);
+
+ TranslationContext translationContext = mock(TranslationContext.class);
+ when(translationContext.getPipelineOptions()).thenReturn(options);
+
+ JavaStream stream = mock(JavaStream.class);
+ PValue mockOutput = mock(PValue.class);
+ when(translationContext.getOutput()).thenReturn(mockOutput);
+ when(translationContext.getSourceStream(any(DataSource.class))).thenReturn(stream);
+
+ translator.translate(transform, translationContext);
+ verify(translationContext).getSourceStream(argThat(new BoundedSourceWrapperMatcher()));
+ verify(translationContext).setOutputStream(mockOutput, stream);
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslatorTest.java
new file mode 100644
index 0000000..f27b568
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslatorTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.source.DataSource;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+
+/** Tests for {@link ReadUnboundedTranslator}. */
+public class ReadUnboundedTranslatorTest {
+
+ class UnboundedSourceWrapperMatcher extends ArgumentMatcher<DataSource> {
+ @Override
+ public boolean matches(Object o) {
+ return o instanceof UnboundedSourceWrapper;
+ }
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testTranslate() {
+ ReadUnboundedTranslator translator = new ReadUnboundedTranslator();
+ GearpumpPipelineOptions options =
+ PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
+ Read.Unbounded transform = mock(Read.Unbounded.class);
+ UnboundedSource source = mock(UnboundedSource.class);
+ when(transform.getSource()).thenReturn(source);
+
+ TranslationContext translationContext = mock(TranslationContext.class);
+ when(translationContext.getPipelineOptions()).thenReturn(options);
+
+ JavaStream stream = mock(JavaStream.class);
+ PValue mockOutput = mock(PValue.class);
+ when(translationContext.getOutput()).thenReturn(mockOutput);
+ when(translationContext.getSourceStream(any(DataSource.class))).thenReturn(stream);
+
+ translator.translate(transform, translationContext);
+ verify(translationContext).getSourceStream(argThat(new UnboundedSourceWrapperMatcher()));
+ verify(translationContext).setOutputStream(mockOutput, stream);
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslatorTest.java
new file mode 100644
index 0000000..06ccaaf
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslatorTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Test;
+
+/** Tests for {@link WindowAssignTranslator}. */
+public class WindowAssignTranslatorTest {
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testAssignWindowsWithSlidingWindow() {
+ WindowFn slidingWindows = SlidingWindows.of(Duration.millis(10)).every(Duration.millis(5));
+ WindowAssignTranslator.AssignWindows<String> assignWindows =
+ new WindowAssignTranslator.AssignWindows(slidingWindows);
+
+ String value = "v1";
+ Instant timestamp = new Instant(1);
+ WindowedValue<String> windowedValue =
+ WindowedValue.timestampedValueInGlobalWindow(value, timestamp);
+ ArrayList<WindowedValue<String>> expected = new ArrayList<>();
+ expected.add(
+ WindowedValue.of(
+ value,
+ timestamp,
+ new IntervalWindow(new Instant(0), new Instant(10)),
+ PaneInfo.NO_FIRING));
+ expected.add(
+ WindowedValue.of(
+ value,
+ timestamp,
+ new IntervalWindow(new Instant(-5), new Instant(5)),
+ PaneInfo.NO_FIRING));
+
+ Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue);
+ assertThat(expected, equalTo(Lists.newArrayList(result)));
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testAssignWindowsWithSessions() {
+ WindowFn slidingWindows = Sessions.withGapDuration(Duration.millis(10));
+ WindowAssignTranslator.AssignWindows<String> assignWindows =
+ new WindowAssignTranslator.AssignWindows(slidingWindows);
+
+ String value = "v1";
+ Instant timestamp = new Instant(1);
+ WindowedValue<String> windowedValue =
+ WindowedValue.timestampedValueInGlobalWindow(value, timestamp);
+ ArrayList<WindowedValue<String>> expected = new ArrayList<>();
+ expected.add(
+ WindowedValue.of(
+ value,
+ timestamp,
+ new IntervalWindow(new Instant(1), new Instant(11)),
+ PaneInfo.NO_FIRING));
+
+ Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue);
+ assertThat(expected, equalTo(Lists.newArrayList(result)));
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testAssignWindowsGlobal() {
+ WindowFn slidingWindows = new GlobalWindows();
+ WindowAssignTranslator.AssignWindows<String> assignWindows =
+ new WindowAssignTranslator.AssignWindows(slidingWindows);
+
+ String value = "v1";
+ Instant timestamp = new Instant(1);
+ WindowedValue<String> windowedValue =
+ WindowedValue.timestampedValueInGlobalWindow(value, timestamp);
+ ArrayList<WindowedValue<String>> expected = new ArrayList<>();
+ expected.add(WindowedValue.timestampedValueInGlobalWindow(value, timestamp));
+
+ Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue);
+ assertThat(expected, equalTo(Lists.newArrayList(result)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
index af5a1d2..b244484 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
@@ -37,16 +37,14 @@ import org.apache.gearpump.streaming.source.Watermark;
import org.junit.Assert;
import org.junit.Test;
-/**
- * Tests for {@link GearpumpSource}.
- */
+/** Tests for {@link GearpumpSource}. */
public class GearpumpSourceTest {
- private static final List<TimestampedValue<String>> TEST_VALUES = Lists.newArrayList(
- TimestampedValue.of("a", new org.joda.time.Instant(Long.MIN_VALUE)),
- TimestampedValue.of("b", new org.joda.time.Instant(0)),
- TimestampedValue.of("c", new org.joda.time.Instant(53)),
- TimestampedValue.of("d", new org.joda.time.Instant(Long.MAX_VALUE - 1))
- );
+ private static final List<TimestampedValue<String>> TEST_VALUES =
+ Lists.newArrayList(
+ TimestampedValue.of("a", new org.joda.time.Instant(Long.MIN_VALUE)),
+ TimestampedValue.of("b", new org.joda.time.Instant(0)),
+ TimestampedValue.of("c", new org.joda.time.Instant(53)),
+ TimestampedValue.of("d", new org.joda.time.Instant(Long.MAX_VALUE - 1)));
private static class SourceForTest<T> extends GearpumpSource<T> {
private ValuesSource<T> valuesSource;
@@ -64,22 +62,24 @@ public class GearpumpSourceTest {
@Test
public void testGearpumpSource() {
- GearpumpPipelineOptions options = PipelineOptionsFactory.create()
- .as(GearpumpPipelineOptions.class);
- ValuesSource<TimestampedValue<String>> valuesSource = new ValuesSource<>(TEST_VALUES,
- TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of()));
+ GearpumpPipelineOptions options =
+ PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
+ ValuesSource<TimestampedValue<String>> valuesSource =
+ new ValuesSource<>(
+ TEST_VALUES, TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of()));
SourceForTest<TimestampedValue<String>> sourceForTest =
- new SourceForTest<>(options, valuesSource);
+ new SourceForTest<>(options, valuesSource);
sourceForTest.open(null, Instant.EPOCH);
- for (TimestampedValue<String> value: TEST_VALUES) {
+ for (TimestampedValue<String> value : TEST_VALUES) {
// Check the watermark first since the Source will advance when it's opened
Instant expectedWaterMark = TranslatorUtils.jodaTimeToJava8Time(value.getTimestamp());
Assert.assertEquals(expectedWaterMark, sourceForTest.getWatermark());
- Message expectedMsg = Message.apply(
- WindowedValue.timestampedValueInGlobalWindow(value, value.getTimestamp()),
- value.getTimestamp().getMillis());
+ Message expectedMsg =
+ Message.apply(
+ WindowedValue.timestampedValueInGlobalWindow(value, value.getTimestamp()),
+ value.getTimestamp().getMillis());
Message message = sourceForTest.read();
Assert.assertEquals(expectedMsg, message);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/f3138dde/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java
index 8c50703..439e1b1 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java
@@ -41,18 +41,16 @@ import org.apache.gearpump.util.Constants;
import org.junit.Assert;
import org.junit.Test;
-/**
- * Tests for {@link ValuesSource}.
- */
+/** Tests for {@link ValuesSource}. */
public class ValueSoureTest {
@Test
public void testValueSource() {
- GearpumpPipelineOptions options = PipelineOptionsFactory.create()
- .as(GearpumpPipelineOptions.class);
+ GearpumpPipelineOptions options =
+ PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
Config config = ClusterConfig.master(null);
- config = config.withValue(Constants.APPLICATION_TOTAL_RETRIES(),
- ConfigValueFactory.fromAnyRef(0));
+ config =
+ config.withValue(Constants.APPLICATION_TOTAL_RETRIES(), ConfigValueFactory.fromAnyRef(0));
EmbeddedCluster cluster = new EmbeddedCluster(config);
cluster.start();
@@ -62,8 +60,7 @@ public class ValueSoureTest {
Pipeline p = Pipeline.create(options);
List<String> values = Lists.newArrayList("1", "2", "3", "4", "5");
ValuesSource<String> source = new ValuesSource<>(values, StringUtf8Coder.of());
- p.apply(Read.from(source))
- .apply(ParDo.of(new ResultCollector()));
+ p.apply(Read.from(source)).apply(ParDo.of(new ResultCollector()));
p.run().waitUntilFinish();
cluster.stop();
[08/50] [abbrv] beam git commit: [BEAM-79] Fix gearpump-runner merge
conflicts and test failure
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
deleted file mode 100644
index 81970e2..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump.translators;
-
-import com.google.common.collect.Iterables;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.transforms.windowing.WindowFn;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
-import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
-import org.joda.time.Instant;
-
-/**
- * {@link Window.Bound} is translated to Gearpump flatMap function.
- */
-@SuppressWarnings("unchecked")
-public class WindowBoundTranslator<T> implements TransformTranslator<Window.Bound<T>> {
-
- private static final long serialVersionUID = -964887482120489061L;
-
- @Override
- public void translate(Window.Bound<T> transform, TranslationContext context) {
- PCollection<T> input = context.getInput(transform);
- JavaStream<WindowedValue<T>> inputStream = context.getInputStream(input);
- WindowingStrategy<?, ?> outputStrategy =
- transform.getOutputStrategyInternal(input.getWindowingStrategy());
- WindowFn<T, BoundedWindow> windowFn = (WindowFn<T, BoundedWindow>) outputStrategy.getWindowFn();
- JavaStream<WindowedValue<T>> outputStream =
- inputStream
- .flatMap(new AssignWindows(windowFn), "assign_windows");
-
- context.setOutputStream(context.getOutput(transform), outputStream);
- }
-
- private static class AssignWindows<T> extends
- FlatMapFunction<WindowedValue<T>, WindowedValue<T>> {
-
- private static final long serialVersionUID = 7284565861938681360L;
- private final WindowFn<T, BoundedWindow> windowFn;
-
- AssignWindows(WindowFn<T, BoundedWindow> windowFn) {
- this.windowFn = windowFn;
- }
-
- @Override
- public Iterator<WindowedValue<T>> flatMap(final WindowedValue<T> value) {
- try {
- Collection<BoundedWindow> windows = windowFn.assignWindows(windowFn.new AssignContext() {
- @Override
- public T element() {
- return value.getValue();
- }
-
- @Override
- public Instant timestamp() {
- return value.getTimestamp();
- }
-
- @Override
- public BoundedWindow window() {
- return Iterables.getOnlyElement(value.getWindows());
- }
- });
- List<WindowedValue<T>> values = new ArrayList<>(windows.size());
- for (BoundedWindow win: windows) {
- values.add(
- WindowedValue.of(value.getValue(), value.getTimestamp(), win, value.getPane()));
- }
- return values.iterator();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index b2c68d6..9941e71 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -33,6 +33,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.beam.runners.core.DoFnRunners;
+import org.apache.beam.runners.core.InMemoryStateInternals;
import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
import org.apache.beam.runners.core.SideInputHandler;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
@@ -48,7 +49,6 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.state.InMemoryStateInternals;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
@@ -134,12 +134,16 @@ public class DoFnFunction<InputT, OutputT> extends
} else {
// side input
PCollectionView<?> sideInput = tagsToSideInputs.get(unionValue.getUnionTag());
- WindowedValue<Iterable<?>> sideInputValue =
- (WindowedValue<Iterable<?>>) unionValue.getValue();
+ WindowedValue<?> sideInputValue =
+ (WindowedValue<?>) unionValue.getValue();
+ Object value = sideInputValue.getValue();
+ if (!(value instanceof Iterable)) {
+ sideInputValue = sideInputValue.withValue(Lists.newArrayList(value));
+ }
if (!sideInputValues.containsKey(sideInput)) {
sideInputValues.put(sideInput, new LinkedList<WindowedValue<Iterable<?>>>());
}
- sideInputValues.get(sideInput).add(sideInputValue);
+ sideInputValues.get(sideInput).add((WindowedValue<Iterable<?>>) sideInputValue);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/UnboundedSourceWrapper.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/UnboundedSourceWrapper.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/UnboundedSourceWrapper.java
index dfdecb2..cb912c1 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/UnboundedSourceWrapper.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/UnboundedSourceWrapper.java
@@ -30,6 +30,7 @@ import org.apache.beam.sdk.options.PipelineOptions;
public class UnboundedSourceWrapper<OutputT, CheckpointMarkT extends UnboundedSource.CheckpointMark>
extends GearpumpSource<OutputT> {
+ private static final long serialVersionUID = -2453956849834747150L;
private final UnboundedSource<OutputT, CheckpointMarkT> source;
public UnboundedSourceWrapper(UnboundedSource<OutputT, CheckpointMarkT> source,
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
index 5db8320..bdfc336 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
@@ -25,12 +25,12 @@ import java.util.List;
import org.apache.beam.runners.core.AggregatorFactory;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners;
+import org.apache.beam.runners.core.ExecutionContext;
import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
import org.apache.beam.runners.core.SimpleDoFnRunner;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.util.ExecutionContext;
import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PCollectionView;
@@ -75,7 +75,7 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
public PushbackSideInputDoFnRunner<InputT, OutputT> createRunner(
ReadyCheckingSideInputReader sideInputReader) {
- DoFnRunner<InputT, OutputT> underlying = DoFnRunners.createDefault(
+ DoFnRunner<InputT, OutputT> underlying = DoFnRunners.simpleRunner(
options, fn, sideInputReader, outputManager, mainOutputTag,
sideOutputTags, stepContext, aggregatorFactory, windowingStrategy);
return PushbackSideInputDoFnRunner.create(underlying, sideInputs, sideInputReader);
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpAggregatorFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpAggregatorFactory.java
index 22ffc4d..3436930 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpAggregatorFactory.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpAggregatorFactory.java
@@ -21,9 +21,9 @@ package org.apache.beam.runners.gearpump.translators.utils;
import java.io.Serializable;
import org.apache.beam.runners.core.AggregatorFactory;
+import org.apache.beam.runners.core.ExecutionContext;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.util.ExecutionContext;
/**
* no-op aggregator factory.
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
index 45f146b..140df2a 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
@@ -21,12 +21,12 @@ package org.apache.beam.runners.gearpump.translators.utils;
import java.io.IOException;
import java.io.Serializable;
+import org.apache.beam.runners.core.ExecutionContext;
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.TimerInternals;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.ExecutionContext;
-import org.apache.beam.sdk.util.TimerInternals;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.state.StateInternals;
import org.apache.beam.sdk.values.TupleTag;
/**
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
index b8a5233..b8f0ccb 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
@@ -38,8 +38,6 @@ import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
import org.apache.gearpump.streaming.dsl.window.impl.Window;
-
-
/**
* Utility methods for translators.
*/
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
index 10976e8..524887d 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
@@ -34,7 +34,6 @@ import org.apache.beam.sdk.values.KV;
import org.apache.gearpump.streaming.dsl.window.impl.Window;
import org.junit.Test;
-
/**
* Tests for {@link TranslatorUtils}.
*/
[02/50] [abbrv] beam git commit: This closes #1935: Support merging
windows in GearpumpRunner
Posted by ke...@apache.org.
This closes #1935: Support merging windows in GearpumpRunner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2d0aed92
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2d0aed92
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2d0aed92
Branch: refs/heads/master
Commit: 2d0aed9221b4e763917b136ce2ecc60d08621a07
Parents: 4001aeb 7af6472
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Feb 15 13:17:30 2017 -0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Feb 15 13:17:30 2017 -0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 5 -
.../gearpump/GearpumpPipelineResult.java | 8 +-
.../beam/runners/gearpump/GearpumpRunner.java | 24 +---
.../translators/GroupByKeyTranslator.java | 133 +++++++++++--------
.../translators/WindowBoundTranslator.java | 53 +-------
.../gearpump/translators/io/GearpumpSource.java | 6 +-
.../translators/utils/DoFnRunnerFactory.java | 1 +
.../translators/utils/TranslatorUtils.java | 20 +++
.../translators/utils/TranslatorUtilsTest.java | 75 +++++++++++
9 files changed, 186 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
[25/50] [abbrv] beam git commit: Update against master changes
Posted by ke...@apache.org.
Update against master changes
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c9aac967
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c9aac967
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c9aac967
Branch: refs/heads/master
Commit: c9aac967bccba5c3e1a0a3e4d84a8def1bfa2581
Parents: 9e6c906
Author: manuzhang <ow...@gmail.com>
Authored: Mon Jun 5 12:38:30 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Jun 5 16:29:25 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 1 +
.../gearpump/GearpumpPipelineTranslator.java | 9 +--
.../beam/runners/gearpump/GearpumpRunner.java | 7 +-
.../CreatePCollectionViewTranslator.java | 45 ------------
.../ParDoSingleOutputTranslator.java | 75 --------------------
.../translators/TranslationContext.java | 5 +-
.../translators/utils/DoFnRunnerFactory.java | 6 +-
.../translators/utils/NoOpStepContext.java | 37 +---------
.../translators/utils/TranslatorUtils.java | 1 -
.../CreatePCollectionViewTranslatorTest.java | 55 --------------
10 files changed, 17 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c9aac967/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index d4dade1..beb7753 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -86,6 +86,7 @@
]
</beamTestPipelineOptions>
</systemPropertyVariables>
+ <threadCount>4</threadCount>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/beam/blob/c9aac967/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
index dc4592c..daf65d9 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
@@ -30,11 +30,9 @@ import org.apache.beam.runners.core.construction.PTransformMatchers;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.runners.gearpump.translators.CreateGearpumpPCollectionViewTranslator;
-import org.apache.beam.runners.gearpump.translators.CreatePCollectionViewTranslator;
import org.apache.beam.runners.gearpump.translators.FlattenPCollectionsTranslator;
import org.apache.beam.runners.gearpump.translators.GroupByKeyTranslator;
import org.apache.beam.runners.gearpump.translators.ParDoMultiOutputTranslator;
-import org.apache.beam.runners.gearpump.translators.ParDoSingleOutputTranslator;
import org.apache.beam.runners.gearpump.translators.ReadBoundedTranslator;
import org.apache.beam.runners.gearpump.translators.ReadUnboundedTranslator;
import org.apache.beam.runners.gearpump.translators.TransformTranslator;
@@ -73,7 +71,7 @@ import org.slf4j.LoggerFactory;
* into Gearpump {@link Graph}.
*/
@SuppressWarnings({"rawtypes", "unchecked"})
-public class GearpumpPipelineTranslator implements Pipeline.PipelineVisitor {
+public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
private static final Logger LOG = LoggerFactory.getLogger(
GearpumpPipelineTranslator.class);
@@ -89,7 +87,6 @@ public class GearpumpPipelineTranslator implements Pipeline.PipelineVisitor {
static {
// register TransformTranslators
- registerTransformTranslator(ParDo.SingleOutput.class, new ParDoSingleOutputTranslator());
registerTransformTranslator(Read.Unbounded.class, new ReadUnboundedTranslator());
registerTransformTranslator(Read.Bounded.class, new ReadBoundedTranslator());
registerTransformTranslator(GroupByKey.class, new GroupByKeyTranslator());
@@ -97,8 +94,6 @@ public class GearpumpPipelineTranslator implements Pipeline.PipelineVisitor {
new FlattenPCollectionsTranslator());
registerTransformTranslator(ParDo.MultiOutput.class, new ParDoMultiOutputTranslator());
registerTransformTranslator(Window.Assign.class, new WindowAssignTranslator());
- registerTransformTranslator(View.CreatePCollectionView.class,
- new CreatePCollectionViewTranslator());
registerTransformTranslator(CreateGearpumpPCollectionView.class,
new CreateGearpumpPCollectionViewTranslator<>());
}
@@ -155,7 +150,7 @@ public class GearpumpPipelineTranslator implements Pipeline.PipelineVisitor {
throw new IllegalStateException(
"no translator registered for " + transform);
}
- translationContext.setCurrentTransform(node);
+ translationContext.setCurrentTransform(node, getPipeline());
translator.translate(transform, translationContext);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/c9aac967/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
index 6df3f2d..30b1935 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
@@ -95,19 +95,22 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
*/
private Config registerSerializers(Config config, Map<String, String> userSerializers) {
Map<String, String> serializers = new HashMap<>();
+ serializers.put("org.apache.beam.sdk.util.WindowedValue$ValueInGlobalWindow", "");
serializers.put("org.apache.beam.sdk.util.WindowedValue$TimestampedValueInSingleWindow", "");
+ serializers.put("org.apache.beam.sdk.util.WindowedValue$TimestampedValueInGlobalWindow", "");
+ serializers.put("org.apache.beam.sdk.util.WindowedValue$TimestampedValueInMultipleWindows", "");
serializers.put("org.apache.beam.sdk.transforms.windowing.PaneInfo", "");
serializers.put("org.apache.beam.sdk.transforms.windowing.PaneInfo$Timing", "");
serializers.put("org.joda.time.Instant", "");
serializers.put("org.apache.beam.sdk.values.KV", "");
serializers.put("org.apache.beam.sdk.transforms.windowing.IntervalWindow", "");
serializers.put("org.apache.beam.sdk.values.TimestampedValue", "");
+
if (userSerializers != null && !userSerializers.isEmpty()) {
serializers.putAll(userSerializers);
}
+
return config.withValue(GEARPUMP_SERIALIZERS, ConfigValueFactory.fromMap(serializers));
}
-
-
}
http://git-wip-us.apache.org/repos/asf/beam/blob/c9aac967/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java
deleted file mode 100644
index da55d70..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump.translators;
-
-import java.util.List;
-
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
-
-/**
- * View.CreatePCollectionView bridges input stream to down stream
- * transforms.
- */
-public class CreatePCollectionViewTranslator<ElemT, ViewT> implements
- TransformTranslator<View.CreatePCollectionView<ElemT, ViewT>> {
-
- private static final long serialVersionUID = -2394386873317515748L;
-
- @Override
- public void translate(View.CreatePCollectionView<ElemT, ViewT> transform,
- TranslationContext context) {
- JavaStream<WindowedValue<List<ElemT>>> inputStream =
- context.getInputStream(context.getInput());
- PCollectionView<ViewT> view = (PCollectionView<ViewT>) context.getOutput();
- context.setOutputStream(view, inputStream);
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/c9aac967/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
deleted file mode 100644
index 6b0e610..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump.translators;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
-import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.PCollection;
-
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TupleTagList;
-import org.apache.beam.sdk.values.WindowingStrategy;
-import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
-
-/**
- * {@link ParDo.SingleOutput} is translated to Gearpump flatMap function
- * with {@link DoFn} wrapped in {@link DoFnFunction}.
- */
-public class ParDoSingleOutputTranslator<InputT, OutputT> implements
- TransformTranslator<ParDo.SingleOutput<InputT, OutputT>> {
-
- private static final long serialVersionUID = -3413205558160983784L;
- private final TupleTag<OutputT> mainOutput = new TupleTag<>();
- private final List<TupleTag<?>> sideOutputs = TupleTagList.empty().getAll();
-
- @Override
- public void translate(ParDo.SingleOutput<InputT, OutputT> transform, TranslationContext context) {
- DoFn<InputT, OutputT> doFn = transform.getFn();
- PCollection<OutputT> output = (PCollection<OutputT>) context.getOutput();
- WindowingStrategy<?, ?> windowingStrategy = output.getWindowingStrategy();
-
- Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
- Map<String, PCollectionView<?>> tagsToSideInputs =
- TranslatorUtils.getTagsToSideInputs(sideInputs);
- JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(
- context.getInput());
- JavaStream<TranslatorUtils.RawUnionValue> unionStream =
- TranslatorUtils.withSideInputStream(context,
- inputStream, tagsToSideInputs);
-
- DoFnFunction<InputT, OutputT> doFnFunction = new DoFnFunction<>(context.getPipelineOptions(),
- doFn, windowingStrategy, sideInputs, tagsToSideInputs,
- mainOutput, sideOutputs);
-
- JavaStream<WindowedValue<OutputT>> outputStream =
- TranslatorUtils.toList(unionStream)
- .flatMap(doFnFunction, transform.getName())
- .map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
-
- context.setOutputStream(context.getOutput(), outputStream);
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/c9aac967/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
index c3db044..4090354 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.values.PValue;
@@ -52,8 +53,8 @@ public class TranslationContext {
this.pipelineOptions = pipelineOptions;
}
- public void setCurrentTransform(TransformHierarchy.Node treeNode) {
- this.currentTransform = treeNode.toAppliedPTransform();
+ public void setCurrentTransform(TransformHierarchy.Node treeNode, Pipeline pipeline) {
+ this.currentTransform = treeNode.toAppliedPTransform(pipeline);
}
public GearpumpPipelineOptions getPipelineOptions() {
http://git-wip-us.apache.org/repos/asf/beam/blob/c9aac967/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
index d38f11b..35cf2b5 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
@@ -24,11 +24,11 @@ import java.util.List;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners;
-import org.apache.beam.runners.core.ExecutionContext;
import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
import org.apache.beam.runners.core.SimpleDoFnRunner;
import org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner;
+import org.apache.beam.runners.core.StepContext;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
@@ -48,7 +48,7 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
private final DoFnRunners.OutputManager outputManager;
private final TupleTag<OutputT> mainOutputTag;
private final List<TupleTag<?>> sideOutputTags;
- private final ExecutionContext.StepContext stepContext;
+ private final StepContext stepContext;
private final WindowingStrategy<?, ?> windowingStrategy;
public DoFnRunnerFactory(
@@ -58,7 +58,7 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
DoFnRunners.OutputManager outputManager,
TupleTag<OutputT> mainOutputTag,
List<TupleTag<?>> sideOutputTags,
- ExecutionContext.StepContext stepContext,
+ StepContext stepContext,
WindowingStrategy<?, ?> windowingStrategy) {
this.fn = doFn;
this.options = pipelineOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/c9aac967/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
index 64fd615..b795ed9 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
@@ -18,47 +18,16 @@
package org.apache.beam.runners.gearpump.translators.utils;
-import java.io.IOException;
import java.io.Serializable;
-import org.apache.beam.runners.core.ExecutionContext;
import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StepContext;
import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
/**
- * serializable {@link ExecutionContext.StepContext} that basically does nothing.
+ * serializable {@link StepContext} that basically does nothing.
*/
-public class NoOpStepContext implements ExecutionContext.StepContext, Serializable {
-
- @Override
- public String getStepName() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public String getTransformName() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void noteOutput(WindowedValue<?> output) {
- }
-
- @Override
- public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {
-
- }
-
- @Override
- public <T, W extends BoundedWindow> void writePCollectionViewData(TupleTag<?> tag,
- Iterable<WindowedValue<T>> data,
- Coder<Iterable<WindowedValue<T>>> dataCoder, W window, Coder<W> windowCoder) throws
- IOException {
- }
+public class NoOpStepContext implements StepContext, Serializable {
@Override
public StateInternals stateInternals() {
http://git-wip-us.apache.org/repos/asf/beam/blob/c9aac967/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
index b8f0ccb..999afae 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
@@ -74,7 +74,6 @@ public class TranslatorUtils {
for (Map.Entry<String, PCollectionView<?>> tagToSideInput: tagsToSideInputs.entrySet()) {
// actually JavaStream<WindowedValue<List<?>>>
- // check CreatePCollectionViewTranslator
JavaStream<WindowedValue<Object>> sideInputStream = context.getInputStream(
tagToSideInput.getValue());
mainStream = mainStream.merge(sideInputStream.map(new ToRawUnionValue<>(
http://git-wip-us.apache.org/repos/asf/beam/blob/c9aac967/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslatorTest.java
deleted file mode 100644
index 42ff14e..0000000
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslatorTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump.translators;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PValue;
-import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
-import org.junit.Test;
-
-/** Tests for {@link CreatePCollectionViewTranslator}. */
-public class CreatePCollectionViewTranslatorTest {
-
- @Test
- @SuppressWarnings({"rawtypes", "unchecked"})
- public void testTranslate() {
- CreatePCollectionViewTranslator translator = new CreatePCollectionViewTranslator();
- View.CreatePCollectionView<String, Iterable<String>> createView =
- mock(View.CreatePCollectionView.class);
-
- JavaStream javaStream = mock(JavaStream.class);
- TranslationContext translationContext = mock(TranslationContext.class);
-
- PValue mockInput = mock(PValue.class);
- when(translationContext.getInput()).thenReturn(mockInput);
- when(translationContext.getInputStream(mockInput)).thenReturn(javaStream);
-
- PCollectionView view = mock(PCollectionView.class);
- when(translationContext.getOutput()).thenReturn(view);
-
- translator.translate(createView, translationContext);
- verify(translationContext, times(1)).setOutputStream(view, javaStream);
- }
-}
[36/50] [abbrv] beam git commit: Merge branch 'master' of
https://github.com/apache/beam into sync-master
Posted by ke...@apache.org.
Merge branch 'master' of https://github.com/apache/beam into sync-master
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f1582570
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f1582570
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f1582570
Branch: refs/heads/master
Commit: f1582570981df349da625fad765d18366089b5dc
Parents: 99f4f8b 893bf42
Author: manuzhang <ow...@gmail.com>
Authored: Fri Jun 30 16:24:24 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Fri Jun 30 16:24:24 2017 +0800
----------------------------------------------------------------------
.../jenkins/common_job_properties.groovy | 4 +-
.../job_beam_PerformanceTests_Python.groovy | 58 ++
pom.xml | 20 +-
.../apache/beam/runners/apex/ApexRunner.java | 2 +-
.../core/construction/SplittableParDo.java | 66 +-
.../WindowingStrategyTranslation.java | 26 +-
.../core/construction/SplittableParDoTest.java | 8 +-
.../beam/runners/core/ReduceFnRunner.java | 20 +-
.../core/triggers/AfterAllStateMachine.java | 25 +-
.../AfterDelayFromFirstElementStateMachine.java | 6 +-
.../core/triggers/AfterFirstStateMachine.java | 20 +-
.../core/triggers/AfterPaneStateMachine.java | 6 +-
.../triggers/AfterWatermarkStateMachine.java | 7 +-
.../triggers/ExecutableTriggerStateMachine.java | 23 +-
.../core/triggers/NeverStateMachine.java | 5 +-
.../core/triggers/TriggerStateMachine.java | 27 -
.../beam/runners/core/ReduceFnRunnerTest.java | 161 +++++
.../triggers/AfterFirstStateMachineTest.java | 5 +-
.../AfterWatermarkStateMachineTest.java | 7 +-
.../core/triggers/StubTriggerStateMachine.java | 7 +-
.../direct/ParDoMultiOverrideFactory.java | 16 +-
.../flink/FlinkStreamingPipelineTranslator.java | 2 +-
runners/google-cloud-dataflow-java/pom.xml | 2 +-
.../dataflow/BatchStatefulParDoOverrides.java | 2 +
.../dataflow/DataflowPipelineTranslator.java | 5 +-
.../beam/runners/dataflow/DataflowRunner.java | 10 +
.../dataflow/SplittableParDoOverrides.java | 2 +-
.../runners/dataflow/DataflowRunnerTest.java | 38 ++
.../src/main/proto/beam_runner_api.proto | 14 +
.../sdk/options/PipelineOptionsFactory.java | 18 +-
.../sdk/options/PipelineOptionsValidator.java | 34 +-
.../sdk/options/ProxyInvocationHandler.java | 4 +
.../apache/beam/sdk/testing/StaticWindows.java | 5 +
.../org/apache/beam/sdk/transforms/Combine.java | 13 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 3 +
.../apache/beam/sdk/transforms/DoFnTester.java | 16 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 14 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 6 +
.../sdk/transforms/reflect/DoFnInvoker.java | 4 +-
.../sdk/transforms/windowing/GlobalWindows.java | 5 +
.../windowing/PartitioningWindowFn.java | 5 +
.../transforms/windowing/SlidingWindows.java | 5 +
.../beam/sdk/transforms/windowing/Window.java | 32 +
.../beam/sdk/transforms/windowing/WindowFn.java | 11 +
.../apache/beam/sdk/util/IdentityWindowFn.java | 5 +
.../beam/sdk/values/PCollectionViews.java | 14 +
.../beam/sdk/values/WindowingStrategy.java | 46 +-
.../options/PipelineOptionsValidatorTest.java | 44 ++
.../sdk/options/ProxyInvocationHandlerTest.java | 7 +
.../apache/beam/sdk/transforms/CombineTest.java | 63 ++
.../beam/sdk/transforms/DoFnTesterTest.java | 32 +
.../windowing/SlidingWindowsTest.java | 30 +-
.../sdk/util/RetryHttpRequestInitializer.java | 148 +++--
.../extensions/gcp/GcpCoreApiSurfaceTest.java | 48 +-
.../util/RetryHttpRequestInitializerTest.java | 31 +-
sdks/java/io/amqp/pom.xml | 100 +++
.../org/apache/beam/sdk/io/amqp/AmqpIO.java | 399 ++++++++++++
.../beam/sdk/io/amqp/AmqpMessageCoder.java | 79 +++
.../amqp/AmqpMessageCoderProviderRegistrar.java | 44 ++
.../apache/beam/sdk/io/amqp/package-info.java | 22 +
.../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 148 +++++
.../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 89 +++
.../beam/sdk/io/cassandra/CassandraIO.java | 2 +-
sdks/java/io/elasticsearch/pom.xml | 8 +
.../sdk/io/elasticsearch/ElasticsearchIO.java | 13 +-
sdks/java/io/google-cloud-platform/pom.xml | 13 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 6 +-
.../io/gcp/bigtable/BigtableServiceImpl.java | 9 +-
.../sdk/io/gcp/spanner/AbstractSpannerFn.java | 58 ++
.../sdk/io/gcp/spanner/CreateTransactionFn.java | 51 ++
.../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 65 ++
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 137 +++++
.../beam/sdk/io/gcp/spanner/SpannerIO.java | 604 +++++++++++++------
.../sdk/io/gcp/spanner/SpannerWriteGroupFn.java | 125 ++++
.../beam/sdk/io/gcp/spanner/Transaction.java | 33 +
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 10 +
.../sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +-
.../io/gcp/bigtable/BigtableTestOptions.java | 5 -
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +-
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 4 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 2 +-
.../sdk/io/gcp/spanner/FakeServiceFactory.java | 82 +++
.../beam/sdk/io/gcp/spanner/RandomUtils.java | 41 ++
.../sdk/io/gcp/spanner/SpannerIOReadTest.java | 281 +++++++++
.../beam/sdk/io/gcp/spanner/SpannerIOTest.java | 317 ----------
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 258 ++++++++
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 166 +++++
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 27 +-
.../hadoop/inputformat/HadoopInputFormatIO.java | 2 +-
.../inputformat/HIFIOWithElasticTest.java | 11 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 2 +-
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 2 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 2 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 2 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 2 +-
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 2 +-
sdks/java/io/pom.xml | 1 +
.../examples/snippets/snippets_test.py | 16 +
.../apache_beam/examples/streaming_wordcount.py | 14 +-
sdks/python/apache_beam/io/filesystem.py | 8 +
sdks/python/apache_beam/io/gcp/pubsub.py | 162 +++--
sdks/python/apache_beam/io/gcp/pubsub_test.py | 89 +--
sdks/python/apache_beam/pipeline.py | 12 +-
sdks/python/apache_beam/pipeline_test.py | 18 +
.../runners/dataflow/dataflow_runner.py | 11 +-
.../runners/dataflow/internal/apiclient.py | 6 +-
.../runners/dataflow/internal/dependency.py | 69 ++-
.../apache_beam/runners/direct/direct_runner.py | 83 ++-
.../runners/direct/evaluation_context.py | 2 +-
.../runners/direct/transform_evaluator.py | 227 ++++++-
sdks/python/apache_beam/runners/direct/util.py | 25 +-
.../runners/direct/watermark_manager.py | 26 +-
.../runners/portability/fn_api_runner.py | 12 +-
.../runners/portability/fn_api_runner_test.py | 23 +-
.../apache_beam/runners/worker/data_plane.py | 10 +-
.../apache_beam/runners/worker/sdk_worker.py | 2 +-
.../apache_beam/testing/test_stream_test.py | 37 +-
sdks/python/apache_beam/transforms/trigger.py | 16 +
121 files changed, 4273 insertions(+), 1071 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/f1582570/pom.xml
----------------------------------------------------------------------
[32/50] [abbrv] beam git commit: Fix PCollectionView translation
Posted by ke...@apache.org.
Fix PCollectionView translation
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/a7b5d981
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/a7b5d981
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/a7b5d981
Branch: refs/heads/master
Commit: a7b5d981a57bae38a5ef70feb0c6bd167ce22ab0
Parents: f61822d
Author: manuzhang <ow...@gmail.com>
Authored: Sat Jun 17 11:11:07 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Jun 19 16:03:19 2017 +0800
----------------------------------------------------------------------
.../gearpump/GearpumpPipelineTranslator.java | 526 -------------------
.../beam/runners/gearpump/GearpumpRunner.java | 1 +
...CreateGearpumpPCollectionViewTranslator.java | 9 +-
.../CreateStreamingGearpumpView.java | 157 ++++++
.../translators/GearpumpPipelineTranslator.java | 143 +++++
.../translators/TranslationContext.java | 3 +-
.../translators/functions/DoFnFunction.java | 1 -
.../gearpump/translators/io/GearpumpSource.java | 2 +-
.../translators/utils/TranslatorUtils.java | 1 +
...teGearpumpPCollectionViewTranslatorTest.java | 7 +-
.../translators/io/GearpumpSourceTest.java | 18 +-
.../translators/utils/TranslatorUtilsTest.java | 5 +-
12 files changed, 327 insertions(+), 546 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
deleted file mode 100644
index 58b44a3..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
+++ /dev/null
@@ -1,526 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump;
-
-import com.google.common.collect.ImmutableList;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.beam.runners.core.construction.PTransformMatchers;
-import org.apache.beam.runners.core.construction.PTransformReplacements;
-import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
-import org.apache.beam.runners.gearpump.translators.CreateGearpumpPCollectionViewTranslator;
-import org.apache.beam.runners.gearpump.translators.FlattenPCollectionsTranslator;
-import org.apache.beam.runners.gearpump.translators.GroupByKeyTranslator;
-import org.apache.beam.runners.gearpump.translators.ParDoMultiOutputTranslator;
-import org.apache.beam.runners.gearpump.translators.ReadBoundedTranslator;
-import org.apache.beam.runners.gearpump.translators.ReadUnboundedTranslator;
-import org.apache.beam.runners.gearpump.translators.TransformTranslator;
-import org.apache.beam.runners.gearpump.translators.TranslationContext;
-import org.apache.beam.runners.gearpump.translators.WindowAssignTranslator;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.ListCoder;
-import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.runners.PTransformOverride;
-import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.InstanceBuilder;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PCollectionViews;
-import org.apache.beam.sdk.values.PValue;
-
-import org.apache.gearpump.util.Graph;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * {@link GearpumpPipelineTranslator} knows how to translate {@link Pipeline} objects
- * into Gearpump {@link Graph}.
- */
-@SuppressWarnings({"rawtypes", "unchecked"})
-public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- GearpumpPipelineTranslator.class);
-
- /**
- * A map from {@link PTransform} subclass to the corresponding
- * {@link TransformTranslator} to use to translate that transform.
- */
- private static final Map<Class<? extends PTransform>, TransformTranslator>
- transformTranslators = new HashMap<>();
-
- private final TranslationContext translationContext;
-
- static {
- // register TransformTranslators
- registerTransformTranslator(Read.Unbounded.class, new ReadUnboundedTranslator());
- registerTransformTranslator(Read.Bounded.class, new ReadBoundedTranslator());
- registerTransformTranslator(GroupByKey.class, new GroupByKeyTranslator());
- registerTransformTranslator(Flatten.PCollections.class,
- new FlattenPCollectionsTranslator());
- registerTransformTranslator(ParDo.MultiOutput.class, new ParDoMultiOutputTranslator());
- registerTransformTranslator(Window.Assign.class, new WindowAssignTranslator());
- registerTransformTranslator(CreateGearpumpPCollectionView.class,
- new CreateGearpumpPCollectionViewTranslator<>());
- }
-
- public GearpumpPipelineTranslator(TranslationContext translationContext) {
- this.translationContext = translationContext;
- }
-
- public void translate(Pipeline pipeline) {
- List<PTransformOverride> overrides =
- ImmutableList.<PTransformOverride>builder()
- .add(PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsMap.class),
- new ReflectiveOneToOneOverrideFactory(StreamingViewAsMap.class)))
- .add(PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsMultimap.class),
- new ReflectiveOneToOneOverrideFactory(StreamingViewAsMultimap.class)))
- .add(PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsSingleton.class),
- new ReflectiveOneToOneOverrideFactory(StreamingViewAsSingleton.class)))
- .add(PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsList.class),
- new ReflectiveOneToOneOverrideFactory(StreamingViewAsList.class)))
- .add(PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsIterable.class),
- new ReflectiveOneToOneOverrideFactory(StreamingViewAsIterable.class)))
- .add(PTransformOverride.of(
- PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
- new ReflectiveOneToOneOverrideFactory(
- StreamingCombineGloballyAsSingletonView.class)))
- .build();
-
- pipeline.replaceAll(overrides);
- pipeline.traverseTopologically(this);
- }
-
- @Override
- public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
- LOG.debug("entering composite transform {}", node.getTransform());
- return CompositeBehavior.ENTER_TRANSFORM;
- }
-
- @Override
- public void leaveCompositeTransform(TransformHierarchy.Node node) {
- LOG.debug("leaving composite transform {}", node.getTransform());
- }
-
- @Override
- public void visitPrimitiveTransform(TransformHierarchy.Node node) {
- LOG.info("visiting transform {}", node.getTransform());
- PTransform transform = node.getTransform();
- TransformTranslator translator = getTransformTranslator(transform.getClass());
- if (null == translator) {
- throw new IllegalStateException(
- "no translator registered for " + transform);
- }
- translationContext.setCurrentTransform(node, getPipeline());
- translator.translate(transform, translationContext);
- }
-
- @Override
- public void visitValue(PValue value, TransformHierarchy.Node producer) {
- LOG.debug("visiting value {}", value);
- }
-
- /**
- * Records that instances of the specified PTransform class
- * should be translated by default by the corresponding
- * {@link TransformTranslator}.
- */
- private static <TransformT extends PTransform> void registerTransformTranslator(
- Class<TransformT> transformClass,
- TransformTranslator<? extends TransformT> transformTranslator) {
- if (transformTranslators.put(transformClass, transformTranslator) != null) {
- throw new IllegalArgumentException(
- "defining multiple translators for " + transformClass);
- }
- }
-
- /**
- * Returns the {@link TransformTranslator} to use for instances of the
- * specified PTransform class, or null if none registered.
- */
- private <TransformT extends PTransform>
- TransformTranslator<TransformT> getTransformTranslator(Class<TransformT> transformClass) {
- return transformTranslators.get(transformClass);
- }
-
- // The following codes are forked from DataflowRunner for View translator
- private static class ReflectiveOneToOneOverrideFactory<
- InputT, OutputT, TransformT extends PTransform<PCollection<InputT>, PCollection<OutputT>>>
- extends SingleInputOutputOverrideFactory<
- PCollection<InputT>, PCollection<OutputT>, TransformT> {
- private final Class<PTransform<PCollection<InputT>, PCollection<OutputT>>> replacement;
-
- private ReflectiveOneToOneOverrideFactory(
- Class<PTransform<PCollection<InputT>, PCollection<OutputT>>> replacement) {
- this.replacement = replacement;
- }
-
- @Override
- public PTransformReplacement<PCollection<InputT>, PCollection<OutputT>> getReplacementTransform(
- AppliedPTransform<PCollection<InputT>, PCollection<OutputT>, TransformT> transform) {
- return PTransformReplacement.of(
- PTransformReplacements.getSingletonMainInput(transform),
- InstanceBuilder.ofType(replacement)
- .withArg(
- (Class<PTransform<PCollection<InputT>, PCollection<OutputT>>>)
- transform.getTransform().getClass(),
- transform.getTransform())
- .build());
- }
- }
-
- /**
- * Specialized implementation for
- * {@link org.apache.beam.sdk.transforms.View.AsMap View.AsMap}
- * for the Gearpump runner.
- */
- private static class StreamingViewAsMap<K, V>
- extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, V>>> {
-
- private static final long serialVersionUID = 4791080760092950304L;
-
- public StreamingViewAsMap(View.AsMap<K, V> transform) {}
-
- @Override
- public PCollectionView<Map<K, V>> expand(PCollection<KV<K, V>> input) {
- PCollectionView<Map<K, V>> view =
- PCollectionViews.mapView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
- try {
- inputCoder.getKeyCoder().verifyDeterministic();
- } catch (Coder.NonDeterministicException e) {
- // throw new RuntimeException(e);
- }
-
- return input
- .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
- .apply(CreateGearpumpPCollectionView.<KV<K, V>, Map<K, V>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsMap";
- }
- }
-
- /**
- * Specialized expansion for {@link
- * org.apache.beam.sdk.transforms.View.AsMultimap View.AsMultimap} for the
- * Gearpump runner.
- */
- private static class StreamingViewAsMultimap<K, V>
- extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, Iterable<V>>>> {
-
- private static final long serialVersionUID = 5854899081751333352L;
-
- public StreamingViewAsMultimap(View.AsMultimap<K, V> transform) {}
-
- @Override
- public PCollectionView<Map<K, Iterable<V>>> expand(PCollection<KV<K, V>> input) {
- PCollectionView<Map<K, Iterable<V>>> view =
- PCollectionViews.multimapView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
- try {
- inputCoder.getKeyCoder().verifyDeterministic();
- } catch (Coder.NonDeterministicException e) {
- // throw new RuntimeException(e);
- }
-
- return input
- .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
- .apply(CreateGearpumpPCollectionView.<KV<K, V>, Map<K, Iterable<V>>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsMultimap";
- }
- }
-
- /**
- * Specialized implementation for
- * {@link org.apache.beam.sdk.transforms.View.AsIterable View.AsIterable} for the
- * Gearpump runner.
- */
- private static class StreamingViewAsIterable<T>
- extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
-
- private static final long serialVersionUID = -3399860618995613421L;
-
- public StreamingViewAsIterable(View.AsIterable<T> transform) {}
-
- @Override
- public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
- PCollectionView<Iterable<T>> view =
- PCollectionViews.iterableView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
- .apply(CreateGearpumpPCollectionView.<T, Iterable<T>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsIterable";
- }
- }
-
- /**
- * Specialized implementation for
- * {@link org.apache.beam.sdk.transforms.View.AsList View.AsList} for the
- * Gearpump runner.
- */
- private static class StreamingViewAsList<T>
- extends PTransform<PCollection<T>, PCollectionView<List<T>>> {
-
- private static final long serialVersionUID = -5018631473886330629L;
-
- public StreamingViewAsList(View.AsList<T> transform) {}
-
- @Override
- public PCollectionView<List<T>> expand(PCollection<T> input) {
- PCollectionView<List<T>> view =
- PCollectionViews.listView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
- .apply(CreateGearpumpPCollectionView.<T, List<T>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsList";
- }
- }
-
-
- private static class StreamingCombineGloballyAsSingletonView<InputT, OutputT>
- extends PTransform<PCollection<InputT>, PCollectionView<OutputT>> {
-
- private static final long serialVersionUID = 9064900748869035738L;
- private final Combine.GloballyAsSingletonView<InputT, OutputT> transform;
-
- public StreamingCombineGloballyAsSingletonView(
- Combine.GloballyAsSingletonView<InputT, OutputT> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollectionView<OutputT> expand(PCollection<InputT> input) {
- PCollection<OutputT> combined =
- input.apply(Combine.globally(transform.getCombineFn())
- .withoutDefaults()
- .withFanout(transform.getFanout()));
-
- PCollectionView<OutputT> view = PCollectionViews.singletonView(
- combined,
- combined.getWindowingStrategy(),
- transform.getInsertDefault(),
- transform.getInsertDefault()
- ? transform.getCombineFn().defaultValue() : null,
- combined.getCoder());
- return combined
- .apply(ParDo.of(new WrapAsList<OutputT>()))
- .apply(CreateGearpumpPCollectionView.<OutputT, OutputT>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingCombineGloballyAsSingletonView";
- }
- }
-
- private static class StreamingViewAsSingleton<T>
- extends PTransform<PCollection<T>, PCollectionView<T>> {
-
- private static final long serialVersionUID = 5870455965625071546L;
- private final View.AsSingleton<T> transform;
-
- public StreamingViewAsSingleton(View.AsSingleton<T> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollectionView<T> expand(PCollection<T> input) {
- Combine.Globally<T, T> combine = Combine.globally(
- new SingletonCombine<>(transform.hasDefaultValue(), transform.defaultValue()));
- if (!transform.hasDefaultValue()) {
- combine = combine.withoutDefaults();
- }
- return input.apply(combine.asSingletonView());
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsSingleton";
- }
-
- private static class SingletonCombine<T> extends Combine.BinaryCombineFn<T> {
- private boolean hasDefaultValue;
- private T defaultValue;
-
- SingletonCombine(boolean hasDefaultValue, T defaultValue) {
- this.hasDefaultValue = hasDefaultValue;
- this.defaultValue = defaultValue;
- }
-
- @Override
- public T apply(T left, T right) {
- throw new IllegalArgumentException("PCollection with more than one element "
- + "accessed as a singleton view. Consider using Combine.globally().asSingleton() to "
- + "combine the PCollection into a single value");
- }
-
- @Override
- public T identity() {
- if (hasDefaultValue) {
- return defaultValue;
- } else {
- throw new IllegalArgumentException(
- "Empty PCollection accessed as a singleton view. "
- + "Consider setting withDefault to provide a default value");
- }
- }
- }
- }
-
- private static class WrapAsList<T> extends DoFn<T, List<T>> {
- @ProcessElement
- public void processElement(ProcessContext c) {
- c.output(Collections.singletonList(c.element()));
- }
- }
-
- /**
- * Creates a primitive {@link PCollectionView}.
- *
- * <p>For internal use only by runner implementors.
- *
- * @param <ElemT> The type of the elements of the input PCollection
- * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
- */
- public static class CreateGearpumpPCollectionView<ElemT, ViewT>
- extends PTransform<PCollection<List<ElemT>>, PCollectionView<ViewT>> {
- private static final long serialVersionUID = -2637073020800540542L;
- private PCollectionView<ViewT> view;
-
- private CreateGearpumpPCollectionView(PCollectionView<ViewT> view) {
- this.view = view;
- }
-
- public static <ElemT, ViewT> CreateGearpumpPCollectionView<ElemT, ViewT> of(
- PCollectionView<ViewT> view) {
- return new CreateGearpumpPCollectionView<>(view);
- }
-
- public PCollectionView<ViewT> getView() {
- return view;
- }
-
- @Override
- public PCollectionView<ViewT> expand(PCollection<List<ElemT>> input) {
- return view;
- }
- }
-
- /**
- * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
- *
- * <p>For internal use by {@link StreamingViewAsMap}, {@link StreamingViewAsMultimap},
- * {@link StreamingViewAsList}, {@link StreamingViewAsIterable}.
- * They require the input {@link PCollection} fits in memory.
- * For a large {@link PCollection} this is expected to crash!
- *
- * @param <T> the type of elements to concatenate.
- */
- private static class Concatenate<T> extends Combine.CombineFn<T, List<T>, List<T>> {
- @Override
- public List<T> createAccumulator() {
- return new ArrayList<>();
- }
-
- @Override
- public List<T> addInput(List<T> accumulator, T input) {
- accumulator.add(input);
- return accumulator;
- }
-
- @Override
- public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
- List<T> result = createAccumulator();
- for (List<T> accumulator : accumulators) {
- result.addAll(accumulator);
- }
- return result;
- }
-
- @Override
- public List<T> extractOutput(List<T> accumulator) {
- return accumulator;
- }
-
- @Override
- public Coder<List<T>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) {
- return ListCoder.of(inputCoder);
- }
-
- @Override
- public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) {
- return ListCoder.of(inputCoder);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
index 30b1935..ae59121 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
@@ -23,6 +23,7 @@ import com.typesafe.config.ConfigValueFactory;
import java.util.HashMap;
import java.util.Map;
+import org.apache.beam.runners.gearpump.translators.GearpumpPipelineTranslator;
import org.apache.beam.runners.gearpump.translators.TranslationContext;
import org.apache.beam.sdk.Pipeline;
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
index d7588c2..559cb28 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
@@ -20,7 +20,6 @@ package org.apache.beam.runners.gearpump.translators;
import java.util.List;
-import org.apache.beam.runners.gearpump.GearpumpPipelineTranslator;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
@@ -30,17 +29,17 @@ import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
* transforms.
*/
public class CreateGearpumpPCollectionViewTranslator<ElemT, ViewT> implements
- TransformTranslator<GearpumpPipelineTranslator.CreateGearpumpPCollectionView<ElemT, ViewT>> {
+ TransformTranslator<CreateStreamingGearpumpView.CreateGearpumpPCollectionView<ElemT, ViewT>> {
private static final long serialVersionUID = -3955521308055056034L;
@Override
public void translate(
- GearpumpPipelineTranslator.CreateGearpumpPCollectionView<ElemT, ViewT> transform,
+ CreateStreamingGearpumpView.CreateGearpumpPCollectionView<ElemT, ViewT> transform,
TranslationContext context) {
JavaStream<WindowedValue<List<ElemT>>> inputStream =
context.getInputStream(context.getInput());
- PCollectionView<ViewT> view = (PCollectionView<ViewT>) context.getOutput();
- context.setOutputStream(view.getPCollection(), inputStream);
+ PCollectionView<ViewT> view = transform.getView();
+ context.setOutputStream(view, inputStream);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateStreamingGearpumpView.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateStreamingGearpumpView.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateStreamingGearpumpView.java
new file mode 100644
index 0000000..60577b3
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateStreamingGearpumpView.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.gearpump.translators;
+
+import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.PTransformOverrideFactory;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Gearpump streaming overrides for various view (side input) transforms. */
+class CreateStreamingGearpumpView<ElemT, ViewT>
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
+ private final PCollectionView<ViewT> view;
+
+ public CreateStreamingGearpumpView(PCollectionView<ViewT> view) {
+ this.view = view;
+ }
+
+ @Override
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
+ input
+ .apply(Combine.globally(new Concatenate<ElemT>()).withoutDefaults())
+ .apply(CreateGearpumpPCollectionView.<ElemT, ViewT>of(view));
+ return input;
+ }
+
+ /**
+ * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
+ *
+ * <p>For internal use by {@link CreateStreamingGearpumpView}. This combiner requires that
+ * the input {@link PCollection} fits in memory. For a large {@link PCollection} this is
+ * expected to crash!
+ *
+ * @param <T> the type of elements to concatenate.
+ */
+ private static class Concatenate<T> extends Combine.CombineFn<T, List<T>, List<T>> {
+ @Override
+ public List<T> createAccumulator() {
+ return new ArrayList<T>();
+ }
+
+ @Override
+ public List<T> addInput(List<T> accumulator, T input) {
+ accumulator.add(input);
+ return accumulator;
+ }
+
+ @Override
+ public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
+ List<T> result = createAccumulator();
+ for (List<T> accumulator : accumulators) {
+ result.addAll(accumulator);
+ }
+ return result;
+ }
+
+ @Override
+ public List<T> extractOutput(List<T> accumulator) {
+ return accumulator;
+ }
+
+ @Override
+ public Coder<List<T>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+
+ @Override
+ public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+ }
+
+ /**
+ * Creates a primitive {@link PCollectionView}.
+ *
+ * <p>For internal use only by runner implementors.
+ *
+ * @param <ElemT> The type of the elements of the input PCollection
+ * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
+ */
+ public static class CreateGearpumpPCollectionView<ElemT, ViewT>
+ extends PTransform<PCollection<List<ElemT>>, PCollection<List<ElemT>>> {
+ private PCollectionView<ViewT> view;
+
+ private CreateGearpumpPCollectionView(PCollectionView<ViewT> view) {
+ this.view = view;
+ }
+
+ public static <ElemT, ViewT> CreateGearpumpPCollectionView<ElemT, ViewT> of(
+ PCollectionView<ViewT> view) {
+ return new CreateGearpumpPCollectionView<>(view);
+ }
+
+ @Override
+ public PCollection<List<ElemT>> expand(PCollection<List<ElemT>> input) {
+ return PCollection.<List<ElemT>>createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded())
+ .setCoder(input.getCoder());
+ }
+
+ public PCollectionView<ViewT> getView() {
+ return view;
+ }
+ }
+
+ public static class Factory<ElemT, ViewT>
+ implements PTransformOverrideFactory<
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>> {
+ public Factory() {}
+
+ @Override
+ public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>> getReplacementTransform(
+ AppliedPTransform<
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>>
+ transform) {
+ return PTransformReplacement.of(
+ (PCollection<ElemT>) Iterables.getOnlyElement(transform.getInputs().values()),
+ new CreateStreamingGearpumpView<ElemT, ViewT>(transform.getTransform().getView()));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollection<ElemT> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java
new file mode 100644
index 0000000..ca98aac
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.beam.runners.core.construction.PTransformMatchers;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.runners.PTransformOverride;
+import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.PValue;
+
+import org.apache.gearpump.util.Graph;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link GearpumpPipelineTranslator} knows how to translate {@link Pipeline} objects
+ * into Gearpump {@link Graph}.
+ */
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ GearpumpPipelineTranslator.class);
+
+ /**
+ * A map from {@link PTransform} subclass to the corresponding
+ * {@link TransformTranslator} to use to translate that transform.
+ */
+ private static final Map<Class<? extends PTransform>, TransformTranslator>
+ transformTranslators = new HashMap<>();
+
+ private final TranslationContext translationContext;
+
+ static {
+ // register TransformTranslators
+ registerTransformTranslator(Read.Unbounded.class, new ReadUnboundedTranslator());
+ registerTransformTranslator(Read.Bounded.class, new ReadBoundedTranslator());
+ registerTransformTranslator(GroupByKey.class, new GroupByKeyTranslator());
+ registerTransformTranslator(Flatten.PCollections.class,
+ new FlattenPCollectionsTranslator());
+ registerTransformTranslator(ParDo.MultiOutput.class, new ParDoMultiOutputTranslator());
+ registerTransformTranslator(Window.Assign.class, new WindowAssignTranslator());
+ registerTransformTranslator(CreateStreamingGearpumpView.CreateGearpumpPCollectionView.class,
+ new CreateGearpumpPCollectionViewTranslator());
+ }
+
+ public GearpumpPipelineTranslator(TranslationContext translationContext) {
+ this.translationContext = translationContext;
+ }
+
+ public void translate(Pipeline pipeline) {
+ List<PTransformOverride> overrides =
+ ImmutableList.<PTransformOverride>builder()
+ .add(PTransformOverride.of(
+ PTransformMatchers.classEqualTo(View.CreatePCollectionView.class),
+ new CreateStreamingGearpumpView.Factory()))
+ .build();
+
+ pipeline.replaceAll(overrides);
+ pipeline.traverseTopologically(this);
+ }
+
+ @Override
+ public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
+ LOG.debug("entering composite transform {}", node.getTransform());
+ return CompositeBehavior.ENTER_TRANSFORM;
+ }
+
+ @Override
+ public void leaveCompositeTransform(TransformHierarchy.Node node) {
+ LOG.debug("leaving composite transform {}", node.getTransform());
+ }
+
+ @Override
+ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
+ LOG.debug("visiting transform {}", node.getTransform());
+ PTransform transform = node.getTransform();
+ TransformTranslator translator = getTransformTranslator(transform.getClass());
+ if (null == translator) {
+ throw new IllegalStateException(
+ "no translator registered for " + transform);
+ }
+ translationContext.setCurrentTransform(node, getPipeline());
+ translator.translate(transform, translationContext);
+ }
+
+ @Override
+ public void visitValue(PValue value, TransformHierarchy.Node producer) {
+ LOG.debug("visiting value {}", value);
+ }
+
+ /**
+ * Records that instances of the specified PTransform class
+ * should be translated by default by the corresponding
+ * {@link TransformTranslator}.
+ */
+ private static <TransformT extends PTransform> void registerTransformTranslator(
+ Class<TransformT> transformClass,
+ TransformTranslator<? extends TransformT> transformTranslator) {
+ if (transformTranslators.put(transformClass, transformTranslator) != null) {
+ throw new IllegalArgumentException(
+ "defining multiple translators for " + transformClass);
+ }
+ }
+
+ /**
+ * Returns the {@link TransformTranslator} to use for instances of the
+ * specified PTransform class, or null if none registered.
+ */
+ private <TransformT extends PTransform>
+ TransformTranslator<TransformT> getTransformTranslator(Class<TransformT> transformClass) {
+ return transformTranslators.get(transformClass);
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
index 64a1e0d..42b7a53 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
@@ -25,6 +25,7 @@ import com.google.common.collect.Iterables;
import java.util.HashMap;
import java.util.Map;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -78,7 +79,7 @@ public class TranslationContext {
}
public PValue getInput() {
- return Iterables.getOnlyElement(getInputs().values());
+ return Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
}
public Map<TupleTag<?>, PValue> getOutputs() {
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index e2777df..b20896a 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -135,7 +135,6 @@ public class DoFnFunction<InputT, OutputT> extends
}
}
-
for (PCollectionView<?> sideInput: sideInputs) {
for (WindowedValue<InputT> value : pushedBackValues) {
for (BoundedWindow win: value.getWindows()) {
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
index 6637a9b..558eb0d 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
@@ -80,7 +80,7 @@ public abstract class GearpumpSource<T> implements DataSource {
org.joda.time.Instant timestamp = reader.getCurrentTimestamp();
message = new DefaultMessage(
WindowedValue.timestampedValueInGlobalWindow(data, timestamp),
- TranslatorUtils.jodaTimeToJava8Time(timestamp));
+ timestamp.getMillis());
}
available = reader.advance();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
index 83fc6e6..b1cd61c 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
@@ -52,6 +52,7 @@ public class TranslatorUtils {
}
public static Window boundedWindowToGearpumpWindow(BoundedWindow window) {
+ // Gearpump window upper bound is exclusive
Instant end = TranslatorUtils.jodaTimeToJava8Time(window.maxTimestamp().plus(1L));
if (window instanceof IntervalWindow) {
IntervalWindow intervalWindow = (IntervalWindow) window;
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java
index b23b0c6..511eed1 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslatorTest.java
@@ -23,7 +23,6 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.gearpump.GearpumpPipelineTranslator;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PValue;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
@@ -38,8 +37,8 @@ public class CreateGearpumpPCollectionViewTranslatorTest {
CreateGearpumpPCollectionViewTranslator translator =
new CreateGearpumpPCollectionViewTranslator();
- GearpumpPipelineTranslator.CreateGearpumpPCollectionView pCollectionView =
- mock(GearpumpPipelineTranslator.CreateGearpumpPCollectionView.class);
+ CreateStreamingGearpumpView.CreateGearpumpPCollectionView pCollectionView =
+ mock(CreateStreamingGearpumpView.CreateGearpumpPCollectionView.class);
JavaStream javaStream = mock(JavaStream.class);
TranslationContext translationContext = mock(TranslationContext.class);
@@ -49,7 +48,7 @@ public class CreateGearpumpPCollectionViewTranslatorTest {
when(translationContext.getInputStream(mockInput)).thenReturn(javaStream);
PCollectionView view = mock(PCollectionView.class);
- when(translationContext.getOutput()).thenReturn(view);
+ when(pCollectionView.getView()).thenReturn(view);
translator.translate(pCollectionView, translationContext);
verify(translationContext, times(1)).setOutputStream(view, javaStream);
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
index 4490737..cc4284f 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
@@ -30,6 +30,7 @@ import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.TimestampedValue;
import org.apache.gearpump.DefaultMessage;
@@ -42,10 +43,11 @@ import org.junit.Test;
public class GearpumpSourceTest {
private static final List<TimestampedValue<String>> TEST_VALUES =
Lists.newArrayList(
- TimestampedValue.of("a", new org.joda.time.Instant(Long.MIN_VALUE)),
+ TimestampedValue.of("a", BoundedWindow.TIMESTAMP_MIN_VALUE),
TimestampedValue.of("b", new org.joda.time.Instant(0)),
TimestampedValue.of("c", new org.joda.time.Instant(53)),
- TimestampedValue.of("d", new org.joda.time.Instant(Long.MAX_VALUE - 1)));
+ TimestampedValue.of("d", BoundedWindow.TIMESTAMP_MAX_VALUE)
+ );
private static class SourceForTest<T> extends GearpumpSource<T> {
private ValuesSource<T> valuesSource;
@@ -72,10 +74,16 @@ public class GearpumpSourceTest {
new SourceForTest<>(options, valuesSource);
sourceForTest.open(null, Instant.EPOCH);
- for (TimestampedValue<String> value : TEST_VALUES) {
+ for (int i = 0; i < TEST_VALUES.size(); i++) {
+ TimestampedValue<String> value = TEST_VALUES.get(i);
+
// Check the watermark first since the Source will advance when it's opened
- Instant expectedWaterMark = TranslatorUtils.jodaTimeToJava8Time(value.getTimestamp());
- Assert.assertEquals(expectedWaterMark, sourceForTest.getWatermark());
+ if (i < TEST_VALUES.size() - 1) {
+ Instant expectedWaterMark = TranslatorUtils.jodaTimeToJava8Time(value.getTimestamp());
+ Assert.assertEquals(expectedWaterMark, sourceForTest.getWatermark());
+ } else {
+ Assert.assertEquals(Watermark.MAX(), sourceForTest.getWatermark());
+ }
Message expectedMsg =
new DefaultMessage(
http://git-wip-us.apache.org/repos/asf/beam/blob/a7b5d981/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
index 524887d..6ebe59b 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtilsTest.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.assertThat;
import com.google.common.collect.Lists;
-import java.time.Duration;
import java.time.Instant;
import java.util.List;
@@ -68,7 +67,7 @@ public class TranslatorUtilsTest {
Instant.ofEpochMilli(Long.MAX_VALUE))));
BoundedWindow globalWindow = GlobalWindow.INSTANCE;
assertThat(TranslatorUtils.boundedWindowToGearpumpWindow(globalWindow),
- equalTo(Window.apply(Instant.ofEpochMilli(Long.MIN_VALUE / 1000),
- Instant.ofEpochMilli(Long.MAX_VALUE / 1000).minus(Duration.ofDays(1)).plusMillis(1))));
+ equalTo(Window.apply(Instant.ofEpochMilli(BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis()),
+ Instant.ofEpochMilli(globalWindow.maxTimestamp().getMillis() + 1))));
}
}
[28/50] [abbrv] beam git commit: Fix side input handling in
DoFnFunction
Posted by ke...@apache.org.
Fix side input handling in DoFnFunction
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7653e7ed
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7653e7ed
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7653e7ed
Branch: refs/heads/master
Commit: 7653e7ed6de3d9db822dcd390d2bf70819954fa5
Parents: 98854d4
Author: manuzhang <ow...@gmail.com>
Authored: Wed Jun 7 14:08:04 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Jun 12 11:45:37 2017 +0800
----------------------------------------------------------------------
.../translators/TranslationContext.java | 2 ++
.../translators/functions/DoFnFunction.java | 23 ++++----------------
.../gearpump/translators/io/GearpumpSource.java | 1 -
.../translators/utils/TranslatorUtils.java | 5 ++---
4 files changed, 8 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/7653e7ed/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
index 4090354..64a1e0d 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
@@ -68,6 +68,8 @@ public class TranslationContext {
public <OutputT> void setOutputStream(PValue output, JavaStream<OutputT> outputStream) {
if (!streams.containsKey(output)) {
streams.put(output, outputStream);
+ } else {
+ throw new RuntimeException("set stream for duplicated output " + output);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/7653e7ed/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index f521d7b..6e4fbeb 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -24,7 +24,6 @@ import com.google.common.collect.Lists;
import java.io.Serializable;
import java.util.Collection;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
@@ -66,7 +65,6 @@ public class DoFnFunction<InputT, OutputT> extends
private transient PushbackSideInputDoFnRunner<InputT, OutputT> doFnRunner;
private transient SideInputHandler sideInputReader;
private transient List<WindowedValue<InputT>> pushedBackValues;
- private transient Map<PCollectionView<?>, List<WindowedValue<Iterable<?>>>> sideInputValues;
private final Collection<PCollectionView<?>> sideInputs;
private final Map<String, PCollectionView<?>> tagsToSideInputs;
private final TupleTag<OutputT> mainOutput;
@@ -109,7 +107,6 @@ public class DoFnFunction<InputT, OutputT> extends
doFnRunner = doFnRunnerFactory.createRunner(sideInputReader);
pushedBackValues = new LinkedList<>();
- sideInputValues = new HashMap<>();
outputManager.setup(mainOutput, sideOutputs);
}
@@ -132,25 +129,14 @@ public class DoFnFunction<InputT, OutputT> extends
} else {
// side input
PCollectionView<?> sideInput = tagsToSideInputs.get(unionValue.getUnionTag());
- WindowedValue<?> sideInputValue =
- (WindowedValue<?>) unionValue.getValue();
- Object value = sideInputValue.getValue();
- if (!(value instanceof Iterable)) {
- sideInputValue = sideInputValue.withValue(Lists.newArrayList(value));
- }
- if (!sideInputValues.containsKey(sideInput)) {
- sideInputValues.put(sideInput, new LinkedList<WindowedValue<Iterable<?>>>());
- }
- sideInputValues.get(sideInput).add((WindowedValue<Iterable<?>>) sideInputValue);
+ WindowedValue<Iterable<?>> sideInputValue =
+ (WindowedValue<Iterable<?>>) unionValue.getValue();
+ sideInputReader.addSideInputValue(sideInput, sideInputValue);
}
}
+
for (PCollectionView<?> sideInput: sideInputs) {
- if (sideInputValues.containsKey(sideInput)) {
- for (WindowedValue<Iterable<?>> value: sideInputValues.get(sideInput)) {
- sideInputReader.addSideInputValue(sideInput, value);
- }
- }
for (WindowedValue<InputT> value : pushedBackValues) {
for (BoundedWindow win: value.getWindows()) {
BoundedWindow sideInputWindow =
@@ -171,7 +157,6 @@ public class DoFnFunction<InputT, OutputT> extends
}
pushedBackValues.clear();
Iterables.addAll(pushedBackValues, nextPushedBackValues);
- sideInputValues.clear();
doFnRunner.finishBundle();
http://git-wip-us.apache.org/repos/asf/beam/blob/7653e7ed/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
index 5e79151..60f319d 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
@@ -28,7 +28,6 @@ import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
-// import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/beam/blob/7653e7ed/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
index 999afae..282f261 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
@@ -70,11 +70,10 @@ public class TranslatorUtils {
JavaStream<WindowedValue<InputT>> inputStream,
Map<String, PCollectionView<?>> tagsToSideInputs) {
JavaStream<RawUnionValue> mainStream =
- inputStream.map(new ToRawUnionValue<InputT>("0"), "map_to_RawUnionValue");
+ inputStream.map(new ToRawUnionValue<>("0"), "map_to_RawUnionValue");
for (Map.Entry<String, PCollectionView<?>> tagToSideInput: tagsToSideInputs.entrySet()) {
- // actually JavaStream<WindowedValue<List<?>>>
- JavaStream<WindowedValue<Object>> sideInputStream = context.getInputStream(
+ JavaStream<WindowedValue<List<?>>> sideInputStream = context.getInputStream(
tagToSideInput.getValue());
mainStream = mainStream.merge(sideInputStream.map(new ToRawUnionValue<>(
tagToSideInput.getKey()), "map_to_RawUnionValue"), "merge_to_MainStream");
[04/50] [abbrv] beam git commit: [BEAM-79] Add SideInput support for
GearpumpRunner
Posted by ke...@apache.org.
[BEAM-79] Add SideInput support for GearpumpRunner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4eb50d15
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4eb50d15
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4eb50d15
Branch: refs/heads/master
Commit: 4eb50d152b91df46bd7f0478650cb4abac3808c6
Parents: 2d0aed9
Author: manuzhang <ow...@gmail.com>
Authored: Tue Feb 14 12:33:31 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Tue Mar 7 22:15:26 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 16 +-
.../gearpump/GearpumpPipelineResult.java | 47 +--
.../gearpump/GearpumpPipelineTranslator.java | 8 +
.../beam/runners/gearpump/GearpumpRunner.java | 373 ++++++++++++++++++-
.../runners/gearpump/TestGearpumpRunner.java | 39 +-
...CreateGearpumpPCollectionViewTranslator.java | 44 +++
.../CreatePCollectionViewTranslator.java | 43 +++
.../translators/CreateValuesTranslator.java | 2 +
.../FlattenPCollectionTranslator.java | 38 ++
.../translators/GroupByKeyTranslator.java | 82 ++--
.../translators/ParDoBoundMultiTranslator.java | 165 +++-----
.../translators/ParDoBoundTranslator.java | 32 +-
.../translators/WindowBoundTranslator.java | 21 +-
.../translators/functions/DoFnFunction.java | 158 ++++++--
.../translators/io/BoundedSourceWrapper.java | 1 +
.../gearpump/translators/io/GearpumpSource.java | 23 +-
.../gearpump/translators/io/ValuesSource.java | 14 +-
.../translators/utils/DoFnRunnerFactory.java | 20 +-
.../translators/utils/NoOpSideInputReader.java | 48 ---
.../translators/utils/TranslatorUtils.java | 147 ++++++++
20 files changed, 1002 insertions(+), 319 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index 6f91c50..6a41dc0 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -75,23 +75,9 @@
<dependenciesToScan>
<dependency>org.apache.beam:beam-sdks-java-core</dependency>
</dependenciesToScan>
- <argLine>-noverify</argLine>
<excludes>
- <!-- side input is not supported in Gearpump -->
<exclude>
- org.apache.beam.sdk.io.gcp.bigquery.BigQueryIOTest,
- org.apache.beam.sdk.io.CountingInputTest,
- org.apache.beam.sdk.io.CountingSourceTest,
- org.apache.beam.sdk.testing.PAssertTest,
- org.apache.beam.sdk.transforms.ApproximateUniqueTest,
- org.apache.beam.sdk.transforms.CombineTest,
- org.apache.beam.sdk.transforms.CombineFnsTest,
- org.apache.beam.sdk.transforms.CountTest,
- org.apache.beam.sdk.transforms.FlattenTest,
- org.apache.beam.sdk.transforms.ParDoTest,
- org.apache.beam.sdk.transforms.SampleTest,
- org.apache.beam.sdk.transforms.ViewTest,
- org.apache.beam.sdk.transforms.join.CoGroupByKeyTest
+ org.apache.beam.sdk.transforms.ParDoTest
</exclude>
</excludes>
<systemPropertyVariables>
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
index a3740b7..8f90898 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
@@ -30,6 +30,7 @@ import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.gearpump.cluster.ApplicationStatus;
import org.apache.gearpump.cluster.MasterToAppMaster.AppMasterData;
import org.apache.gearpump.cluster.client.ClientContext;
+import org.apache.gearpump.cluster.client.RunningApplication;
import org.joda.time.Duration;
import scala.collection.JavaConverters;
@@ -41,13 +42,11 @@ import scala.collection.Seq;
public class GearpumpPipelineResult implements PipelineResult {
private final ClientContext client;
- private final int appId;
- private final Duration defaultWaitDuration = Duration.standardSeconds(30);
- private final Duration defaultWaitInterval = Duration.standardSeconds(5);
+ private final RunningApplication app;
- public GearpumpPipelineResult(ClientContext client, int appId) {
+ public GearpumpPipelineResult(ClientContext client, RunningApplication app) {
this.client = client;
- this.appId = appId;
+ this.app = app;
}
@Override
@@ -57,38 +56,19 @@ public class GearpumpPipelineResult implements PipelineResult {
@Override
public State cancel() throws IOException {
- client.shutdown(appId);
+ app.shutDown();
return State.CANCELLED;
}
@Override
public State waitUntilFinish(Duration duration) {
- long start = System.currentTimeMillis();
- do {
- try {
- Thread.sleep(defaultWaitInterval.getMillis());
- } catch (Exception e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- if (e instanceof RuntimeException) {
- throw (RuntimeException) e;
- }
- throw new RuntimeException(e);
- }
- } while (State.RUNNING == getGearpumpState()
- && (System.currentTimeMillis() - start) < duration.getMillis());
-
- if (State.RUNNING == getGearpumpState()) {
- return State.DONE;
- } else {
- return State.FAILED;
- }
+ return waitUntilFinish();
}
@Override
public State waitUntilFinish() {
- return waitUntilFinish(defaultWaitDuration);
+ app.waitUntilFinish();
+ return State.DONE;
}
@Override
@@ -109,18 +89,19 @@ public class GearpumpPipelineResult implements PipelineResult {
List<AppMasterData> apps =
JavaConverters.<AppMasterData>seqAsJavaListConverter(
(Seq<AppMasterData>) client.listApps().appMasters()).asJava();
- for (AppMasterData app: apps) {
- if (app.appId() == appId) {
- status = app.status();
+ for (AppMasterData appData: apps) {
+ if (appData.appId() == app.appId()) {
+ status = appData.status();
}
}
if (null == status || status instanceof ApplicationStatus.NONEXIST$) {
return State.UNKNOWN;
} else if (status instanceof ApplicationStatus.ACTIVE$) {
return State.RUNNING;
+ } else if (status instanceof ApplicationStatus.SUCCEEDED$) {
+ return State.DONE;
} else {
- return State.STOPPED;
+ return State.FAILED;
}
}
-
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
index 20624ed..4cc060c 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
@@ -20,6 +20,9 @@ package org.apache.beam.runners.gearpump;
import java.util.HashMap;
import java.util.Map;
+
+import org.apache.beam.runners.gearpump.translators.CreateGearpumpPCollectionViewTranslator;
+import org.apache.beam.runners.gearpump.translators.CreatePCollectionViewTranslator;
import org.apache.beam.runners.gearpump.translators.CreateValuesTranslator;
import org.apache.beam.runners.gearpump.translators.FlattenPCollectionTranslator;
import org.apache.beam.runners.gearpump.translators.GroupByKeyTranslator;
@@ -38,6 +41,7 @@ import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.PValue;
@@ -75,6 +79,10 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
registerTransformTranslator(ParDo.BoundMulti.class, new ParDoBoundMultiTranslator());
registerTransformTranslator(Window.Bound.class, new WindowBoundTranslator());
registerTransformTranslator(Create.Values.class, new CreateValuesTranslator());
+ registerTransformTranslator(View.CreatePCollectionView.class,
+ new CreatePCollectionViewTranslator());
+ registerTransformTranslator(GearpumpRunner.CreateGearpumpPCollectionView.class,
+ new CreateGearpumpPCollectionViewTranslator<>());
}
public GearpumpPipelineTranslator(TranslationContext translationContext) {
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
index 9ca1eb2..72f2126 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
@@ -17,29 +17,45 @@
*/
package org.apache.beam.runners.gearpump;
+import com.google.common.collect.ImmutableMap;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigValueFactory;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.beam.runners.gearpump.translators.TranslationContext;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.ListCoder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.util.InstanceBuilder;
+import org.apache.beam.sdk.util.PCollectionViews;
+import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
import org.apache.gearpump.cluster.ClusterConfig;
import org.apache.gearpump.cluster.UserConfig;
import org.apache.gearpump.cluster.client.ClientContext;
+import org.apache.gearpump.cluster.client.RunningApplication;
import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStreamApp;
@@ -56,8 +72,21 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
private static final String GEARPUMP_SERIALIZERS = "gearpump.serializers";
private static final String DEFAULT_APPNAME = "beam_gearpump_app";
+ /** Custom transforms implementations. */
+ private final Map<Class<?>, Class<?>> overrides;
+
public GearpumpRunner(GearpumpPipelineOptions options) {
this.options = options;
+
+ ImmutableMap.Builder<Class<?>, Class<?>> builder = ImmutableMap.builder();
+ builder.put(Combine.GloballyAsSingletonView.class,
+ StreamingCombineGloballyAsSingletonView.class);
+ builder.put(View.AsMap.class, StreamingViewAsMap.class);
+ builder.put(View.AsMultimap.class, StreamingViewAsMultimap.class);
+ builder.put(View.AsSingleton.class, StreamingViewAsSingleton.class);
+ builder.put(View.AsList.class, StreamingViewAsList.class);
+ builder.put(View.AsIterable.class, StreamingViewAsIterable.class);
+ overrides = builder.build();
}
public static GearpumpRunner fromOptions(PipelineOptions options) {
@@ -69,15 +98,23 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
public <OutputT extends POutput, InputT extends PInput> OutputT apply(
PTransform<InputT, OutputT> transform, InputT input) {
- if (Flatten.FlattenPCollectionList.class.equals(transform.getClass())
+ if (overrides.containsKey(transform.getClass())) {
+
+ Class<PTransform<InputT, OutputT>> transformClass =
+ (Class<PTransform<InputT, OutputT>>) transform.getClass();
+
+ Class<PTransform<InputT, OutputT>> customTransformClass =
+ (Class<PTransform<InputT, OutputT>>) overrides.get(transform.getClass());
+
+ PTransform<InputT, OutputT> customTransform =
+ InstanceBuilder.ofType(customTransformClass)
+ .withArg(transformClass, transform)
+ .build();
+
+ return Pipeline.applyTransform(input, customTransform);
+ } else if (Flatten.FlattenPCollectionList.class.equals(transform.getClass())
&& ((PCollectionList<?>) input).size() == 0) {
return (OutputT) Pipeline.applyTransform(input.getPipeline().begin(), Create.of());
- } else if (Create.Values.class.equals(transform.getClass())) {
- return (OutputT) PCollection
- .<OutputT>createPrimitiveOutputInternal(
- input.getPipeline(),
- WindowingStrategy.globalDefault(),
- PCollection.IsBounded.BOUNDED);
} else {
return super.apply(transform, input);
}
@@ -99,9 +136,9 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
TranslationContext translationContext = new TranslationContext(streamApp, options);
GearpumpPipelineTranslator translator = new GearpumpPipelineTranslator(translationContext);
translator.translate(pipeline);
- int appId = streamApp.submit().appId();
+ RunningApplication app = streamApp.submit();
- return new GearpumpPipelineResult(clientContext, appId);
+ return new GearpumpPipelineResult(clientContext, app);
}
private ClientContext getClientContext(GearpumpPipelineOptions options, Config config) {
@@ -131,4 +168,320 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
return config.withValue(GEARPUMP_SERIALIZERS, ConfigValueFactory.fromMap(serializers));
}
+
+
+ // The following codes are forked from DataflowRunner for View translator
+ /**
+ * Specialized implementation for
+ * {@link org.apache.beam.sdk.transforms.View.AsMap View.AsMap}
+ * for the Gearpump runner.
+ */
+ private static class StreamingViewAsMap<K, V>
+ extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, V>>> {
+
+ private static final long serialVersionUID = 4791080760092950304L;
+
+ public StreamingViewAsMap(View.AsMap<K, V> transform) {}
+
+ @Override
+ public PCollectionView<Map<K, V>> expand(PCollection<KV<K, V>> input) {
+ PCollectionView<Map<K, V>> view =
+ PCollectionViews.mapView(
+ input.getPipeline(),
+ input.getWindowingStrategy(),
+ input.getCoder());
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
+ try {
+ inputCoder.getKeyCoder().verifyDeterministic();
+ } catch (Coder.NonDeterministicException e) {
+ // throw new RuntimeException(e);
+ }
+
+ return input
+ .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
+ .apply(CreateGearpumpPCollectionView.<KV<K, V>, Map<K, V>>of(view));
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingViewAsMap";
+ }
+ }
+
+ /**
+ * Specialized expansion for {@link
+ * org.apache.beam.sdk.transforms.View.AsMultimap View.AsMultimap} for the
+ * Gearpump runner.
+ */
+ private static class StreamingViewAsMultimap<K, V>
+ extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, Iterable<V>>>> {
+
+ private static final long serialVersionUID = 5854899081751333352L;
+
+ public StreamingViewAsMultimap(View.AsMultimap<K, V> transform) {}
+
+ @Override
+ public PCollectionView<Map<K, Iterable<V>>> expand(PCollection<KV<K, V>> input) {
+ PCollectionView<Map<K, Iterable<V>>> view =
+ PCollectionViews.multimapView(
+ input.getPipeline(),
+ input.getWindowingStrategy(),
+ input.getCoder());
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
+ try {
+ inputCoder.getKeyCoder().verifyDeterministic();
+ } catch (Coder.NonDeterministicException e) {
+ // throw new RuntimeException(e);
+ }
+
+ return input
+ .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
+ .apply(CreateGearpumpPCollectionView.<KV<K, V>, Map<K, Iterable<V>>>of(view));
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingViewAsMultimap";
+ }
+ }
+
+ /**
+ * Specialized implementation for
+ * {@link org.apache.beam.sdk.transforms.View.AsIterable View.AsIterable} for the
+ * Gearpump runner.
+ */
+ private static class StreamingViewAsIterable<T>
+ extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
+
+ private static final long serialVersionUID = -3399860618995613421L;
+
+ public StreamingViewAsIterable(View.AsIterable<T> transform) {}
+
+ @Override
+ public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
+ PCollectionView<Iterable<T>> view =
+ PCollectionViews.iterableView(
+ input.getPipeline(),
+ input.getWindowingStrategy(),
+ input.getCoder());
+
+ return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
+ .apply(CreateGearpumpPCollectionView.<T, Iterable<T>>of(view));
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingViewAsIterable";
+ }
+ }
+
+ /**
+ * Specialized implementation for
+ * {@link org.apache.beam.sdk.transforms.View.AsList View.AsList} for the
+ * Gearpump runner.
+ */
+ private static class StreamingViewAsList<T>
+ extends PTransform<PCollection<T>, PCollectionView<List<T>>> {
+
+ private static final long serialVersionUID = -5018631473886330629L;
+
+ public StreamingViewAsList(View.AsList<T> transform) {}
+
+ @Override
+ public PCollectionView<List<T>> expand(PCollection<T> input) {
+ PCollectionView<List<T>> view =
+ PCollectionViews.listView(
+ input.getPipeline(),
+ input.getWindowingStrategy(),
+ input.getCoder());
+
+ return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
+ .apply(CreateGearpumpPCollectionView.<T, List<T>>of(view));
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingViewAsList";
+ }
+ }
+ private static class StreamingCombineGloballyAsSingletonView<InputT, OutputT>
+ extends PTransform<PCollection<InputT>, PCollectionView<OutputT>> {
+
+ private static final long serialVersionUID = 9064900748869035738L;
+ private final Combine.GloballyAsSingletonView<InputT, OutputT> transform;
+
+ public StreamingCombineGloballyAsSingletonView(
+ Combine.GloballyAsSingletonView<InputT, OutputT> transform) {
+ this.transform = transform;
+ }
+
+ @Override
+ public PCollectionView<OutputT> expand(PCollection<InputT> input) {
+ PCollection<OutputT> combined =
+ input.apply(Combine.globally(transform.getCombineFn())
+ .withoutDefaults()
+ .withFanout(transform.getFanout()));
+
+ PCollectionView<OutputT> view = PCollectionViews.singletonView(
+ combined.getPipeline(),
+ combined.getWindowingStrategy(),
+ transform.getInsertDefault(),
+ transform.getInsertDefault()
+ ? transform.getCombineFn().defaultValue() : null,
+ combined.getCoder());
+ return combined
+ .apply(ParDo.of(new WrapAsList<OutputT>()))
+ .apply(CreateGearpumpPCollectionView.<OutputT, OutputT>of(view));
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingCombineGloballyAsSingletonView";
+ }
+ }
+
+ private static class StreamingViewAsSingleton<T>
+ extends PTransform<PCollection<T>, PCollectionView<T>> {
+
+ private static final long serialVersionUID = 5870455965625071546L;
+ private final View.AsSingleton<T> transform;
+
+ public StreamingViewAsSingleton(View.AsSingleton<T> transform) {
+ this.transform = transform;
+ }
+
+ @Override
+ public PCollectionView<T> expand(PCollection<T> input) {
+ Combine.Globally<T, T> combine = Combine.globally(
+ new SingletonCombine<>(transform.hasDefaultValue(), transform.defaultValue()));
+ if (!transform.hasDefaultValue()) {
+ combine = combine.withoutDefaults();
+ }
+ return input.apply(combine.asSingletonView());
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingViewAsSingleton";
+ }
+
+ private static class SingletonCombine<T> extends Combine.BinaryCombineFn<T> {
+ private boolean hasDefaultValue;
+ private T defaultValue;
+
+ SingletonCombine(boolean hasDefaultValue, T defaultValue) {
+ this.hasDefaultValue = hasDefaultValue;
+ this.defaultValue = defaultValue;
+ }
+
+ @Override
+ public T apply(T left, T right) {
+ throw new IllegalArgumentException("PCollection with more than one element "
+ + "accessed as a singleton view. Consider using Combine.globally().asSingleton() to "
+ + "combine the PCollection into a single value");
+ }
+
+ @Override
+ public T identity() {
+ if (hasDefaultValue) {
+ return defaultValue;
+ } else {
+ throw new IllegalArgumentException(
+ "Empty PCollection accessed as a singleton view. "
+ + "Consider setting withDefault to provide a default value");
+ }
+ }
+ }
+ }
+
+ private static class WrapAsList<T> extends DoFn<T, List<T>> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(Collections.singletonList(c.element()));
+ }
+ }
+
+ /**
+ * Creates a primitive {@link PCollectionView}.
+ *
+ * <p>For internal use only by runner implementors.
+ *
+ * @param <ElemT> The type of the elements of the input PCollection
+ * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
+ */
+ public static class CreateGearpumpPCollectionView<ElemT, ViewT>
+ extends PTransform<PCollection<List<ElemT>>, PCollectionView<ViewT>> {
+ private static final long serialVersionUID = -2637073020800540542L;
+ private PCollectionView<ViewT> view;
+
+ private CreateGearpumpPCollectionView(PCollectionView<ViewT> view) {
+ this.view = view;
+ }
+
+ public static <ElemT, ViewT> CreateGearpumpPCollectionView<ElemT, ViewT> of(
+ PCollectionView<ViewT> view) {
+ return new CreateGearpumpPCollectionView<>(view);
+ }
+
+ public PCollectionView<ViewT> getView() {
+ return view;
+ }
+
+ @Override
+ public PCollectionView<ViewT> expand(PCollection<List<ElemT>> input) {
+ return view;
+ }
+ }
+
+ /**
+ * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
+ *
+ * <p>For internal use by {@link StreamingViewAsMap}, {@link StreamingViewAsMultimap},
+ * {@link StreamingViewAsList}, {@link StreamingViewAsIterable}.
+ * They require the input {@link PCollection} fits in memory.
+ * For a large {@link PCollection} this is expected to crash!
+ *
+ * @param <T> the type of elements to concatenate.
+ */
+ private static class Concatenate<T> extends Combine.CombineFn<T, List<T>, List<T>> {
+ @Override
+ public List<T> createAccumulator() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public List<T> addInput(List<T> accumulator, T input) {
+ accumulator.add(input);
+ return accumulator;
+ }
+
+ @Override
+ public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
+ List<T> result = createAccumulator();
+ for (List<T> accumulator : accumulators) {
+ result.addAll(accumulator);
+ }
+ return result;
+ }
+
+ @Override
+ public List<T> extractOutput(List<T> accumulator) {
+ return accumulator;
+ }
+
+ @Override
+ public Coder<List<T>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+
+ @Override
+ public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
index ee31fb5..c96bcb1 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
@@ -18,8 +18,9 @@
package org.apache.beam.runners.gearpump;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigValueFactory;
import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
import org.apache.beam.sdk.runners.PipelineRunner;
@@ -27,7 +28,9 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
+import org.apache.gearpump.cluster.ClusterConfig;
import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
+import org.apache.gearpump.util.Constants;
/**
* Gearpump {@link PipelineRunner} for tests, which uses {@link EmbeddedCluster}.
@@ -38,7 +41,10 @@ public class TestGearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
private final EmbeddedCluster cluster;
private TestGearpumpRunner(GearpumpPipelineOptions options) {
- cluster = EmbeddedCluster.apply();
+ Config config = ClusterConfig.master(null);
+ config = config.withValue(Constants.APPLICATION_TOTAL_RETRIES(),
+ ConfigValueFactory.fromAnyRef(0));
+ cluster = new EmbeddedCluster(config);
cluster.start();
options.setEmbeddedCluster(cluster);
delegate = GearpumpRunner.fromOptions(options);
@@ -52,12 +58,31 @@ public class TestGearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
@Override
public GearpumpPipelineResult run(Pipeline pipeline) {
- GearpumpPipelineResult result = delegate.run(pipeline);
- PipelineResult.State state = result.waitUntilFinish();
- cluster.stop();
- assert(state == PipelineResult.State.DONE);
+ try {
+ GearpumpPipelineResult result = delegate.run(pipeline);
+ result.waitUntilFinish();
+ cluster.stop();
+ return result;
+ } catch (Throwable e) {
+ // copied from TestFlinkRunner to pull out AssertionError
+ // which is wrapped in UserCodeException
+ Throwable cause = e;
+ Throwable oldCause;
+ do {
+ if (cause.getCause() == null) {
+ break;
+ }
- return result;
+ oldCause = cause;
+ cause = cause.getCause();
+
+ } while (!oldCause.equals(cause));
+ if (cause instanceof AssertionError) {
+ throw (AssertionError) cause;
+ } else {
+ throw e;
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
new file mode 100644
index 0000000..d05c89d
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import java.util.List;
+
+import org.apache.beam.runners.gearpump.GearpumpRunner;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+
+
+/**
+ * CreateGearpumpPCollectionView bridges input stream to down stream
+ * transforms.
+ */
+public class CreateGearpumpPCollectionViewTranslator<ElemT, ViewT> implements
+ TransformTranslator<GearpumpRunner.CreateGearpumpPCollectionView<ElemT, ViewT>> {
+
+ @Override
+ public void translate(GearpumpRunner.CreateGearpumpPCollectionView<ElemT, ViewT> transform,
+ TranslationContext context) {
+ JavaStream<WindowedValue<List<ElemT>>> inputStream =
+ context.getInputStream(context.getInput(transform));
+ PCollectionView<ViewT> view = transform.getView();
+ context.setOutputStream(view, inputStream);
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java
new file mode 100644
index 0000000..e9e2e5d
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import java.util.List;
+
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+
+/**
+ * View.CreatePCollectionView bridges input stream to down stream
+ * transforms.
+ */
+public class CreatePCollectionViewTranslator<ElemT, ViewT> implements
+ TransformTranslator<View.CreatePCollectionView<ElemT, ViewT>> {
+
+ @Override
+ public void translate(View.CreatePCollectionView<ElemT, ViewT> transform,
+ TranslationContext context) {
+ JavaStream<WindowedValue<List<ElemT>>> inputStream =
+ context.getInputStream(context.getInput(transform));
+ PCollectionView<ViewT> view = transform.getView();
+ context.setOutputStream(view, inputStream);
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateValuesTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateValuesTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateValuesTranslator.java
index 452127a..e5dc6dd 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateValuesTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateValuesTranslator.java
@@ -33,6 +33,8 @@ import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
*/
public class CreateValuesTranslator<T> implements TransformTranslator<Create.Values<T>> {
+ private static final long serialVersionUID = 5411841848199229738L;
+
@Override
public void translate(Create.Values<T> transform, TranslationContext context) {
try {
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionTranslator.java
index b740ab5..27e54b8 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionTranslator.java
@@ -18,11 +18,22 @@
package org.apache.beam.runners.gearpump.translators;
+import com.google.common.collect.Lists;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
+import org.apache.beam.runners.gearpump.translators.io.ValuesSource;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+
+
/**
* Flatten.FlattenPCollectionList is translated to Gearpump merge function.
* Note only two-way merge is working now
@@ -30,17 +41,44 @@ import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
public class FlattenPCollectionTranslator<T> implements
TransformTranslator<Flatten.FlattenPCollectionList<T>> {
+ private static final long serialVersionUID = -5552148802472944759L;
+
@Override
public void translate(Flatten.FlattenPCollectionList<T> transform, TranslationContext context) {
JavaStream<T> merged = null;
+ Set<PCollection<T>> unique = new HashSet<>();
for (PCollection<T> collection : context.getInput(transform).getAll()) {
+ unique.add(collection);
JavaStream<T> inputStream = context.getInputStream(collection);
if (null == merged) {
merged = inputStream;
} else {
+ // duplicate edges are not allowed in Gearpump graph
+ // so we route through a dummy node
+ if (unique.contains(collection)) {
+ inputStream = inputStream.map(new DummyFunction<T>(), "dummy");
+ }
+
merged = merged.merge(inputStream, transform.getName());
}
}
+
+ if (null == merged) {
+ UnboundedSourceWrapper<String, ?> unboundedSourceWrapper = new UnboundedSourceWrapper<>(
+ new ValuesSource<>(Lists.newArrayList("dummy"),
+ StringUtf8Coder.of()), context.getPipelineOptions());
+ merged = context.getSourceStream(unboundedSourceWrapper);
+ }
context.setOutputStream(context.getOutput(transform), merged);
}
+
+ private static class DummyFunction<T> extends MapFunction<T, T> {
+
+ private static final long serialVersionUID = 5454396869997290471L;
+
+ @Override
+ public T map(T t) {
+ return t;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
index 69a1d11..df8bfe9 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
@@ -40,8 +40,8 @@ import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.gearpump.streaming.dsl.api.functions.FoldFunction;
import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
-import org.apache.gearpump.streaming.dsl.api.functions.ReduceFunction;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
import org.apache.gearpump.streaming.dsl.javaapi.functions.GroupByFunction;
import org.apache.gearpump.streaming.dsl.window.api.Discarding$;
@@ -49,12 +49,16 @@ import org.apache.gearpump.streaming.dsl.window.api.EventTimeTrigger$;
import org.apache.gearpump.streaming.dsl.window.api.WindowFunction;
import org.apache.gearpump.streaming.dsl.window.api.Windows;
import org.apache.gearpump.streaming.dsl.window.impl.Window;
+import org.joda.time.Instant;
/**
* {@link GroupByKey} is translated to Gearpump groupBy function.
*/
@SuppressWarnings({"rawtypes", "unchecked"})
public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKey<K, V>> {
+
+ private static final long serialVersionUID = -8742202583992787659L;
+
@Override
public void translate(GroupByKey<K, V> transform, TranslationContext context) {
PCollection<KV<K, V>> input = context.getInput(transform);
@@ -66,15 +70,14 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
input.getWindowingStrategy().getOutputTimeFn();
WindowFn<KV<K, V>, BoundedWindow> windowFn = (WindowFn<KV<K, V>, BoundedWindow>)
input.getWindowingStrategy().getWindowFn();
- JavaStream<WindowedValue<KV<K, Iterable<V>>>> outputStream = inputStream
+ JavaStream<WindowedValue<KV<K, List<V>>>> outputStream = inputStream
.window(Windows.apply(
new GearpumpWindowFn(windowFn.isNonMerging()),
EventTimeTrigger$.MODULE$, Discarding$.MODULE$), "assign_window")
.groupBy(new GroupByFn<K, V>(inputKeyCoder), parallelism, "group_by_Key_and_Window")
- .map(new ValueToIterable<K, V>(), "map_value_to_iterable")
.map(new KeyedByTimestamp<K, V>((OutputTimeFn<? super BoundedWindow>)
input.getWindowingStrategy().getOutputTimeFn()), "keyed_by_timestamp")
- .reduce(new Merge<>(windowFn, outputTimeFn), "merge")
+ .fold(new Merge<>(windowFn, outputTimeFn), "merge")
.map(new Values<K, V>(), "values");
context.setOutputStream(context.getOutput(transform), outputStream);
@@ -115,6 +118,7 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
private static class GroupByFn<K, V> extends
GroupByFunction<WindowedValue<KV<K, V>>, ByteBuffer> {
+ private static final long serialVersionUID = -807905402490735530L;
private final Coder<K> keyCoder;
GroupByFn(Coder<K> keyCoder) {
@@ -122,7 +126,7 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
}
@Override
- public ByteBuffer apply(WindowedValue<KV<K, V>> wv) {
+ public ByteBuffer groupBy(WindowedValue<KV<K, V>> wv) {
try {
return ByteBuffer.wrap(CoderUtils.encodeToByteArray(keyCoder, wv.getValue().getKey()));
} catch (CoderException e) {
@@ -131,19 +135,9 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
}
}
- private static class ValueToIterable<K, V>
- extends MapFunction<WindowedValue<KV<K, V>>, WindowedValue<KV<K, Iterable<V>>>> {
-
- @Override
- public WindowedValue<KV<K, Iterable<V>>> apply(WindowedValue<KV<K, V>> wv) {
- Iterable<V> values = Lists.newArrayList(wv.getValue().getValue());
- return wv.withValue(KV.of(wv.getValue().getKey(), values));
- }
- }
-
private static class KeyedByTimestamp<K, V>
- extends MapFunction<WindowedValue<KV<K, Iterable<V>>>,
- KV<org.joda.time.Instant, WindowedValue<KV<K, Iterable<V>>>>> {
+ extends MapFunction<WindowedValue<KV<K, V>>,
+ KV<Instant, WindowedValue<KV<K, V>>>> {
private final OutputTimeFn<? super BoundedWindow> outputTimeFn;
@@ -152,16 +146,17 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
}
@Override
- public KV<org.joda.time.Instant, WindowedValue<KV<K, Iterable<V>>>> apply(
- WindowedValue<KV<K, Iterable<V>>> wv) {
- org.joda.time.Instant timestamp = outputTimeFn.assignOutputTime(wv.getTimestamp(),
+ public KV<org.joda.time.Instant, WindowedValue<KV<K, V>>> map(
+ WindowedValue<KV<K, V>> wv) {
+ Instant timestamp = outputTimeFn.assignOutputTime(wv.getTimestamp(),
Iterables.getOnlyElement(wv.getWindows()));
return KV.of(timestamp, wv);
}
}
private static class Merge<K, V> extends
- ReduceFunction<KV<org.joda.time.Instant, WindowedValue<KV<K, Iterable<V>>>>> {
+ FoldFunction<KV<Instant, WindowedValue<KV<K, V>>>,
+ KV<Instant, WindowedValue<KV<K, List<V>>>>> {
private final WindowFn<KV<K, V>, BoundedWindow> windowFn;
private final OutputTimeFn<? super BoundedWindow> outputTimeFn;
@@ -173,14 +168,28 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
}
@Override
- public KV<org.joda.time.Instant, WindowedValue<KV<K, Iterable<V>>>> apply(
- KV<org.joda.time.Instant, WindowedValue<KV<K, Iterable<V>>>> kv1,
- KV<org.joda.time.Instant, WindowedValue<KV<K, Iterable<V>>>> kv2) {
- org.joda.time.Instant t1 = kv1.getKey();
- org.joda.time.Instant t2 = kv2.getKey();
+ public KV<Instant, WindowedValue<KV<K, List<V>>>> init() {
+ return KV.of(null, null);
+ }
+
+ @Override
+ public KV<Instant, WindowedValue<KV<K, List<V>>>> fold(
+ KV<Instant, WindowedValue<KV<K, List<V>>>> accum,
+ KV<Instant, WindowedValue<KV<K, V>>> iter) {
+ if (accum.getKey() == null) {
+ WindowedValue<KV<K, V>> wv = iter.getValue();
+ KV<K, V> kv = wv.getValue();
+ V v = kv.getValue();
+ List<V> nv = Lists.newArrayList(v);
+ return KV.of(iter.getKey(), wv.withValue(KV.of(kv.getKey(), nv)));
+ }
+
+ Instant t1 = accum.getKey();
+ Instant t2 = iter.getKey();
- final WindowedValue<KV<K, Iterable<V>>> wv1 = kv1.getValue();
- final WindowedValue<KV<K, Iterable<V>>> wv2 = kv2.getValue();
+ final WindowedValue<KV<K, List<V>>> wv1 = accum.getValue();
+ final WindowedValue<KV<K, V>> wv2 = iter.getValue();
+ wv1.getValue().getValue().add(wv2.getValue().getValue());
final List<BoundedWindow> mergedWindows = new ArrayList<>();
if (!windowFn.isNonMerging()) {
@@ -208,23 +217,22 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
mergedWindows.addAll(wv1.getWindows());
}
- org.joda.time.Instant timestamp = outputTimeFn.combine(t1, t2);
+ Instant timestamp = outputTimeFn.combine(t1, t2);
return KV.of(timestamp,
- WindowedValue.of(KV.of(wv1.getValue().getKey(),
- Iterables.concat(wv1.getValue().getValue(), wv2.getValue().getValue())), timestamp,
+ WindowedValue.of(wv1.getValue(), timestamp,
mergedWindows, wv1.getPane()));
}
}
private static class Values<K, V> extends
- MapFunction<KV<org.joda.time.Instant, WindowedValue<KV<K, Iterable<V>>>>,
- WindowedValue<KV<K, Iterable<V>>>> {
+ MapFunction<KV<Instant, WindowedValue<KV<K, List<V>>>>,
+ WindowedValue<KV<K, List<V>>>> {
@Override
- public WindowedValue<KV<K, Iterable<V>>> apply(KV<org.joda.time.Instant,
- WindowedValue<KV<K, Iterable<V>>>> kv) {
- org.joda.time.Instant timestamp = kv.getKey();
- WindowedValue<KV<K, Iterable<V>>> wv = kv.getValue();
+ public WindowedValue<KV<K, List<V>>> map(KV<org.joda.time.Instant,
+ WindowedValue<KV<K, List<V>>>> kv) {
+ Instant timestamp = kv.getKey();
+ WindowedValue<KV<K, List<V>>> wv = kv.getValue();
return WindowedValue.of(wv.getValue(), timestamp, wv.getWindows(), wv.getPane());
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
index bf7073b..8c57019 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
@@ -18,158 +18,93 @@
package org.apache.beam.runners.gearpump.translators;
+
+import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
-import java.util.Iterator;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
-import org.apache.beam.runners.core.DoFnRunner;
-import org.apache.beam.runners.core.DoFnRunners;
-import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
-import org.apache.beam.runners.gearpump.translators.utils.DoFnRunnerFactory;
-import org.apache.beam.runners.gearpump.translators.utils.NoOpAggregatorFactory;
-import org.apache.beam.runners.gearpump.translators.utils.NoOpSideInputReader;
-import org.apache.beam.runners.gearpump.translators.utils.NoOpStepContext;
+import javax.annotation.Nullable;
+
+import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
-import org.apache.beam.sdk.util.SideInputReader;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TupleTagList;
import org.apache.gearpump.streaming.dsl.api.functions.FilterFunction;
-import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
-import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
/**
* {@link ParDo.BoundMulti} is translated to Gearpump flatMap function
- * with {@link DoFn} wrapped in {@link DoFnMultiFunction}. The outputs are
+ * with {@link DoFn} wrapped in {@link DoFnFunction}. The outputs are
* further filtered with Gearpump filter function by output tag
*/
@SuppressWarnings({"rawtypes", "unchecked"})
public class ParDoBoundMultiTranslator<InputT, OutputT> implements
TransformTranslator<ParDo.BoundMulti<InputT, OutputT>> {
+ private static final long serialVersionUID = -6023461558200028849L;
+
@Override
public void translate(ParDo.BoundMulti<InputT, OutputT> transform, TranslationContext context) {
PCollection<InputT> inputT = (PCollection<InputT>) context.getInput(transform);
JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(inputT);
- Map<TupleTag<?>, PCollection<?>> outputs = context.getOutput(transform).getAll();
+ Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
+ Map<String, PCollectionView<?>> tagsToSideInputs =
+ TranslatorUtils.getTagsToSideInputs(sideInputs);
- JavaStream<WindowedValue<KV<TupleTag<OutputT>, OutputT>>> outputStream = inputStream.flatMap(
- new DoFnMultiFunction<>(
- context.getPipelineOptions(),
- transform.getFn(),
- transform.getMainOutputTag(),
- transform.getSideOutputTags(),
- inputT.getWindowingStrategy(),
- new NoOpSideInputReader()
- ), transform.getName());
- for (Map.Entry<TupleTag<?>, PCollection<?>> output : outputs.entrySet()) {
+ Map<TupleTag<?>, PCollection<?>> outputs = context.getOutput(transform).getAll();
+ final TupleTag<OutputT> mainOutput = transform.getMainOutputTag();
+ List<TupleTag<?>> sideOutputs = Lists.newLinkedList(Sets.filter(outputs.keySet(),
+ new Predicate<TupleTag<?>>() {
+ @Override
+ public boolean apply(@Nullable TupleTag<?> tupleTag) {
+ return tupleTag != null && !tupleTag.getId().equals(mainOutput.getId());
+ }
+ }));
+
+ JavaStream<TranslatorUtils.RawUnionValue> unionStream = TranslatorUtils.withSideInputStream(
+ context, inputStream, tagsToSideInputs);
+
+ JavaStream<TranslatorUtils.RawUnionValue> outputStream =
+ TranslatorUtils.toList(unionStream).flatMap(
+ new DoFnFunction<>(
+ context.getPipelineOptions(),
+ transform.getFn(),
+ inputT.getWindowingStrategy(),
+ sideInputs,
+ tagsToSideInputs,
+ mainOutput,
+ sideOutputs), transform.getName());
+ for (Map.Entry<TupleTag<?>, PCollection<?>> output: outputs.entrySet()) {
+ output.getValue().getCoder();
JavaStream<WindowedValue<OutputT>> taggedStream = outputStream
- .filter(new FilterByOutputTag<>((TupleTag<OutputT>) output.getKey())
- , "filter_by_output_tag")
- .map(new ExtractOutput<OutputT>(), "extract output");
-
+ .filter(new FilterByOutputTag(output.getKey().getId()),
+ "filter_by_output_tag")
+ .map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
context.setOutputStream(output.getValue(), taggedStream);
}
}
- /**
- * Gearpump {@link FlatMapFunction} wrapper over Beam {@link DoFnMultiFunction}.
- */
- private static class DoFnMultiFunction<InputT, OutputT>
- extends FlatMapFunction<WindowedValue<InputT>, WindowedValue<KV<TupleTag<OutputT>, OutputT>>>
- implements DoFnRunners.OutputManager {
-
- private final DoFnRunnerFactory<InputT, OutputT> doFnRunnerFactory;
- private DoFnRunner<InputT, OutputT> doFnRunner;
- private final DoFn<InputT, OutputT> doFn;
- private List<WindowedValue<KV<TupleTag<OutputT>, OutputT>>> outputs;
-
- public DoFnMultiFunction(
- GearpumpPipelineOptions pipelineOptions,
- DoFn<InputT, OutputT> doFn,
- TupleTag<OutputT> mainOutputTag,
- TupleTagList sideOutputTags,
- WindowingStrategy<?, ?> windowingStrategy,
- SideInputReader sideInputReader) {
- this.doFn = doFn;
- this.doFnRunnerFactory = new DoFnRunnerFactory<>(
- pipelineOptions,
- doFn,
- sideInputReader,
- this,
- mainOutputTag,
- sideOutputTags.getAll(),
- new NoOpStepContext(),
- new NoOpAggregatorFactory(),
- windowingStrategy
- );
- }
-
- @Override
- public void setup() {
- DoFnInvokers.invokerFor(doFn).invokeSetup();
- }
-
- @Override
- public void teardown() {
- DoFnInvokers.invokerFor(doFn).invokeTeardown();
- }
-
- @Override
- public Iterator<WindowedValue<KV<TupleTag<OutputT>, OutputT>>> apply(WindowedValue<InputT> wv) {
- outputs = Lists.newArrayList();
-
- if (null == doFnRunner) {
- doFnRunner = doFnRunnerFactory.createRunner();
- }
- doFnRunner.startBundle();
- doFnRunner.processElement(wv);
- doFnRunner.finishBundle();
-
- return outputs.iterator();
- }
-
- @Override
- public <T> void output(TupleTag<T> tag, WindowedValue<T> output) {
- KV<TupleTag<OutputT>, OutputT> kv = KV.of((TupleTag<OutputT>) tag,
- (OutputT) output.getValue());
- outputs.add(WindowedValue.of(kv, output.getTimestamp(),
- output.getWindows(), output.getPane()));
- }
- }
-
- private static class FilterByOutputTag<OutputT> extends
- FilterFunction<WindowedValue<KV<TupleTag<OutputT>, OutputT>>> {
+ private static class FilterByOutputTag extends FilterFunction<TranslatorUtils.RawUnionValue> {
- private final TupleTag<OutputT> tupleTag;
+ private static final long serialVersionUID = 7276155265895637526L;
+ private final String tag;
- public FilterByOutputTag(TupleTag<OutputT> tupleTag) {
- this.tupleTag = tupleTag;
+ FilterByOutputTag(String tag) {
+ this.tag = tag;
}
@Override
- public boolean apply(WindowedValue<KV<TupleTag<OutputT>, OutputT>> wv) {
- return wv.getValue().getKey().equals(tupleTag);
- }
- }
-
- private static class ExtractOutput<OutputT> extends
- MapFunction<WindowedValue<KV<TupleTag<OutputT>, OutputT>>, WindowedValue<OutputT>> {
-
- @Override
- public WindowedValue<OutputT> apply(WindowedValue<KV<TupleTag<OutputT>, OutputT>> wv) {
- // System.out.println(wv.getValue().getKey() + ":" + wv.getValue().getValue());
- return WindowedValue.of(wv.getValue().getValue(), wv.getTimestamp(),
- wv.getWindows(), wv.getPane());
+ public boolean filter(TranslatorUtils.RawUnionValue value) {
+ return value.getUnionTag().equals(tag);
}
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
index 689bc08..efae938 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
@@ -18,14 +18,21 @@
package org.apache.beam.runners.gearpump.translators;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
-import org.apache.beam.runners.gearpump.translators.utils.NoOpSideInputReader;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
@@ -36,18 +43,33 @@ import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
public class ParDoBoundTranslator<InputT, OutputT> implements
TransformTranslator<ParDo.Bound<InputT, OutputT>> {
+ private static final long serialVersionUID = -3413205558160983784L;
+ private final TupleTag<OutputT> mainOutput = new TupleTag<>();
+ private final List<TupleTag<?>> sideOutputs = TupleTagList.empty().getAll();
+
@Override
public void translate(ParDo.Bound<InputT, OutputT> transform, TranslationContext context) {
DoFn<InputT, OutputT> doFn = transform.getFn();
PCollection<OutputT> output = context.getOutput(transform);
WindowingStrategy<?, ?> windowingStrategy = output.getWindowingStrategy();
+ Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
+ Map<String, PCollectionView<?>> tagsToSideInputs =
+ TranslatorUtils.getTagsToSideInputs(sideInputs);
+ JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(
+ context.getInput(transform));
+ JavaStream<TranslatorUtils.RawUnionValue> unionStream =
+ TranslatorUtils.withSideInputStream(context,
+ inputStream, tagsToSideInputs);
+
DoFnFunction<InputT, OutputT> doFnFunction = new DoFnFunction<>(context.getPipelineOptions(),
- doFn, windowingStrategy, new NoOpSideInputReader());
- JavaStream<WindowedValue<InputT>> inputStream =
- context.getInputStream(context.getInput(transform));
+ doFn, windowingStrategy, sideInputs, tagsToSideInputs,
+ mainOutput, sideOutputs);
+
JavaStream<WindowedValue<OutputT>> outputStream =
- inputStream.flatMap(doFnFunction, transform.getName());
+ TranslatorUtils.toList(unionStream)
+ .flatMap(doFnFunction, transform.getName())
+ .map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
context.setOutputStream(context.getOutput(transform), outputStream);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
index c0de2df..81970e2 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowBoundTranslator.java
@@ -20,7 +20,10 @@ package org.apache.beam.runners.gearpump.translators;
import com.google.common.collect.Iterables;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.Window;
@@ -28,8 +31,8 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
import org.joda.time.Instant;
/**
@@ -38,6 +41,8 @@ import org.joda.time.Instant;
@SuppressWarnings("unchecked")
public class WindowBoundTranslator<T> implements TransformTranslator<Window.Bound<T>> {
+ private static final long serialVersionUID = -964887482120489061L;
+
@Override
public void translate(Window.Bound<T> transform, TranslationContext context) {
PCollection<T> input = context.getInput(transform);
@@ -47,14 +52,15 @@ public class WindowBoundTranslator<T> implements TransformTranslator<Window.Bou
WindowFn<T, BoundedWindow> windowFn = (WindowFn<T, BoundedWindow>) outputStrategy.getWindowFn();
JavaStream<WindowedValue<T>> outputStream =
inputStream
- .map(new AssignWindows(windowFn), "assign_windows");
+ .flatMap(new AssignWindows(windowFn), "assign_windows");
context.setOutputStream(context.getOutput(transform), outputStream);
}
private static class AssignWindows<T> extends
- MapFunction<WindowedValue<T>, WindowedValue<T>> {
+ FlatMapFunction<WindowedValue<T>, WindowedValue<T>> {
+ private static final long serialVersionUID = 7284565861938681360L;
private final WindowFn<T, BoundedWindow> windowFn;
AssignWindows(WindowFn<T, BoundedWindow> windowFn) {
@@ -62,7 +68,7 @@ public class WindowBoundTranslator<T> implements TransformTranslator<Window.Bou
}
@Override
- public WindowedValue<T> apply(final WindowedValue<T> value) {
+ public Iterator<WindowedValue<T>> flatMap(final WindowedValue<T> value) {
try {
Collection<BoundedWindow> windows = windowFn.assignWindows(windowFn.new AssignContext() {
@Override
@@ -80,7 +86,12 @@ public class WindowBoundTranslator<T> implements TransformTranslator<Window.Bou
return Iterables.getOnlyElement(value.getWindows());
}
});
- return WindowedValue.of(value.getValue(), value.getTimestamp(), windows, value.getPane());
+ List<WindowedValue<T>> values = new ArrayList<>(windows.size());
+ for (BoundedWindow win: windows) {
+ values.add(
+ WindowedValue.of(value.getValue(), value.getTimestamp(), win, value.getPane()));
+ }
+ return values.iterator();
} catch (Exception e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index a66d3a4..b2c68d6 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -18,90 +18,190 @@
package org.apache.beam.runners.gearpump.translators.functions;
+import com.google.common.collect.Iterables;
+
import com.google.common.collect.Lists;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
-import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners;
+import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
+import org.apache.beam.runners.core.SideInputHandler;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
import org.apache.beam.runners.gearpump.translators.utils.DoFnRunnerFactory;
import org.apache.beam.runners.gearpump.translators.utils.NoOpAggregatorFactory;
import org.apache.beam.runners.gearpump.translators.utils.NoOpStepContext;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils.RawUnionValue;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
-import org.apache.beam.sdk.util.SideInputReader;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingStrategy;
+
+import org.apache.beam.sdk.util.state.InMemoryStateInternals;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TupleTagList;
import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
/**
* Gearpump {@link FlatMapFunction} wrapper over Beam {@link DoFn}.
*/
+@SuppressWarnings("unchecked")
public class DoFnFunction<InputT, OutputT> extends
- FlatMapFunction<WindowedValue<InputT>, WindowedValue<OutputT>> implements
- DoFnRunners.OutputManager {
+ FlatMapFunction<List<RawUnionValue>, RawUnionValue> {
- private final TupleTag<OutputT> mainTag = new TupleTag<OutputT>() {};
- private List<WindowedValue<OutputT>> outputs = Lists.newArrayList();
+ private static final long serialVersionUID = -5701440128544343353L;
private final DoFnRunnerFactory<InputT, OutputT> doFnRunnerFactory;
- private DoFnRunner<InputT, OutputT> doFnRunner;
private final DoFn<InputT, OutputT> doFn;
+ private transient DoFnInvoker<InputT, OutputT> doFnInvoker;
+ private transient PushbackSideInputDoFnRunner<InputT, OutputT> doFnRunner;
+ private transient SideInputHandler sideInputReader;
+ private transient List<WindowedValue<InputT>> pushedBackValues;
+ private transient Map<PCollectionView<?>, List<WindowedValue<Iterable<?>>>> sideInputValues;
+ private final Collection<PCollectionView<?>> sideInputs;
+ private final Map<String, PCollectionView<?>> tagsToSideInputs;
+ private final TupleTag<OutputT> mainOutput;
+ private final List<TupleTag<?>> sideOutputs;
+ private final DoFnOutputManager outputManager;
public DoFnFunction(
GearpumpPipelineOptions pipelineOptions,
DoFn<InputT, OutputT> doFn,
WindowingStrategy<?, ?> windowingStrategy,
- SideInputReader sideInputReader) {
+ Collection<PCollectionView<?>> sideInputs,
+ Map<String, PCollectionView<?>> sideInputTagMapping,
+ TupleTag<OutputT> mainOutput,
+ List<TupleTag<?>> sideOutputs) {
this.doFn = doFn;
+ this.outputManager = new DoFnOutputManager();
this.doFnRunnerFactory = new DoFnRunnerFactory<>(
pipelineOptions,
doFn,
- sideInputReader,
- this,
- mainTag,
- TupleTagList.empty().getAll(),
+ sideInputs,
+ outputManager,
+ mainOutput,
+ sideOutputs,
new NoOpStepContext(),
new NoOpAggregatorFactory(),
windowingStrategy
);
+ this.sideInputs = sideInputs;
+ this.tagsToSideInputs = sideInputTagMapping;
+ this.mainOutput = mainOutput;
+ this.sideOutputs = sideOutputs;
}
@Override
public void setup() {
- DoFnInvokers.invokerFor(doFn).invokeSetup();
+ sideInputReader = new SideInputHandler(sideInputs,
+ InMemoryStateInternals.<Void>forKey(null));
+ doFnInvoker = DoFnInvokers.invokerFor(doFn);
+ doFnInvoker.invokeSetup();
+
+ doFnRunner = doFnRunnerFactory.createRunner(sideInputReader);
+
+ pushedBackValues = new LinkedList<>();
+ sideInputValues = new HashMap<>();
+ outputManager.setup(mainOutput, sideOutputs);
}
@Override
public void teardown() {
- DoFnInvokers.invokerFor(doFn).invokeTeardown();
+ doFnInvoker.invokeTeardown();
}
@Override
- public Iterator<WindowedValue<OutputT>> apply(WindowedValue<InputT> value) {
- outputs = Lists.newArrayList();
+ public Iterator<TranslatorUtils.RawUnionValue> flatMap(List<RawUnionValue> inputs) {
+ outputManager.clear();
+
+ doFnRunner.startBundle();
- if (null == doFnRunner) {
- doFnRunner = doFnRunnerFactory.createRunner();
+ for (RawUnionValue unionValue: inputs) {
+ final String tag = unionValue.getUnionTag();
+ if (tag.equals("0")) {
+ // main input
+ pushedBackValues.add((WindowedValue<InputT>) unionValue.getValue());
+ } else {
+ // side input
+ PCollectionView<?> sideInput = tagsToSideInputs.get(unionValue.getUnionTag());
+ WindowedValue<Iterable<?>> sideInputValue =
+ (WindowedValue<Iterable<?>>) unionValue.getValue();
+ if (!sideInputValues.containsKey(sideInput)) {
+ sideInputValues.put(sideInput, new LinkedList<WindowedValue<Iterable<?>>>());
+ }
+ sideInputValues.get(sideInput).add(sideInputValue);
+ }
}
- doFnRunner.startBundle();
- doFnRunner.processElement(value);
+ for (PCollectionView<?> sideInput: sideInputs) {
+ if (sideInputValues.containsKey(sideInput)) {
+ for (WindowedValue<Iterable<?>> value: sideInputValues.get(sideInput)) {
+ sideInputReader.addSideInputValue(sideInput, value);
+ }
+ }
+ for (WindowedValue<InputT> value : pushedBackValues) {
+ for (BoundedWindow win: value.getWindows()) {
+ BoundedWindow sideInputWindow =
+ sideInput.getWindowingStrategyInternal().getWindowFn().getSideInputWindow(win);
+ if (!sideInputReader.isReady(sideInput, sideInputWindow)) {
+ Object emptyValue = WindowedValue.of(
+ Lists.newArrayList(), value.getTimestamp(), sideInputWindow, value.getPane());
+ sideInputReader.addSideInputValue(sideInput, (WindowedValue<Iterable<?>>) emptyValue);
+ }
+ }
+ }
+ }
+
+ List<WindowedValue<InputT>> nextPushedBackValues = new LinkedList<>();
+ for (WindowedValue<InputT> value : pushedBackValues) {
+ Iterable<WindowedValue<InputT>> values = doFnRunner.processElementInReadyWindows(value);
+ Iterables.addAll(nextPushedBackValues, values);
+ }
+ pushedBackValues.clear();
+ Iterables.addAll(pushedBackValues, nextPushedBackValues);
+ sideInputValues.clear();
+
doFnRunner.finishBundle();
- return outputs.iterator();
+ return outputManager.getOutputs();
}
- @SuppressWarnings({"rawtypes", "unchecked"})
- @Override
- public <T> void output(TupleTag<T> tag, WindowedValue<T> output) {
- if (mainTag.equals(tag)) {
- outputs.add((WindowedValue<OutputT>) output);
- } else {
- throw new RuntimeException("output is not of main tag");
+ private static class DoFnOutputManager implements DoFnRunners.OutputManager, Serializable {
+
+ private static final long serialVersionUID = 4967375172737408160L;
+ private transient List<RawUnionValue> outputs;
+ private transient Set<TupleTag<?>> outputTags;
+
+ @Override
+ public <T> void output(TupleTag<T> outputTag, WindowedValue<T> output) {
+ if (outputTags.contains(outputTag)) {
+ outputs.add(new RawUnionValue(outputTag.getId(), output));
+ }
+ }
+
+ void setup(TupleTag<?> mainOutput, List<TupleTag<?>> sideOutputs) {
+ outputs = new LinkedList<>();
+ outputTags = new HashSet<>();
+ outputTags.add(mainOutput);
+ outputTags.addAll(sideOutputs);
+ }
+
+ void clear() {
+ outputs.clear();
+ }
+
+ Iterator<RawUnionValue> getOutputs() {
+ return outputs.iterator();
}
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/BoundedSourceWrapper.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/BoundedSourceWrapper.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/BoundedSourceWrapper.java
index f889101..2c18735 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/BoundedSourceWrapper.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/BoundedSourceWrapper.java
@@ -29,6 +29,7 @@ import org.apache.beam.sdk.options.PipelineOptions;
*/
public class BoundedSourceWrapper<T> extends GearpumpSource<T> {
+ private static final long serialVersionUID = 8199570485738786123L;
private final BoundedSource<T> source;
public BoundedSourceWrapper(BoundedSource<T> source, PipelineOptions options) {
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
index 3d0d7c8..c079603 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
@@ -28,10 +28,13 @@ import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
+// import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.gearpump.Message;
import org.apache.gearpump.streaming.source.DataSource;
+import org.apache.gearpump.streaming.source.Watermark;
import org.apache.gearpump.streaming.task.TaskContext;
/**
@@ -74,11 +77,11 @@ public abstract class GearpumpSource<T> implements DataSource {
if (available) {
T data = reader.getCurrent();
org.joda.time.Instant timestamp = reader.getCurrentTimestamp();
- available = reader.advance();
message = Message.apply(
- WindowedValue.valueInGlobalWindow(data),
+ WindowedValue.timestampedValueInGlobalWindow(data, timestamp),
timestamp.getMillis());
}
+ available = reader.advance();
} catch (Exception e) {
close();
throw new RuntimeException(e);
@@ -100,11 +103,19 @@ public abstract class GearpumpSource<T> implements DataSource {
@Override
public Instant getWatermark() {
if (reader instanceof UnboundedSource.UnboundedReader) {
- return TranslatorUtils.jodaTimeToJava8Time(
- ((UnboundedSource.UnboundedReader) reader).getWatermark());
+ org.joda.time.Instant watermark =
+ ((UnboundedSource.UnboundedReader) reader).getWatermark();
+ if (watermark == BoundedWindow.TIMESTAMP_MAX_VALUE) {
+ return Watermark.MAX();
+ } else {
+ return TranslatorUtils.jodaTimeToJava8Time(watermark);
+ }
} else {
- return Instant.now();
+ if (available) {
+ return TranslatorUtils.jodaTimeToJava8Time(reader.getCurrentTimestamp());
+ } else {
+ return Watermark.MAX();
+ }
}
}
-
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
index f5a5eb4..e0488cd 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
@@ -33,6 +33,7 @@ import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.TimestampedValue;
import org.joda.time.Instant;
/**
@@ -40,6 +41,7 @@ import org.joda.time.Instant;
*/
public class ValuesSource<T> extends UnboundedSource<T, UnboundedSource.CheckpointMark> {
+ private static final long serialVersionUID = 9113026175795235710L;
private final byte[] values;
private final IterableCoder<T> iterableCoder;
@@ -135,7 +137,7 @@ public class ValuesSource<T> extends UnboundedSource<T, UnboundedSource.Checkpoi
@Override
public Instant getCurrentTimestamp() throws NoSuchElementException {
- return Instant.now();
+ return getTimestamp(current);
}
@Override
@@ -145,7 +147,7 @@ public class ValuesSource<T> extends UnboundedSource<T, UnboundedSource.Checkpoi
@Override
public Instant getWatermark() {
if (iterator.hasNext()) {
- return Instant.now();
+ return getTimestamp(current);
} else {
return BoundedWindow.TIMESTAMP_MAX_VALUE;
}
@@ -160,5 +162,13 @@ public class ValuesSource<T> extends UnboundedSource<T, UnboundedSource.Checkpoi
public UnboundedSource<T, ?> getCurrentSource() {
return source;
}
+
+ private Instant getTimestamp(Object value) {
+ if (value instanceof TimestampedValue) {
+ return ((TimestampedValue) value).getTimestamp();
+ } else {
+ return Instant.now();
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
index aaefb88..5db8320 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
@@ -19,18 +19,21 @@
package org.apache.beam.runners.gearpump.translators.utils;
import java.io.Serializable;
+import java.util.Collection;
import java.util.List;
import org.apache.beam.runners.core.AggregatorFactory;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners;
+import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
import org.apache.beam.runners.core.SimpleDoFnRunner;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.util.ExecutionContext;
-import org.apache.beam.sdk.util.SideInputReader;
+import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
/**
@@ -38,10 +41,10 @@ import org.apache.beam.sdk.values.TupleTag;
*/
public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
- private static final long serialVersionUID = 1083167395296383469L;
+ private static final long serialVersionUID = -4109539010014189725L;
private final DoFn<InputT, OutputT> fn;
private final transient PipelineOptions options;
- private final SideInputReader sideInputReader;
+ private final Collection<PCollectionView<?>> sideInputs;
private final DoFnRunners.OutputManager outputManager;
private final TupleTag<OutputT> mainOutputTag;
private final List<TupleTag<?>> sideOutputTags;
@@ -52,7 +55,7 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
public DoFnRunnerFactory(
GearpumpPipelineOptions pipelineOptions,
DoFn<InputT, OutputT> doFn,
- SideInputReader sideInputReader,
+ Collection<PCollectionView<?>> sideInputs,
DoFnRunners.OutputManager outputManager,
TupleTag<OutputT> mainOutputTag,
List<TupleTag<?>> sideOutputTags,
@@ -61,7 +64,7 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
WindowingStrategy<?, ?> windowingStrategy) {
this.fn = doFn;
this.options = pipelineOptions;
- this.sideInputReader = sideInputReader;
+ this.sideInputs = sideInputs;
this.outputManager = outputManager;
this.mainOutputTag = mainOutputTag;
this.sideOutputTags = sideOutputTags;
@@ -70,9 +73,12 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
this.windowingStrategy = windowingStrategy;
}
- public DoFnRunner<InputT, OutputT> createRunner() {
- return DoFnRunners.createDefault(options, fn, sideInputReader, outputManager, mainOutputTag,
+ public PushbackSideInputDoFnRunner<InputT, OutputT> createRunner(
+ ReadyCheckingSideInputReader sideInputReader) {
+ DoFnRunner<InputT, OutputT> underlying = DoFnRunners.createDefault(
+ options, fn, sideInputReader, outputManager, mainOutputTag,
sideOutputTags, stepContext, aggregatorFactory, windowingStrategy);
+ return PushbackSideInputDoFnRunner.create(underlying, sideInputs, sideInputReader);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpSideInputReader.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpSideInputReader.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpSideInputReader.java
deleted file mode 100644
index d1a9198..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpSideInputReader.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump.translators.utils;
-
-import java.io.Serializable;
-
-import javax.annotation.Nullable;
-
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.values.PCollectionView;
-
-/**
- * no-op side input reader.
- */
-public class NoOpSideInputReader implements SideInputReader, Serializable {
- @Nullable
- @Override
- public <T> T get(PCollectionView<T> view, BoundedWindow window) {
- return null;
- }
-
- @Override
- public <T> boolean contains(PCollectionView<T> view) {
- return false;
- }
-
- @Override
- public boolean isEmpty() {
- return false;
- }
-}
[47/50] [abbrv] beam git commit: Override maven-enforcer-plugin for
precommit tests
Posted by ke...@apache.org.
Override maven-enforcer-plugin for precommit tests
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/18daed90
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/18daed90
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/18daed90
Branch: refs/heads/master
Commit: 18daed908d47aa2605507e7c91f42695dd8fae9f
Parents: a282165
Author: manuzhang <ow...@gmail.com>
Authored: Thu Jul 27 08:54:18 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Thu Jul 27 08:54:18 2017 +0800
----------------------------------------------------------------------
examples/java/pom.xml | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/18daed90/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index bd18889..6a1ce99 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -189,6 +189,28 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <enforceBytecodeVersion>
+ <maxJdkVersion>1.8</maxJdkVersion>
+ </enforceBytecodeVersion>
+ <requireJavaVersion>
+ <version>[1.8,)</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<useManifestOnlyJar>false</useManifestOnlyJar>
[23/50] [abbrv] beam git commit: This closes #3172: Sync
gearpump-runner with master
Posted by ke...@apache.org.
This closes #3172: Sync gearpump-runner with master
Update gearpump-runner against master changes
Use stable naming strategy for ByteBuddy invokers
Translate a Pipeline in SdkComponents
[TRIVIAL] runners-core: delete placeholder
Fixes an accidentally found bug in SimpleDoFnRunner
Removes OldDoFn and its kin from runners-core
Bump Dataflow containers to 0512
Improve Pruning performed by the DirectRunnerApiSurfaceTest
Adding support for subnetwork in Python Pipelineoptions
Use built-in cmp python function in comparing datastore paths
ApexRunner SDF support
Fix documentation for the shard_template_name
[BEAM-2299] Run maven install on Windows machine for build/test coverage on Windows
Remove "Dataflow" from apache_beam __init__.py file
Moving the data file for trigger tests to testing/data
Fix GcsResourceIdTest in postcommits
readAvros should't have proto Message upper bound
Reduce Log Level of PubsubUnboundedSource
[BEAM-2290] Fix issue where timestamps weren't set when using CompressedSource
[BEAM-2279] Fix archetype breakages
internal comments
Fix shading of guava testlib
Rename FileSystems.setDefaultConfigInWorkers
[BEAM-2277] HadoopFileSystem: normalize implementation
Mark FileSystem and related as Experimental
[BEAM-2277] Add ResourceIdTester and test existing ResourceId implementations
Remove '/' entirely from determining FileSystem scheme
[BEAM-2279] Add HDFS support to Spark runner profiles in archetypes and examples
[BEAM-2277] Fix URI_SCHEME_PATTERN in FileSystems
BigtableIO should use AutoValue for read and write
[BEAM-2153] Move connection management in JmsIO.write() to setup/teardown methods
Mark More values methods Internal
Rename filesink to filebasedsink
Enable SerializableCoder to Serialize with Generic Types
Remove unused test data
Fix due to GBKO name change.
Don't deploy jdk1.8-tests module
Remove some internal details from the public API.
Move assert_that, equal_to, is_empty to apache_beam.testing.util
[BEAM-1345] Clearly delineate public api in apache_beam/typehints.
[BEAM-1345] Mark apache_beam/internal as internal.
[BEAM-1345] Annotate public members of pvalue.
Add internal comments to metrics
[BEAM-1340] Add __all__ tags to modules in package apache_beam/transforms
[BEAM-2256] Add the last previous range filter
Use a consistent calculation for GC Time
fix lint error in fake_datastore.py
Add __all__ tags to modules in package apache_beam/testing
[BEAM-1340] Adds __all__ tags to classes in package apache_beam/io.
[BEAM-1345] Clearly delineate public api in apache_beam/coders.
[BEAM-1345] Clearly delineate public api in runners package.
[BEAM-1345] Mark Pipeline as public.
[BEAM-1345] Clearly delineate public API in apache_beam/options
Mark internal modules in python datastoreio
[BEAM-2260] Improve construction-time errors for Text and AvroIO
[BEAM-2179] Archetype generate-sources.sh cleanup the existing sources before rsync
[BEAM-1345] Mark windowed value as experimental
Add internal usage only comments to util/
Remove protobuf and http-client dependency from runners/google-cloud-dataflow
minor typo fix in comment
Add support for local execution to PubsubIO using the google cloud emulator
[BEAM-2150] Relax regex to support wildcard globbing for GCS
bump time of precommits
[BEAM-2244] Move details of Metrics to Runners Core
Correct javadoc for mobile gaming examples
Update SDK Coders to return the Empty List from getCoderArguments
Skip generating empty jars for parent poms
Fix a typo in TestDataflowRunnerTest
Re-enable UsesTimersInParDo tests in Dataflow runner
TestDataflowRunner: throw AssertionError only when assertion known failed
Allow any throwable in PAssert to constitute adequate failure
[BEAM-2242] Ensure that jars are shaded correctly by running the jar plugin before the shade plugin
[BEAM-2240] Always augment exception with step name.
Adds dependency on findbugs to examples/java
Splits WriteBundles into windowed/unwindowed versions
Simpler code for setting shard numbers on results in FileBasedSink
Implement dynamic-sharding for windowed file outputs, and add an integration test.
Renames FileBasedSink inner classes
[BEAM-2250] remove experimental and internal things from pydoc
[BEAM-2249] Correctly handle partial reads in AvroSource
Use text output for first two mobile gaming examples
Remove verifyDeterministic from StructuredCoder
Update Coder Documentation
Improve DirectRunner Javadoc
[BEAM-2211] Delete deprecated NoopPathValidator
Remove Timer.cancel() from user-facing API
Remove Readme files.
Renames some python classes and functions that were unnecessarily public.
Mark PipelineVisitor and AppliedPTransform as internal.
Mark PValue and PValueBase Internal
[BEAM-2236] Move test utilities out of python core
Include 'sun.reflect' in GcpCoreApiSurfaceTest
Fix checkstyle error
Shade dependencies in sdks/core
Remove trailing whitespace
Add per-runner profile to Java 8 examples
Register TestSparkPipelineOptions only in src/test to avoid hard hamcrest dep
Update Apache Beam Python version to 2.1.0.dev
Shade JSR305 in the DirectRunner
Remove hadoop io readme
...
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/99221e73
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/99221e73
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/99221e73
Branch: refs/heads/master
Commit: 99221e73991002b8ed95cd5649fadee1d14bba4d
Parents: 58546ac 6c06967
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu May 18 13:07:02 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu May 18 13:07:02 2017 -0700
----------------------------------------------------------------------
.gitignore | 1 +
.../jenkins/common_job_properties.groovy | 29 +-
.../job_beam_PerformanceTests_Dataflow.groovy | 3 +
...job_beam_PostCommit_Java_MavenInstall.groovy | 8 +-
..._PostCommit_Java_MavenInstall_Windows.groovy | 45 ++
...tCommit_Java_ValidatesRunner_Dataflow.groovy | 2 +-
.../job_beam_PreCommit_Java_MavenInstall.groovy | 5 +-
.../job_beam_Release_NightlySnapshot.groovy | 2 +-
README.md | 4 +-
examples/java/README.md | 64 +-
examples/java/pom.xml | 53 +-
.../apache/beam/examples/WindowedWordCount.java | 9 +-
.../beam/examples/common/ExampleUtils.java | 6 +-
.../examples/common/WriteOneFilePerWindow.java | 72 +-
.../org/apache/beam/examples/complete/README.md | 2 +-
.../apache/beam/examples/complete/TfIdf.java | 2 +-
.../org/apache/beam/examples/cookbook/README.md | 2 +-
.../beam/examples/WindowedWordCountIT.java | 26 +-
.../beam/examples/complete/TfIdfTest.java | 2 +-
examples/java8/pom.xml | 124 ++-
.../beam/examples/complete/game/GameStats.java | 15 +-
.../examples/complete/game/HourlyTeamScore.java | 58 +-
.../examples/complete/game/LeaderBoard.java | 62 +-
.../beam/examples/complete/game/README.md | 131 ---
.../beam/examples/complete/game/UserScore.java | 74 +-
.../complete/game/utils/WriteToBigQuery.java | 32 +-
.../complete/game/utils/WriteToText.java | 184 +++++
.../game/utils/WriteWindowedToBigQuery.java | 9 +-
examples/pom.xml | 2 +-
pom.xml | 71 +-
runners/apex/README.md | 76 --
runners/apex/pom.xml | 10 +-
.../beam/runners/apex/ApexPipelineOptions.java | 5 +
.../apache/beam/runners/apex/ApexRunner.java | 51 +-
.../beam/runners/apex/ApexRunnerRegistrar.java | 3 +-
.../beam/runners/apex/ApexRunnerResult.java | 2 -
.../beam/runners/apex/ApexYarnLauncher.java | 2 -
.../beam/runners/apex/TestApexRunner.java | 10 +-
.../translation/ApexPipelineTranslator.java | 23 +
.../apex/translation/ParDoTranslator.java | 68 +-
.../translation/ReadUnboundedTranslator.java | 1 -
.../apex/translation/TransformTranslator.java | 2 -
.../apex/translation/TranslationContext.java | 97 ++-
.../operators/ApexGroupByKeyOperator.java | 10 +-
.../operators/ApexParDoOperator.java | 131 ++-
.../operators/ApexProcessFnOperator.java | 8 +-
.../ApexReadUnboundedInputOperator.java | 2 -
.../operators/ApexTimerInternals.java | 23 +-
.../translation/utils/ApexStateInternals.java | 24 +-
.../apex/translation/utils/ApexStreamTuple.java | 16 +-
.../utils/CoderAdapterStreamCodec.java | 2 -
.../utils/SerializablePipelineOptions.java | 19 +-
.../translation/utils/StateInternalsProxy.java | 11 +-
.../utils/ValueAndCoderKryoSerializable.java | 2 -
.../apex/translation/utils/ValuesSource.java | 2 -
.../beam/runners/apex/ApexRunnerTest.java | 49 +-
.../beam/runners/apex/ApexYarnLauncherTest.java | 2 -
.../apex/examples/UnboundedTextSource.java | 2 -
.../runners/apex/examples/WordCountTest.java | 2 -
.../translation/ApexGroupByKeyOperatorTest.java | 4 +-
.../FlattenPCollectionTranslatorTest.java | 13 +-
.../apex/translation/ParDoTranslatorTest.java | 2 +-
.../operators/ApexTimerInternalsTest.java | 10 +-
.../utils/ApexStateInternalsTest.java | 12 +-
.../translation/utils/CollectionSource.java | 2 -
.../translation/utils/PipelineOptionsTest.java | 98 ++-
.../test/resources/beam-runners-apex.properties | 2 +-
runners/core-construction-java/pom.xml | 2 +-
.../DeduplicatedFlattenFactory.java | 2 +-
.../EmptyFlattenAsCreateFactory.java | 2 +-
.../core/construction/ForwardingPTransform.java | 6 +-
.../runners/core/construction/PCollections.java | 2 +-
.../core/construction/PTransformMatchers.java | 2 +-
.../construction/PTransformReplacements.java | 2 +-
.../runners/core/construction/PTransforms.java | 19 +-
.../core/construction/PrimitiveCreate.java | 4 +-
.../core/construction/SdkComponents.java | 54 +-
.../runners/core/construction/Triggers.java | 4 +-
.../UnboundedReadFromBoundedSource.java | 16 +-
.../UnsupportedOverrideFactory.java | 2 +-
.../core/construction/WindowingStrategies.java | 4 +-
.../runners/core/metrics/MetricFiltering.java | 102 +++
.../beam/runners/core/metrics/MetricKey.java | 43 +
.../beam/runners/core/metrics/package-info.java | 22 +
.../runners/core/construction/CodersTest.java | 18 +-
.../EmptyFlattenAsCreateFactoryTest.java | 2 +-
.../core/construction/PCollectionsTest.java | 17 +-
.../construction/PTransformMatchersTest.java | 34 +-
.../PTransformReplacementsTest.java | 2 +-
.../core/construction/PTransformsTest.java | 2 +-
.../construction/ReplacementOutputsTest.java | 2 +-
.../core/construction/SdkComponentsTest.java | 106 ++-
.../SingleInputOutputOverrideFactoryTest.java | 2 +-
.../construction/WindowingStrategiesTest.java | 4 +-
.../core/metrics/MetricFilteringTest.java | 148 ++++
runners/core-java/pom.xml | 2 +-
.../beam/runners/core/AssignWindowsDoFn.java | 78 --
.../beam/runners/core/BaseExecutionContext.java | 13 +-
.../apache/beam/runners/core/DoFnAdapters.java | 310 -------
.../apache/beam/runners/core/DoFnRunner.java | 2 +-
.../apache/beam/runners/core/DoFnRunners.java | 52 +-
.../core/ElementAndRestrictionCoder.java | 29 +-
.../beam/runners/core/ExecutionContext.java | 10 +-
.../runners/core/GlobalCombineFnRunner.java | 1 -
.../runners/core/GlobalCombineFnRunners.java | 41 +-
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 113 ---
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 94 ---
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 9 +-
.../core/GroupAlsoByWindowsAggregators.java | 26 +
.../runners/core/GroupAlsoByWindowsDoFn.java | 39 -
.../core/GroupByKeyViaGroupByKeyOnly.java | 2 +-
.../runners/core/InMemoryStateInternals.java | 22 +-
.../runners/core/InMemoryTimerInternals.java | 2 +-
.../beam/runners/core/KeyedWorkItemCoder.java | 22 +-
.../core/LateDataDroppingDoFnRunner.java | 9 +-
.../apache/beam/runners/core/LateDataUtils.java | 37 +-
.../runners/core/MergingActiveWindowSet.java | 2 +-
.../beam/runners/core/MergingStateAccessor.java | 2 +-
.../apache/beam/runners/core/NonEmptyPanes.java | 8 +-
.../beam/runners/core/NullSideInputReader.java | 61 ++
.../org/apache/beam/runners/core/OldDoFn.java | 335 --------
...eBoundedSplittableProcessElementInvoker.java | 18 +-
.../beam/runners/core/PaneInfoTracker.java | 4 +-
.../beam/runners/core/ProcessFnRunner.java | 3 +-
.../core/PushbackSideInputDoFnRunner.java | 2 +-
.../core/ReadyCheckingSideInputReader.java | 34 +
.../org/apache/beam/runners/core/ReduceFn.java | 6 +-
.../runners/core/ReduceFnContextFactory.java | 13 +-
.../beam/runners/core/ReduceFnRunner.java | 50 +-
.../beam/runners/core/SideInputHandler.java | 15 +-
.../beam/runners/core/SideInputReader.java | 47 ++
.../beam/runners/core/SimpleDoFnRunner.java | 223 ++++-
.../beam/runners/core/SimpleOldDoFnRunner.java | 500 ------------
.../core/SimplePushbackSideInputDoFnRunner.java | 3 +-
.../beam/runners/core/SplittableParDo.java | 50 +-
.../apache/beam/runners/core/StateAccessor.java | 2 +-
.../beam/runners/core/StateInternals.java | 4 +-
.../apache/beam/runners/core/StateMerging.java | 14 +-
.../apache/beam/runners/core/StateTable.java | 4 +-
.../org/apache/beam/runners/core/StateTag.java | 16 +-
.../org/apache/beam/runners/core/StateTags.java | 20 +-
.../beam/runners/core/StatefulDoFnRunner.java | 14 +-
.../beam/runners/core/SystemReduceFn.java | 8 +-
.../core/TestInMemoryStateInternals.java | 4 +-
.../beam/runners/core/TimerInternals.java | 30 +-
.../core/UnsupportedSideInputReader.java | 1 -
.../apache/beam/runners/core/WatermarkHold.java | 15 +-
.../core/WindowingInternalsAdapters.java | 75 --
.../beam/runners/core/metrics/CounterCell.java | 86 ++
.../beam/runners/core/metrics/DirtyState.java | 99 +++
.../runners/core/metrics/DistributionCell.java | 80 ++
.../runners/core/metrics/DistributionData.java | 62 ++
.../beam/runners/core/metrics/GaugeCell.java | 78 ++
.../beam/runners/core/metrics/GaugeData.java | 83 ++
.../beam/runners/core/metrics/MetricCell.java | 42 +
.../runners/core/metrics/MetricUpdates.java | 79 ++
.../core/metrics/MetricsContainerImpl.java | 188 +++++
.../core/metrics/MetricsContainerStepMap.java | 495 ++++++++++++
.../beam/runners/core/metrics/MetricsMap.java | 88 ++
.../beam/runners/core/metrics/package-info.java | 22 +
.../AfterDelayFromFirstElementStateMachine.java | 6 +-
.../core/triggers/AfterPaneStateMachine.java | 2 +-
.../AfterProcessingTimeStateMachine.java | 2 +-
...rSynchronizedProcessingTimeStateMachine.java | 2 +-
.../triggers/AfterWatermarkStateMachine.java | 2 +-
.../triggers/DefaultTriggerStateMachine.java | 2 +-
.../triggers/ReshuffleTriggerStateMachine.java | 2 +-
.../core/triggers/TriggerStateMachine.java | 2 +-
.../TriggerStateMachineContextFactory.java | 6 +-
.../triggers/TriggerStateMachineRunner.java | 4 +-
runners/core-java/src/test/java/.placeholder | 0
...roupAlsoByWindowViaOutputBufferDoFnTest.java | 109 ---
.../core/GroupAlsoByWindowsProperties.java | 744 -----------------
.../core/InMemoryStateInternalsTest.java | 16 +-
.../core/InMemoryTimerInternalsTest.java | 2 +-
.../runners/core/KeyedWorkItemCoderTest.java | 2 +-
.../core/LateDataDroppingDoFnRunnerTest.java | 11 +-
.../beam/runners/core/LateDataUtilsTest.java | 90 +++
.../apache/beam/runners/core/NoOpOldDoFn.java | 65 --
.../apache/beam/runners/core/OldDoFnTest.java | 51 --
...ndedSplittableProcessElementInvokerTest.java | 1 -
.../beam/runners/core/ReduceFnRunnerTest.java | 41 +-
.../beam/runners/core/ReduceFnTester.java | 8 +-
.../beam/runners/core/SideInputHandlerTest.java | 2 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 17 +-
.../runners/core/SimpleOldDoFnRunnerTest.java | 86 --
.../SimplePushbackSideInputDoFnRunnerTest.java | 3 +-
.../beam/runners/core/SplittableParDoTest.java | 7 +-
.../runners/core/StatefulDoFnRunnerTest.java | 19 +-
.../beam/runners/core/TimerInternalsTest.java | 2 +-
.../runners/core/metrics/CounterCellTest.java | 56 ++
.../runners/core/metrics/DirtyStateTest.java | 56 ++
.../core/metrics/DistributionCellTest.java | 54 ++
.../runners/core/metrics/GaugeCellTest.java | 51 ++
.../core/metrics/MetricUpdateMatchers.java | 82 ++
.../core/metrics/MetricsContainerImplTest.java | 130 +++
.../metrics/MetricsContainerStepMapTest.java | 272 +++++++
.../runners/core/metrics/MetricsMapTest.java | 103 +++
.../triggers/TriggerStateMachineTester.java | 6 +-
.../core/triggers/TriggerStateMachinesTest.java | 2 +-
runners/direct-java/pom.xml | 100 ++-
.../direct/AbstractModelEnforcement.java | 1 -
.../direct/BoundedReadEvaluatorFactory.java | 17 +-
.../beam/runners/direct/BundleFactory.java | 3 +-
.../org/apache/beam/runners/direct/Clock.java | 2 +-
.../runners/direct/CloningBundleFactory.java | 2 -
.../beam/runners/direct/CommittedBundle.java | 82 ++
.../beam/runners/direct/CommittedResult.java | 3 +-
.../beam/runners/direct/CompletionCallback.java | 9 +-
.../CopyOnAccessInMemoryStateInternals.java | 20 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 2 +-
.../apache/beam/runners/direct/DirectGraph.java | 14 +-
.../beam/runners/direct/DirectGraphVisitor.java | 2 +-
.../beam/runners/direct/DirectGroupByKey.java | 2 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 2 +-
.../beam/runners/direct/DirectMetrics.java | 25 +-
.../beam/runners/direct/DirectOptions.java | 2 +-
.../beam/runners/direct/DirectRegistrar.java | 12 +-
.../beam/runners/direct/DirectRunner.java | 158 +---
.../runners/direct/DirectTimerInternals.java | 2 +-
.../runners/direct/DisplayDataValidator.java | 6 +-
.../runners/direct/DoFnLifecycleManager.java | 2 +-
.../beam/runners/direct/EmptyInputProvider.java | 3 +-
.../runners/direct/EmptyTransformEvaluator.java | 50 --
.../beam/runners/direct/EvaluationContext.java | 14 +-
.../runners/direct/ExecutorServiceFactory.java | 2 +-
.../direct/ExecutorServiceParallelExecutor.java | 42 +-
.../runners/direct/FlattenEvaluatorFactory.java | 4 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 18 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 4 +-
.../ImmutabilityCheckingBundleFactory.java | 2 -
.../direct/ImmutabilityEnforcementFactory.java | 3 +-
.../direct/ImmutableListBundleFactory.java | 2 -
.../beam/runners/direct/ModelEnforcement.java | 12 +-
.../runners/direct/ModelEnforcementFactory.java | 5 +-
.../beam/runners/direct/NanosOffsetClock.java | 2 +-
.../runners/direct/PCollectionViewWindow.java | 67 ++
.../runners/direct/PCollectionViewWriter.java | 34 +
.../beam/runners/direct/ParDoEvaluator.java | 7 +-
.../runners/direct/ParDoEvaluatorFactory.java | 3 +-
.../direct/ParDoMultiOverrideFactory.java | 7 +-
.../direct/PassthroughTransformEvaluator.java | 3 +-
.../beam/runners/direct/PipelineExecutor.java | 3 +-
.../beam/runners/direct/RootInputProvider.java | 3 +-
.../runners/direct/RootProviderRegistry.java | 3 +-
.../beam/runners/direct/SideInputContainer.java | 7 +-
...littableProcessElementsEvaluatorFactory.java | 18 +-
.../direct/StatefulParDoEvaluatorFactory.java | 7 +-
.../apache/beam/runners/direct/StepAndKey.java | 2 +-
.../runners/direct/StepTransformResult.java | 7 +-
.../direct/TestStreamEvaluatorFactory.java | 6 +-
.../beam/runners/direct/TransformEvaluator.java | 3 +-
.../direct/TransformEvaluatorFactory.java | 8 +-
.../direct/TransformEvaluatorRegistry.java | 3 +-
.../beam/runners/direct/TransformExecutor.java | 25 +-
.../beam/runners/direct/TransformResult.java | 7 +-
.../direct/UnboundedReadEvaluatorFactory.java | 4 +-
.../beam/runners/direct/UncommittedBundle.java | 57 ++
.../runners/direct/ViewEvaluatorFactory.java | 5 +-
.../runners/direct/ViewOverrideFactory.java | 2 +-
.../direct/WatermarkCallbackExecutor.java | 4 +-
.../beam/runners/direct/WatermarkManager.java | 7 +-
.../runners/direct/WindowEvaluatorFactory.java | 4 +-
.../direct/WriteWithShardingFactory.java | 2 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 4 +-
.../direct/CloningBundleFactoryTest.java | 32 +-
.../runners/direct/CommittedResultTest.java | 16 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 16 +-
.../runners/direct/DirectGraphVisitorTest.java | 2 +-
.../beam/runners/direct/DirectGraphs.java | 2 +-
.../DirectGroupByKeyOverrideFactoryTest.java | 2 +-
.../beam/runners/direct/DirectMetricsTest.java | 15 +-
.../direct/DirectRunnerApiSurfaceTest.java | 68 ++
.../beam/runners/direct/DirectRunnerTest.java | 11 +-
.../direct/DirectTimerInternalsTest.java | 2 +-
...leManagerRemovingTransformEvaluatorTest.java | 2 +-
.../runners/direct/EvaluationContextTest.java | 13 +-
.../direct/FlattenEvaluatorFactoryTest.java | 4 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 2 -
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 2 -
.../ImmutabilityCheckingBundleFactoryTest.java | 2 -
.../ImmutabilityEnforcementFactoryTest.java | 3 +-
.../direct/ImmutableListBundleFactoryTest.java | 2 -
.../direct/KeyedPValueTrackingVisitorTest.java | 2 +-
.../apache/beam/runners/direct/MockClock.java | 2 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 5 +-
.../runners/direct/SideInputContainerTest.java | 8 +-
.../StatefulParDoEvaluatorFactoryTest.java | 14 +-
.../runners/direct/StepTransformResultTest.java | 3 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 3 +-
.../runners/direct/TransformExecutorTest.java | 8 +-
.../UnboundedReadEvaluatorFactoryTest.java | 13 +-
.../direct/ViewEvaluatorFactoryTest.java | 6 +-
.../runners/direct/ViewOverrideFactoryTest.java | 6 +-
.../direct/WatermarkCallbackExecutorTest.java | 4 +-
.../runners/direct/WatermarkManagerTest.java | 6 +-
.../direct/WindowEvaluatorFactoryTest.java | 2 -
.../direct/WriteWithShardingFactoryTest.java | 61 +-
runners/flink/pom.xml | 7 +-
.../flink/FlinkBatchTransformTranslators.java | 4 +-
.../flink/FlinkBatchTranslationContext.java | 4 +-
.../flink/FlinkDetachedRunnerResult.java | 1 -
.../apache/beam/runners/flink/FlinkRunner.java | 5 +-
.../runners/flink/FlinkRunnerRegistrar.java | 2 +-
.../beam/runners/flink/FlinkRunnerResult.java | 8 +-
.../flink/FlinkStreamingPipelineTranslator.java | 2 +-
.../FlinkStreamingTransformTranslators.java | 84 +-
.../flink/FlinkStreamingTranslationContext.java | 2 +-
.../flink/FlinkStreamingViewOverrides.java | 2 +-
.../beam/runners/flink/TestFlinkRunner.java | 2 +-
.../metrics/DoFnRunnerWithMetricsUpdate.java | 17 +-
.../flink/metrics/FlinkMetricContainer.java | 276 ++-----
.../flink/metrics/FlinkMetricResults.java | 146 ----
.../flink/metrics/MetricsAccumulator.java | 60 ++
.../flink/metrics/ReaderInvocationUtil.java | 12 +-
.../functions/AbstractFlinkCombineRunner.java | 4 +-
.../functions/FlinkDoFnFunction.java | 3 +-
.../FlinkMergingNonShuffleReduceFunction.java | 2 +-
.../functions/FlinkPartialReduceFunction.java | 2 +-
.../functions/FlinkReduceFunction.java | 2 +-
.../functions/FlinkSideInputReader.java | 4 +-
.../functions/FlinkStatefulDoFnFunction.java | 3 +-
.../functions/HashingFlinkCombineRunner.java | 4 +-
.../functions/SortingFlinkCombineRunner.java | 4 +-
.../translation/types/CoderTypeSerializer.java | 4 +-
.../types/EncodedValueSerializer.java | 2 -
.../utils/SerializedPipelineOptions.java | 20 +-
.../translation/wrappers/SourceInputFormat.java | 8 +-
.../wrappers/streaming/DoFnOperator.java | 58 +-
.../streaming/SingletonKeyedWorkItemCoder.java | 20 +-
.../streaming/SplittableDoFnOperator.java | 30 +-
.../wrappers/streaming/WindowDoFnOperator.java | 2 +-
.../streaming/io/BoundedSourceWrapper.java | 8 +-
.../wrappers/streaming/io/DedupingOperator.java | 187 +++++
.../streaming/io/UnboundedSourceWrapper.java | 24 +-
.../state/FlinkBroadcastStateInternals.java | 20 +-
.../state/FlinkKeyGroupStateInternals.java | 29 +-
.../state/FlinkSplitStateInternals.java | 20 +-
.../streaming/state/FlinkStateInternals.java | 22 +-
.../beam/runners/flink/PipelineOptionsTest.java | 89 +-
.../flink/streaming/DedupingOperatorTest.java | 131 +++
.../flink/streaming/DoFnOperatorTest.java | 16 +-
.../FlinkBroadcastStateInternalsTest.java | 10 +-
.../FlinkKeyGroupStateInternalsTest.java | 4 +-
.../streaming/FlinkSplitStateInternalsTest.java | 4 +-
.../streaming/FlinkStateInternalsTest.java | 12 +-
.../streaming/UnboundedSourceWrapperTest.java | 29 +-
runners/gearpump/pom.xml | 2 +-
.../gearpump/GearpumpPipelineTranslator.java | 4 +-
.../beam/runners/gearpump/GearpumpRunner.java | 2 +-
.../gearpump/GearpumpRunnerRegistrar.java | 2 +-
.../runners/gearpump/TestGearpumpRunner.java | 2 +-
.../ParDoSingleOutputTranslator.java | 2 +-
.../translators/TranslationContext.java | 2 +-
.../translators/WindowAssignTranslator.java | 2 +-
.../translators/functions/DoFnFunction.java | 2 +-
.../translators/utils/DoFnRunnerFactory.java | 4 +-
runners/google-cloud-dataflow-java/pom.xml | 76 +-
.../beam/runners/dataflow/AssignWindows.java | 2 +-
.../dataflow/BatchStatefulParDoOverrides.java | 4 +-
.../runners/dataflow/BatchViewOverrides.java | 32 +-
.../beam/runners/dataflow/DataflowMetrics.java | 4 +-
.../runners/dataflow/DataflowPipelineJob.java | 20 +-
.../dataflow/DataflowPipelineRegistrar.java | 4 +-
.../dataflow/DataflowPipelineTranslator.java | 40 +-
.../beam/runners/dataflow/DataflowRunner.java | 110 +--
.../dataflow/PrimitiveParDoSingleFactory.java | 2 +-
.../beam/runners/dataflow/ReadTranslator.java | 22 +-
.../dataflow/ReshuffleOverrideFactory.java | 8 +-
.../dataflow/StreamingViewOverrides.java | 2 +-
.../dataflow/TestDataflowPipelineOptions.java | 28 +
.../runners/dataflow/TestDataflowRunner.java | 388 +++++++++
.../runners/dataflow/TransformTranslator.java | 2 +-
.../dataflow/internal/CustomSources.java | 18 +-
.../runners/dataflow/internal/IsmFormat.java | 99 ++-
.../options/DataflowPipelineOptions.java | 18 +-
.../testing/TestDataflowPipelineOptions.java | 28 -
.../dataflow/testing/TestDataflowRunner.java | 326 --------
.../runners/dataflow/testing/package-info.java | 24 -
.../util/AvroCoderCloudObjectTranslator.java | 2 -
.../beam/runners/dataflow/util/CloudObject.java | 1 -
.../runners/dataflow/util/CloudObjectKinds.java | 2 -
.../dataflow/util/CloudObjectTranslator.java | 2 -
.../dataflow/util/CloudObjectTranslators.java | 23 +-
.../runners/dataflow/util/CloudObjects.java | 1 -
.../CoderCloudObjectTranslatorRegistrar.java | 1 -
.../dataflow/util/DataflowTemplateJob.java | 1 -
...aultCoderCloudObjectTranslatorRegistrar.java | 2 -
.../beam/runners/dataflow/util/DoFnInfo.java | 2 +-
.../beam/runners/dataflow/util/GcsStager.java | 26 +-
.../beam/runners/dataflow/util/PackageUtil.java | 41 +-
.../runners/dataflow/util/PropertyNames.java | 112 +++
.../runners/dataflow/util/RandomAccessData.java | 26 +-
.../SerializableCoderCloudObjectTranslator.java | 2 -
.../beam/runners/dataflow/util/Serializer.java | 262 ------
.../beam/runners/dataflow/util/Structs.java | 372 +++++++++
.../BatchStatefulParDoOverridesTest.java | 6 +-
.../runners/dataflow/DataflowMetricsTest.java | 7 +-
.../dataflow/DataflowPipelineJobTest.java | 18 +-
.../dataflow/DataflowPipelineRegistrarTest.java | 2 +-
.../DataflowPipelineTranslatorTest.java | 133 ++-
.../runners/dataflow/DataflowRunnerTest.java | 90 +--
.../PrimitiveParDoSingleFactoryTest.java | 2 +-
.../dataflow/TestDataflowRunnerTest.java | 634 +++++++++++++++
.../options/DataflowPipelineOptionsTest.java | 21 +-
.../options/DataflowProfilingOptionsTest.java | 4 +-
.../DataflowWorkerLoggingOptionsTest.java | 4 +-
.../testing/TestDataflowRunnerTest.java | 661 ---------------
.../transforms/DataflowGroupByKeyTest.java | 4 +-
.../dataflow/transforms/DataflowViewTest.java | 4 +-
.../runners/dataflow/util/CloudObjectsTest.java | 9 +-
.../runners/dataflow/util/PackageUtilTest.java | 145 ++--
.../dataflow/util/RandomAccessDataTest.java | 6 +-
.../beam/runners/dataflow/util/StructsTest.java | 206 +++++
runners/pom.xml | 2 +-
runners/spark/README.md | 124 ---
runners/spark/pom.xml | 54 +-
.../beam/runners/spark/SparkPipelineResult.java | 8 +-
.../apache/beam/runners/spark/SparkRunner.java | 8 +-
.../beam/runners/spark/SparkRunnerDebugger.java | 7 +-
.../runners/spark/SparkRunnerRegistrar.java | 2 +-
.../beam/runners/spark/TestSparkRunner.java | 83 +-
.../spark/aggregators/NamedAggregators.java | 6 +-
.../beam/runners/spark/coders/CoderHelpers.java | 1 -
.../spark/coders/StatelessJavaSerializer.java | 1 -
.../beam/runners/spark/io/CreateStream.java | 2 +-
.../beam/runners/spark/io/MicrobatchSource.java | 2 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 177 ++--
.../runners/spark/io/SparkUnboundedSource.java | 20 +-
.../runners/spark/metrics/AggregatorMetric.java | 1 -
.../spark/metrics/MetricsAccumulator.java | 20 +-
.../spark/metrics/MetricsAccumulatorParam.java | 20 +-
.../runners/spark/metrics/SparkBeamMetric.java | 11 +-
.../spark/metrics/SparkBeamMetricSource.java | 3 +-
.../spark/metrics/SparkMetricResults.java | 172 ----
.../spark/metrics/SparkMetricsContainer.java | 174 ----
.../SparkGroupAlsoByWindowViaWindowSet.java | 17 +-
.../spark/stateful/SparkStateInternals.java | 20 +-
.../spark/stateful/SparkTimerInternals.java | 2 +-
.../spark/stateful/StateSpecFunctions.java | 9 +-
.../translation/DoFnRunnerWithMetrics.java | 11 +-
.../spark/translation/EvaluationContext.java | 18 +-
.../translation/GroupCombineFunctions.java | 5 +-
.../spark/translation/MultiDoFnFunction.java | 8 +-
.../translation/SparkAbstractCombineFn.java | 4 +-
.../spark/translation/SparkAssignWindowFn.java | 4 +-
.../spark/translation/SparkGlobalCombineFn.java | 2 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 6 +-
.../spark/translation/SparkKeyedCombineFn.java | 2 +-
.../spark/translation/SparkRuntimeContext.java | 26 +-
.../translation/StorageLevelPTransform.java | 2 +-
.../spark/translation/TransformTranslator.java | 8 +-
.../spark/translation/TranslationUtils.java | 3 +-
.../SparkRunnerStreamingContextFactory.java | 2 +-
.../streaming/StreamingTransformTranslator.java | 8 +-
.../spark/util/GlobalWatermarkHolder.java | 1 -
.../util/SinglePrimitiveOutputPTransform.java | 2 +-
.../spark/util/SparkSideInputReader.java | 4 +-
.../apache/beam/runners/spark/CacheTest.java | 14 +-
.../beam/runners/spark/ForceStreamingTest.java | 18 +-
.../apache/beam/runners/spark/PipelineRule.java | 109 ---
.../runners/spark/ProvidedSparkContextTest.java | 10 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 15 +-
.../beam/runners/spark/StreamingTest.java | 23 +
.../TestSparkPipelineOptionsRegistrar.java | 36 +
.../metrics/sink/SparkMetricsSinkTest.java | 12 +-
.../beam/runners/spark/io/AvroPipelineTest.java | 10 +-
.../beam/runners/spark/io/NumShardsTest.java | 6 +-
.../spark/io/ReaderToIteratorAdapterTest.java | 145 ++++
.../translation/SparkRuntimeContextTest.java | 122 +++
.../spark/translation/StorageLevelTest.java | 31 +-
.../translation/streaming/CreateStreamTest.java | 53 +-
.../ResumeFromCheckpointStreamingTest.java | 64 +-
.../streaming/StreamingSourceMetricsTest.java | 16 +-
.../streaming/TrackStreamingSourcesTest.java | 4 +-
sdks/common/fn-api/pom.xml | 2 +-
sdks/common/pom.xml | 2 +-
sdks/common/runner-api/pom.xml | 5 +-
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/findbugs-filter.xml | 12 +-
sdks/java/core/pom.xml | 93 ++-
.../main/java/org/apache/beam/sdk/Pipeline.java | 101 ++-
.../org/apache/beam/sdk/PipelineResult.java | 11 +-
.../org/apache/beam/sdk/PipelineRunner.java | 61 ++
.../beam/sdk/annotations/Experimental.java | 35 +-
.../apache/beam/sdk/annotations/Internal.java | 6 +-
.../org/apache/beam/sdk/coders/AtomicCoder.java | 85 ++
.../org/apache/beam/sdk/coders/AvroCoder.java | 70 +-
.../apache/beam/sdk/coders/BigDecimalCoder.java | 25 +-
.../beam/sdk/coders/BigEndianIntegerCoder.java | 10 +-
.../beam/sdk/coders/BigEndianLongCoder.java | 12 +-
.../apache/beam/sdk/coders/BigIntegerCoder.java | 21 +-
.../org/apache/beam/sdk/coders/BitSetCoder.java | 15 +-
.../apache/beam/sdk/coders/ByteArrayCoder.java | 41 +-
.../org/apache/beam/sdk/coders/ByteCoder.java | 10 +-
.../sdk/coders/CannotProvideCoderException.java | 2 +-
.../java/org/apache/beam/sdk/coders/Coder.java | 178 +++-
.../apache/beam/sdk/coders/CoderFactories.java | 290 -------
.../apache/beam/sdk/coders/CoderFactory.java | 44 -
.../apache/beam/sdk/coders/CoderProvider.java | 19 +-
.../beam/sdk/coders/CoderProviderRegistrar.java | 42 +
.../apache/beam/sdk/coders/CoderProviders.java | 240 +++---
.../apache/beam/sdk/coders/CoderRegistrar.java | 45 --
.../apache/beam/sdk/coders/CoderRegistry.java | 618 +++++---------
.../apache/beam/sdk/coders/CollectionCoder.java | 22 -
.../org/apache/beam/sdk/coders/CustomCoder.java | 26 +-
.../apache/beam/sdk/coders/DefaultCoder.java | 119 ++-
.../apache/beam/sdk/coders/DelegateCoder.java | 11 +
.../org/apache/beam/sdk/coders/DoubleCoder.java | 10 +-
.../apache/beam/sdk/coders/DurationCoder.java | 18 +-
.../apache/beam/sdk/coders/InstantCoder.java | 18 +-
.../apache/beam/sdk/coders/IterableCoder.java | 22 -
.../beam/sdk/coders/IterableLikeCoder.java | 35 +-
.../org/apache/beam/sdk/coders/KvCoder.java | 51 +-
.../beam/sdk/coders/LengthPrefixCoder.java | 28 +-
.../org/apache/beam/sdk/coders/ListCoder.java | 24 +-
.../org/apache/beam/sdk/coders/MapCoder.java | 47 +-
.../apache/beam/sdk/coders/NullableCoder.java | 31 +-
.../beam/sdk/coders/SerializableCoder.java | 66 +-
.../org/apache/beam/sdk/coders/SetCoder.java | 25 -
.../beam/sdk/coders/StringDelegateCoder.java | 12 +-
.../apache/beam/sdk/coders/StringUtf8Coder.java | 29 +-
.../beam/sdk/coders/StructuralByteArray.java | 4 +-
.../apache/beam/sdk/coders/StructuredCoder.java | 74 +-
.../beam/sdk/coders/TextualIntegerCoder.java | 17 +-
.../org/apache/beam/sdk/coders/VarIntCoder.java | 10 +-
.../apache/beam/sdk/coders/VarLongCoder.java | 15 +-
.../org/apache/beam/sdk/coders/VoidCoder.java | 12 +-
.../apache/beam/sdk/coders/package-info.java | 2 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 219 +++--
.../java/org/apache/beam/sdk/io/AvroSink.java | 34 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 23 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 4 +-
.../apache/beam/sdk/io/CompressedSource.java | 6 +
.../beam/sdk/io/DefaultFilenamePolicy.java | 169 ++++
.../org/apache/beam/sdk/io/FileBasedSink.java | 803 ++++++++++---------
.../org/apache/beam/sdk/io/FileBasedSource.java | 39 +-
.../java/org/apache/beam/sdk/io/FileSystem.java | 3 +
.../apache/beam/sdk/io/FileSystemRegistrar.java | 3 +
.../org/apache/beam/sdk/io/FileSystems.java | 65 +-
.../beam/sdk/io/LocalFileSystemRegistrar.java | 3 +
.../org/apache/beam/sdk/io/LocalResources.java | 3 +
.../main/java/org/apache/beam/sdk/io/Read.java | 8 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 153 ++--
.../java/org/apache/beam/sdk/io/TextIO.java | 255 +++---
.../java/org/apache/beam/sdk/io/TextSink.java | 32 +-
.../org/apache/beam/sdk/io/UnboundedSource.java | 23 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 285 ++++---
.../org/apache/beam/sdk/io/fs/ResourceId.java | 3 +
.../apache/beam/sdk/metrics/CounterCell.java | 67 --
.../org/apache/beam/sdk/metrics/DirtyState.java | 98 ---
.../beam/sdk/metrics/DistributionCell.java | 64 --
.../beam/sdk/metrics/DistributionData.java | 60 --
.../beam/sdk/metrics/DistributionResult.java | 3 +
.../org/apache/beam/sdk/metrics/GaugeCell.java | 54 --
.../org/apache/beam/sdk/metrics/GaugeData.java | 81 --
.../apache/beam/sdk/metrics/GaugeResult.java | 3 +
.../org/apache/beam/sdk/metrics/Metric.java | 7 +-
.../org/apache/beam/sdk/metrics/MetricCell.java | 41 -
.../beam/sdk/metrics/MetricFiltering.java | 99 ---
.../org/apache/beam/sdk/metrics/MetricKey.java | 41 -
.../apache/beam/sdk/metrics/MetricUpdates.java | 78 --
.../org/apache/beam/sdk/metrics/Metrics.java | 6 +-
.../beam/sdk/metrics/MetricsContainer.java | 129 +--
.../beam/sdk/metrics/MetricsEnvironment.java | 14 +-
.../org/apache/beam/sdk/metrics/MetricsMap.java | 86 --
.../apache/beam/sdk/metrics/SinkMetrics.java | 4 +
.../apache/beam/sdk/metrics/SourceMetrics.java | 3 +
.../DefaultPipelineOptionsRegistrar.java | 3 +-
.../beam/sdk/options/PipelineOptionSpec.java | 4 +-
.../beam/sdk/options/PipelineOptions.java | 5 +-
.../sdk/options/PipelineOptionsFactory.java | 2 +-
.../apache/beam/sdk/options/ValueProvider.java | 15 +-
.../beam/sdk/options/ValueProviderUtils.java | 60 --
.../apache/beam/sdk/options/ValueProviders.java | 61 ++
.../java/org/apache/beam/sdk/package-info.java | 2 +-
.../beam/sdk/runners/AppliedPTransform.java | 73 ++
.../beam/sdk/runners/PTransformMatcher.java | 7 +-
.../beam/sdk/runners/PTransformOverride.java | 6 +-
.../sdk/runners/PTransformOverrideFactory.java | 8 +-
.../apache/beam/sdk/runners/PipelineRunner.java | 65 --
.../sdk/runners/PipelineRunnerRegistrar.java | 5 +
.../beam/sdk/runners/TransformHierarchy.java | 15 +-
.../apache/beam/sdk/runners/package-info.java | 4 +-
.../org/apache/beam/sdk/state/BagState.java | 36 +
.../apache/beam/sdk/state/CombiningState.java | 55 ++
.../apache/beam/sdk/state/GroupingState.java | 44 +
.../org/apache/beam/sdk/state/MapState.java | 73 ++
.../apache/beam/sdk/state/ReadableState.java | 51 ++
.../apache/beam/sdk/state/ReadableStates.java | 44 +
.../org/apache/beam/sdk/state/SetState.java | 47 ++
.../java/org/apache/beam/sdk/state/State.java | 36 +
.../org/apache/beam/sdk/state/StateBinder.java | 70 ++
.../org/apache/beam/sdk/state/StateContext.java | 46 ++
.../apache/beam/sdk/state/StateContexts.java | 51 ++
.../org/apache/beam/sdk/state/StateSpec.java | 63 ++
.../org/apache/beam/sdk/state/StateSpecs.java | 683 ++++++++++++++++
.../org/apache/beam/sdk/state/TimeDomain.java | 49 ++
.../java/org/apache/beam/sdk/state/Timer.java | 70 ++
.../org/apache/beam/sdk/state/TimerSpec.java | 30 +
.../org/apache/beam/sdk/state/TimerSpecs.java | 39 +
.../java/org/apache/beam/sdk/state/Timers.java | 58 ++
.../org/apache/beam/sdk/state/ValueState.java | 35 +
.../beam/sdk/state/WatermarkHoldState.java | 41 +
.../org/apache/beam/sdk/state/package-info.java | 22 +
.../beam/sdk/testing/CoderProperties.java | 6 +-
.../apache/beam/sdk/testing/CrashingRunner.java | 2 +-
.../beam/sdk/testing/FileChecksumMatcher.java | 2 +-
.../beam/sdk/testing/MatcherDeserializer.java | 4 +-
.../beam/sdk/testing/MatcherSerializer.java | 4 +-
.../apache/beam/sdk/testing/NeedsRunner.java | 2 +-
.../org/apache/beam/sdk/testing/PAssert.java | 4 +-
.../apache/beam/sdk/testing/TestPipeline.java | 77 +-
.../beam/sdk/testing/TestPipelineOptions.java | 2 -
.../org/apache/beam/sdk/testing/TestStream.java | 4 +-
.../apache/beam/sdk/testing/UsesMapState.java | 2 +-
.../apache/beam/sdk/testing/UsesSetState.java | 2 +-
.../apache/beam/sdk/transforms/Aggregator.java | 34 -
.../beam/sdk/transforms/AppliedPTransform.java | 73 --
.../sdk/transforms/ApproximateQuantiles.java | 90 ++-
.../org/apache/beam/sdk/transforms/Combine.java | 44 +-
.../beam/sdk/transforms/CombineFnBase.java | 26 +-
.../apache/beam/sdk/transforms/CombineFns.java | 25 +-
.../beam/sdk/transforms/CombineWithContext.java | 3 +-
.../org/apache/beam/sdk/transforms/Count.java | 12 +-
.../org/apache/beam/sdk/transforms/Create.java | 76 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 91 ++-
.../apache/beam/sdk/transforms/DoFnTester.java | 66 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 2 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 2 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 71 +-
.../org/apache/beam/sdk/transforms/Latest.java | 15 +-
.../beam/sdk/transforms/Materialization.java | 8 +-
.../beam/sdk/transforms/Materializations.java | 11 +-
.../org/apache/beam/sdk/transforms/Mean.java | 19 +-
.../apache/beam/sdk/transforms/PTransform.java | 10 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 20 +-
.../org/apache/beam/sdk/transforms/Regex.java | 1 -
.../beam/sdk/transforms/ReifyTimestamps.java | 79 ++
.../apache/beam/sdk/transforms/Reshuffle.java | 97 +++
.../org/apache/beam/sdk/transforms/Top.java | 46 +-
.../org/apache/beam/sdk/transforms/View.java | 40 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 6 +-
.../apache/beam/sdk/transforms/WithKeys.java | 4 +-
.../beam/sdk/transforms/join/CoGbkResult.java | 20 +-
.../beam/sdk/transforms/join/UnionCoder.java | 29 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 31 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 20 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 23 +-
.../sdk/transforms/reflect/DoFnSignature.java | 50 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 66 +-
.../reflect/StableInvokerNamingStrategy.java | 54 ++
.../sdk/transforms/reflect/package-info.java | 5 +-
.../beam/sdk/transforms/windowing/AfterAll.java | 8 +-
.../sdk/transforms/windowing/AfterEach.java | 2 +-
.../sdk/transforms/windowing/AfterFirst.java | 5 +-
.../sdk/transforms/windowing/AfterPane.java | 5 +-
.../windowing/AfterProcessingTime.java | 6 +-
.../transforms/windowing/AfterWatermark.java | 6 +-
.../transforms/windowing/DefaultTrigger.java | 10 +-
.../sdk/transforms/windowing/GlobalWindow.java | 10 +-
.../sdk/transforms/windowing/GlobalWindows.java | 5 +-
.../transforms/windowing/IntervalWindow.java | 19 +-
.../transforms/windowing/InvalidWindows.java | 2 +-
.../MergeOverlappingIntervalWindows.java | 6 +-
.../beam/sdk/transforms/windowing/Never.java | 4 +-
.../windowing/NonMergingWindowFn.java | 2 +-
.../transforms/windowing/OrFinallyTrigger.java | 7 +-
.../beam/sdk/transforms/windowing/PaneInfo.java | 18 +-
.../sdk/transforms/windowing/Repeatedly.java | 6 +-
.../transforms/windowing/ReshuffleTrigger.java | 58 ++
.../beam/sdk/transforms/windowing/Sessions.java | 13 +-
.../transforms/windowing/TimestampCombiner.java | 5 +
.../windowing/TimestampTransform.java | 41 +-
.../beam/sdk/transforms/windowing/Trigger.java | 18 +-
.../beam/sdk/transforms/windowing/Window.java | 16 +-
.../beam/sdk/transforms/windowing/WindowFn.java | 10 +-
.../transforms/windowing/WindowMappingFn.java | 12 +-
.../org/apache/beam/sdk/util/ApiSurface.java | 10 +-
.../apache/beam/sdk/util/AppliedCombineFn.java | 1 +
.../java/org/apache/beam/sdk/util/BackOff.java | 81 ++
.../org/apache/beam/sdk/util/BackOffUtils.java | 57 ++
.../org/apache/beam/sdk/util/BitSetCoder.java | 21 +-
.../org/apache/beam/sdk/util/ClassPath.java | 1 +
.../apache/beam/sdk/util/CloudKnownType.java | 143 ----
.../org/apache/beam/sdk/util/CloudObject.java | 187 -----
.../org/apache/beam/sdk/util/CoderUtils.java | 127 +--
.../beam/sdk/util/CombineContextFactory.java | 27 +-
.../org/apache/beam/sdk/util/CombineFnUtil.java | 2 +-
.../beam/sdk/util/ExplicitShardedFile.java | 3 -
.../beam/sdk/util/FileIOChannelFactory.java | 223 -----
.../sdk/util/FileIOChannelFactoryRegistrar.java | 38 -
.../org/apache/beam/sdk/util/FluentBackoff.java | 1 -
.../apache/beam/sdk/util/IOChannelFactory.java | 127 ---
.../sdk/util/IOChannelFactoryRegistrar.java | 48 --
.../apache/beam/sdk/util/IOChannelUtils.java | 280 -------
.../apache/beam/sdk/util/IdentityWindowFn.java | 1 +
.../apache/beam/sdk/util/NoopPathValidator.java | 49 --
.../beam/sdk/util/NullSideInputReader.java | 61 --
.../beam/sdk/util/NumberedShardedFile.java | 7 +-
.../beam/sdk/util/PCollectionViewWindow.java | 67 --
.../apache/beam/sdk/util/PCollectionViews.java | 496 ------------
.../org/apache/beam/sdk/util/PathValidator.java | 48 --
.../org/apache/beam/sdk/util/PropertyNames.java | 112 ---
.../sdk/util/ReadyCheckingSideInputReader.java | 34 -
.../apache/beam/sdk/util/ReifyTimestamps.java | 76 --
.../org/apache/beam/sdk/util/ReleaseInfo.java | 70 +-
.../org/apache/beam/sdk/util/Reshuffle.java | 88 --
.../apache/beam/sdk/util/ReshuffleTrigger.java | 52 --
.../org/apache/beam/sdk/util/Serializer.java | 147 ----
.../org/apache/beam/sdk/util/ShardedFile.java | 2 -
.../apache/beam/sdk/util/SideInputReader.java | 47 --
.../java/org/apache/beam/sdk/util/Sleeper.java | 48 ++
.../java/org/apache/beam/sdk/util/Structs.java | 371 ---------
.../beam/sdk/util/SystemDoFnInternal.java | 3 -
.../org/apache/beam/sdk/util/TimeDomain.java | 43 -
.../java/org/apache/beam/sdk/util/Timer.java | 67 --
.../org/apache/beam/sdk/util/TimerSpec.java | 30 -
.../org/apache/beam/sdk/util/TimerSpecs.java | 41 -
.../java/org/apache/beam/sdk/util/Timers.java | 58 --
.../sdk/util/UploadIdResponseInterceptor.java | 60 --
.../apache/beam/sdk/util/ValueWithRecordId.java | 134 ----
.../java/org/apache/beam/sdk/util/Values.java | 88 --
.../org/apache/beam/sdk/util/WindowedValue.java | 93 +--
.../apache/beam/sdk/util/WindowingStrategy.java | 283 -------
.../apache/beam/sdk/util/state/BagState.java | 28 -
.../beam/sdk/util/state/CombiningState.java | 53 --
.../beam/sdk/util/state/GroupingState.java | 42 -
.../apache/beam/sdk/util/state/MapState.java | 79 --
.../beam/sdk/util/state/ReadableState.java | 51 --
.../beam/sdk/util/state/ReadableStates.java | 45 --
.../apache/beam/sdk/util/state/SetState.java | 45 --
.../org/apache/beam/sdk/util/state/State.java | 32 -
.../apache/beam/sdk/util/state/StateBinder.java | 66 --
.../beam/sdk/util/state/StateContext.java | 43 -
.../beam/sdk/util/state/StateContexts.java | 51 --
.../apache/beam/sdk/util/state/StateSpec.java | 53 --
.../apache/beam/sdk/util/state/StateSpecs.java | 629 ---------------
.../apache/beam/sdk/util/state/ValueState.java | 37 -
.../beam/sdk/util/state/WatermarkHoldState.java | 42 -
.../beam/sdk/util/state/package-info.java | 22 -
.../java/org/apache/beam/sdk/values/PBegin.java | 26 +-
.../org/apache/beam/sdk/values/PCollection.java | 162 +++-
.../apache/beam/sdk/values/PCollectionList.java | 14 +-
.../beam/sdk/values/PCollectionTuple.java | 24 +-
.../apache/beam/sdk/values/PCollectionView.java | 33 +-
.../beam/sdk/values/PCollectionViews.java | 495 ++++++++++++
.../java/org/apache/beam/sdk/values/PDone.java | 12 +-
.../org/apache/beam/sdk/values/POutput.java | 25 +-
.../java/org/apache/beam/sdk/values/PValue.java | 12 +-
.../org/apache/beam/sdk/values/PValueBase.java | 61 +-
.../apache/beam/sdk/values/TaggedPValue.java | 6 +-
.../beam/sdk/values/TimestampedValue.java | 30 +-
.../org/apache/beam/sdk/values/TupleTag.java | 22 -
.../org/apache/beam/sdk/values/TypedPValue.java | 208 -----
.../beam/sdk/values/ValueInSingleWindow.java | 29 +-
.../beam/sdk/values/ValueWithRecordId.java | 149 ++++
.../beam/sdk/values/WindowingStrategy.java | 283 +++++++
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 1 -
.../java/org/apache/beam/sdk/PipelineTest.java | 3 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 11 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 5 +-
.../beam/sdk/coders/BigIntegerCoderTest.java | 5 +-
.../beam/sdk/coders/CoderFactoriesTest.java | 100 ---
.../beam/sdk/coders/CoderProvidersTest.java | 82 +-
.../beam/sdk/coders/CoderRegistryTest.java | 191 ++---
.../apache/beam/sdk/coders/CustomCoderTest.java | 4 +-
.../beam/sdk/coders/DefaultCoderTest.java | 65 +-
.../beam/sdk/coders/DelegateCoderTest.java | 25 -
.../beam/sdk/coders/IterableCoderTest.java | 17 -
.../beam/sdk/coders/LengthPrefixCoderTest.java | 24 +-
.../apache/beam/sdk/coders/ListCoderTest.java | 17 -
.../apache/beam/sdk/coders/MapCoderTest.java | 20 -
.../beam/sdk/coders/NullableCoderTest.java | 31 +-
.../beam/sdk/coders/SerializableCoderTest.java | 47 +-
.../beam/sdk/coders/StructuredCoderTest.java | 12 +-
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 101 ++-
.../beam/sdk/io/CompressedSourceTest.java | 25 +-
.../beam/sdk/io/DefaultFilenamePolicyTest.java | 55 ++
.../sdk/io/DrunkWritableByteChannelFactory.java | 1 -
.../apache/beam/sdk/io/FileBasedSinkTest.java | 460 +++++------
.../apache/beam/sdk/io/LocalResourceIdTest.java | 6 +
.../java/org/apache/beam/sdk/io/SimpleSink.java | 31 +-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 2 -
.../java/org/apache/beam/sdk/io/TextIOTest.java | 57 +-
.../org/apache/beam/sdk/io/WriteFilesTest.java | 41 +-
.../apache/beam/sdk/io/fs/ResourceIdTester.java | 150 ++++
.../beam/sdk/metrics/CounterCellTest.java | 55 --
.../apache/beam/sdk/metrics/DirtyStateTest.java | 56 --
.../beam/sdk/metrics/DistributionCellTest.java | 53 --
.../apache/beam/sdk/metrics/GaugeCellTest.java | 48 --
.../beam/sdk/metrics/MetricFilteringTest.java | 145 ----
.../apache/beam/sdk/metrics/MetricMatchers.java | 242 ------
.../beam/sdk/metrics/MetricResultsMatchers.java | 190 +++++
.../beam/sdk/metrics/MetricsContainerTest.java | 129 ---
.../sdk/metrics/MetricsEnvironmentTest.java | 23 +-
.../apache/beam/sdk/metrics/MetricsMapTest.java | 103 ---
.../apache/beam/sdk/metrics/MetricsTest.java | 47 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 4 +-
.../beam/sdk/options/PipelineOptionsTest.java | 1 -
.../sdk/options/ProxyInvocationHandlerTest.java | 4 +-
.../beam/sdk/options/ValueProviderTest.java | 23 +-
.../sdk/options/ValueProviderUtilsTest.java | 78 --
.../beam/sdk/options/ValueProvidersTest.java | 79 ++
.../beam/sdk/runners/PipelineRunnerTest.java | 1 +
.../sdk/runners/TransformHierarchyTest.java | 5 +-
.../beam/sdk/runners/TransformTreeTest.java | 2 +-
.../beam/sdk/testing/CoderPropertiesTest.java | 73 +-
.../beam/sdk/testing/CrashingRunnerTest.java | 2 +-
.../beam/sdk/testing/ExpectedLogsTest.java | 2 +-
.../sdk/testing/FastNanoClockAndSleeper.java | 47 --
.../testing/FastNanoClockAndSleeperTest.java | 47 --
.../sdk/testing/FileChecksumMatcherTest.java | 12 +-
.../apache/beam/sdk/testing/PAssertTest.java | 14 +-
.../sdk/testing/PCollectionViewTesting.java | 2 +-
.../sdk/testing/SerializableMatchersTest.java | 9 +-
.../beam/sdk/testing/SystemNanoTimeSleeper.java | 2 +-
.../beam/sdk/testing/TestPipelineTest.java | 6 +-
.../beam/sdk/testing/WindowSupplierTest.java | 10 +-
.../sdk/transforms/ApproximateUniqueTest.java | 2 +-
.../beam/sdk/transforms/CombineFnsTest.java | 30 +-
.../apache/beam/sdk/transforms/CombineTest.java | 39 +-
.../apache/beam/sdk/transforms/CreateTest.java | 25 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 14 +-
.../sdk/transforms/FlatMapElementsTest.java | 4 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 15 +-
.../beam/sdk/transforms/LatestFnTest.java | 2 +-
.../beam/sdk/transforms/MapElementsTest.java | 4 +-
.../beam/sdk/transforms/ParDoLifecycleTest.java | 16 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 258 +++---
.../apache/beam/sdk/transforms/RegexTest.java | 2 -
.../sdk/transforms/ReifyTimestampsTest.java | 140 ++++
.../beam/sdk/transforms/ReshuffleTest.java | 293 +++++++
.../beam/sdk/transforms/SplittableDoFnTest.java | 4 +-
.../apache/beam/sdk/transforms/ViewTest.java | 17 +-
.../display/DisplayDataEvaluator.java | 2 +-
.../transforms/reflect/DoFnInvokersTest.java | 73 +-
.../DoFnSignaturesSplittableDoFnTest.java | 4 +-
.../transforms/reflect/DoFnSignaturesTest.java | 35 +-
.../transforms/reflect/OnTimerInvokersTest.java | 38 +-
.../testhelper/DoFnInvokersTestHelper.java | 6 +-
.../transforms/windowing/GlobalWindowTest.java | 2 +-
.../windowing/ReshuffleTriggerTest.java | 42 +
.../sdk/transforms/windowing/WindowTest.java | 4 +-
...BufferedElementCountingOutputStreamTest.java | 5 +-
.../apache/beam/sdk/util/CoderUtilsTest.java | 8 +-
.../apache/beam/sdk/util/CombineFnUtilTest.java | 2 +-
.../util/FileIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/FileIOChannelFactoryTest.java | 242 ------
.../apache/beam/sdk/util/FluentBackoffTest.java | 1 -
.../beam/sdk/util/IOChannelUtilsTest.java | 118 ---
.../beam/sdk/util/NumberedShardedFileTest.java | 14 +-
.../beam/sdk/util/ReifyTimestampsTest.java | 109 ---
.../org/apache/beam/sdk/util/ReshuffleTest.java | 270 -------
.../beam/sdk/util/ReshuffleTriggerTest.java | 44 -
.../beam/sdk/util/SerializableUtilsTest.java | 8 +-
.../apache/beam/sdk/util/SerializerTest.java | 162 ----
.../org/apache/beam/sdk/util/StructsTest.java | 206 -----
.../util/UploadIdResponseInterceptorTest.java | 98 ---
.../beam/sdk/util/ValueWithRecordIdTest.java | 34 -
.../beam/sdk/values/PCollectionTupleTest.java | 1 -
.../apache/beam/sdk/values/TypedPValueTest.java | 28 +-
.../beam/sdk/values/ValueWithRecordIdTest.java | 34 +
.../google-cloud-platform-core/pom.xml | 8 +-
.../sdk/extensions/gcp/options/GcpOptions.java | 9 +-
.../sdk/extensions/gcp/options/GcsOptions.java | 4 +-
.../gcp/storage/GcsCreateOptions.java | 56 ++
.../extensions/gcp/storage/GcsFileSystem.java | 20 +-
.../gcp/storage/GcsFileSystemRegistrar.java | 5 +-
.../gcp/storage/GcsPathValidator.java | 105 +++
.../gcp/storage/NoopPathValidator.java | 53 ++
.../extensions/gcp/storage/PathValidator.java | 59 ++
.../apache/beam/sdk/util/BackOffAdapter.java | 43 +
.../beam/sdk/util/GcsIOChannelFactory.java | 111 ---
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 -
.../apache/beam/sdk/util/GcsPathValidator.java | 98 ---
.../java/org/apache/beam/sdk/util/GcsUtil.java | 71 +-
.../sdk/util/UploadIdResponseInterceptor.java | 60 ++
.../extensions/gcp/GcpCoreApiSurfaceTest.java | 3 +-
.../extensions/gcp/options/GcpOptionsTest.java | 4 +-
.../gcp/options/GoogleApiDebugOptionsTest.java | 8 +-
.../gcp/storage/GcsFileSystemTest.java | 30 +-
.../gcp/storage/GcsPathValidatorTest.java | 107 +++
.../gcp/storage/GcsResourceIdTest.java | 9 +
.../beam/sdk/util/FastNanoClockAndSleeper.java | 47 ++
.../sdk/util/FastNanoClockAndSleeperTest.java | 47 ++
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 -
.../beam/sdk/util/GcsPathValidatorTest.java | 106 ---
.../org/apache/beam/sdk/util/GcsUtilTest.java | 80 +-
.../util/UploadIdResponseInterceptorTest.java | 98 +++
sdks/java/extensions/jackson/pom.xml | 2 +-
sdks/java/extensions/join-library/README.md | 42 -
sdks/java/extensions/join-library/pom.xml | 2 +-
sdks/java/extensions/pom.xml | 2 +-
sdks/java/extensions/protobuf/pom.xml | 2 +-
.../extensions/protobuf/ByteStringCoder.java | 24 +-
.../sdk/extensions/protobuf/ProtoCoder.java | 85 +-
.../ProtobufCoderProviderRegistrar.java | 41 +
.../protobuf/ProtobufCoderRegistrar.java | 39 -
.../protobuf/ByteStringCoderTest.java | 10 +-
.../sdk/extensions/protobuf/ProtoCoderTest.java | 7 +-
sdks/java/extensions/sorter/README.md | 42 -
sdks/java/extensions/sorter/pom.xml | 2 +-
.../extensions/sorter/ExternalSorterTest.java | 1 -
sdks/java/harness/pom.xml | 12 +-
.../org/apache/beam/fn/harness/FnHarness.java | 8 +-
.../harness/control/ProcessBundleHandler.java | 4 +-
.../BeamFnDataBufferingOutboundObserver.java | 3 +-
.../harness/data/BeamFnDataInboundObserver.java | 3 +-
.../beam/runners/core/BeamFnDataReadRunner.java | 19 +-
.../runners/core/BeamFnDataWriteRunner.java | 20 +-
.../control/ProcessBundleHandlerTest.java | 55 +-
...BeamFnDataBufferingOutboundObserverTest.java | 3 +-
.../data/BeamFnDataInboundObserverTest.java | 3 +-
.../runners/core/BeamFnDataReadRunnerTest.java | 14 +-
.../runners/core/BeamFnDataWriteRunnerTest.java | 14 +-
sdks/java/io/common/pom.xml | 2 +-
.../apache/beam/sdk/io/common/HashingFn.java | 2 -
sdks/java/io/elasticsearch/pom.xml | 2 +-
.../sdk/io/elasticsearch/ElasticsearchIO.java | 12 +-
sdks/java/io/google-cloud-platform/pom.xml | 24 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 253 ++++--
.../BigQueryCoderProviderRegistrar.java | 40 +
.../io/gcp/bigquery/BigQueryCoderRegistrar.java | 39 -
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 141 ++--
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 562 +++++++------
.../io/gcp/bigquery/BigQueryQuerySource.java | 44 +-
.../io/gcp/bigquery/BigQueryServicesImpl.java | 101 +--
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 46 +-
.../gcp/bigquery/BigQueryTableRowIterator.java | 7 +-
.../io/gcp/bigquery/BigQueryTableSource.java | 17 +-
.../sdk/io/gcp/bigquery/CalculateSchemas.java | 78 ++
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 66 +-
.../io/gcp/bigquery/DynamicDestinations.java | 178 ++++
.../bigquery/DynamicDestinationsHelpers.java | 193 +++++
.../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 36 +-
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 22 +-
.../sdk/io/gcp/bigquery/StreamingInserts.java | 59 +-
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 4 +-
.../io/gcp/bigquery/StreamingWriteTables.java | 10 +-
.../sdk/io/gcp/bigquery/TableDestination.java | 14 +-
.../io/gcp/bigquery/TableDestinationCoder.java | 27 +-
.../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 19 +-
.../sdk/io/gcp/bigquery/TableRowJsonCoder.java | 19 +-
.../sdk/io/gcp/bigquery/TableRowWriter.java | 59 +-
.../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 2 +-
.../io/gcp/bigquery/WriteBundlesToFiles.java | 190 +++--
.../bigquery/WriteGroupedRecordsToFiles.java | 68 ++
.../sdk/io/gcp/bigquery/WritePartition.java | 86 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 10 +-
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 23 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 160 ++--
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 245 +++---
.../io/gcp/bigtable/BigtableServiceImpl.java | 9 -
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 19 +-
.../pubsub/PubsubCoderProviderRegistrar.java | 37 +
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 78 +-
.../beam/sdk/io/gcp/pubsub/PubsubMessage.java | 61 ++
.../pubsub/PubsubMessagePayloadOnlyCoder.java | 32 +-
.../PubsubMessageWithAttributesCoder.java | 40 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 40 +-
.../io/gcp/pubsub/PubsubUnboundedSource.java | 51 +-
.../BigQueryCoderProviderRegistrarTest.java | 40 +
.../bigquery/BigQueryCoderRegistrarTest.java | 40 -
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 561 ++++++++-----
.../gcp/bigquery/BigQueryServicesImplTest.java | 24 +-
.../io/gcp/bigquery/FakeBigQueryServices.java | 2 -
.../sdk/io/gcp/bigquery/FakeJobService.java | 55 +-
.../sdk/io/gcp/bigquery/TableContainer.java | 1 -
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 6 +-
.../io/gcp/pubsub/PubsubUnboundedSinkTest.java | 41 +-
.../gcp/pubsub/PubsubUnboundedSourceTest.java | 4 +-
.../sdk/io/gcp/testing/BigqueryMatcher.java | 4 +-
.../sdk/io/gcp/testing/BigqueryMatcherTest.java | 7 +-
sdks/java/io/hadoop-common/pom.xml | 9 +-
.../beam/sdk/io/hadoop/WritableCoder.java | 78 +-
.../beam/sdk/io/hadoop/WritableCoderTest.java | 10 +
sdks/java/io/hadoop-file-system/README.md | 43 -
sdks/java/io/hadoop-file-system/pom.xml | 45 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 478 -----------
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 625 ---------------
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 32 +-
.../sdk/io/hdfs/HadoopFileSystemOptions.java | 76 +-
.../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 3 +
.../beam/sdk/io/hdfs/HadoopResourceId.java | 16 +-
.../java/org/apache/beam/sdk/io/hdfs/Sink.java | 195 -----
.../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 -
.../java/org/apache/beam/sdk/io/hdfs/Write.java | 585 --------------
.../apache/beam/sdk/io/hdfs/package-info.java | 3 +-
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 172 ----
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 231 ------
.../sdk/io/hdfs/HadoopFileSystemModuleTest.java | 2 +-
.../io/hdfs/HadoopFileSystemOptionsTest.java | 125 +++
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 5 +-
.../beam/sdk/io/hdfs/HadoopResourceIdTest.java | 71 ++
sdks/java/io/hadoop/README.md | 167 ----
sdks/java/io/hadoop/input-format/pom.xml | 2 +-
.../ConfigurableEmployeeInputFormat.java | 1 -
.../hadoop/inputformat/EmployeeInputFormat.java | 1 -
.../ReuseObjectsEmployeeInputFormat.java | 1 -
.../hadoop/inputformat/TestEmployeeDataSet.java | 2 -
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 30 +-
.../inputformat/HIFIOWithElasticTest.java | 1 -
.../HIFIOWithEmbeddedCassandraTest.java | 2 -
.../integration/tests/HIFIOCassandraIT.java | 2 -
.../integration/tests/HIFIOElasticIT.java | 1 -
sdks/java/io/hadoop/pom.xml | 2 +-
sdks/java/io/hbase/pom.xml | 27 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 7 +-
.../beam/sdk/io/hbase/HBaseMutationCoder.java | 10 +-
.../beam/sdk/io/hbase/HBaseResultCoder.java | 8 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 4 +-
sdks/java/io/jdbc/pom.xml | 2 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 13 +-
sdks/java/io/jdbc/src/test/README.md | 32 -
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 1 -
sdks/java/io/jms/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 17 +-
.../org/apache/beam/sdk/io/jms/JmsRecord.java | 1 -
sdks/java/io/kafka/pom.xml | 2 +-
.../beam/sdk/io/kafka/KafkaCheckpointMark.java | 2 -
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 348 +++-----
.../apache/beam/sdk/io/kafka/KafkaRecord.java | 1 -
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 19 +-
.../CoderBasedKafkaDeserializer.java | 71 --
.../CoderBasedKafkaSerializer.java | 73 --
.../serialization/InstantDeserializer.java | 1 -
.../kafka/serialization/InstantSerializer.java | 1 -
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 166 +---
sdks/java/io/kinesis/pom.xml | 2 +-
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 42 +-
sdks/java/io/mongodb/pom.xml | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 7 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 57 +-
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 3 -
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 20 +-
sdks/java/io/mqtt/pom.xml | 2 +-
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 3 -
sdks/java/io/pom.xml | 2 +-
sdks/java/io/xml/pom.xml | 2 +-
.../org/apache/beam/sdk/io/xml/JAXBCoder.java | 63 +-
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 19 +-
.../org/apache/beam/sdk/io/xml/XmlSink.java | 24 +-
.../apache/beam/sdk/io/xml/JAXBCoderTest.java | 25 +-
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 52 +-
sdks/java/java8tests/pom.xml | 2 +-
.../beam/sdk/transforms/DistinctJava8Test.java | 8 -
.../beam/sdk/transforms/FilterJava8Test.java | 2 +-
.../beam/sdk/transforms/PartitionJava8Test.java | 2 +-
.../beam/sdk/transforms/WithKeysJava8Test.java | 9 -
sdks/java/javadoc/README.md | 23 -
sdks/java/javadoc/ant.xml | 5 +-
sdks/java/javadoc/pom.xml | 2 +-
.../examples-java8/generate-sources.sh | 12 +-
.../maven-archetypes/examples-java8/pom.xml | 108 +--
.../main/resources/archetype-resources/pom.xml | 23 +-
.../examples/generate-sources.sh | 8 +-
sdks/java/maven-archetypes/examples/pom.xml | 67 +-
.../main/resources/archetype-resources/pom.xml | 16 +-
sdks/java/maven-archetypes/pom.xml | 22 +-
sdks/java/maven-archetypes/starter/pom.xml | 19 +-
.../resources/projects/basic/reference/pom.xml | 2 +-
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 20 +-
sdks/python/README.md | 298 -------
sdks/python/apache_beam/__init__.py | 2 +-
sdks/python/apache_beam/coders/coder_impl.py | 59 +-
sdks/python/apache_beam/coders/coders.py | 32 +-
sdks/python/apache_beam/coders/coders_test.py | 11 +-
.../apache_beam/coders/coders_test_common.py | 2 +-
sdks/python/apache_beam/coders/observable.py | 9 +-
sdks/python/apache_beam/coders/slow_stream.py | 21 +-
.../apache_beam/coders/standard_coders_test.py | 4 +-
sdks/python/apache_beam/coders/stream.pyx | 5 +
sdks/python/apache_beam/coders/typecoders.py | 7 +-
.../examples/complete/autocomplete.py | 4 +-
.../examples/complete/autocomplete_test.py | 6 +-
.../examples/complete/estimate_pi.py | 4 +-
.../examples/complete/estimate_pi_test.py | 6 +-
.../examples/complete/game/README.md | 69 --
.../examples/complete/game/hourly_team_score.py | 6 +-
.../complete/game/hourly_team_score_test.py | 6 +-
.../examples/complete/game/user_score.py | 4 +-
.../examples/complete/game/user_score_test.py | 6 +-
.../apache_beam/examples/complete/tfidf.py | 4 +-
.../apache_beam/examples/complete/tfidf_test.py | 6 +-
.../examples/complete/top_wikipedia_sessions.py | 4 +-
.../complete/top_wikipedia_sessions_test.py | 6 +-
.../examples/cookbook/bigquery_side_input.py | 4 +-
.../cookbook/bigquery_side_input_test.py | 6 +-
.../cookbook/bigquery_tornadoes_it_test.py | 4 +-
.../cookbook/bigquery_tornadoes_test.py | 8 +-
.../apache_beam/examples/cookbook/coders.py | 4 +-
.../examples/cookbook/coders_test.py | 6 +-
.../examples/cookbook/combiners_test.py | 8 +-
.../examples/cookbook/custom_ptransform.py | 2 +-
.../examples/cookbook/custom_ptransform_test.py | 6 +-
.../examples/cookbook/datastore_wordcount.py | 19 +-
.../examples/cookbook/filters_test.py | 14 +-
.../examples/cookbook/group_with_coder.py | 4 +-
.../examples/cookbook/mergecontacts.py | 18 +-
.../examples/cookbook/multiple_output_pardo.py | 4 +-
.../apache_beam/examples/snippets/snippets.py | 53 +-
.../examples/snippets/snippets_test.py | 49 +-
sdks/python/apache_beam/examples/wordcount.py | 42 +-
.../apache_beam/examples/wordcount_debugging.py | 10 +-
.../apache_beam/examples/wordcount_it_test.py | 6 +-
.../apache_beam/examples/wordcount_minimal.py | 4 +-
sdks/python/apache_beam/internal/__init__.py | 2 +
.../python/apache_beam/internal/gcp/__init__.py | 2 +
sdks/python/apache_beam/internal/gcp/auth.py | 50 +-
.../apache_beam/internal/gcp/json_value.py | 14 +-
.../apache_beam/internal/gcp/json_value_test.py | 4 +-
sdks/python/apache_beam/internal/pickler.py | 28 +-
sdks/python/apache_beam/internal/util.py | 21 +-
sdks/python/apache_beam/io/__init__.py | 2 +-
sdks/python/apache_beam/io/avroio.py | 17 +-
sdks/python/apache_beam/io/avroio_test.py | 6 +-
sdks/python/apache_beam/io/concat_source.py | 12 +-
.../python/apache_beam/io/concat_source_test.py | 6 +-
sdks/python/apache_beam/io/filebasedsink.py | 299 +++++++
.../python/apache_beam/io/filebasedsink_test.py | 303 +++++++
sdks/python/apache_beam/io/filebasedsource.py | 8 +-
.../apache_beam/io/filebasedsource_test.py | 10 +-
sdks/python/apache_beam/io/fileio.py | 297 -------
sdks/python/apache_beam/io/fileio_test.py | 304 -------
sdks/python/apache_beam/io/filesystem.py | 19 +
sdks/python/apache_beam/io/filesystems.py | 38 +-
sdks/python/apache_beam/io/filesystems_test.py | 15 +-
sdks/python/apache_beam/io/filesystems_util.py | 36 -
sdks/python/apache_beam/io/gcp/bigquery.py | 10 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 2 +-
.../io/gcp/datastore/v1/fake_datastore.py | 14 +-
.../apache_beam/io/gcp/datastore/v1/helper.py | 35 +-
.../io/gcp/datastore/v1/helper_test.py | 26 +-
.../io/gcp/datastore/v1/query_splitter_test.py | 37 +-
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 8 +
.../apache_beam/io/gcp/gcsfilesystem_test.py | 5 +
sdks/python/apache_beam/io/gcp/gcsio.py | 9 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 2 +
.../io/gcp/tests/bigquery_matcher.py | 5 +-
.../io/gcp/tests/bigquery_matcher_test.py | 2 +-
sdks/python/apache_beam/io/iobase.py | 23 +-
sdks/python/apache_beam/io/localfilesystem.py | 7 +
.../apache_beam/io/localfilesystem_test.py | 4 +
sdks/python/apache_beam/io/range_trackers.py | 7 +-
sdks/python/apache_beam/io/source_test_utils.py | 8 +
sdks/python/apache_beam/io/sources_test.py | 6 +-
sdks/python/apache_beam/io/textio.py | 19 +-
sdks/python/apache_beam/io/textio_test.py | 7 +-
sdks/python/apache_beam/io/tfrecordio.py | 19 +-
sdks/python/apache_beam/io/tfrecordio_test.py | 26 +-
sdks/python/apache_beam/metrics/__init__.py | 1 +
sdks/python/apache_beam/metrics/cells.py | 50 +-
sdks/python/apache_beam/metrics/execution.py | 7 +-
sdks/python/apache_beam/metrics/metric.py | 4 +
sdks/python/apache_beam/metrics/metricbase.py | 2 +
sdks/python/apache_beam/options/__init__.py | 16 +
.../apache_beam/options/pipeline_options.py | 650 +++++++++++++++
.../options/pipeline_options_test.py | 240 ++++++
.../options/pipeline_options_validator.py | 201 +++++
.../options/pipeline_options_validator_test.py | 343 ++++++++
.../apache_beam/options/value_provider.py | 111 +++
.../apache_beam/options/value_provider_test.py | 145 ++++
sdks/python/apache_beam/pipeline.py | 33 +-
sdks/python/apache_beam/pipeline_test.py | 8 +-
sdks/python/apache_beam/pvalue.py | 18 +-
sdks/python/apache_beam/pvalue_test.py | 2 +-
sdks/python/apache_beam/runners/api/__init__.py | 4 +-
.../apache_beam/runners/api/beam_fn_api_pb2.py | 2 +
.../runners/api/beam_fn_api_pb2_grpc.py | 2 +
.../runners/api/beam_runner_api_pb2.py | 2 +
sdks/python/apache_beam/runners/common.pxd | 6 +-
sdks/python/apache_beam/runners/common.py | 99 ++-
sdks/python/apache_beam/runners/common_test.py | 58 ++
.../apache_beam/runners/dataflow/__init__.py | 9 +
.../runners/dataflow/dataflow_metrics.py | 114 ++-
.../runners/dataflow/dataflow_metrics_test.py | 197 +++--
.../runners/dataflow/dataflow_runner.py | 11 +-
.../runners/dataflow/dataflow_runner_test.py | 9 +-
.../runners/dataflow/internal/apiclient.py | 17 +-
.../runners/dataflow/internal/apiclient_test.py | 23 +-
.../runners/dataflow/internal/dependency.py | 22 +-
.../dataflow/internal/dependency_test.py | 6 +-
.../runners/dataflow/internal/names.py | 8 +-
.../runners/dataflow/native_io/iobase.py | 2 +
.../runners/dataflow/template_runner_test.py | 2 +-
.../runners/dataflow/test_dataflow_runner.py | 5 +-
.../apache_beam/runners/direct/__init__.py | 6 +-
.../runners/direct/bundle_factory.py | 16 +-
sdks/python/apache_beam/runners/direct/clock.py | 5 +-
.../consumer_tracking_pipeline_visitor.py | 4 +-
.../apache_beam/runners/direct/direct_runner.py | 7 +-
.../apache_beam/runners/direct/executor.py | 67 +-
.../runners/direct/transform_evaluator.py | 20 +-
.../runners/direct/transform_result.py | 4 +-
.../runners/direct/watermark_manager.py | 14 +-
.../apache_beam/runners/pipeline_context.py | 10 +-
.../apache_beam/runners/portability/__init__.py | 2 +
.../runners/portability/fn_api_runner.py | 2 +
.../portability/maptask_executor_runner.py | 9 +-
.../portability/maptask_executor_runner_test.py | 35 +-
sdks/python/apache_beam/runners/runner.py | 11 +-
sdks/python/apache_beam/runners/runner_test.py | 6 +-
.../apache_beam/runners/worker/__init__.py | 2 +
.../apache_beam/runners/worker/data_plane.py | 2 +
.../apache_beam/runners/worker/log_handler.py | 2 +
.../python/apache_beam/runners/worker/logger.py | 2 +
.../apache_beam/runners/worker/opcounters.py | 2 +
.../runners/worker/operation_specs.py | 2 +
.../apache_beam/runners/worker/sdk_worker.py | 4 +
.../runners/worker/sdk_worker_main.py | 2 +
.../apache_beam/runners/worker/sideinputs.py | 2 +
.../runners/worker/statesampler_fake.py | 2 +
sdks/python/apache_beam/test_pipeline.py | 163 ----
sdks/python/apache_beam/test_pipeline_test.py | 112 ---
sdks/python/apache_beam/testing/__init__.py | 16 +
.../testing/data/standard_coders.yaml | 196 +++++
.../testing/data/trigger_transcripts.yaml | 224 ++++++
.../apache_beam/testing/pipeline_verifiers.py | 154 ++++
.../testing/pipeline_verifiers_test.py | 148 ++++
.../python/apache_beam/testing/test_pipeline.py | 168 ++++
.../apache_beam/testing/test_pipeline_test.py | 112 +++
sdks/python/apache_beam/testing/test_stream.py | 175 ++++
.../apache_beam/testing/test_stream_test.py | 83 ++
sdks/python/apache_beam/testing/test_utils.py | 73 ++
sdks/python/apache_beam/testing/util.py | 107 +++
sdks/python/apache_beam/testing/util_test.py | 50 ++
sdks/python/apache_beam/tests/__init__.py | 16 -
sdks/python/apache_beam/tests/data/README.md | 20 -
.../apache_beam/tests/data/privatekey.p12 | Bin 2452 -> 0 bytes
.../apache_beam/tests/data/standard_coders.yaml | 196 -----
.../apache_beam/tests/pipeline_verifiers.py | 146 ----
.../tests/pipeline_verifiers_test.py | 148 ----
sdks/python/apache_beam/tests/test_utils.py | 69 --
sdks/python/apache_beam/transforms/__init__.py | 2 +-
.../apache_beam/transforms/combiners_test.py | 4 +-
sdks/python/apache_beam/transforms/core.py | 96 ++-
.../apache_beam/transforms/create_test.py | 5 +-
.../apache_beam/transforms/cy_combiners.py | 5 +-
sdks/python/apache_beam/transforms/display.py | 2 +-
.../apache_beam/transforms/display_test.py | 2 +-
.../python/apache_beam/transforms/ptransform.py | 45 +-
.../apache_beam/transforms/ptransform_test.py | 22 +-
.../python/apache_beam/transforms/sideinputs.py | 2 +
.../apache_beam/transforms/sideinputs_test.py | 4 +-
sdks/python/apache_beam/transforms/timeutil.py | 33 +-
sdks/python/apache_beam/transforms/trigger.py | 118 +--
.../apache_beam/transforms/trigger_test.py | 50 +-
.../transforms/trigger_transcripts.yaml | 224 ------
sdks/python/apache_beam/transforms/util.py | 79 --
sdks/python/apache_beam/transforms/util_test.py | 50 --
sdks/python/apache_beam/transforms/window.py | 51 +-
.../apache_beam/transforms/window_test.py | 12 +-
.../transforms/write_ptransform_test.py | 4 +-
sdks/python/apache_beam/typehints/decorators.py | 17 +-
sdks/python/apache_beam/typehints/opcodes.py | 2 +
.../apache_beam/typehints/trivial_inference.py | 2 +
sdks/python/apache_beam/typehints/typecheck.py | 17 +-
.../typehints/typed_pipeline_test.py | 6 +-
sdks/python/apache_beam/typehints/typehints.py | 18 +
.../apache_beam/typehints/typehints_test.py | 19 +-
sdks/python/apache_beam/utils/__init__.py | 5 +-
sdks/python/apache_beam/utils/annotations.py | 4 +-
sdks/python/apache_beam/utils/counters.py | 5 +-
.../apache_beam/utils/pipeline_options.py | 627 ---------------
.../apache_beam/utils/pipeline_options_test.py | 240 ------
.../utils/pipeline_options_validator.py | 199 -----
.../utils/pipeline_options_validator_test.py | 343 --------
sdks/python/apache_beam/utils/processes.py | 6 +-
sdks/python/apache_beam/utils/profiler.py | 5 +-
sdks/python/apache_beam/utils/proto_utils.py | 2 +
sdks/python/apache_beam/utils/retry.py | 2 +
sdks/python/apache_beam/utils/test_stream.py | 163 ----
.../apache_beam/utils/test_stream_test.py | 83 --
sdks/python/apache_beam/utils/timestamp.py | 5 +-
sdks/python/apache_beam/utils/urns.py | 2 +
sdks/python/apache_beam/utils/value_provider.py | 103 ---
.../apache_beam/utils/value_provider_test.py | 145 ----
sdks/python/apache_beam/utils/windowed_value.py | 4 +-
.../apache_beam/utils/windowed_value_test.py | 2 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/generate_pydoc.sh | 15 +-
sdks/python/pom.xml | 2 +-
sdks/python/setup.py | 4 +-
sdks/python/tox.ini | 5 +
1287 files changed, 28743 insertions(+), 31376 deletions(-)
----------------------------------------------------------------------
[12/50] [abbrv] beam git commit: This closes #2302: Add unit tests to
Gearpump runner
Posted by ke...@apache.org.
This closes #2302: Add unit tests to Gearpump runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f4f23330
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f4f23330
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f4f23330
Branch: refs/heads/master
Commit: f4f233304836871884378c2a897ebcab07f08b2d
Parents: 555842a eb0d333
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Mar 29 21:03:08 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Mar 29 21:03:08 2017 -0700
----------------------------------------------------------------------
examples/java/pom.xml | 12 +++
pom.xml | 6 ++
runners/gearpump/README.md | 41 ++++++++-
runners/gearpump/pom.xml | 2 -
.../gearpump/GearpumpRunnerRegistrar.java | 4 +-
.../translators/WindowAssignTranslator.java | 2 +-
.../gearpump/translators/io/ValuesSource.java | 2 -
.../gearpump/GearpumpRunnerRegistrarTest.java | 55 ++++++++++++
.../runners/gearpump/PipelineOptionsTest.java | 73 ++++++++++++++++
.../translators/io/GearpumpSourceTest.java | 90 ++++++++++++++++++++
.../gearpump/translators/io/ValueSoureTest.java | 82 ++++++++++++++++++
11 files changed, 362 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
[03/50] [abbrv] beam git commit: [BEAM-79] Add SideInput support for
GearpumpRunner
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/4eb50d15/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
index 656fc6a..b8a5233 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
@@ -18,14 +18,28 @@
package org.apache.beam.runners.gearpump.translators.utils;
+import com.google.common.collect.Lists;
+
import java.time.Instant;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.runners.gearpump.translators.TranslationContext;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollectionView;
+
+import org.apache.gearpump.streaming.dsl.api.functions.FoldFunction;
+import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
import org.apache.gearpump.streaming.dsl.window.impl.Window;
+
/**
* Utility methods for translators.
*/
@@ -52,4 +66,137 @@ public class TranslatorUtils {
throw new RuntimeException("unknown window " + window.getClass().getName());
}
}
+
+ public static <InputT> JavaStream<RawUnionValue> withSideInputStream(
+ TranslationContext context,
+ JavaStream<WindowedValue<InputT>> inputStream,
+ Map<String, PCollectionView<?>> tagsToSideInputs) {
+ JavaStream<RawUnionValue> mainStream =
+ inputStream.map(new ToRawUnionValue<InputT>("0"), "map_to_RawUnionValue");
+
+ for (Map.Entry<String, PCollectionView<?>> tagToSideInput: tagsToSideInputs.entrySet()) {
+ // actually JavaStream<WindowedValue<List<?>>>
+ // check CreatePCollectionViewTranslator
+ JavaStream<WindowedValue<Object>> sideInputStream = context.getInputStream(
+ tagToSideInput.getValue());
+ mainStream = mainStream.merge(sideInputStream.map(new ToRawUnionValue<>(
+ tagToSideInput.getKey()), "map_to_RawUnionValue"), "merge_to_MainStream");
+ }
+ return mainStream;
+ }
+
+ public static Map<String, PCollectionView<?>> getTagsToSideInputs(
+ Collection<PCollectionView<?>> sideInputs) {
+ Map<String, PCollectionView<?>> tagsToSideInputs = new HashMap<>();
+ // tag 0 is reserved for main input
+ int tag = 1;
+ for (PCollectionView<?> sideInput: sideInputs) {
+ tagsToSideInputs.put(tag + "", sideInput);
+ tag++;
+ }
+ return tagsToSideInputs;
+ }
+
+ public static JavaStream<List<RawUnionValue>> toList(JavaStream<RawUnionValue> stream) {
+ return stream.fold(new FoldFunction<RawUnionValue, List<RawUnionValue>>() {
+
+ @Override
+ public List<RawUnionValue> init() {
+ return Lists.newArrayList();
+ }
+
+ @Override
+ public List<RawUnionValue> fold(List<RawUnionValue> accumulator,
+ RawUnionValue rawUnionValue) {
+ accumulator.add(rawUnionValue);
+ return accumulator;
+ }
+ }, "fold_to_iterable");
+ }
+
+ /**
+ * Converts @link{RawUnionValue} to @link{WindowedValue}.
+ */
+ public static class FromRawUnionValue<OutputT> extends
+ MapFunction<RawUnionValue, WindowedValue<OutputT>> {
+
+ private static final long serialVersionUID = -4764968219713478955L;
+
+ @Override
+ public WindowedValue<OutputT> map(RawUnionValue value) {
+ return (WindowedValue<OutputT>) value.getValue();
+ }
+ }
+
+ private static class ToRawUnionValue<T> extends
+ MapFunction<WindowedValue<T>, RawUnionValue> {
+
+ private static final long serialVersionUID = 8648852871014813583L;
+ private final String tag;
+
+ ToRawUnionValue(String tag) {
+ this.tag = tag;
+ }
+
+ @Override
+ public RawUnionValue map(WindowedValue<T> windowedValue) {
+ return new RawUnionValue(tag, windowedValue);
+ }
+ }
+
+
+ /**
+ * This is copied from org.apache.beam.sdk.transforms.join.RawUnionValue.
+ */
+ public static class RawUnionValue {
+ private final String unionTag;
+ private final Object value;
+
+ /**
+ * Constructs a partial union from the given union tag and value.
+ */
+ public RawUnionValue(String unionTag, Object value) {
+ this.unionTag = unionTag;
+ this.value = value;
+ }
+
+ public String getUnionTag() {
+ return unionTag;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return unionTag + ":" + value;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ RawUnionValue that = (RawUnionValue) o;
+
+ if (unionTag != that.unionTag) {
+ return false;
+ }
+ return value != null ? value.equals(that.value) : that.value == null;
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = unionTag.hashCode();
+ result = 31 * result + value.hashCode();
+ return result;
+ }
+ }
+
}
[21/50] [abbrv] beam git commit: Merge branch 'master' of
https://github.com/apache/beam into sync-master
Posted by ke...@apache.org.
Merge branch 'master' of https://github.com/apache/beam into sync-master
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/bc8da292
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/bc8da292
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/bc8da292
Branch: refs/heads/master
Commit: bc8da29299d153f5491380135ad7c7194ec8971b
Parents: 58546ac 5e3c5c6
Author: manuzhang <ow...@gmail.com>
Authored: Thu May 18 10:53:16 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Thu May 18 10:53:16 2017 +0800
----------------------------------------------------------------------
.gitignore | 1 +
.../jenkins/common_job_properties.groovy | 29 +-
.../job_beam_PerformanceTests_Dataflow.groovy | 3 +
...job_beam_PostCommit_Java_MavenInstall.groovy | 8 +-
..._PostCommit_Java_MavenInstall_Windows.groovy | 45 ++
...tCommit_Java_ValidatesRunner_Dataflow.groovy | 2 +-
.../job_beam_PreCommit_Java_MavenInstall.groovy | 5 +-
.../job_beam_Release_NightlySnapshot.groovy | 2 +-
README.md | 4 +-
examples/java/README.md | 64 +-
examples/java/pom.xml | 53 +-
.../apache/beam/examples/WindowedWordCount.java | 9 +-
.../beam/examples/common/ExampleUtils.java | 6 +-
.../examples/common/WriteOneFilePerWindow.java | 72 +-
.../org/apache/beam/examples/complete/README.md | 2 +-
.../apache/beam/examples/complete/TfIdf.java | 2 +-
.../org/apache/beam/examples/cookbook/README.md | 2 +-
.../beam/examples/WindowedWordCountIT.java | 26 +-
.../beam/examples/complete/TfIdfTest.java | 2 +-
examples/java8/pom.xml | 124 ++-
.../beam/examples/complete/game/GameStats.java | 15 +-
.../examples/complete/game/HourlyTeamScore.java | 58 +-
.../examples/complete/game/LeaderBoard.java | 62 +-
.../beam/examples/complete/game/README.md | 131 ---
.../beam/examples/complete/game/UserScore.java | 74 +-
.../complete/game/utils/WriteToBigQuery.java | 32 +-
.../complete/game/utils/WriteToText.java | 184 +++++
.../game/utils/WriteWindowedToBigQuery.java | 9 +-
examples/pom.xml | 2 +-
pom.xml | 71 +-
runners/apex/README.md | 76 --
runners/apex/pom.xml | 10 +-
.../beam/runners/apex/ApexPipelineOptions.java | 5 +
.../apache/beam/runners/apex/ApexRunner.java | 51 +-
.../beam/runners/apex/ApexRunnerRegistrar.java | 3 +-
.../beam/runners/apex/ApexRunnerResult.java | 2 -
.../beam/runners/apex/ApexYarnLauncher.java | 2 -
.../beam/runners/apex/TestApexRunner.java | 10 +-
.../translation/ApexPipelineTranslator.java | 23 +
.../apex/translation/ParDoTranslator.java | 68 +-
.../translation/ReadUnboundedTranslator.java | 1 -
.../apex/translation/TransformTranslator.java | 2 -
.../apex/translation/TranslationContext.java | 97 ++-
.../operators/ApexGroupByKeyOperator.java | 10 +-
.../operators/ApexParDoOperator.java | 131 ++-
.../operators/ApexProcessFnOperator.java | 8 +-
.../ApexReadUnboundedInputOperator.java | 2 -
.../operators/ApexTimerInternals.java | 23 +-
.../translation/utils/ApexStateInternals.java | 24 +-
.../apex/translation/utils/ApexStreamTuple.java | 16 +-
.../utils/CoderAdapterStreamCodec.java | 2 -
.../utils/SerializablePipelineOptions.java | 19 +-
.../translation/utils/StateInternalsProxy.java | 11 +-
.../utils/ValueAndCoderKryoSerializable.java | 2 -
.../apex/translation/utils/ValuesSource.java | 2 -
.../beam/runners/apex/ApexRunnerTest.java | 49 +-
.../beam/runners/apex/ApexYarnLauncherTest.java | 2 -
.../apex/examples/UnboundedTextSource.java | 2 -
.../runners/apex/examples/WordCountTest.java | 2 -
.../translation/ApexGroupByKeyOperatorTest.java | 4 +-
.../FlattenPCollectionTranslatorTest.java | 13 +-
.../apex/translation/ParDoTranslatorTest.java | 2 +-
.../operators/ApexTimerInternalsTest.java | 10 +-
.../utils/ApexStateInternalsTest.java | 12 +-
.../translation/utils/CollectionSource.java | 2 -
.../translation/utils/PipelineOptionsTest.java | 98 ++-
.../test/resources/beam-runners-apex.properties | 2 +-
runners/core-construction-java/pom.xml | 2 +-
.../DeduplicatedFlattenFactory.java | 2 +-
.../EmptyFlattenAsCreateFactory.java | 2 +-
.../core/construction/ForwardingPTransform.java | 6 +-
.../runners/core/construction/PCollections.java | 2 +-
.../core/construction/PTransformMatchers.java | 2 +-
.../construction/PTransformReplacements.java | 2 +-
.../runners/core/construction/PTransforms.java | 19 +-
.../core/construction/PrimitiveCreate.java | 4 +-
.../core/construction/SdkComponents.java | 54 +-
.../runners/core/construction/Triggers.java | 4 +-
.../UnboundedReadFromBoundedSource.java | 16 +-
.../UnsupportedOverrideFactory.java | 2 +-
.../core/construction/WindowingStrategies.java | 4 +-
.../runners/core/metrics/MetricFiltering.java | 102 +++
.../beam/runners/core/metrics/MetricKey.java | 43 +
.../beam/runners/core/metrics/package-info.java | 22 +
.../runners/core/construction/CodersTest.java | 18 +-
.../EmptyFlattenAsCreateFactoryTest.java | 2 +-
.../core/construction/PCollectionsTest.java | 17 +-
.../construction/PTransformMatchersTest.java | 34 +-
.../PTransformReplacementsTest.java | 2 +-
.../core/construction/PTransformsTest.java | 2 +-
.../construction/ReplacementOutputsTest.java | 2 +-
.../core/construction/SdkComponentsTest.java | 106 ++-
.../SingleInputOutputOverrideFactoryTest.java | 2 +-
.../construction/WindowingStrategiesTest.java | 4 +-
.../core/metrics/MetricFilteringTest.java | 148 ++++
runners/core-java/pom.xml | 2 +-
.../beam/runners/core/AssignWindowsDoFn.java | 78 --
.../beam/runners/core/BaseExecutionContext.java | 13 +-
.../apache/beam/runners/core/DoFnAdapters.java | 310 -------
.../apache/beam/runners/core/DoFnRunner.java | 2 +-
.../apache/beam/runners/core/DoFnRunners.java | 52 +-
.../core/ElementAndRestrictionCoder.java | 29 +-
.../beam/runners/core/ExecutionContext.java | 10 +-
.../runners/core/GlobalCombineFnRunner.java | 1 -
.../runners/core/GlobalCombineFnRunners.java | 41 +-
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 113 ---
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 94 ---
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 9 +-
.../core/GroupAlsoByWindowsAggregators.java | 26 +
.../runners/core/GroupAlsoByWindowsDoFn.java | 39 -
.../core/GroupByKeyViaGroupByKeyOnly.java | 2 +-
.../runners/core/InMemoryStateInternals.java | 22 +-
.../runners/core/InMemoryTimerInternals.java | 2 +-
.../beam/runners/core/KeyedWorkItemCoder.java | 22 +-
.../core/LateDataDroppingDoFnRunner.java | 9 +-
.../apache/beam/runners/core/LateDataUtils.java | 37 +-
.../runners/core/MergingActiveWindowSet.java | 2 +-
.../beam/runners/core/MergingStateAccessor.java | 2 +-
.../apache/beam/runners/core/NonEmptyPanes.java | 8 +-
.../beam/runners/core/NullSideInputReader.java | 61 ++
.../org/apache/beam/runners/core/OldDoFn.java | 335 --------
...eBoundedSplittableProcessElementInvoker.java | 18 +-
.../beam/runners/core/PaneInfoTracker.java | 4 +-
.../beam/runners/core/ProcessFnRunner.java | 3 +-
.../core/PushbackSideInputDoFnRunner.java | 2 +-
.../core/ReadyCheckingSideInputReader.java | 34 +
.../org/apache/beam/runners/core/ReduceFn.java | 6 +-
.../runners/core/ReduceFnContextFactory.java | 13 +-
.../beam/runners/core/ReduceFnRunner.java | 50 +-
.../beam/runners/core/SideInputHandler.java | 15 +-
.../beam/runners/core/SideInputReader.java | 47 ++
.../beam/runners/core/SimpleDoFnRunner.java | 223 ++++-
.../beam/runners/core/SimpleOldDoFnRunner.java | 500 ------------
.../core/SimplePushbackSideInputDoFnRunner.java | 3 +-
.../beam/runners/core/SplittableParDo.java | 50 +-
.../apache/beam/runners/core/StateAccessor.java | 2 +-
.../beam/runners/core/StateInternals.java | 4 +-
.../apache/beam/runners/core/StateMerging.java | 14 +-
.../apache/beam/runners/core/StateTable.java | 4 +-
.../org/apache/beam/runners/core/StateTag.java | 16 +-
.../org/apache/beam/runners/core/StateTags.java | 20 +-
.../beam/runners/core/StatefulDoFnRunner.java | 14 +-
.../beam/runners/core/SystemReduceFn.java | 8 +-
.../core/TestInMemoryStateInternals.java | 4 +-
.../beam/runners/core/TimerInternals.java | 30 +-
.../core/UnsupportedSideInputReader.java | 1 -
.../apache/beam/runners/core/WatermarkHold.java | 15 +-
.../core/WindowingInternalsAdapters.java | 75 --
.../beam/runners/core/metrics/CounterCell.java | 86 ++
.../beam/runners/core/metrics/DirtyState.java | 99 +++
.../runners/core/metrics/DistributionCell.java | 80 ++
.../runners/core/metrics/DistributionData.java | 62 ++
.../beam/runners/core/metrics/GaugeCell.java | 78 ++
.../beam/runners/core/metrics/GaugeData.java | 83 ++
.../beam/runners/core/metrics/MetricCell.java | 42 +
.../runners/core/metrics/MetricUpdates.java | 79 ++
.../core/metrics/MetricsContainerImpl.java | 188 +++++
.../core/metrics/MetricsContainerStepMap.java | 495 ++++++++++++
.../beam/runners/core/metrics/MetricsMap.java | 88 ++
.../beam/runners/core/metrics/package-info.java | 22 +
.../AfterDelayFromFirstElementStateMachine.java | 6 +-
.../core/triggers/AfterPaneStateMachine.java | 2 +-
.../AfterProcessingTimeStateMachine.java | 2 +-
...rSynchronizedProcessingTimeStateMachine.java | 2 +-
.../triggers/AfterWatermarkStateMachine.java | 2 +-
.../triggers/DefaultTriggerStateMachine.java | 2 +-
.../triggers/ReshuffleTriggerStateMachine.java | 2 +-
.../core/triggers/TriggerStateMachine.java | 2 +-
.../TriggerStateMachineContextFactory.java | 6 +-
.../triggers/TriggerStateMachineRunner.java | 4 +-
runners/core-java/src/test/java/.placeholder | 0
...roupAlsoByWindowViaOutputBufferDoFnTest.java | 109 ---
.../core/GroupAlsoByWindowsProperties.java | 744 -----------------
.../core/InMemoryStateInternalsTest.java | 16 +-
.../core/InMemoryTimerInternalsTest.java | 2 +-
.../runners/core/KeyedWorkItemCoderTest.java | 2 +-
.../core/LateDataDroppingDoFnRunnerTest.java | 11 +-
.../beam/runners/core/LateDataUtilsTest.java | 90 +++
.../apache/beam/runners/core/NoOpOldDoFn.java | 65 --
.../apache/beam/runners/core/OldDoFnTest.java | 51 --
...ndedSplittableProcessElementInvokerTest.java | 1 -
.../beam/runners/core/ReduceFnRunnerTest.java | 41 +-
.../beam/runners/core/ReduceFnTester.java | 8 +-
.../beam/runners/core/SideInputHandlerTest.java | 2 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 17 +-
.../runners/core/SimpleOldDoFnRunnerTest.java | 86 --
.../SimplePushbackSideInputDoFnRunnerTest.java | 3 +-
.../beam/runners/core/SplittableParDoTest.java | 7 +-
.../runners/core/StatefulDoFnRunnerTest.java | 19 +-
.../beam/runners/core/TimerInternalsTest.java | 2 +-
.../runners/core/metrics/CounterCellTest.java | 56 ++
.../runners/core/metrics/DirtyStateTest.java | 56 ++
.../core/metrics/DistributionCellTest.java | 54 ++
.../runners/core/metrics/GaugeCellTest.java | 51 ++
.../core/metrics/MetricUpdateMatchers.java | 82 ++
.../core/metrics/MetricsContainerImplTest.java | 130 +++
.../metrics/MetricsContainerStepMapTest.java | 272 +++++++
.../runners/core/metrics/MetricsMapTest.java | 103 +++
.../triggers/TriggerStateMachineTester.java | 6 +-
.../core/triggers/TriggerStateMachinesTest.java | 2 +-
runners/direct-java/pom.xml | 100 ++-
.../direct/AbstractModelEnforcement.java | 1 -
.../direct/BoundedReadEvaluatorFactory.java | 17 +-
.../beam/runners/direct/BundleFactory.java | 3 +-
.../org/apache/beam/runners/direct/Clock.java | 2 +-
.../runners/direct/CloningBundleFactory.java | 2 -
.../beam/runners/direct/CommittedBundle.java | 82 ++
.../beam/runners/direct/CommittedResult.java | 3 +-
.../beam/runners/direct/CompletionCallback.java | 9 +-
.../CopyOnAccessInMemoryStateInternals.java | 20 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 2 +-
.../apache/beam/runners/direct/DirectGraph.java | 14 +-
.../beam/runners/direct/DirectGraphVisitor.java | 2 +-
.../beam/runners/direct/DirectGroupByKey.java | 2 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 2 +-
.../beam/runners/direct/DirectMetrics.java | 25 +-
.../beam/runners/direct/DirectOptions.java | 2 +-
.../beam/runners/direct/DirectRegistrar.java | 12 +-
.../beam/runners/direct/DirectRunner.java | 158 +---
.../runners/direct/DirectTimerInternals.java | 2 +-
.../runners/direct/DisplayDataValidator.java | 6 +-
.../runners/direct/DoFnLifecycleManager.java | 2 +-
.../beam/runners/direct/EmptyInputProvider.java | 3 +-
.../runners/direct/EmptyTransformEvaluator.java | 50 --
.../beam/runners/direct/EvaluationContext.java | 14 +-
.../runners/direct/ExecutorServiceFactory.java | 2 +-
.../direct/ExecutorServiceParallelExecutor.java | 42 +-
.../runners/direct/FlattenEvaluatorFactory.java | 4 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 18 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 4 +-
.../ImmutabilityCheckingBundleFactory.java | 2 -
.../direct/ImmutabilityEnforcementFactory.java | 3 +-
.../direct/ImmutableListBundleFactory.java | 2 -
.../beam/runners/direct/ModelEnforcement.java | 12 +-
.../runners/direct/ModelEnforcementFactory.java | 5 +-
.../beam/runners/direct/NanosOffsetClock.java | 2 +-
.../runners/direct/PCollectionViewWindow.java | 67 ++
.../runners/direct/PCollectionViewWriter.java | 34 +
.../beam/runners/direct/ParDoEvaluator.java | 7 +-
.../runners/direct/ParDoEvaluatorFactory.java | 3 +-
.../direct/ParDoMultiOverrideFactory.java | 7 +-
.../direct/PassthroughTransformEvaluator.java | 3 +-
.../beam/runners/direct/PipelineExecutor.java | 3 +-
.../beam/runners/direct/RootInputProvider.java | 3 +-
.../runners/direct/RootProviderRegistry.java | 3 +-
.../beam/runners/direct/SideInputContainer.java | 7 +-
...littableProcessElementsEvaluatorFactory.java | 18 +-
.../direct/StatefulParDoEvaluatorFactory.java | 7 +-
.../apache/beam/runners/direct/StepAndKey.java | 2 +-
.../runners/direct/StepTransformResult.java | 7 +-
.../direct/TestStreamEvaluatorFactory.java | 6 +-
.../beam/runners/direct/TransformEvaluator.java | 3 +-
.../direct/TransformEvaluatorFactory.java | 8 +-
.../direct/TransformEvaluatorRegistry.java | 3 +-
.../beam/runners/direct/TransformExecutor.java | 25 +-
.../beam/runners/direct/TransformResult.java | 7 +-
.../direct/UnboundedReadEvaluatorFactory.java | 4 +-
.../beam/runners/direct/UncommittedBundle.java | 57 ++
.../runners/direct/ViewEvaluatorFactory.java | 5 +-
.../runners/direct/ViewOverrideFactory.java | 2 +-
.../direct/WatermarkCallbackExecutor.java | 4 +-
.../beam/runners/direct/WatermarkManager.java | 7 +-
.../runners/direct/WindowEvaluatorFactory.java | 4 +-
.../direct/WriteWithShardingFactory.java | 2 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 4 +-
.../direct/CloningBundleFactoryTest.java | 32 +-
.../runners/direct/CommittedResultTest.java | 16 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 16 +-
.../runners/direct/DirectGraphVisitorTest.java | 2 +-
.../beam/runners/direct/DirectGraphs.java | 2 +-
.../DirectGroupByKeyOverrideFactoryTest.java | 2 +-
.../beam/runners/direct/DirectMetricsTest.java | 15 +-
.../direct/DirectRunnerApiSurfaceTest.java | 68 ++
.../beam/runners/direct/DirectRunnerTest.java | 11 +-
.../direct/DirectTimerInternalsTest.java | 2 +-
...leManagerRemovingTransformEvaluatorTest.java | 2 +-
.../runners/direct/EvaluationContextTest.java | 13 +-
.../direct/FlattenEvaluatorFactoryTest.java | 4 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 2 -
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 2 -
.../ImmutabilityCheckingBundleFactoryTest.java | 2 -
.../ImmutabilityEnforcementFactoryTest.java | 3 +-
.../direct/ImmutableListBundleFactoryTest.java | 2 -
.../direct/KeyedPValueTrackingVisitorTest.java | 2 +-
.../apache/beam/runners/direct/MockClock.java | 2 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 5 +-
.../runners/direct/SideInputContainerTest.java | 8 +-
.../StatefulParDoEvaluatorFactoryTest.java | 14 +-
.../runners/direct/StepTransformResultTest.java | 3 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 3 +-
.../runners/direct/TransformExecutorTest.java | 8 +-
.../UnboundedReadEvaluatorFactoryTest.java | 13 +-
.../direct/ViewEvaluatorFactoryTest.java | 6 +-
.../runners/direct/ViewOverrideFactoryTest.java | 6 +-
.../direct/WatermarkCallbackExecutorTest.java | 4 +-
.../runners/direct/WatermarkManagerTest.java | 6 +-
.../direct/WindowEvaluatorFactoryTest.java | 2 -
.../direct/WriteWithShardingFactoryTest.java | 61 +-
runners/flink/pom.xml | 7 +-
.../flink/FlinkBatchTransformTranslators.java | 4 +-
.../flink/FlinkBatchTranslationContext.java | 4 +-
.../flink/FlinkDetachedRunnerResult.java | 1 -
.../apache/beam/runners/flink/FlinkRunner.java | 5 +-
.../runners/flink/FlinkRunnerRegistrar.java | 2 +-
.../beam/runners/flink/FlinkRunnerResult.java | 8 +-
.../flink/FlinkStreamingPipelineTranslator.java | 2 +-
.../FlinkStreamingTransformTranslators.java | 84 +-
.../flink/FlinkStreamingTranslationContext.java | 2 +-
.../flink/FlinkStreamingViewOverrides.java | 2 +-
.../beam/runners/flink/TestFlinkRunner.java | 2 +-
.../metrics/DoFnRunnerWithMetricsUpdate.java | 17 +-
.../flink/metrics/FlinkMetricContainer.java | 276 ++-----
.../flink/metrics/FlinkMetricResults.java | 146 ----
.../flink/metrics/MetricsAccumulator.java | 60 ++
.../flink/metrics/ReaderInvocationUtil.java | 12 +-
.../functions/AbstractFlinkCombineRunner.java | 4 +-
.../functions/FlinkDoFnFunction.java | 3 +-
.../FlinkMergingNonShuffleReduceFunction.java | 2 +-
.../functions/FlinkPartialReduceFunction.java | 2 +-
.../functions/FlinkReduceFunction.java | 2 +-
.../functions/FlinkSideInputReader.java | 4 +-
.../functions/FlinkStatefulDoFnFunction.java | 3 +-
.../functions/HashingFlinkCombineRunner.java | 4 +-
.../functions/SortingFlinkCombineRunner.java | 4 +-
.../translation/types/CoderTypeSerializer.java | 4 +-
.../types/EncodedValueSerializer.java | 2 -
.../utils/SerializedPipelineOptions.java | 20 +-
.../translation/wrappers/SourceInputFormat.java | 8 +-
.../wrappers/streaming/DoFnOperator.java | 58 +-
.../streaming/SingletonKeyedWorkItemCoder.java | 20 +-
.../streaming/SplittableDoFnOperator.java | 30 +-
.../wrappers/streaming/WindowDoFnOperator.java | 2 +-
.../streaming/io/BoundedSourceWrapper.java | 8 +-
.../wrappers/streaming/io/DedupingOperator.java | 187 +++++
.../streaming/io/UnboundedSourceWrapper.java | 24 +-
.../state/FlinkBroadcastStateInternals.java | 20 +-
.../state/FlinkKeyGroupStateInternals.java | 29 +-
.../state/FlinkSplitStateInternals.java | 20 +-
.../streaming/state/FlinkStateInternals.java | 22 +-
.../beam/runners/flink/PipelineOptionsTest.java | 89 +-
.../flink/streaming/DedupingOperatorTest.java | 131 +++
.../flink/streaming/DoFnOperatorTest.java | 16 +-
.../FlinkBroadcastStateInternalsTest.java | 10 +-
.../FlinkKeyGroupStateInternalsTest.java | 4 +-
.../streaming/FlinkSplitStateInternalsTest.java | 4 +-
.../streaming/FlinkStateInternalsTest.java | 12 +-
.../streaming/UnboundedSourceWrapperTest.java | 29 +-
runners/google-cloud-dataflow-java/pom.xml | 76 +-
.../beam/runners/dataflow/AssignWindows.java | 2 +-
.../dataflow/BatchStatefulParDoOverrides.java | 4 +-
.../runners/dataflow/BatchViewOverrides.java | 32 +-
.../beam/runners/dataflow/DataflowMetrics.java | 4 +-
.../runners/dataflow/DataflowPipelineJob.java | 20 +-
.../dataflow/DataflowPipelineRegistrar.java | 4 +-
.../dataflow/DataflowPipelineTranslator.java | 40 +-
.../beam/runners/dataflow/DataflowRunner.java | 110 +--
.../dataflow/PrimitiveParDoSingleFactory.java | 2 +-
.../beam/runners/dataflow/ReadTranslator.java | 22 +-
.../dataflow/ReshuffleOverrideFactory.java | 8 +-
.../dataflow/StreamingViewOverrides.java | 2 +-
.../dataflow/TestDataflowPipelineOptions.java | 28 +
.../runners/dataflow/TestDataflowRunner.java | 388 +++++++++
.../runners/dataflow/TransformTranslator.java | 2 +-
.../dataflow/internal/CustomSources.java | 18 +-
.../runners/dataflow/internal/IsmFormat.java | 99 ++-
.../options/DataflowPipelineOptions.java | 18 +-
.../testing/TestDataflowPipelineOptions.java | 28 -
.../dataflow/testing/TestDataflowRunner.java | 326 --------
.../runners/dataflow/testing/package-info.java | 24 -
.../util/AvroCoderCloudObjectTranslator.java | 2 -
.../beam/runners/dataflow/util/CloudObject.java | 1 -
.../runners/dataflow/util/CloudObjectKinds.java | 2 -
.../dataflow/util/CloudObjectTranslator.java | 2 -
.../dataflow/util/CloudObjectTranslators.java | 23 +-
.../runners/dataflow/util/CloudObjects.java | 1 -
.../CoderCloudObjectTranslatorRegistrar.java | 1 -
.../dataflow/util/DataflowTemplateJob.java | 1 -
...aultCoderCloudObjectTranslatorRegistrar.java | 2 -
.../beam/runners/dataflow/util/DoFnInfo.java | 2 +-
.../beam/runners/dataflow/util/GcsStager.java | 26 +-
.../beam/runners/dataflow/util/PackageUtil.java | 41 +-
.../runners/dataflow/util/PropertyNames.java | 112 +++
.../runners/dataflow/util/RandomAccessData.java | 26 +-
.../SerializableCoderCloudObjectTranslator.java | 2 -
.../beam/runners/dataflow/util/Serializer.java | 262 ------
.../beam/runners/dataflow/util/Structs.java | 372 +++++++++
.../BatchStatefulParDoOverridesTest.java | 6 +-
.../runners/dataflow/DataflowMetricsTest.java | 7 +-
.../dataflow/DataflowPipelineJobTest.java | 18 +-
.../dataflow/DataflowPipelineRegistrarTest.java | 2 +-
.../DataflowPipelineTranslatorTest.java | 133 ++-
.../runners/dataflow/DataflowRunnerTest.java | 90 +--
.../PrimitiveParDoSingleFactoryTest.java | 2 +-
.../dataflow/TestDataflowRunnerTest.java | 634 +++++++++++++++
.../options/DataflowPipelineOptionsTest.java | 21 +-
.../options/DataflowProfilingOptionsTest.java | 4 +-
.../DataflowWorkerLoggingOptionsTest.java | 4 +-
.../testing/TestDataflowRunnerTest.java | 661 ---------------
.../transforms/DataflowGroupByKeyTest.java | 4 +-
.../dataflow/transforms/DataflowViewTest.java | 4 +-
.../runners/dataflow/util/CloudObjectsTest.java | 9 +-
.../runners/dataflow/util/PackageUtilTest.java | 145 ++--
.../dataflow/util/RandomAccessDataTest.java | 6 +-
.../beam/runners/dataflow/util/StructsTest.java | 206 +++++
runners/pom.xml | 2 +-
runners/spark/README.md | 124 ---
runners/spark/pom.xml | 54 +-
.../beam/runners/spark/SparkPipelineResult.java | 8 +-
.../apache/beam/runners/spark/SparkRunner.java | 8 +-
.../beam/runners/spark/SparkRunnerDebugger.java | 7 +-
.../runners/spark/SparkRunnerRegistrar.java | 2 +-
.../beam/runners/spark/TestSparkRunner.java | 83 +-
.../spark/aggregators/NamedAggregators.java | 6 +-
.../beam/runners/spark/coders/CoderHelpers.java | 1 -
.../spark/coders/StatelessJavaSerializer.java | 1 -
.../beam/runners/spark/io/CreateStream.java | 2 +-
.../beam/runners/spark/io/MicrobatchSource.java | 2 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 177 ++--
.../runners/spark/io/SparkUnboundedSource.java | 20 +-
.../runners/spark/metrics/AggregatorMetric.java | 1 -
.../spark/metrics/MetricsAccumulator.java | 20 +-
.../spark/metrics/MetricsAccumulatorParam.java | 20 +-
.../runners/spark/metrics/SparkBeamMetric.java | 11 +-
.../spark/metrics/SparkBeamMetricSource.java | 3 +-
.../spark/metrics/SparkMetricResults.java | 172 ----
.../spark/metrics/SparkMetricsContainer.java | 174 ----
.../SparkGroupAlsoByWindowViaWindowSet.java | 17 +-
.../spark/stateful/SparkStateInternals.java | 20 +-
.../spark/stateful/SparkTimerInternals.java | 2 +-
.../spark/stateful/StateSpecFunctions.java | 9 +-
.../translation/DoFnRunnerWithMetrics.java | 11 +-
.../spark/translation/EvaluationContext.java | 18 +-
.../translation/GroupCombineFunctions.java | 5 +-
.../spark/translation/MultiDoFnFunction.java | 8 +-
.../translation/SparkAbstractCombineFn.java | 4 +-
.../spark/translation/SparkAssignWindowFn.java | 4 +-
.../spark/translation/SparkGlobalCombineFn.java | 2 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 6 +-
.../spark/translation/SparkKeyedCombineFn.java | 2 +-
.../spark/translation/SparkRuntimeContext.java | 26 +-
.../translation/StorageLevelPTransform.java | 2 +-
.../spark/translation/TransformTranslator.java | 8 +-
.../spark/translation/TranslationUtils.java | 3 +-
.../SparkRunnerStreamingContextFactory.java | 2 +-
.../streaming/StreamingTransformTranslator.java | 8 +-
.../spark/util/GlobalWatermarkHolder.java | 1 -
.../util/SinglePrimitiveOutputPTransform.java | 2 +-
.../spark/util/SparkSideInputReader.java | 4 +-
.../apache/beam/runners/spark/CacheTest.java | 14 +-
.../beam/runners/spark/ForceStreamingTest.java | 18 +-
.../apache/beam/runners/spark/PipelineRule.java | 109 ---
.../runners/spark/ProvidedSparkContextTest.java | 10 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 15 +-
.../beam/runners/spark/StreamingTest.java | 23 +
.../TestSparkPipelineOptionsRegistrar.java | 36 +
.../metrics/sink/SparkMetricsSinkTest.java | 12 +-
.../beam/runners/spark/io/AvroPipelineTest.java | 10 +-
.../beam/runners/spark/io/NumShardsTest.java | 6 +-
.../spark/io/ReaderToIteratorAdapterTest.java | 145 ++++
.../translation/SparkRuntimeContextTest.java | 122 +++
.../spark/translation/StorageLevelTest.java | 31 +-
.../translation/streaming/CreateStreamTest.java | 53 +-
.../ResumeFromCheckpointStreamingTest.java | 64 +-
.../streaming/StreamingSourceMetricsTest.java | 16 +-
.../streaming/TrackStreamingSourcesTest.java | 4 +-
sdks/common/fn-api/pom.xml | 2 +-
sdks/common/pom.xml | 2 +-
sdks/common/runner-api/pom.xml | 5 +-
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/findbugs-filter.xml | 12 +-
sdks/java/core/pom.xml | 93 ++-
.../main/java/org/apache/beam/sdk/Pipeline.java | 101 ++-
.../org/apache/beam/sdk/PipelineResult.java | 11 +-
.../org/apache/beam/sdk/PipelineRunner.java | 61 ++
.../beam/sdk/annotations/Experimental.java | 35 +-
.../apache/beam/sdk/annotations/Internal.java | 6 +-
.../org/apache/beam/sdk/coders/AtomicCoder.java | 85 ++
.../org/apache/beam/sdk/coders/AvroCoder.java | 70 +-
.../apache/beam/sdk/coders/BigDecimalCoder.java | 25 +-
.../beam/sdk/coders/BigEndianIntegerCoder.java | 10 +-
.../beam/sdk/coders/BigEndianLongCoder.java | 12 +-
.../apache/beam/sdk/coders/BigIntegerCoder.java | 21 +-
.../org/apache/beam/sdk/coders/BitSetCoder.java | 15 +-
.../apache/beam/sdk/coders/ByteArrayCoder.java | 41 +-
.../org/apache/beam/sdk/coders/ByteCoder.java | 10 +-
.../sdk/coders/CannotProvideCoderException.java | 2 +-
.../java/org/apache/beam/sdk/coders/Coder.java | 178 +++-
.../apache/beam/sdk/coders/CoderFactories.java | 290 -------
.../apache/beam/sdk/coders/CoderFactory.java | 44 -
.../apache/beam/sdk/coders/CoderProvider.java | 19 +-
.../beam/sdk/coders/CoderProviderRegistrar.java | 42 +
.../apache/beam/sdk/coders/CoderProviders.java | 240 +++---
.../apache/beam/sdk/coders/CoderRegistrar.java | 45 --
.../apache/beam/sdk/coders/CoderRegistry.java | 618 +++++---------
.../apache/beam/sdk/coders/CollectionCoder.java | 22 -
.../org/apache/beam/sdk/coders/CustomCoder.java | 26 +-
.../apache/beam/sdk/coders/DefaultCoder.java | 119 ++-
.../apache/beam/sdk/coders/DelegateCoder.java | 11 +
.../org/apache/beam/sdk/coders/DoubleCoder.java | 10 +-
.../apache/beam/sdk/coders/DurationCoder.java | 18 +-
.../apache/beam/sdk/coders/InstantCoder.java | 18 +-
.../apache/beam/sdk/coders/IterableCoder.java | 22 -
.../beam/sdk/coders/IterableLikeCoder.java | 35 +-
.../org/apache/beam/sdk/coders/KvCoder.java | 51 +-
.../beam/sdk/coders/LengthPrefixCoder.java | 28 +-
.../org/apache/beam/sdk/coders/ListCoder.java | 24 +-
.../org/apache/beam/sdk/coders/MapCoder.java | 47 +-
.../apache/beam/sdk/coders/NullableCoder.java | 31 +-
.../beam/sdk/coders/SerializableCoder.java | 66 +-
.../org/apache/beam/sdk/coders/SetCoder.java | 25 -
.../beam/sdk/coders/StringDelegateCoder.java | 12 +-
.../apache/beam/sdk/coders/StringUtf8Coder.java | 29 +-
.../beam/sdk/coders/StructuralByteArray.java | 4 +-
.../apache/beam/sdk/coders/StructuredCoder.java | 74 +-
.../beam/sdk/coders/TextualIntegerCoder.java | 17 +-
.../org/apache/beam/sdk/coders/VarIntCoder.java | 10 +-
.../apache/beam/sdk/coders/VarLongCoder.java | 15 +-
.../org/apache/beam/sdk/coders/VoidCoder.java | 12 +-
.../apache/beam/sdk/coders/package-info.java | 2 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 219 +++--
.../java/org/apache/beam/sdk/io/AvroSink.java | 34 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 23 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 4 +-
.../apache/beam/sdk/io/CompressedSource.java | 6 +
.../beam/sdk/io/DefaultFilenamePolicy.java | 169 ++++
.../org/apache/beam/sdk/io/FileBasedSink.java | 803 ++++++++++---------
.../org/apache/beam/sdk/io/FileBasedSource.java | 39 +-
.../java/org/apache/beam/sdk/io/FileSystem.java | 3 +
.../apache/beam/sdk/io/FileSystemRegistrar.java | 3 +
.../org/apache/beam/sdk/io/FileSystems.java | 65 +-
.../beam/sdk/io/LocalFileSystemRegistrar.java | 3 +
.../org/apache/beam/sdk/io/LocalResources.java | 3 +
.../main/java/org/apache/beam/sdk/io/Read.java | 8 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 153 ++--
.../java/org/apache/beam/sdk/io/TextIO.java | 255 +++---
.../java/org/apache/beam/sdk/io/TextSink.java | 32 +-
.../org/apache/beam/sdk/io/UnboundedSource.java | 23 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 285 ++++---
.../org/apache/beam/sdk/io/fs/ResourceId.java | 3 +
.../apache/beam/sdk/metrics/CounterCell.java | 67 --
.../org/apache/beam/sdk/metrics/DirtyState.java | 98 ---
.../beam/sdk/metrics/DistributionCell.java | 64 --
.../beam/sdk/metrics/DistributionData.java | 60 --
.../beam/sdk/metrics/DistributionResult.java | 3 +
.../org/apache/beam/sdk/metrics/GaugeCell.java | 54 --
.../org/apache/beam/sdk/metrics/GaugeData.java | 81 --
.../apache/beam/sdk/metrics/GaugeResult.java | 3 +
.../org/apache/beam/sdk/metrics/Metric.java | 7 +-
.../org/apache/beam/sdk/metrics/MetricCell.java | 41 -
.../beam/sdk/metrics/MetricFiltering.java | 99 ---
.../org/apache/beam/sdk/metrics/MetricKey.java | 41 -
.../apache/beam/sdk/metrics/MetricUpdates.java | 78 --
.../org/apache/beam/sdk/metrics/Metrics.java | 6 +-
.../beam/sdk/metrics/MetricsContainer.java | 129 +--
.../beam/sdk/metrics/MetricsEnvironment.java | 14 +-
.../org/apache/beam/sdk/metrics/MetricsMap.java | 86 --
.../apache/beam/sdk/metrics/SinkMetrics.java | 4 +
.../apache/beam/sdk/metrics/SourceMetrics.java | 3 +
.../DefaultPipelineOptionsRegistrar.java | 3 +-
.../beam/sdk/options/PipelineOptionSpec.java | 4 +-
.../beam/sdk/options/PipelineOptions.java | 5 +-
.../sdk/options/PipelineOptionsFactory.java | 2 +-
.../apache/beam/sdk/options/ValueProvider.java | 15 +-
.../beam/sdk/options/ValueProviderUtils.java | 60 --
.../apache/beam/sdk/options/ValueProviders.java | 61 ++
.../java/org/apache/beam/sdk/package-info.java | 2 +-
.../beam/sdk/runners/AppliedPTransform.java | 73 ++
.../beam/sdk/runners/PTransformMatcher.java | 7 +-
.../beam/sdk/runners/PTransformOverride.java | 6 +-
.../sdk/runners/PTransformOverrideFactory.java | 8 +-
.../apache/beam/sdk/runners/PipelineRunner.java | 65 --
.../sdk/runners/PipelineRunnerRegistrar.java | 5 +
.../beam/sdk/runners/TransformHierarchy.java | 15 +-
.../apache/beam/sdk/runners/package-info.java | 4 +-
.../org/apache/beam/sdk/state/BagState.java | 36 +
.../apache/beam/sdk/state/CombiningState.java | 55 ++
.../apache/beam/sdk/state/GroupingState.java | 44 +
.../org/apache/beam/sdk/state/MapState.java | 73 ++
.../apache/beam/sdk/state/ReadableState.java | 51 ++
.../apache/beam/sdk/state/ReadableStates.java | 44 +
.../org/apache/beam/sdk/state/SetState.java | 47 ++
.../java/org/apache/beam/sdk/state/State.java | 36 +
.../org/apache/beam/sdk/state/StateBinder.java | 70 ++
.../org/apache/beam/sdk/state/StateContext.java | 46 ++
.../apache/beam/sdk/state/StateContexts.java | 51 ++
.../org/apache/beam/sdk/state/StateSpec.java | 63 ++
.../org/apache/beam/sdk/state/StateSpecs.java | 683 ++++++++++++++++
.../org/apache/beam/sdk/state/TimeDomain.java | 49 ++
.../java/org/apache/beam/sdk/state/Timer.java | 70 ++
.../org/apache/beam/sdk/state/TimerSpec.java | 30 +
.../org/apache/beam/sdk/state/TimerSpecs.java | 39 +
.../java/org/apache/beam/sdk/state/Timers.java | 58 ++
.../org/apache/beam/sdk/state/ValueState.java | 35 +
.../beam/sdk/state/WatermarkHoldState.java | 41 +
.../org/apache/beam/sdk/state/package-info.java | 22 +
.../beam/sdk/testing/CoderProperties.java | 6 +-
.../apache/beam/sdk/testing/CrashingRunner.java | 2 +-
.../beam/sdk/testing/FileChecksumMatcher.java | 2 +-
.../beam/sdk/testing/MatcherDeserializer.java | 4 +-
.../beam/sdk/testing/MatcherSerializer.java | 4 +-
.../apache/beam/sdk/testing/NeedsRunner.java | 2 +-
.../org/apache/beam/sdk/testing/PAssert.java | 4 +-
.../apache/beam/sdk/testing/TestPipeline.java | 77 +-
.../beam/sdk/testing/TestPipelineOptions.java | 2 -
.../org/apache/beam/sdk/testing/TestStream.java | 4 +-
.../apache/beam/sdk/testing/UsesMapState.java | 2 +-
.../apache/beam/sdk/testing/UsesSetState.java | 2 +-
.../apache/beam/sdk/transforms/Aggregator.java | 34 -
.../beam/sdk/transforms/AppliedPTransform.java | 73 --
.../sdk/transforms/ApproximateQuantiles.java | 90 ++-
.../org/apache/beam/sdk/transforms/Combine.java | 44 +-
.../beam/sdk/transforms/CombineFnBase.java | 26 +-
.../apache/beam/sdk/transforms/CombineFns.java | 25 +-
.../beam/sdk/transforms/CombineWithContext.java | 3 +-
.../org/apache/beam/sdk/transforms/Count.java | 12 +-
.../org/apache/beam/sdk/transforms/Create.java | 76 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 91 ++-
.../apache/beam/sdk/transforms/DoFnTester.java | 66 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 2 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 2 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 71 +-
.../org/apache/beam/sdk/transforms/Latest.java | 15 +-
.../beam/sdk/transforms/Materialization.java | 8 +-
.../beam/sdk/transforms/Materializations.java | 11 +-
.../org/apache/beam/sdk/transforms/Mean.java | 19 +-
.../apache/beam/sdk/transforms/PTransform.java | 10 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 20 +-
.../org/apache/beam/sdk/transforms/Regex.java | 1 -
.../beam/sdk/transforms/ReifyTimestamps.java | 79 ++
.../apache/beam/sdk/transforms/Reshuffle.java | 97 +++
.../org/apache/beam/sdk/transforms/Top.java | 46 +-
.../org/apache/beam/sdk/transforms/View.java | 40 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 6 +-
.../apache/beam/sdk/transforms/WithKeys.java | 4 +-
.../beam/sdk/transforms/join/CoGbkResult.java | 20 +-
.../beam/sdk/transforms/join/UnionCoder.java | 29 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 31 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 20 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 23 +-
.../sdk/transforms/reflect/DoFnSignature.java | 50 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 66 +-
.../reflect/StableInvokerNamingStrategy.java | 54 ++
.../sdk/transforms/reflect/package-info.java | 5 +-
.../beam/sdk/transforms/windowing/AfterAll.java | 8 +-
.../sdk/transforms/windowing/AfterEach.java | 2 +-
.../sdk/transforms/windowing/AfterFirst.java | 5 +-
.../sdk/transforms/windowing/AfterPane.java | 5 +-
.../windowing/AfterProcessingTime.java | 6 +-
.../transforms/windowing/AfterWatermark.java | 6 +-
.../transforms/windowing/DefaultTrigger.java | 10 +-
.../sdk/transforms/windowing/GlobalWindow.java | 10 +-
.../sdk/transforms/windowing/GlobalWindows.java | 5 +-
.../transforms/windowing/IntervalWindow.java | 19 +-
.../transforms/windowing/InvalidWindows.java | 2 +-
.../MergeOverlappingIntervalWindows.java | 6 +-
.../beam/sdk/transforms/windowing/Never.java | 4 +-
.../windowing/NonMergingWindowFn.java | 2 +-
.../transforms/windowing/OrFinallyTrigger.java | 7 +-
.../beam/sdk/transforms/windowing/PaneInfo.java | 18 +-
.../sdk/transforms/windowing/Repeatedly.java | 6 +-
.../transforms/windowing/ReshuffleTrigger.java | 58 ++
.../beam/sdk/transforms/windowing/Sessions.java | 13 +-
.../transforms/windowing/TimestampCombiner.java | 5 +
.../windowing/TimestampTransform.java | 41 +-
.../beam/sdk/transforms/windowing/Trigger.java | 18 +-
.../beam/sdk/transforms/windowing/Window.java | 16 +-
.../beam/sdk/transforms/windowing/WindowFn.java | 10 +-
.../transforms/windowing/WindowMappingFn.java | 12 +-
.../org/apache/beam/sdk/util/ApiSurface.java | 10 +-
.../apache/beam/sdk/util/AppliedCombineFn.java | 1 +
.../java/org/apache/beam/sdk/util/BackOff.java | 81 ++
.../org/apache/beam/sdk/util/BackOffUtils.java | 57 ++
.../org/apache/beam/sdk/util/BitSetCoder.java | 21 +-
.../org/apache/beam/sdk/util/ClassPath.java | 1 +
.../apache/beam/sdk/util/CloudKnownType.java | 143 ----
.../org/apache/beam/sdk/util/CloudObject.java | 187 -----
.../org/apache/beam/sdk/util/CoderUtils.java | 127 +--
.../beam/sdk/util/CombineContextFactory.java | 27 +-
.../org/apache/beam/sdk/util/CombineFnUtil.java | 2 +-
.../beam/sdk/util/ExplicitShardedFile.java | 3 -
.../beam/sdk/util/FileIOChannelFactory.java | 223 -----
.../sdk/util/FileIOChannelFactoryRegistrar.java | 38 -
.../org/apache/beam/sdk/util/FluentBackoff.java | 1 -
.../apache/beam/sdk/util/IOChannelFactory.java | 127 ---
.../sdk/util/IOChannelFactoryRegistrar.java | 48 --
.../apache/beam/sdk/util/IOChannelUtils.java | 280 -------
.../apache/beam/sdk/util/IdentityWindowFn.java | 1 +
.../apache/beam/sdk/util/NoopPathValidator.java | 49 --
.../beam/sdk/util/NullSideInputReader.java | 61 --
.../beam/sdk/util/NumberedShardedFile.java | 7 +-
.../beam/sdk/util/PCollectionViewWindow.java | 67 --
.../apache/beam/sdk/util/PCollectionViews.java | 496 ------------
.../org/apache/beam/sdk/util/PathValidator.java | 48 --
.../org/apache/beam/sdk/util/PropertyNames.java | 112 ---
.../sdk/util/ReadyCheckingSideInputReader.java | 34 -
.../apache/beam/sdk/util/ReifyTimestamps.java | 76 --
.../org/apache/beam/sdk/util/ReleaseInfo.java | 70 +-
.../org/apache/beam/sdk/util/Reshuffle.java | 88 --
.../apache/beam/sdk/util/ReshuffleTrigger.java | 52 --
.../org/apache/beam/sdk/util/Serializer.java | 147 ----
.../org/apache/beam/sdk/util/ShardedFile.java | 2 -
.../apache/beam/sdk/util/SideInputReader.java | 47 --
.../java/org/apache/beam/sdk/util/Sleeper.java | 48 ++
.../java/org/apache/beam/sdk/util/Structs.java | 371 ---------
.../beam/sdk/util/SystemDoFnInternal.java | 3 -
.../org/apache/beam/sdk/util/TimeDomain.java | 43 -
.../java/org/apache/beam/sdk/util/Timer.java | 67 --
.../org/apache/beam/sdk/util/TimerSpec.java | 30 -
.../org/apache/beam/sdk/util/TimerSpecs.java | 41 -
.../java/org/apache/beam/sdk/util/Timers.java | 58 --
.../sdk/util/UploadIdResponseInterceptor.java | 60 --
.../apache/beam/sdk/util/ValueWithRecordId.java | 134 ----
.../java/org/apache/beam/sdk/util/Values.java | 88 --
.../org/apache/beam/sdk/util/WindowedValue.java | 93 +--
.../apache/beam/sdk/util/WindowingStrategy.java | 283 -------
.../apache/beam/sdk/util/state/BagState.java | 28 -
.../beam/sdk/util/state/CombiningState.java | 53 --
.../beam/sdk/util/state/GroupingState.java | 42 -
.../apache/beam/sdk/util/state/MapState.java | 79 --
.../beam/sdk/util/state/ReadableState.java | 51 --
.../beam/sdk/util/state/ReadableStates.java | 45 --
.../apache/beam/sdk/util/state/SetState.java | 45 --
.../org/apache/beam/sdk/util/state/State.java | 32 -
.../apache/beam/sdk/util/state/StateBinder.java | 66 --
.../beam/sdk/util/state/StateContext.java | 43 -
.../beam/sdk/util/state/StateContexts.java | 51 --
.../apache/beam/sdk/util/state/StateSpec.java | 53 --
.../apache/beam/sdk/util/state/StateSpecs.java | 629 ---------------
.../apache/beam/sdk/util/state/ValueState.java | 37 -
.../beam/sdk/util/state/WatermarkHoldState.java | 42 -
.../beam/sdk/util/state/package-info.java | 22 -
.../java/org/apache/beam/sdk/values/PBegin.java | 26 +-
.../org/apache/beam/sdk/values/PCollection.java | 162 +++-
.../apache/beam/sdk/values/PCollectionList.java | 14 +-
.../beam/sdk/values/PCollectionTuple.java | 24 +-
.../apache/beam/sdk/values/PCollectionView.java | 33 +-
.../beam/sdk/values/PCollectionViews.java | 495 ++++++++++++
.../java/org/apache/beam/sdk/values/PDone.java | 12 +-
.../org/apache/beam/sdk/values/POutput.java | 25 +-
.../java/org/apache/beam/sdk/values/PValue.java | 12 +-
.../org/apache/beam/sdk/values/PValueBase.java | 61 +-
.../apache/beam/sdk/values/TaggedPValue.java | 6 +-
.../beam/sdk/values/TimestampedValue.java | 30 +-
.../org/apache/beam/sdk/values/TupleTag.java | 22 -
.../org/apache/beam/sdk/values/TypedPValue.java | 208 -----
.../beam/sdk/values/ValueInSingleWindow.java | 29 +-
.../beam/sdk/values/ValueWithRecordId.java | 149 ++++
.../beam/sdk/values/WindowingStrategy.java | 283 +++++++
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 1 -
.../java/org/apache/beam/sdk/PipelineTest.java | 3 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 11 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 5 +-
.../beam/sdk/coders/BigIntegerCoderTest.java | 5 +-
.../beam/sdk/coders/CoderFactoriesTest.java | 100 ---
.../beam/sdk/coders/CoderProvidersTest.java | 82 +-
.../beam/sdk/coders/CoderRegistryTest.java | 191 ++---
.../apache/beam/sdk/coders/CustomCoderTest.java | 4 +-
.../beam/sdk/coders/DefaultCoderTest.java | 65 +-
.../beam/sdk/coders/DelegateCoderTest.java | 25 -
.../beam/sdk/coders/IterableCoderTest.java | 17 -
.../beam/sdk/coders/LengthPrefixCoderTest.java | 24 +-
.../apache/beam/sdk/coders/ListCoderTest.java | 17 -
.../apache/beam/sdk/coders/MapCoderTest.java | 20 -
.../beam/sdk/coders/NullableCoderTest.java | 31 +-
.../beam/sdk/coders/SerializableCoderTest.java | 47 +-
.../beam/sdk/coders/StructuredCoderTest.java | 12 +-
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 101 ++-
.../beam/sdk/io/CompressedSourceTest.java | 25 +-
.../beam/sdk/io/DefaultFilenamePolicyTest.java | 55 ++
.../sdk/io/DrunkWritableByteChannelFactory.java | 1 -
.../apache/beam/sdk/io/FileBasedSinkTest.java | 460 +++++------
.../apache/beam/sdk/io/LocalResourceIdTest.java | 6 +
.../java/org/apache/beam/sdk/io/SimpleSink.java | 31 +-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 2 -
.../java/org/apache/beam/sdk/io/TextIOTest.java | 57 +-
.../org/apache/beam/sdk/io/WriteFilesTest.java | 41 +-
.../apache/beam/sdk/io/fs/ResourceIdTester.java | 150 ++++
.../beam/sdk/metrics/CounterCellTest.java | 55 --
.../apache/beam/sdk/metrics/DirtyStateTest.java | 56 --
.../beam/sdk/metrics/DistributionCellTest.java | 53 --
.../apache/beam/sdk/metrics/GaugeCellTest.java | 48 --
.../beam/sdk/metrics/MetricFilteringTest.java | 145 ----
.../apache/beam/sdk/metrics/MetricMatchers.java | 242 ------
.../beam/sdk/metrics/MetricResultsMatchers.java | 190 +++++
.../beam/sdk/metrics/MetricsContainerTest.java | 129 ---
.../sdk/metrics/MetricsEnvironmentTest.java | 23 +-
.../apache/beam/sdk/metrics/MetricsMapTest.java | 103 ---
.../apache/beam/sdk/metrics/MetricsTest.java | 47 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 4 +-
.../beam/sdk/options/PipelineOptionsTest.java | 1 -
.../sdk/options/ProxyInvocationHandlerTest.java | 4 +-
.../beam/sdk/options/ValueProviderTest.java | 23 +-
.../sdk/options/ValueProviderUtilsTest.java | 78 --
.../beam/sdk/options/ValueProvidersTest.java | 79 ++
.../beam/sdk/runners/PipelineRunnerTest.java | 1 +
.../sdk/runners/TransformHierarchyTest.java | 5 +-
.../beam/sdk/runners/TransformTreeTest.java | 2 +-
.../beam/sdk/testing/CoderPropertiesTest.java | 73 +-
.../beam/sdk/testing/CrashingRunnerTest.java | 2 +-
.../beam/sdk/testing/ExpectedLogsTest.java | 2 +-
.../sdk/testing/FastNanoClockAndSleeper.java | 47 --
.../testing/FastNanoClockAndSleeperTest.java | 47 --
.../sdk/testing/FileChecksumMatcherTest.java | 12 +-
.../apache/beam/sdk/testing/PAssertTest.java | 14 +-
.../sdk/testing/PCollectionViewTesting.java | 2 +-
.../sdk/testing/SerializableMatchersTest.java | 9 +-
.../beam/sdk/testing/SystemNanoTimeSleeper.java | 2 +-
.../beam/sdk/testing/TestPipelineTest.java | 6 +-
.../beam/sdk/testing/WindowSupplierTest.java | 10 +-
.../sdk/transforms/ApproximateUniqueTest.java | 2 +-
.../beam/sdk/transforms/CombineFnsTest.java | 30 +-
.../apache/beam/sdk/transforms/CombineTest.java | 39 +-
.../apache/beam/sdk/transforms/CreateTest.java | 25 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 14 +-
.../sdk/transforms/FlatMapElementsTest.java | 4 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 15 +-
.../beam/sdk/transforms/LatestFnTest.java | 2 +-
.../beam/sdk/transforms/MapElementsTest.java | 4 +-
.../beam/sdk/transforms/ParDoLifecycleTest.java | 16 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 258 +++---
.../apache/beam/sdk/transforms/RegexTest.java | 2 -
.../sdk/transforms/ReifyTimestampsTest.java | 140 ++++
.../beam/sdk/transforms/ReshuffleTest.java | 293 +++++++
.../beam/sdk/transforms/SplittableDoFnTest.java | 4 +-
.../apache/beam/sdk/transforms/ViewTest.java | 17 +-
.../display/DisplayDataEvaluator.java | 2 +-
.../transforms/reflect/DoFnInvokersTest.java | 73 +-
.../DoFnSignaturesSplittableDoFnTest.java | 4 +-
.../transforms/reflect/DoFnSignaturesTest.java | 35 +-
.../transforms/reflect/OnTimerInvokersTest.java | 38 +-
.../testhelper/DoFnInvokersTestHelper.java | 6 +-
.../transforms/windowing/GlobalWindowTest.java | 2 +-
.../windowing/ReshuffleTriggerTest.java | 42 +
.../sdk/transforms/windowing/WindowTest.java | 4 +-
...BufferedElementCountingOutputStreamTest.java | 5 +-
.../apache/beam/sdk/util/CoderUtilsTest.java | 8 +-
.../apache/beam/sdk/util/CombineFnUtilTest.java | 2 +-
.../util/FileIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/FileIOChannelFactoryTest.java | 242 ------
.../apache/beam/sdk/util/FluentBackoffTest.java | 1 -
.../beam/sdk/util/IOChannelUtilsTest.java | 118 ---
.../beam/sdk/util/NumberedShardedFileTest.java | 14 +-
.../beam/sdk/util/ReifyTimestampsTest.java | 109 ---
.../org/apache/beam/sdk/util/ReshuffleTest.java | 270 -------
.../beam/sdk/util/ReshuffleTriggerTest.java | 44 -
.../beam/sdk/util/SerializableUtilsTest.java | 8 +-
.../apache/beam/sdk/util/SerializerTest.java | 162 ----
.../org/apache/beam/sdk/util/StructsTest.java | 206 -----
.../util/UploadIdResponseInterceptorTest.java | 98 ---
.../beam/sdk/util/ValueWithRecordIdTest.java | 34 -
.../beam/sdk/values/PCollectionTupleTest.java | 1 -
.../apache/beam/sdk/values/TypedPValueTest.java | 28 +-
.../beam/sdk/values/ValueWithRecordIdTest.java | 34 +
.../google-cloud-platform-core/pom.xml | 8 +-
.../sdk/extensions/gcp/options/GcpOptions.java | 9 +-
.../sdk/extensions/gcp/options/GcsOptions.java | 4 +-
.../gcp/storage/GcsCreateOptions.java | 56 ++
.../extensions/gcp/storage/GcsFileSystem.java | 20 +-
.../gcp/storage/GcsFileSystemRegistrar.java | 5 +-
.../gcp/storage/GcsPathValidator.java | 105 +++
.../gcp/storage/NoopPathValidator.java | 53 ++
.../extensions/gcp/storage/PathValidator.java | 59 ++
.../apache/beam/sdk/util/BackOffAdapter.java | 43 +
.../beam/sdk/util/GcsIOChannelFactory.java | 111 ---
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 -
.../apache/beam/sdk/util/GcsPathValidator.java | 98 ---
.../java/org/apache/beam/sdk/util/GcsUtil.java | 71 +-
.../sdk/util/UploadIdResponseInterceptor.java | 60 ++
.../extensions/gcp/GcpCoreApiSurfaceTest.java | 3 +-
.../extensions/gcp/options/GcpOptionsTest.java | 4 +-
.../gcp/options/GoogleApiDebugOptionsTest.java | 8 +-
.../gcp/storage/GcsFileSystemTest.java | 30 +-
.../gcp/storage/GcsPathValidatorTest.java | 107 +++
.../gcp/storage/GcsResourceIdTest.java | 9 +
.../beam/sdk/util/FastNanoClockAndSleeper.java | 47 ++
.../sdk/util/FastNanoClockAndSleeperTest.java | 47 ++
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 -
.../beam/sdk/util/GcsPathValidatorTest.java | 106 ---
.../org/apache/beam/sdk/util/GcsUtilTest.java | 80 +-
.../util/UploadIdResponseInterceptorTest.java | 98 +++
sdks/java/extensions/jackson/pom.xml | 2 +-
sdks/java/extensions/join-library/README.md | 42 -
sdks/java/extensions/join-library/pom.xml | 2 +-
sdks/java/extensions/pom.xml | 2 +-
sdks/java/extensions/protobuf/pom.xml | 2 +-
.../extensions/protobuf/ByteStringCoder.java | 24 +-
.../sdk/extensions/protobuf/ProtoCoder.java | 85 +-
.../ProtobufCoderProviderRegistrar.java | 41 +
.../protobuf/ProtobufCoderRegistrar.java | 39 -
.../protobuf/ByteStringCoderTest.java | 10 +-
.../sdk/extensions/protobuf/ProtoCoderTest.java | 7 +-
sdks/java/extensions/sorter/README.md | 42 -
sdks/java/extensions/sorter/pom.xml | 2 +-
.../extensions/sorter/ExternalSorterTest.java | 1 -
sdks/java/harness/pom.xml | 12 +-
.../org/apache/beam/fn/harness/FnHarness.java | 8 +-
.../harness/control/ProcessBundleHandler.java | 4 +-
.../BeamFnDataBufferingOutboundObserver.java | 3 +-
.../harness/data/BeamFnDataInboundObserver.java | 3 +-
.../beam/runners/core/BeamFnDataReadRunner.java | 19 +-
.../runners/core/BeamFnDataWriteRunner.java | 20 +-
.../control/ProcessBundleHandlerTest.java | 55 +-
...BeamFnDataBufferingOutboundObserverTest.java | 3 +-
.../data/BeamFnDataInboundObserverTest.java | 3 +-
.../runners/core/BeamFnDataReadRunnerTest.java | 14 +-
.../runners/core/BeamFnDataWriteRunnerTest.java | 14 +-
sdks/java/io/common/pom.xml | 2 +-
.../apache/beam/sdk/io/common/HashingFn.java | 2 -
sdks/java/io/elasticsearch/pom.xml | 2 +-
.../sdk/io/elasticsearch/ElasticsearchIO.java | 12 +-
sdks/java/io/google-cloud-platform/pom.xml | 24 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 253 ++++--
.../BigQueryCoderProviderRegistrar.java | 40 +
.../io/gcp/bigquery/BigQueryCoderRegistrar.java | 39 -
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 141 ++--
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 562 +++++++------
.../io/gcp/bigquery/BigQueryQuerySource.java | 44 +-
.../io/gcp/bigquery/BigQueryServicesImpl.java | 101 +--
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 46 +-
.../gcp/bigquery/BigQueryTableRowIterator.java | 7 +-
.../io/gcp/bigquery/BigQueryTableSource.java | 17 +-
.../sdk/io/gcp/bigquery/CalculateSchemas.java | 78 ++
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 66 +-
.../io/gcp/bigquery/DynamicDestinations.java | 178 ++++
.../bigquery/DynamicDestinationsHelpers.java | 193 +++++
.../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 36 +-
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 22 +-
.../sdk/io/gcp/bigquery/StreamingInserts.java | 59 +-
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 4 +-
.../io/gcp/bigquery/StreamingWriteTables.java | 10 +-
.../sdk/io/gcp/bigquery/TableDestination.java | 14 +-
.../io/gcp/bigquery/TableDestinationCoder.java | 27 +-
.../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 19 +-
.../sdk/io/gcp/bigquery/TableRowJsonCoder.java | 19 +-
.../sdk/io/gcp/bigquery/TableRowWriter.java | 59 +-
.../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 2 +-
.../io/gcp/bigquery/WriteBundlesToFiles.java | 190 +++--
.../bigquery/WriteGroupedRecordsToFiles.java | 68 ++
.../sdk/io/gcp/bigquery/WritePartition.java | 86 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 10 +-
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 23 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 160 ++--
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 245 +++---
.../io/gcp/bigtable/BigtableServiceImpl.java | 9 -
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 19 +-
.../pubsub/PubsubCoderProviderRegistrar.java | 37 +
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 78 +-
.../beam/sdk/io/gcp/pubsub/PubsubMessage.java | 61 ++
.../pubsub/PubsubMessagePayloadOnlyCoder.java | 32 +-
.../PubsubMessageWithAttributesCoder.java | 40 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 40 +-
.../io/gcp/pubsub/PubsubUnboundedSource.java | 51 +-
.../BigQueryCoderProviderRegistrarTest.java | 40 +
.../bigquery/BigQueryCoderRegistrarTest.java | 40 -
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 561 ++++++++-----
.../gcp/bigquery/BigQueryServicesImplTest.java | 24 +-
.../io/gcp/bigquery/FakeBigQueryServices.java | 2 -
.../sdk/io/gcp/bigquery/FakeJobService.java | 55 +-
.../sdk/io/gcp/bigquery/TableContainer.java | 1 -
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 6 +-
.../io/gcp/pubsub/PubsubUnboundedSinkTest.java | 41 +-
.../gcp/pubsub/PubsubUnboundedSourceTest.java | 4 +-
.../sdk/io/gcp/testing/BigqueryMatcher.java | 4 +-
.../sdk/io/gcp/testing/BigqueryMatcherTest.java | 7 +-
sdks/java/io/hadoop-common/pom.xml | 9 +-
.../beam/sdk/io/hadoop/WritableCoder.java | 78 +-
.../beam/sdk/io/hadoop/WritableCoderTest.java | 10 +
sdks/java/io/hadoop-file-system/README.md | 43 -
sdks/java/io/hadoop-file-system/pom.xml | 45 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 478 -----------
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 625 ---------------
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 32 +-
.../sdk/io/hdfs/HadoopFileSystemOptions.java | 76 +-
.../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 3 +
.../beam/sdk/io/hdfs/HadoopResourceId.java | 16 +-
.../java/org/apache/beam/sdk/io/hdfs/Sink.java | 195 -----
.../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 -
.../java/org/apache/beam/sdk/io/hdfs/Write.java | 585 --------------
.../apache/beam/sdk/io/hdfs/package-info.java | 3 +-
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 172 ----
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 231 ------
.../sdk/io/hdfs/HadoopFileSystemModuleTest.java | 2 +-
.../io/hdfs/HadoopFileSystemOptionsTest.java | 125 +++
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 5 +-
.../beam/sdk/io/hdfs/HadoopResourceIdTest.java | 71 ++
sdks/java/io/hadoop/README.md | 167 ----
sdks/java/io/hadoop/input-format/pom.xml | 2 +-
.../ConfigurableEmployeeInputFormat.java | 1 -
.../hadoop/inputformat/EmployeeInputFormat.java | 1 -
.../ReuseObjectsEmployeeInputFormat.java | 1 -
.../hadoop/inputformat/TestEmployeeDataSet.java | 2 -
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 30 +-
.../inputformat/HIFIOWithElasticTest.java | 1 -
.../HIFIOWithEmbeddedCassandraTest.java | 2 -
.../integration/tests/HIFIOCassandraIT.java | 2 -
.../integration/tests/HIFIOElasticIT.java | 1 -
sdks/java/io/hadoop/pom.xml | 2 +-
sdks/java/io/hbase/pom.xml | 27 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 7 +-
.../beam/sdk/io/hbase/HBaseMutationCoder.java | 10 +-
.../beam/sdk/io/hbase/HBaseResultCoder.java | 8 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 4 +-
sdks/java/io/jdbc/pom.xml | 2 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 13 +-
sdks/java/io/jdbc/src/test/README.md | 32 -
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 1 -
sdks/java/io/jms/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 17 +-
.../org/apache/beam/sdk/io/jms/JmsRecord.java | 1 -
sdks/java/io/kafka/pom.xml | 2 +-
.../beam/sdk/io/kafka/KafkaCheckpointMark.java | 2 -
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 348 +++-----
.../apache/beam/sdk/io/kafka/KafkaRecord.java | 1 -
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 19 +-
.../CoderBasedKafkaDeserializer.java | 71 --
.../CoderBasedKafkaSerializer.java | 73 --
.../serialization/InstantDeserializer.java | 1 -
.../kafka/serialization/InstantSerializer.java | 1 -
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 166 +---
sdks/java/io/kinesis/pom.xml | 2 +-
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 42 +-
sdks/java/io/mongodb/pom.xml | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 7 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 57 +-
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 3 -
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 20 +-
sdks/java/io/mqtt/pom.xml | 2 +-
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 3 -
sdks/java/io/pom.xml | 2 +-
sdks/java/io/xml/pom.xml | 2 +-
.../org/apache/beam/sdk/io/xml/JAXBCoder.java | 63 +-
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 19 +-
.../org/apache/beam/sdk/io/xml/XmlSink.java | 24 +-
.../apache/beam/sdk/io/xml/JAXBCoderTest.java | 25 +-
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 52 +-
sdks/java/java8tests/pom.xml | 2 +-
.../beam/sdk/transforms/DistinctJava8Test.java | 8 -
.../beam/sdk/transforms/FilterJava8Test.java | 2 +-
.../beam/sdk/transforms/PartitionJava8Test.java | 2 +-
.../beam/sdk/transforms/WithKeysJava8Test.java | 9 -
sdks/java/javadoc/README.md | 23 -
sdks/java/javadoc/ant.xml | 5 +-
sdks/java/javadoc/pom.xml | 2 +-
.../examples-java8/generate-sources.sh | 12 +-
.../maven-archetypes/examples-java8/pom.xml | 108 +--
.../main/resources/archetype-resources/pom.xml | 23 +-
.../examples/generate-sources.sh | 8 +-
sdks/java/maven-archetypes/examples/pom.xml | 67 +-
.../main/resources/archetype-resources/pom.xml | 16 +-
sdks/java/maven-archetypes/pom.xml | 22 +-
sdks/java/maven-archetypes/starter/pom.xml | 19 +-
.../resources/projects/basic/reference/pom.xml | 2 +-
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 20 +-
sdks/python/README.md | 298 -------
sdks/python/apache_beam/__init__.py | 2 +-
sdks/python/apache_beam/coders/coder_impl.py | 59 +-
sdks/python/apache_beam/coders/coders.py | 32 +-
sdks/python/apache_beam/coders/coders_test.py | 11 +-
.../apache_beam/coders/coders_test_common.py | 2 +-
sdks/python/apache_beam/coders/observable.py | 9 +-
sdks/python/apache_beam/coders/slow_stream.py | 21 +-
.../apache_beam/coders/standard_coders_test.py | 4 +-
sdks/python/apache_beam/coders/stream.pyx | 5 +
sdks/python/apache_beam/coders/typecoders.py | 7 +-
.../examples/complete/autocomplete.py | 4 +-
.../examples/complete/autocomplete_test.py | 6 +-
.../examples/complete/estimate_pi.py | 4 +-
.../examples/complete/estimate_pi_test.py | 6 +-
.../examples/complete/game/README.md | 69 --
.../examples/complete/game/hourly_team_score.py | 6 +-
.../complete/game/hourly_team_score_test.py | 6 +-
.../examples/complete/game/user_score.py | 4 +-
.../examples/complete/game/user_score_test.py | 6 +-
.../apache_beam/examples/complete/tfidf.py | 4 +-
.../apache_beam/examples/complete/tfidf_test.py | 6 +-
.../examples/complete/top_wikipedia_sessions.py | 4 +-
.../complete/top_wikipedia_sessions_test.py | 6 +-
.../examples/cookbook/bigquery_side_input.py | 4 +-
.../cookbook/bigquery_side_input_test.py | 6 +-
.../cookbook/bigquery_tornadoes_it_test.py | 4 +-
.../cookbook/bigquery_tornadoes_test.py | 8 +-
.../apache_beam/examples/cookbook/coders.py | 4 +-
.../examples/cookbook/coders_test.py | 6 +-
.../examples/cookbook/combiners_test.py | 8 +-
.../examples/cookbook/custom_ptransform.py | 2 +-
.../examples/cookbook/custom_ptransform_test.py | 6 +-
.../examples/cookbook/datastore_wordcount.py | 19 +-
.../examples/cookbook/filters_test.py | 14 +-
.../examples/cookbook/group_with_coder.py | 4 +-
.../examples/cookbook/mergecontacts.py | 18 +-
.../examples/cookbook/multiple_output_pardo.py | 4 +-
.../apache_beam/examples/snippets/snippets.py | 53 +-
.../examples/snippets/snippets_test.py | 49 +-
sdks/python/apache_beam/examples/wordcount.py | 42 +-
.../apache_beam/examples/wordcount_debugging.py | 10 +-
.../apache_beam/examples/wordcount_it_test.py | 6 +-
.../apache_beam/examples/wordcount_minimal.py | 4 +-
sdks/python/apache_beam/internal/__init__.py | 2 +
.../python/apache_beam/internal/gcp/__init__.py | 2 +
sdks/python/apache_beam/internal/gcp/auth.py | 50 +-
.../apache_beam/internal/gcp/json_value.py | 14 +-
.../apache_beam/internal/gcp/json_value_test.py | 4 +-
sdks/python/apache_beam/internal/pickler.py | 28 +-
sdks/python/apache_beam/internal/util.py | 21 +-
sdks/python/apache_beam/io/__init__.py | 2 +-
sdks/python/apache_beam/io/avroio.py | 17 +-
sdks/python/apache_beam/io/avroio_test.py | 6 +-
sdks/python/apache_beam/io/concat_source.py | 12 +-
.../python/apache_beam/io/concat_source_test.py | 6 +-
sdks/python/apache_beam/io/filebasedsink.py | 299 +++++++
.../python/apache_beam/io/filebasedsink_test.py | 303 +++++++
sdks/python/apache_beam/io/filebasedsource.py | 8 +-
.../apache_beam/io/filebasedsource_test.py | 10 +-
sdks/python/apache_beam/io/fileio.py | 297 -------
sdks/python/apache_beam/io/fileio_test.py | 304 -------
sdks/python/apache_beam/io/filesystem.py | 19 +
sdks/python/apache_beam/io/filesystems.py | 38 +-
sdks/python/apache_beam/io/filesystems_test.py | 15 +-
sdks/python/apache_beam/io/filesystems_util.py | 36 -
sdks/python/apache_beam/io/gcp/bigquery.py | 10 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 2 +-
.../io/gcp/datastore/v1/fake_datastore.py | 14 +-
.../apache_beam/io/gcp/datastore/v1/helper.py | 35 +-
.../io/gcp/datastore/v1/helper_test.py | 26 +-
.../io/gcp/datastore/v1/query_splitter_test.py | 37 +-
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 8 +
.../apache_beam/io/gcp/gcsfilesystem_test.py | 5 +
sdks/python/apache_beam/io/gcp/gcsio.py | 9 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 2 +
.../io/gcp/tests/bigquery_matcher.py | 5 +-
.../io/gcp/tests/bigquery_matcher_test.py | 2 +-
sdks/python/apache_beam/io/iobase.py | 23 +-
sdks/python/apache_beam/io/localfilesystem.py | 7 +
.../apache_beam/io/localfilesystem_test.py | 4 +
sdks/python/apache_beam/io/range_trackers.py | 7 +-
sdks/python/apache_beam/io/source_test_utils.py | 8 +
sdks/python/apache_beam/io/sources_test.py | 6 +-
sdks/python/apache_beam/io/textio.py | 19 +-
sdks/python/apache_beam/io/textio_test.py | 7 +-
sdks/python/apache_beam/io/tfrecordio.py | 19 +-
sdks/python/apache_beam/io/tfrecordio_test.py | 26 +-
sdks/python/apache_beam/metrics/__init__.py | 1 +
sdks/python/apache_beam/metrics/cells.py | 50 +-
sdks/python/apache_beam/metrics/execution.py | 7 +-
sdks/python/apache_beam/metrics/metric.py | 4 +
sdks/python/apache_beam/metrics/metricbase.py | 2 +
sdks/python/apache_beam/options/__init__.py | 16 +
.../apache_beam/options/pipeline_options.py | 650 +++++++++++++++
.../options/pipeline_options_test.py | 240 ++++++
.../options/pipeline_options_validator.py | 201 +++++
.../options/pipeline_options_validator_test.py | 343 ++++++++
.../apache_beam/options/value_provider.py | 111 +++
.../apache_beam/options/value_provider_test.py | 145 ++++
sdks/python/apache_beam/pipeline.py | 33 +-
sdks/python/apache_beam/pipeline_test.py | 8 +-
sdks/python/apache_beam/pvalue.py | 18 +-
sdks/python/apache_beam/pvalue_test.py | 2 +-
sdks/python/apache_beam/runners/api/__init__.py | 4 +-
.../apache_beam/runners/api/beam_fn_api_pb2.py | 2 +
.../runners/api/beam_fn_api_pb2_grpc.py | 2 +
.../runners/api/beam_runner_api_pb2.py | 2 +
sdks/python/apache_beam/runners/common.pxd | 6 +-
sdks/python/apache_beam/runners/common.py | 99 ++-
sdks/python/apache_beam/runners/common_test.py | 58 ++
.../apache_beam/runners/dataflow/__init__.py | 9 +
.../runners/dataflow/dataflow_metrics.py | 114 ++-
.../runners/dataflow/dataflow_metrics_test.py | 197 +++--
.../runners/dataflow/dataflow_runner.py | 11 +-
.../runners/dataflow/dataflow_runner_test.py | 9 +-
.../runners/dataflow/internal/apiclient.py | 17 +-
.../runners/dataflow/internal/apiclient_test.py | 23 +-
.../runners/dataflow/internal/dependency.py | 22 +-
.../dataflow/internal/dependency_test.py | 6 +-
.../runners/dataflow/internal/names.py | 8 +-
.../runners/dataflow/native_io/iobase.py | 2 +
.../runners/dataflow/template_runner_test.py | 2 +-
.../runners/dataflow/test_dataflow_runner.py | 5 +-
.../apache_beam/runners/direct/__init__.py | 6 +-
.../runners/direct/bundle_factory.py | 16 +-
sdks/python/apache_beam/runners/direct/clock.py | 5 +-
.../consumer_tracking_pipeline_visitor.py | 4 +-
.../apache_beam/runners/direct/direct_runner.py | 7 +-
.../apache_beam/runners/direct/executor.py | 67 +-
.../runners/direct/transform_evaluator.py | 20 +-
.../runners/direct/transform_result.py | 4 +-
.../runners/direct/watermark_manager.py | 14 +-
.../apache_beam/runners/pipeline_context.py | 10 +-
.../apache_beam/runners/portability/__init__.py | 2 +
.../runners/portability/fn_api_runner.py | 2 +
.../portability/maptask_executor_runner.py | 9 +-
.../portability/maptask_executor_runner_test.py | 35 +-
sdks/python/apache_beam/runners/runner.py | 11 +-
sdks/python/apache_beam/runners/runner_test.py | 6 +-
.../apache_beam/runners/worker/__init__.py | 2 +
.../apache_beam/runners/worker/data_plane.py | 2 +
.../apache_beam/runners/worker/log_handler.py | 2 +
.../python/apache_beam/runners/worker/logger.py | 2 +
.../apache_beam/runners/worker/opcounters.py | 2 +
.../runners/worker/operation_specs.py | 2 +
.../apache_beam/runners/worker/sdk_worker.py | 4 +
.../runners/worker/sdk_worker_main.py | 2 +
.../apache_beam/runners/worker/sideinputs.py | 2 +
.../runners/worker/statesampler_fake.py | 2 +
sdks/python/apache_beam/test_pipeline.py | 163 ----
sdks/python/apache_beam/test_pipeline_test.py | 112 ---
sdks/python/apache_beam/testing/__init__.py | 16 +
.../testing/data/standard_coders.yaml | 196 +++++
.../testing/data/trigger_transcripts.yaml | 224 ++++++
.../apache_beam/testing/pipeline_verifiers.py | 154 ++++
.../testing/pipeline_verifiers_test.py | 148 ++++
.../python/apache_beam/testing/test_pipeline.py | 168 ++++
.../apache_beam/testing/test_pipeline_test.py | 112 +++
sdks/python/apache_beam/testing/test_stream.py | 175 ++++
.../apache_beam/testing/test_stream_test.py | 83 ++
sdks/python/apache_beam/testing/test_utils.py | 73 ++
sdks/python/apache_beam/testing/util.py | 107 +++
sdks/python/apache_beam/testing/util_test.py | 50 ++
sdks/python/apache_beam/tests/__init__.py | 16 -
sdks/python/apache_beam/tests/data/README.md | 20 -
.../apache_beam/tests/data/privatekey.p12 | Bin 2452 -> 0 bytes
.../apache_beam/tests/data/standard_coders.yaml | 196 -----
.../apache_beam/tests/pipeline_verifiers.py | 146 ----
.../tests/pipeline_verifiers_test.py | 148 ----
sdks/python/apache_beam/tests/test_utils.py | 69 --
sdks/python/apache_beam/transforms/__init__.py | 2 +-
.../apache_beam/transforms/combiners_test.py | 4 +-
sdks/python/apache_beam/transforms/core.py | 96 ++-
.../apache_beam/transforms/create_test.py | 5 +-
.../apache_beam/transforms/cy_combiners.py | 5 +-
sdks/python/apache_beam/transforms/display.py | 2 +-
.../apache_beam/transforms/display_test.py | 2 +-
.../python/apache_beam/transforms/ptransform.py | 45 +-
.../apache_beam/transforms/ptransform_test.py | 22 +-
.../python/apache_beam/transforms/sideinputs.py | 2 +
.../apache_beam/transforms/sideinputs_test.py | 4 +-
sdks/python/apache_beam/transforms/timeutil.py | 33 +-
sdks/python/apache_beam/transforms/trigger.py | 118 +--
.../apache_beam/transforms/trigger_test.py | 50 +-
.../transforms/trigger_transcripts.yaml | 224 ------
sdks/python/apache_beam/transforms/util.py | 79 --
sdks/python/apache_beam/transforms/util_test.py | 50 --
sdks/python/apache_beam/transforms/window.py | 51 +-
.../apache_beam/transforms/window_test.py | 12 +-
.../transforms/write_ptransform_test.py | 4 +-
sdks/python/apache_beam/typehints/decorators.py | 17 +-
sdks/python/apache_beam/typehints/opcodes.py | 2 +
.../apache_beam/typehints/trivial_inference.py | 2 +
sdks/python/apache_beam/typehints/typecheck.py | 17 +-
.../typehints/typed_pipeline_test.py | 6 +-
sdks/python/apache_beam/typehints/typehints.py | 18 +
.../apache_beam/typehints/typehints_test.py | 19 +-
sdks/python/apache_beam/utils/__init__.py | 5 +-
sdks/python/apache_beam/utils/annotations.py | 4 +-
sdks/python/apache_beam/utils/counters.py | 5 +-
.../apache_beam/utils/pipeline_options.py | 627 ---------------
.../apache_beam/utils/pipeline_options_test.py | 240 ------
.../utils/pipeline_options_validator.py | 199 -----
.../utils/pipeline_options_validator_test.py | 343 --------
sdks/python/apache_beam/utils/processes.py | 6 +-
sdks/python/apache_beam/utils/profiler.py | 5 +-
sdks/python/apache_beam/utils/proto_utils.py | 2 +
sdks/python/apache_beam/utils/retry.py | 2 +
sdks/python/apache_beam/utils/test_stream.py | 163 ----
.../apache_beam/utils/test_stream_test.py | 83 --
sdks/python/apache_beam/utils/timestamp.py | 5 +-
sdks/python/apache_beam/utils/urns.py | 2 +
sdks/python/apache_beam/utils/value_provider.py | 103 ---
.../apache_beam/utils/value_provider_test.py | 145 ----
sdks/python/apache_beam/utils/windowed_value.py | 4 +-
.../apache_beam/utils/windowed_value_test.py | 2 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/generate_pydoc.sh | 15 +-
sdks/python/pom.xml | 2 +-
sdks/python/setup.py | 4 +-
sdks/python/tox.ini | 5 +
1277 files changed, 28731 insertions(+), 31364 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/bc8da292/examples/java/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/bc8da292/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/bc8da292/runners/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/bc8da292/sdks/java/pom.xml
----------------------------------------------------------------------
[44/50] [abbrv] beam git commit: Add beam-runners-gearpump dependency
to javadoc
Posted by ke...@apache.org.
Add beam-runners-gearpump dependency to javadoc
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/49d4ed59
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/49d4ed59
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/49d4ed59
Branch: refs/heads/master
Commit: 49d4ed59567852e978d14729c89b9ea91bb96c32
Parents: daa7566
Author: manuzhang <ow...@gmail.com>
Authored: Fri Jul 21 22:33:36 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Sat Jul 22 08:30:18 2017 +0800
----------------------------------------------------------------------
sdks/java/javadoc/pom.xml | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/49d4ed59/sdks/java/javadoc/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/javadoc/pom.xml b/sdks/java/javadoc/pom.xml
index 51109fb..e1adb79 100644
--- a/sdks/java/javadoc/pom.xml
+++ b/sdks/java/javadoc/pom.xml
@@ -79,6 +79,11 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gearpump</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
</dependency>
[07/50] [abbrv] beam git commit: Merge branch 'master' of
https://github.com/apache/incubator-beam into gearpump-runner
Posted by ke...@apache.org.
Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3f917987
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3f917987
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3f917987
Branch: refs/heads/master
Commit: 3f917987f02b6e28f99e40c711a7289b2d4c8e6b
Parents: 15a8ad6 781e417
Author: manuzhang <ow...@gmail.com>
Authored: Sun Mar 12 18:41:51 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Sun Mar 12 18:41:51 2017 +0800
----------------------------------------------------------------------
.gitignore | 10 +
.jenkins/common_job_properties.groovy | 98 +-
...job_beam_PostCommit_Java_MavenInstall.groovy | 4 +-
...ostCommit_Java_RunnableOnService_Apex.groovy | 8 +-
...ommit_Java_RunnableOnService_Dataflow.groovy | 8 +-
...stCommit_Java_RunnableOnService_Flink.groovy | 8 +-
...ommit_Java_RunnableOnService_Gearpump.groovy | 12 +-
...stCommit_Java_RunnableOnService_Spark.groovy | 10 +-
.../job_beam_PostCommit_Python_Verify.groovy | 8 +-
.../job_beam_PreCommit_Java_MavenInstall.groovy | 6 +-
.../job_beam_PreCommit_Website_Stage.groovy | 80 +
.jenkins/job_beam_PreCommit_Website_Test.groovy | 65 +
.../job_beam_Release_NightlySnapshot.groovy | 9 +-
.jenkins/job_seed.groovy | 24 +-
.travis.yml | 24 +-
.travis/README.md | 2 +-
DISCLAIMER | 10 -
NOTICE | 4 +-
README.md | 57 +-
examples/java/README.md | 16 +-
examples/java/pom.xml | 24 +-
.../beam/examples/DebuggingWordCount.java | 4 +-
.../org/apache/beam/examples/WordCount.java | 6 +-
.../beam/examples/complete/AutoComplete.java | 4 +-
.../org/apache/beam/examples/complete/README.md | 14 +-
.../apache/beam/examples/complete/TfIdf.java | 2 +-
.../examples/complete/TopWikipediaSessions.java | 27 +-
.../examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 2 +-
.../org/apache/beam/examples/cookbook/README.md | 14 +-
.../beam/examples/cookbook/TriggerExample.java | 4 +-
.../beam/examples/WindowedWordCountIT.java | 16 +-
.../org/apache/beam/examples/WordCountTest.java | 7 +-
.../examples/complete/AutoCompleteTest.java | 11 +-
.../beam/examples/complete/TfIdfTest.java | 6 +-
.../complete/TopWikipediaSessionsTest.java | 7 +-
.../examples/cookbook/DistinctExampleTest.java | 9 +-
.../examples/cookbook/JoinExamplesTest.java | 6 +-
.../examples/cookbook/TriggerExampleTest.java | 6 +-
examples/java8/pom.xml | 3 +-
.../beam/examples/complete/game/GameStats.java | 7 +-
.../examples/complete/game/LeaderBoard.java | 5 +-
.../beam/examples/complete/game/UserScore.java | 2 +-
.../examples/MinimalWordCountJava8Test.java | 6 +-
.../examples/complete/game/GameStatsTest.java | 7 +-
.../complete/game/HourlyTeamScoreTest.java | 5 +-
.../examples/complete/game/LeaderBoardTest.java | 11 +-
.../examples/complete/game/UserScoreTest.java | 10 +-
examples/pom.xml | 16 +-
pom.xml | 319 +-
runners/apex/README.md | 4 +-
runners/apex/pom.xml | 32 +-
.../beam/runners/apex/ApexPipelineOptions.java | 7 +-
.../apache/beam/runners/apex/ApexRunner.java | 144 +-
.../beam/runners/apex/ApexYarnLauncher.java | 29 +-
.../beam/runners/apex/TestApexRunner.java | 10 -
.../translation/ApexPipelineTranslator.java | 17 +-
.../translation/CreateValuesTranslator.java | 25 +-
.../FlattenPCollectionTranslator.java | 37 +-
.../apex/translation/GroupByKeyTranslator.java | 2 +-
.../translation/ParDoBoundMultiTranslator.java | 32 +-
.../apex/translation/ParDoBoundTranslator.java | 9 +-
.../apex/translation/TranslationContext.java | 43 +-
.../translation/WindowAssignTranslator.java | 78 +
.../apex/translation/WindowBoundTranslator.java | 78 -
.../operators/ApexFlattenOperator.java | 4 +-
.../operators/ApexGroupByKeyOperator.java | 22 +-
.../operators/ApexParDoOperator.java | 12 +-
.../ApexReadUnboundedInputOperator.java | 17 +-
.../translation/utils/ApexStateInternals.java | 30 +-
.../apex/translation/utils/NoOpStepContext.java | 7 +-
.../beam/runners/apex/ApexRunnerTest.java | 76 +
.../beam/runners/apex/ApexYarnLauncherTest.java | 9 +-
.../runners/apex/examples/WordCountTest.java | 2 +-
.../FlattenPCollectionTranslatorTest.java | 24 +-
.../translation/ParDoBoundTranslatorTest.java | 38 +-
.../translation/ReadUnboundTranslatorTest.java | 8 +-
.../utils/ApexStateInternalsTest.java | 14 +-
.../test/resources/beam-runners-apex.properties | 20 +
runners/core-construction-java/pom.xml | 138 +
.../EmptyFlattenAsCreateFactory.java | 71 +
.../core/construction/PTransformMatchers.java | 192 +
.../core/construction/PrimitiveCreate.java | 77 +
.../core/construction/ReplacementOutputs.java | 105 +
.../SingleInputOutputOverrideFactory.java | 50 +
.../UnsupportedOverrideFactory.java | 71 +
.../runners/core/construction/package-info.java | 22 +
.../construction/PTransformMatchersTest.java | 425 ++
.../construction/ReplacementOutputsTest.java | 254 +
.../SingleInputOutputOverrideFactoryTest.java | 114 +
.../UnsupportedOverrideFactoryTest.java | 65 +
runners/core-java/pom.xml | 24 +-
.../beam/runners/core/AggregatorFactory.java | 1 -
.../beam/runners/core/AssignWindowsDoFn.java | 3 +-
.../beam/runners/core/BaseExecutionContext.java | 174 +
.../apache/beam/runners/core/DoFnAdapters.java | 323 ++
.../apache/beam/runners/core/DoFnRunner.java | 21 -
.../apache/beam/runners/core/DoFnRunners.java | 158 +-
.../beam/runners/core/ExecutionContext.java | 100 +
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 114 +
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 17 +-
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 154 +
.../runners/core/GroupAlsoByWindowsDoFn.java | 5 +-
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 134 -
.../runners/core/InMemoryStateInternals.java | 643 +++
.../runners/core/InMemoryTimerInternals.java | 300 ++
.../apache/beam/runners/core/KeyedWorkItem.java | 2 +-
.../beam/runners/core/KeyedWorkItemCoder.java | 8 +-
.../beam/runners/core/KeyedWorkItems.java | 2 +-
.../core/LateDataDroppingDoFnRunner.java | 2 -
.../apache/beam/runners/core/LateDataUtils.java | 88 +
.../runners/core/MergingActiveWindowSet.java | 4 -
.../beam/runners/core/MergingStateAccessor.java | 41 +
.../apache/beam/runners/core/NonEmptyPanes.java | 7 +-
.../org/apache/beam/runners/core/OldDoFn.java | 471 ++
...eBoundedSplittableProcessElementInvoker.java | 285 ++
.../beam/runners/core/PaneInfoTracker.java | 4 -
.../runners/core/PerKeyCombineFnRunner.java | 70 -
.../runners/core/PerKeyCombineFnRunners.java | 101 -
.../org/apache/beam/runners/core/ReduceFn.java | 2 -
.../runners/core/ReduceFnContextFactory.java | 11 +-
.../beam/runners/core/ReduceFnRunner.java | 6 +-
.../beam/runners/core/SideInputHandler.java | 4 -
.../beam/runners/core/SimpleDoFnRunner.java | 190 +-
.../beam/runners/core/SimpleOldDoFnRunner.java | 12 +-
.../beam/runners/core/SplittableParDo.java | 233 +-
.../core/SplittableProcessElementInvoker.java | 65 +
.../apache/beam/runners/core/StateAccessor.java | 38 +
.../beam/runners/core/StateInternals.java | 59 +
.../runners/core/StateInternalsFactory.java | 35 +
.../apache/beam/runners/core/StateMerging.java | 309 ++
.../beam/runners/core/StateNamespace.java | 56 +
.../runners/core/StateNamespaceForTest.java | 65 +
.../beam/runners/core/StateNamespaces.java | 278 ++
.../apache/beam/runners/core/StateTable.java | 84 +
.../org/apache/beam/runners/core/StateTag.java | 125 +
.../org/apache/beam/runners/core/StateTags.java | 382 ++
.../beam/runners/core/StatefulDoFnRunner.java | 171 +
.../beam/runners/core/SystemReduceFn.java | 5 -
.../core/TestInMemoryStateInternals.java | 63 +
.../beam/runners/core/TimerInternals.java | 286 ++
.../runners/core/TimerInternalsFactory.java | 35 +
.../core/UnboundedReadFromBoundedSource.java | 14 +-
.../core/UnsupportedSideInputReader.java | 52 +
.../apache/beam/runners/core/WatermarkHold.java | 9 +-
.../beam/runners/core/WindowingInternals.java | 81 +
.../core/WindowingInternalsAdapters.java | 1 -
.../core/triggers/AfterAllStateMachine.java | 2 +-
.../AfterDelayFromFirstElementStateMachine.java | 16 +-
.../core/triggers/AfterFirstStateMachine.java | 2 +-
.../core/triggers/AfterPaneStateMachine.java | 12 +-
.../AfterProcessingTimeStateMachine.java | 2 +
...rSynchronizedProcessingTimeStateMachine.java | 7 +-
.../triggers/AfterWatermarkStateMachine.java | 15 +-
.../core/triggers/OrFinallyStateMachine.java | 2 +-
.../triggers/ReshuffleTriggerStateMachine.java | 6 +-
.../core/triggers/TriggerStateMachine.java | 6 +-
.../TriggerStateMachineContextFactory.java | 12 +-
.../triggers/TriggerStateMachineRunner.java | 8 +-
.../core/triggers/TriggerStateMachines.java | 250 +-
.../core/DoFnDelegatingAggregatorTest.java | 144 +
...roupAlsoByWindowViaOutputBufferDoFnTest.java | 109 +
.../core/GroupAlsoByWindowsProperties.java | 7 +-
...oupAlsoByWindowsViaOutputBufferDoFnTest.java | 110 -
.../core/InMemoryStateInternalsTest.java | 574 +++
.../core/InMemoryTimerInternalsTest.java | 195 +
.../runners/core/KeyedWorkItemCoderTest.java | 9 +-
.../core/LateDataDroppingDoFnRunnerTest.java | 3 +-
.../core/MergingActiveWindowSetTest.java | 2 -
.../apache/beam/runners/core/NoOpOldDoFn.java | 72 +
.../beam/runners/core/OldDoFnContextTest.java | 72 +
.../apache/beam/runners/core/OldDoFnTest.java | 192 +
...ndedSplittableProcessElementInvokerTest.java | 146 +
.../core/PushbackSideInputDoFnRunnerTest.java | 8 +-
.../beam/runners/core/ReduceFnRunnerTest.java | 12 +-
.../beam/runners/core/ReduceFnTester.java | 19 +-
.../beam/runners/core/SideInputHandlerTest.java | 1 -
.../beam/runners/core/SimpleDoFnRunnerTest.java | 6 +-
.../runners/core/SimpleOldDoFnRunnerTest.java | 4 +-
.../beam/runners/core/SplittableParDoTest.java | 254 +-
.../beam/runners/core/StateNamespacesTest.java | 130 +
.../apache/beam/runners/core/StateTagTest.java | 205 +
.../runners/core/StatefulDoFnRunnerTest.java | 347 ++
.../beam/runners/core/TimerInternalsTest.java | 105 +
.../UnboundedReadFromBoundedSourceTest.java | 18 +-
.../beam/runners/core/WindowMatchers.java | 3 +-
...chronizedProcessingTimeStateMachineTest.java | 3 +-
.../AfterWatermarkStateMachineTest.java | 19 +
.../core/triggers/NeverStateMachineTest.java | 1 -
.../ReshuffleTriggerStateMachineTest.java | 8 +-
.../triggers/TriggerStateMachineTester.java | 16 +-
.../core/triggers/TriggerStateMachinesTest.java | 167 +-
runners/direct-java/pom.xml | 27 +-
.../runners/direct/AggregatorContainer.java | 2 +-
.../direct/BoundedReadEvaluatorFactory.java | 10 +-
.../CopyOnAccessInMemoryStateInternals.java | 68 +-
.../runners/direct/DirectExecutionContext.java | 8 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 4 +-
.../beam/runners/direct/DirectGraphVisitor.java | 33 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 3 +-
.../beam/runners/direct/DirectMetrics.java | 31 +-
.../beam/runners/direct/DirectRunner.java | 114 +-
.../runners/direct/DirectTimerInternals.java | 14 +-
...ecycleManagerRemovingTransformEvaluator.java | 19 +-
.../beam/runners/direct/EmptyInputProvider.java | 4 +-
.../beam/runners/direct/EvaluationContext.java | 40 +-
.../direct/ExecutorServiceParallelExecutor.java | 14 +-
.../runners/direct/FlattenEvaluatorFactory.java | 8 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 38 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 10 +-
.../direct/ImmutableListBundleFactory.java | 6 +
.../direct/KeyedPValueTrackingVisitor.java | 52 +-
.../beam/runners/direct/ParDoEvaluator.java | 29 +-
.../runners/direct/ParDoEvaluatorFactory.java | 25 +-
.../direct/ParDoMultiOverrideFactory.java | 154 +-
.../ParDoSingleViaMultiOverrideFactory.java | 5 +-
.../runners/direct/RootProviderRegistry.java | 4 +-
...littableProcessElementsEvaluatorFactory.java | 84 +-
.../direct/StatefulParDoEvaluatorFactory.java | 72 +-
.../direct/TestStreamEvaluatorFactory.java | 44 +-
.../direct/TransformEvaluatorRegistry.java | 6 +-
.../direct/UnboundedReadEvaluatorFactory.java | 22 +-
.../runners/direct/ViewEvaluatorFactory.java | 16 +-
.../beam/runners/direct/WatermarkManager.java | 107 +-
.../runners/direct/WindowEvaluatorFactory.java | 17 +-
.../direct/WriteWithShardingFactory.java | 148 +-
.../runners/direct/AggregatorContainerTest.java | 18 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 18 +-
.../direct/CloningBundleFactoryTest.java | 16 +-
.../runners/direct/CommittedResultTest.java | 23 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 79 +-
.../runners/direct/DirectGraphVisitorTest.java | 55 +-
.../DirectGroupByKeyOverrideFactoryTest.java | 51 +
.../beam/runners/direct/DirectMetricsTest.java | 128 +-
.../beam/runners/direct/DirectRunnerTest.java | 8 +-
.../direct/DirectTimerInternalsTest.java | 4 +-
...leManagerRemovingTransformEvaluatorTest.java | 103 +-
.../runners/direct/EvaluationContextTest.java | 30 +-
.../direct/FlattenEvaluatorFactoryTest.java | 8 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 5 +-
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 5 +-
.../ImmutabilityCheckingBundleFactoryTest.java | 6 +-
.../ImmutabilityEnforcementFactoryTest.java | 3 +-
.../direct/ImmutableListBundleFactoryTest.java | 37 +-
.../direct/KeyedPValueTrackingVisitorTest.java | 167 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 9 +-
.../direct/ParDoMultiOverrideFactoryTest.java | 45 +
.../ParDoSingleViaMultiOverrideFactoryTest.java | 46 +
.../runners/direct/SideInputContainerTest.java | 5 +-
.../StatefulParDoEvaluatorFactoryTest.java | 129 +-
.../runners/direct/StepTransformResultTest.java | 5 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 38 +-
.../runners/direct/TransformExecutorTest.java | 4 +-
.../UnboundedReadEvaluatorFactoryTest.java | 22 +-
.../direct/ViewEvaluatorFactoryTest.java | 18 +-
.../direct/WatermarkCallbackExecutorTest.java | 5 +-
.../runners/direct/WatermarkManagerTest.java | 37 +-
.../direct/WindowEvaluatorFactoryTest.java | 87 +-
.../direct/WriteWithShardingFactoryTest.java | 154 +-
runners/flink/README.md | 250 -
runners/flink/examples/pom.xml | 2 +-
.../beam/runners/flink/examples/WordCount.java | 2 +-
.../flink/examples/streaming/AutoComplete.java | 4 +-
.../examples/streaming/KafkaIOExamples.java | 4 +-
.../KafkaWindowedWordCountExample.java | 2 +-
.../examples/streaming/WindowedWordCount.java | 2 +-
runners/flink/pom.xml | 4 +-
runners/flink/runner/pom.xml | 56 +-
.../flink/DefaultParallelismFactory.java | 3 +-
.../flink/FlinkBatchPipelineTranslator.java | 139 +
.../flink/FlinkBatchTransformTranslators.java | 797 ++++
.../flink/FlinkBatchTranslationContext.java | 154 +
.../FlinkPipelineExecutionEnvironment.java | 11 +-
.../runners/flink/FlinkPipelineOptions.java | 6 +-
.../runners/flink/FlinkPipelineTranslator.java | 53 +
.../apache/beam/runners/flink/FlinkRunner.java | 395 +-
.../flink/FlinkStreamingPipelineTranslator.java | 230 +
.../FlinkStreamingTransformTranslators.java | 1043 +++++
.../flink/FlinkStreamingTranslationContext.java | 130 +
.../flink/FlinkStreamingViewOverrides.java | 372 ++
.../flink/PipelineTranslationOptimizer.java | 72 +
.../beam/runners/flink/TestFlinkRunner.java | 24 +-
.../beam/runners/flink/TranslationMode.java | 31 +
.../FlinkBatchPipelineTranslator.java | 143 -
.../FlinkBatchTransformTranslators.java | 745 ---
.../FlinkBatchTranslationContext.java | 145 -
.../translation/FlinkPipelineTranslator.java | 53 -
.../FlinkStreamingPipelineTranslator.java | 154 -
.../FlinkStreamingTransformTranslators.java | 978 ----
.../FlinkStreamingTranslationContext.java | 110 -
.../PipelineTranslationOptimizer.java | 73 -
.../flink/translation/TranslationMode.java | 31 -
.../functions/FlinkAggregatorFactory.java | 53 +
.../functions/FlinkAssignWindows.java | 2 +-
.../functions/FlinkDoFnFunction.java | 136 +-
.../FlinkMergingNonShuffleReduceFunction.java | 57 +-
.../FlinkMergingPartialReduceFunction.java | 37 +-
.../functions/FlinkMergingReduceFunction.java | 31 +-
.../functions/FlinkMultiOutputDoFnFunction.java | 126 -
.../FlinkMultiOutputProcessContext.java | 118 -
.../FlinkMultiOutputPruningFunction.java | 2 +-
.../functions/FlinkNoElementAssignContext.java | 68 -
.../functions/FlinkNoOpStepContext.java | 73 +
.../functions/FlinkPartialReduceFunction.java | 45 +-
.../functions/FlinkProcessContextBase.java | 267 --
.../functions/FlinkReduceFunction.java | 41 +-
.../functions/FlinkSideInputReader.java | 80 +
.../FlinkSingleOutputProcessContext.java | 69 -
.../functions/FlinkStatefulDoFnFunction.java | 198 +
.../utils/SerializedPipelineOptions.java | 4 +-
.../wrappers/streaming/DoFnOperator.java | 671 ++-
.../wrappers/streaming/FlinkStateInternals.java | 1037 -----
.../streaming/KvToByteBufferKeySelector.java | 56 +
.../streaming/SingletonKeyedWorkItem.java | 2 +-
.../streaming/SingletonKeyedWorkItemCoder.java | 10 +-
.../wrappers/streaming/WindowDoFnOperator.java | 457 +-
.../streaming/io/BoundedSourceWrapper.java | 7 +-
.../streaming/io/UnboundedFlinkSink.java | 6 +
.../streaming/io/UnboundedSourceWrapper.java | 159 +-
.../state/FlinkBroadcastStateInternals.java | 865 ++++
.../state/FlinkKeyGroupStateInternals.java | 487 ++
.../state/FlinkSplitStateInternals.java | 260 ++
.../streaming/state/FlinkStateInternals.java | 1053 +++++
.../state/KeyGroupCheckpointedOperator.java | 35 +
.../state/KeyGroupRestoringOperator.java | 32 +
.../wrappers/streaming/state/package-info.java | 22 +
.../beam/runners/flink/PipelineOptionsTest.java | 42 +-
.../beam/runners/flink/WriteSinkITCase.java | 5 +-
.../flink/streaming/DoFnOperatorTest.java | 337 +-
.../FlinkBroadcastStateInternalsTest.java | 245 +
.../FlinkKeyGroupStateInternalsTest.java | 262 ++
.../streaming/FlinkSplitStateInternalsTest.java | 101 +
.../streaming/FlinkStateInternalsTest.java | 39 +-
.../streaming/UnboundedSourceWrapperTest.java | 570 ++-
runners/google-cloud-dataflow-java/pom.xml | 40 +-
.../beam/runners/dataflow/AssignWindows.java | 89 +
.../dataflow/BatchStatefulParDoOverrides.java | 283 ++
.../runners/dataflow/BatchViewOverrides.java | 1391 ++++++
.../dataflow/DataflowAggregatorTransforms.java | 79 +
.../beam/runners/dataflow/DataflowClient.java | 44 +-
.../dataflow/DataflowMetricUpdateExtractor.java | 109 +
.../runners/dataflow/DataflowPipelineJob.java | 4 +-
.../dataflow/DataflowPipelineTranslator.java | 606 +--
.../beam/runners/dataflow/DataflowRunner.java | 2466 ++--------
.../DataflowUnboundedReadFromBoundedSource.java | 547 +++
.../beam/runners/dataflow/ReadTranslator.java | 102 +
.../dataflow/StreamingViewOverrides.java | 110 +
.../runners/dataflow/TransformTranslator.java | 131 +
.../dataflow/internal/AssignWindows.java | 89 -
.../dataflow/internal/CustomSources.java | 12 +-
.../internal/DataflowAggregatorTransforms.java | 79 -
.../internal/DataflowMetricUpdateExtractor.java | 109 -
.../DataflowUnboundedReadFromBoundedSource.java | 556 ---
.../runners/dataflow/internal/IsmFormat.java | 20 +-
.../dataflow/internal/ReadTranslator.java | 107 -
.../options/DataflowPipelineOptions.java | 36 +-
.../DataflowPipelineWorkerPoolOptions.java | 16 +-
.../dataflow/testing/TestDataflowRunner.java | 17 +-
.../beam/runners/dataflow/util/DoFnInfo.java | 75 +-
.../beam/runners/dataflow/util/GcsStager.java | 18 +-
.../beam/runners/dataflow/util/PackageUtil.java | 363 +-
.../beam/runners/dataflow/dataflow.properties | 6 +-
.../BatchStatefulParDoOverridesTest.java | 169 +
.../dataflow/BatchViewOverridesTest.java | 633 +++
.../dataflow/DataflowPipelineJobTest.java | 184 +-
.../DataflowPipelineTranslatorTest.java | 145 +-
.../runners/dataflow/DataflowRunnerTest.java | 614 +--
...aflowUnboundedReadFromBoundedSourceTest.java | 79 +
...aflowUnboundedReadFromBoundedSourceTest.java | 83 -
.../DataflowPipelineDebugOptionsTest.java | 2 +-
.../options/DataflowPipelineOptionsTest.java | 20 +-
.../options/DataflowProfilingOptionsTest.java | 4 +-
.../testing/TestDataflowRunnerTest.java | 53 +-
.../runners/dataflow/util/PackageUtilTest.java | 99 +-
runners/pom.xml | 17 +-
runners/spark/README.md | 8 +-
runners/spark/pom.xml | 70 +-
.../spark/SparkNativePipelineVisitor.java | 198 +
.../runners/spark/SparkPipelineOptions.java | 6 +-
.../beam/runners/spark/SparkPipelineResult.java | 67 +-
.../apache/beam/runners/spark/SparkRunner.java | 124 +-
.../beam/runners/spark/SparkRunnerDebugger.java | 137 +
.../runners/spark/TestSparkPipelineOptions.java | 61 +
.../beam/runners/spark/TestSparkRunner.java | 204 +-
.../spark/aggregators/AccumulatorSingleton.java | 53 -
.../aggregators/AggregatorsAccumulator.java | 131 +
.../spark/aggregators/NamedAggregators.java | 12 +-
.../spark/aggregators/SparkAggregators.java | 26 +-
.../aggregators/metrics/AggregatorMetric.java | 44 -
.../metrics/AggregatorMetricSource.java | 50 -
.../metrics/WithNamedAggregatorsSupport.java | 174 -
.../spark/aggregators/metrics/sink/CsvSink.java | 39 -
.../aggregators/metrics/sink/GraphiteSink.java | 39 -
.../aggregators/metrics/sink/package-info.java | 23 -
.../coders/BeamSparkRunnerRegistrator.java | 48 +-
.../beam/runners/spark/coders/CoderHelpers.java | 23 +
.../runners/spark/coders/NullWritableCoder.java | 76 -
.../spark/coders/StatelessJavaSerializer.java | 97 +
.../runners/spark/coders/WritableCoder.java | 122 -
.../beam/runners/spark/examples/WordCount.java | 2 +-
.../apache/beam/runners/spark/io/ConsoleIO.java | 4 +-
.../beam/runners/spark/io/CreateStream.java | 198 +-
.../beam/runners/spark/io/MicrobatchSource.java | 9 +-
.../runners/spark/io/SparkUnboundedSource.java | 166 +-
.../runners/spark/metrics/AggregatorMetric.java | 43 +
.../spark/metrics/AggregatorMetricSource.java | 51 +
.../runners/spark/metrics/CompositeSource.java | 49 +
.../spark/metrics/MetricsAccumulator.java | 132 +
.../spark/metrics/MetricsAccumulatorParam.java | 42 +
.../runners/spark/metrics/SparkBeamMetric.java | 69 +
.../spark/metrics/SparkBeamMetricSource.java | 51 +
.../spark/metrics/SparkMetricResults.java | 181 +
.../spark/metrics/SparkMetricsContainer.java | 147 +
.../spark/metrics/WithMetricsSupport.java | 209 +
.../runners/spark/metrics/package-info.java | 20 +
.../runners/spark/metrics/sink/CsvSink.java | 38 +
.../spark/metrics/sink/GraphiteSink.java | 38 +
.../spark/metrics/sink/package-info.java | 22 +
.../SparkGroupAlsoByWindowViaWindowSet.java | 431 ++
.../spark/stateful/SparkStateInternals.java | 418 ++
.../spark/stateful/SparkTimerInternals.java | 193 +
.../spark/stateful/StateSpecFunctions.java | 60 +-
.../spark/translation/BoundedDataset.java | 3 +-
.../runners/spark/translation/DoFnFunction.java | 41 +-
.../translation/DoFnRunnerWithMetrics.java | 91 +
.../spark/translation/EvaluationContext.java | 89 +-
.../translation/GroupCombineFunctions.java | 259 +-
.../spark/translation/MultiDoFnFunction.java | 39 +-
.../translation/SparkAbstractCombineFn.java | 12 +-
.../spark/translation/SparkContextFactory.java | 13 +-
.../spark/translation/SparkGlobalCombineFn.java | 13 +-
.../translation/SparkGroupAlsoByWindowFn.java | 214 -
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 179 +
.../spark/translation/SparkKeyedCombineFn.java | 13 +-
.../spark/translation/SparkPCollectionView.java | 99 +
.../spark/translation/SparkProcessContext.java | 8 +-
.../spark/translation/SparkRuntimeContext.java | 84 +-
.../spark/translation/TransformEvaluator.java | 1 +
.../spark/translation/TransformTranslator.java | 467 +-
.../spark/translation/TranslationUtils.java | 93 +-
.../spark/translation/streaming/Checkpoint.java | 137 +
.../SparkRunnerStreamingContextFactory.java | 57 +-
.../streaming/StreamingTransformTranslator.java | 492 +-
.../translation/streaming/UnboundedDataset.java | 63 +-
.../runners/spark/util/BroadcastHelper.java | 127 -
.../spark/util/GlobalWatermarkHolder.java | 200 +
.../runners/spark/util/SideInputBroadcast.java | 77 +
.../spark/util/SparkSideInputReader.java | 8 +-
.../beam/runners/spark/ClearWatermarksRule.java | 37 +
.../beam/runners/spark/ForceStreamingTest.java | 60 +-
.../spark/GlobalWatermarkHolderTest.java | 151 +
.../apache/beam/runners/spark/PipelineRule.java | 109 +
.../runners/spark/ProvidedSparkContextTest.java | 72 +-
.../runners/spark/ReuseSparkContextRule.java | 46 +
.../runners/spark/SparkPipelineStateTest.java | 27 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 180 +
.../spark/aggregators/ClearAggregatorsRule.java | 5 +-
.../metrics/sink/InMemoryMetrics.java | 10 +-
.../metrics/sink/NamedAggregatorsTest.java | 25 +-
.../coders/BeamSparkRunnerRegistratorTest.java | 57 -
.../runners/spark/coders/WritableCoderTest.java | 45 -
.../beam/runners/spark/io/AvroPipelineTest.java | 6 +-
.../beam/runners/spark/io/NumShardsTest.java | 6 +-
.../io/hadoop/HadoopFileFormatPipelineTest.java | 8 +-
.../spark/metrics/SparkBeamMetricTest.java | 60 +
.../spark/translation/StorageLevelTest.java | 8 +-
.../translation/streaming/CreateStreamTest.java | 376 ++
.../streaming/EmptyStreamAssertionTest.java | 87 -
.../streaming/FlattenStreamingTest.java | 103 -
.../streaming/KafkaStreamingTest.java | 215 -
.../ResumeFromCheckpointStreamingTest.java | 308 +-
.../streaming/SimpleStreamingWordCountTest.java | 84 -
.../streaming/TrackStreamingSourcesTest.java | 171 +
.../utils/KafkaWriteOnBatchCompleted.java | 105 -
.../streaming/utils/PAssertStreaming.java | 121 -
.../utils/SparkTestPipelineOptions.java | 42 -
.../SparkTestPipelineOptionsForStreaming.java | 37 -
.../spark/src/test/resources/log4j.properties | 30 +
.../spark/src/test/resources/metrics.properties | 10 +-
sdks/common/fn-api/pom.xml | 109 +
.../fn-api/src/main/proto/beam_fn_api.proto | 771 +++
.../org/apache/beam/fn/v1/standard_coders.yaml | 195 +
sdks/common/pom.xml | 39 +
sdks/common/runner-api/pom.xml | 89 +
.../src/main/proto/beam_runner_api.proto | 711 +++
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/findbugs-filter.xml | 95 +-
sdks/java/core/pom.xml | 28 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 75 +-
.../beam/sdk/annotations/Experimental.java | 5 +-
.../org/apache/beam/sdk/coders/AtomicCoder.java | 2 +-
.../org/apache/beam/sdk/coders/AvroCoder.java | 30 +-
.../apache/beam/sdk/coders/BigDecimalCoder.java | 6 +-
.../beam/sdk/coders/BigEndianIntegerCoder.java | 7 +
.../beam/sdk/coders/BigEndianLongCoder.java | 7 +
.../apache/beam/sdk/coders/ByteArrayCoder.java | 7 +
.../org/apache/beam/sdk/coders/ByteCoder.java | 7 +
.../apache/beam/sdk/coders/ByteStringCoder.java | 8 +
.../java/org/apache/beam/sdk/coders/Coder.java | 7 +
.../apache/beam/sdk/coders/CollectionCoder.java | 12 +-
.../org/apache/beam/sdk/coders/CustomCoder.java | 18 +-
.../apache/beam/sdk/coders/DelegateCoder.java | 29 +-
.../org/apache/beam/sdk/coders/DoubleCoder.java | 7 +
.../apache/beam/sdk/coders/DurationCoder.java | 8 +
.../apache/beam/sdk/coders/InstantCoder.java | 7 +
.../apache/beam/sdk/coders/IterableCoder.java | 12 +-
.../beam/sdk/coders/IterableLikeCoder.java | 6 +-
.../org/apache/beam/sdk/coders/JAXBCoder.java | 48 +-
.../org/apache/beam/sdk/coders/KvCoder.java | 35 +-
.../beam/sdk/coders/LengthPrefixCoder.java | 145 +
.../org/apache/beam/sdk/coders/ListCoder.java | 7 +
.../org/apache/beam/sdk/coders/MapCoder.java | 62 +-
.../apache/beam/sdk/coders/NullableCoder.java | 6 +
.../beam/sdk/coders/SerializableCoder.java | 17 +-
.../org/apache/beam/sdk/coders/SetCoder.java | 12 +-
.../apache/beam/sdk/coders/StandardCoder.java | 42 +-
.../beam/sdk/coders/StringDelegateCoder.java | 16 +-
.../apache/beam/sdk/coders/StringUtf8Coder.java | 18 +-
.../beam/sdk/coders/TableRowJsonCoder.java | 7 +
.../beam/sdk/coders/TextualIntegerCoder.java | 8 +
.../org/apache/beam/sdk/coders/VarIntCoder.java | 10 +-
.../apache/beam/sdk/coders/VarLongCoder.java | 7 +
.../org/apache/beam/sdk/coders/VoidCoder.java | 7 +
.../beam/sdk/coders/protobuf/ProtoCoder.java | 8 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 4 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 5 -
.../sdk/io/BoundedReadFromUnboundedSource.java | 79 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 8 -
.../apache/beam/sdk/io/CompressedSource.java | 40 +-
.../org/apache/beam/sdk/io/CountingSource.java | 5 -
.../org/apache/beam/sdk/io/FileBasedSink.java | 22 +
.../java/org/apache/beam/sdk/io/FileSystem.java | 115 +-
.../org/apache/beam/sdk/io/FileSystems.java | 88 +-
.../org/apache/beam/sdk/io/LocalFileSystem.java | 195 +-
.../org/apache/beam/sdk/io/LocalResourceId.java | 136 +
.../java/org/apache/beam/sdk/io/PubsubIO.java | 1142 ++---
.../apache/beam/sdk/io/PubsubUnboundedSink.java | 88 +-
.../beam/sdk/io/PubsubUnboundedSource.java | 104 +-
.../main/java/org/apache/beam/sdk/io/Read.java | 7 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 401 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 706 +--
.../java/org/apache/beam/sdk/io/XmlSource.java | 5 -
.../apache/beam/sdk/io/fs/CreateOptions.java | 60 +
.../org/apache/beam/sdk/io/fs/MatchResult.java | 125 +
.../apache/beam/sdk/io/fs/ResolveOptions.java | 41 +
.../org/apache/beam/sdk/io/fs/ResourceId.java | 85 +
.../org/apache/beam/sdk/io/fs/package-info.java | 22 +
.../beam/sdk/metrics/DistributionData.java | 3 +-
.../org/apache/beam/sdk/metrics/MetricKey.java | 3 +-
.../apache/beam/sdk/metrics/MetricUpdates.java | 3 +-
.../org/apache/beam/sdk/options/GcpOptions.java | 25 +-
.../org/apache/beam/sdk/options/GcsOptions.java | 4 +-
.../beam/sdk/options/PipelineOptions.java | 2 +-
.../sdk/options/PipelineOptionsFactory.java | 10 +-
.../apache/beam/sdk/options/ValueProvider.java | 6 +-
.../beam/sdk/runners/PTransformMatcher.java | 32 +
.../sdk/runners/PTransformOverrideFactory.java | 31 +
.../apache/beam/sdk/runners/PipelineRunner.java | 14 -
.../beam/sdk/runners/TransformHierarchy.java | 285 +-
.../apache/beam/sdk/testing/Annotations.java | 72 +
.../beam/sdk/testing/CoderProperties.java | 85 +-
.../testing/FlattenWithHeterogeneousCoders.java | 29 +
.../org/apache/beam/sdk/testing/PAssert.java | 259 +-
.../apache/beam/sdk/testing/RegexMatcher.java | 49 +
.../beam/sdk/testing/RunnableOnService.java | 14 +-
.../beam/sdk/testing/SourceTestUtils.java | 5 -
.../apache/beam/sdk/testing/TestPipeline.java | 111 +-
.../org/apache/beam/sdk/testing/TestStream.java | 18 +-
.../beam/sdk/testing/UsesAttemptedMetrics.java | 28 +
.../beam/sdk/testing/UsesCommittedMetrics.java | 28 +
.../apache/beam/sdk/testing/UsesMapState.java | 25 +
.../apache/beam/sdk/testing/UsesMetrics.java | 24 -
.../apache/beam/sdk/testing/UsesSetState.java | 25 +
.../apache/beam/sdk/testing/UsesTestStream.java | 24 +
.../sdk/testing/UsesUnboundedPCollections.java | 23 +
.../beam/sdk/testing/ValueInSingleWindow.java | 6 +-
.../apache/beam/sdk/transforms/Aggregator.java | 19 -
.../sdk/transforms/AggregatorRetriever.java | 13 +-
.../beam/sdk/transforms/AppliedPTransform.java | 31 +-
.../org/apache/beam/sdk/transforms/Combine.java | 204 +-
.../apache/beam/sdk/transforms/CombineFns.java | 14 +-
.../org/apache/beam/sdk/transforms/Count.java | 28 +-
.../org/apache/beam/sdk/transforms/Create.java | 88 +-
.../sdk/transforms/DelegatingAggregator.java | 2 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 77 +-
.../beam/sdk/transforms/DoFnAdapters.java | 504 --
.../apache/beam/sdk/transforms/DoFnTester.java | 77 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 15 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 10 +-
.../org/apache/beam/sdk/transforms/Latest.java | 80 +-
.../org/apache/beam/sdk/transforms/Max.java | 124 +-
.../org/apache/beam/sdk/transforms/Mean.java | 27 +-
.../org/apache/beam/sdk/transforms/Min.java | 122 +-
.../org/apache/beam/sdk/transforms/OldDoFn.java | 758 ---
.../apache/beam/sdk/transforms/PTransform.java | 9 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 55 +-
.../org/apache/beam/sdk/transforms/Regex.java | 589 ++-
.../org/apache/beam/sdk/transforms/Sample.java | 121 +-
.../beam/sdk/transforms/SimpleFunction.java | 44 +-
.../org/apache/beam/sdk/transforms/Sum.java | 57 +-
.../apache/beam/sdk/transforms/ToString.java | 181 +
.../org/apache/beam/sdk/transforms/Top.java | 27 +-
.../org/apache/beam/sdk/transforms/View.java | 10 +
.../sdk/transforms/display/DisplayData.java | 6 +-
.../beam/sdk/transforms/join/CoGbkResult.java | 35 +-
.../transforms/join/KeyedPCollectionTuple.java | 41 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 16 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 4 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 42 +-
.../sdk/transforms/reflect/DoFnInvokers.java | 142 +-
.../sdk/transforms/reflect/DoFnSignature.java | 71 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 82 +-
.../transforms/splittabledofn/OffsetRange.java | 71 +
.../splittabledofn/OffsetRangeTracker.java | 75 +
.../splittabledofn/RestrictionTracker.java | 2 +-
.../beam/sdk/transforms/windowing/AfterAll.java | 7 +
.../windowing/AfterDelayFromFirstElement.java | 251 -
.../sdk/transforms/windowing/AfterEach.java | 7 +
.../sdk/transforms/windowing/AfterFirst.java | 7 +
.../sdk/transforms/windowing/AfterPane.java | 10 -
.../windowing/AfterProcessingTime.java | 105 +-
.../AfterSynchronizedProcessingTime.java | 31 +-
.../sdk/transforms/windowing/BoundedWindow.java | 31 +
.../sdk/transforms/windowing/GlobalWindow.java | 6 +
.../sdk/transforms/windowing/GlobalWindows.java | 10 +
.../transforms/windowing/IntervalWindow.java | 19 +-
.../sdk/transforms/windowing/OutputTimeFns.java | 45 +
.../beam/sdk/transforms/windowing/PaneInfo.java | 2 -
.../windowing/TimestampTransform.java | 65 +
.../beam/sdk/transforms/windowing/Triggers.java | 320 ++
.../beam/sdk/transforms/windowing/Window.java | 96 +-
.../org/apache/beam/sdk/util/ApiSurface.java | 446 +-
.../beam/sdk/util/BaseExecutionContext.java | 174 -
.../org/apache/beam/sdk/util/CoderUtils.java | 30 +-
.../beam/sdk/util/CombineContextFactory.java | 18 -
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 +
.../util/EmptyOnDeserializationThreadLocal.java | 39 +
.../apache/beam/sdk/util/ExecutionContext.java | 100 -
.../apache/beam/sdk/util/GcpProjectUtil.java | 2 +-
.../apache/beam/sdk/util/GcsPathValidator.java | 3 +-
.../java/org/apache/beam/sdk/util/GcsUtil.java | 334 +-
.../org/apache/beam/sdk/util/NameUtils.java | 167 +
.../org/apache/beam/sdk/util/PropertyNames.java | 1 +
.../org/apache/beam/sdk/util/PubsubClient.java | 28 +-
.../apache/beam/sdk/util/PubsubGrpcClient.java | 6 +-
.../apache/beam/sdk/util/PubsubJsonClient.java | 4 +-
.../apache/beam/sdk/util/PubsubTestClient.java | 6 +-
.../org/apache/beam/sdk/util/StringUtils.java | 100 -
.../java/org/apache/beam/sdk/util/Timer.java | 11 +
.../apache/beam/sdk/util/TimerInternals.java | 273 --
.../org/apache/beam/sdk/util/WindowedValue.java | 23 +-
.../beam/sdk/util/WindowingInternals.java | 82 -
.../beam/sdk/util/WindowingStrategies.java | 266 ++
.../apache/beam/sdk/util/WindowingStrategy.java | 53 +-
.../beam/sdk/util/common/ReflectHelpers.java | 16 +-
.../sdk/util/state/InMemoryStateInternals.java | 430 --
.../sdk/util/state/InMemoryTimerInternals.java | 275 --
.../apache/beam/sdk/util/state/MapState.java | 93 +
.../sdk/util/state/MergingStateAccessor.java | 40 -
.../beam/sdk/util/state/ReadableState.java | 4 +-
.../apache/beam/sdk/util/state/SetState.java | 71 +
.../beam/sdk/util/state/StateAccessor.java | 37 -
.../apache/beam/sdk/util/state/StateBinder.java | 6 +
.../beam/sdk/util/state/StateContexts.java | 63 -
.../beam/sdk/util/state/StateInternals.java | 57 -
.../sdk/util/state/StateInternalsFactory.java | 35 -
.../beam/sdk/util/state/StateMerging.java | 259 --
.../beam/sdk/util/state/StateNamespace.java | 56 -
.../sdk/util/state/StateNamespaceForTest.java | 65 -
.../beam/sdk/util/state/StateNamespaces.java | 278 --
.../apache/beam/sdk/util/state/StateSpecs.java | 155 +-
.../apache/beam/sdk/util/state/StateTable.java | 82 -
.../apache/beam/sdk/util/state/StateTag.java | 111 -
.../apache/beam/sdk/util/state/StateTags.java | 290 --
.../util/state/TestInMemoryStateInternals.java | 61 -
.../sdk/util/state/TimerInternalsFactory.java | 36 -
.../java/org/apache/beam/sdk/values/PBegin.java | 9 +-
.../apache/beam/sdk/values/PCollectionList.java | 62 +-
.../beam/sdk/values/PCollectionTuple.java | 33 +-
.../java/org/apache/beam/sdk/values/PDone.java | 4 +-
.../java/org/apache/beam/sdk/values/PInput.java | 13 +-
.../org/apache/beam/sdk/values/POutput.java | 24 +-
.../beam/sdk/values/POutputValueBase.java | 4 +-
.../java/org/apache/beam/sdk/values/PValue.java | 24 +-
.../org/apache/beam/sdk/values/PValueBase.java | 18 +-
.../apache/beam/sdk/values/TaggedPValue.java | 42 +
.../beam/sdk/values/TimestampedValue.java | 10 +-
.../apache/beam/sdk/values/TupleTagList.java | 6 +
.../org/apache/beam/sdk/values/TypedPValue.java | 78 +-
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 62 +
.../sdk/AggregatorPipelineExtractorTest.java | 16 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 37 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 18 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 46 +-
.../sdk/coders/BigEndianIntegerCoderTest.java | 9 +
.../beam/sdk/coders/BigEndianLongCoderTest.java | 9 +
.../beam/sdk/coders/ByteArrayCoderTest.java | 6 +
.../apache/beam/sdk/coders/ByteCoderTest.java | 9 +
.../beam/sdk/coders/ByteStringCoderTest.java | 8 +
.../beam/sdk/coders/CoderRegistryTest.java | 12 +-
.../org/apache/beam/sdk/coders/CoderTest.java | 8 +
.../beam/sdk/coders/CollectionCoderTest.java | 16 +
.../apache/beam/sdk/coders/CommonCoderTest.java | 351 ++
.../beam/sdk/coders/DefaultCoderTest.java | 4 +-
.../beam/sdk/coders/DelegateCoderTest.java | 35 +-
.../apache/beam/sdk/coders/DoubleCoderTest.java | 9 +
.../beam/sdk/coders/DurationCoderTest.java | 10 +
.../beam/sdk/coders/InstantCoderTest.java | 9 +
.../beam/sdk/coders/IterableCoderTest.java | 27 +-
.../apache/beam/sdk/coders/JAXBCoderTest.java | 26 +-
.../org/apache/beam/sdk/coders/KvCoderTest.java | 29 +
.../beam/sdk/coders/LengthPrefixCoderTest.java | 129 +
.../apache/beam/sdk/coders/ListCoderTest.java | 16 +-
.../apache/beam/sdk/coders/MapCoderTest.java | 21 +-
.../beam/sdk/coders/NullableCoderTest.java | 12 +
.../beam/sdk/coders/SerializableCoderTest.java | 16 +-
.../apache/beam/sdk/coders/SetCoderTest.java | 16 +
.../beam/sdk/coders/StandardCoderTest.java | 40 +
.../sdk/coders/StringDelegateCoderTest.java | 11 +
.../beam/sdk/coders/StringUtf8CoderTest.java | 9 +
.../beam/sdk/coders/TableRowJsonCoderTest.java | 9 +
.../sdk/coders/TextualIntegerCoderTest.java | 9 +
.../apache/beam/sdk/coders/VarIntCoderTest.java | 9 +
.../beam/sdk/coders/VarLongCoderTest.java | 9 +
.../apache/beam/sdk/coders/VoidCoderTest.java | 40 +
.../beam/sdk/io/AvroIOGeneratedClassTest.java | 285 --
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 18 +-
.../apache/beam/sdk/io/AvroIOTransformTest.java | 324 ++
.../io/BoundedReadFromUnboundedSourceTest.java | 6 +-
.../beam/sdk/io/CompressedSourceTest.java | 89 +-
.../apache/beam/sdk/io/CountingInputTest.java | 12 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 13 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 17 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 9 +-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 61 +-
.../apache/beam/sdk/io/LocalFileSystemTest.java | 318 ++
.../apache/beam/sdk/io/LocalResourceIdTest.java | 226 +
.../beam/sdk/io/OffsetBasedSourceTest.java | 5 -
.../org/apache/beam/sdk/io/PubsubIOTest.java | 86 +-
.../beam/sdk/io/PubsubUnboundedSinkTest.java | 43 +-
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 22 +-
.../java/org/apache/beam/sdk/io/ReadTest.java | 5 -
.../java/org/apache/beam/sdk/io/TextIOTest.java | 222 +-
.../java/org/apache/beam/sdk/io/WriteTest.java | 158 +-
.../org/apache/beam/sdk/io/XmlSourceTest.java | 10 +-
.../apache/beam/sdk/metrics/MetricMatchers.java | 192 +-
.../apache/beam/sdk/metrics/MetricsTest.java | 124 +-
.../apache/beam/sdk/options/GcpOptionsTest.java | 32 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 6 +-
.../beam/sdk/options/PipelineOptionsTest.java | 3 +-
.../sdk/options/ProxyInvocationHandlerTest.java | 5 +-
.../beam/sdk/options/ValueProviderTest.java | 36 +-
.../sdk/options/ValueProviderUtilsTest.java | 2 +-
.../sdk/runners/TransformHierarchyTest.java | 320 +-
.../beam/sdk/runners/TransformTreeTest.java | 34 +-
.../beam/sdk/testing/GatherAllPanesTest.java | 7 +-
.../apache/beam/sdk/testing/PAssertTest.java | 118 +-
.../beam/sdk/testing/TestPipelineTest.java | 504 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 45 +-
.../testing/ValueInSingleWindowCoderTest.java | 7 +
.../transforms/ApproximateQuantilesTest.java | 12 +-
.../sdk/transforms/ApproximateUniqueTest.java | 487 +-
.../beam/sdk/transforms/CombineFnsTest.java | 25 +-
.../apache/beam/sdk/transforms/CombineTest.java | 174 +-
.../apache/beam/sdk/transforms/CountTest.java | 15 +-
.../apache/beam/sdk/transforms/CreateTest.java | 119 +-
.../beam/sdk/transforms/DistinctTest.java | 12 +-
.../DoFnDelegatingAggregatorTest.java | 142 -
.../apache/beam/sdk/transforms/DoFnTest.java | 19 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 12 +-
.../apache/beam/sdk/transforms/FilterTest.java | 18 +-
.../sdk/transforms/FlatMapElementsTest.java | 10 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 74 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 32 +-
.../apache/beam/sdk/transforms/KeysTest.java | 9 +-
.../apache/beam/sdk/transforms/KvSwapTest.java | 22 +-
.../apache/beam/sdk/transforms/LatestTest.java | 23 +-
.../beam/sdk/transforms/MapElementsTest.java | 14 +-
.../org/apache/beam/sdk/transforms/MaxTest.java | 20 +-
.../apache/beam/sdk/transforms/MeanTest.java | 7 +-
.../org/apache/beam/sdk/transforms/MinTest.java | 21 +-
.../apache/beam/sdk/transforms/NoOpOldDoFn.java | 71 -
.../beam/sdk/transforms/OldDoFnContextTest.java | 69 -
.../apache/beam/sdk/transforms/OldDoFnTest.java | 188 -
.../beam/sdk/transforms/ParDoLifecycleTest.java | 17 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 704 ++-
.../beam/sdk/transforms/PartitionTest.java | 8 +-
.../apache/beam/sdk/transforms/RegexTest.java | 148 +-
.../apache/beam/sdk/transforms/SampleTest.java | 419 +-
.../beam/sdk/transforms/SimpleFunctionTest.java | 43 +
.../beam/sdk/transforms/SimpleStatsFnsTest.java | 36 +-
.../beam/sdk/transforms/SplittableDoFnTest.java | 83 +-
.../org/apache/beam/sdk/transforms/SumTest.java | 24 +-
.../beam/sdk/transforms/ToStringTest.java | 125 +
.../org/apache/beam/sdk/transforms/TopTest.java | 33 +-
.../apache/beam/sdk/transforms/ValuesTest.java | 7 +-
.../apache/beam/sdk/transforms/ViewTest.java | 108 +-
.../beam/sdk/transforms/WithKeysTest.java | 8 +-
.../beam/sdk/transforms/WithTimestampsTest.java | 9 +-
.../display/DisplayDataEvaluator.java | 8 +-
.../sdk/transforms/display/DisplayDataTest.java | 15 +
.../transforms/join/CoGbkResultCoderTest.java | 10 +-
.../sdk/transforms/join/CoGroupByKeyTest.java | 11 +-
.../sdk/transforms/join/UnionCoderTest.java | 17 +-
.../transforms/reflect/DoFnInvokersTest.java | 70 +-
.../DoFnSignaturesProcessElementTest.java | 40 +-
.../DoFnSignaturesSplittableDoFnTest.java | 3 +-
.../transforms/reflect/DoFnSignaturesTest.java | 6 +-
.../splittabledofn/OffsetRangeTrackerTest.java | 111 +
.../windowing/AfterProcessingTimeTest.java | 2 +-
.../AfterSynchronizedProcessingTimeTest.java | 2 +-
.../transforms/windowing/GlobalWindowTest.java | 64 +
.../transforms/windowing/OutputTimeFnsTest.java | 51 +
.../sdk/transforms/windowing/TriggersTest.java | 100 +
.../sdk/transforms/windowing/WindowTest.java | 204 +-
.../sdk/transforms/windowing/WindowingTest.java | 15 +-
.../apache/beam/sdk/util/ApiSurfaceTest.java | 152 +-
.../apache/beam/sdk/util/CombineFnUtilTest.java | 8 +-
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 +
.../beam/sdk/util/FileIOChannelFactoryTest.java | 13 +-
.../beam/sdk/util/GcsPathValidatorTest.java | 17 +-
.../org/apache/beam/sdk/util/GcsUtilTest.java | 86 +-
.../org/apache/beam/sdk/util/NameUtilsTest.java | 177 +
.../beam/sdk/util/PubsubGrpcClientTest.java | 8 +-
.../beam/sdk/util/PubsubJsonClientTest.java | 3 +-
.../beam/sdk/util/PubsubTestClientTest.java | 4 +-
.../org/apache/beam/sdk/util/ReshuffleTest.java | 11 +-
.../beam/sdk/util/SerializableUtilsTest.java | 4 +-
.../apache/beam/sdk/util/StringUtilsTest.java | 100 -
.../beam/sdk/util/TimerInternalsTest.java | 101 -
.../beam/sdk/util/ValueWithRecordIdTest.java | 34 +
.../apache/beam/sdk/util/WindowedValueTest.java | 23 +
.../beam/sdk/util/WindowingStrategiesTest.java | 91 +
.../util/state/InMemoryStateInternalsTest.java | 348 --
.../util/state/InMemoryTimerInternalsTest.java | 153 -
.../sdk/util/state/StateNamespacesTest.java | 130 -
.../beam/sdk/util/state/StateTagTest.java | 173 -
.../beam/sdk/values/PCollectionListTest.java | 117 +
.../beam/sdk/values/PCollectionTupleTest.java | 80 +-
.../org/apache/beam/sdk/values/PDoneTest.java | 9 +-
.../beam/sdk/values/TimestampedValueTest.java | 19 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 17 +-
sdks/java/extensions/jackson/pom.xml | 125 +
.../beam/sdk/extensions/jackson/AsJsons.java | 76 +
.../beam/sdk/extensions/jackson/ParseJsons.java | 75 +
.../sdk/extensions/jackson/package-info.java | 22 +
.../jackson/JacksonTransformsTest.java | 242 +
sdks/java/extensions/join-library/README.md | 10 -
sdks/java/extensions/join-library/pom.xml | 3 +-
.../extensions/joinlibrary/InnerJoinTest.java | 23 +-
.../joinlibrary/OuterLeftJoinTest.java | 31 +-
.../joinlibrary/OuterRightJoinTest.java | 31 +-
sdks/java/extensions/pom.xml | 3 +-
sdks/java/extensions/sorter/pom.xml | 5 +-
.../sorter/BufferedExternalSorter.java | 23 +-
.../sorter/BufferedExternalSorterTest.java | 46 +-
.../sdk/extensions/sorter/SortValuesTest.java | 9 +-
sdks/java/harness/pom.xml | 177 +
.../org/apache/beam/fn/harness/FnHarness.java | 131 +
.../harness/channel/ManagedChannelFactory.java | 80 +
.../harness/channel/SocketAddressFactory.java | 64 +
.../beam/fn/harness/channel/package-info.java | 22 +
.../fn/harness/control/BeamFnControlClient.java | 166 +
.../harness/control/ProcessBundleHandler.java | 334 ++
.../fn/harness/control/RegisterHandler.java | 92 +
.../beam/fn/harness/control/package-info.java | 22 +
.../BeamFnDataBufferingOutboundObserver.java | 135 +
.../beam/fn/harness/data/BeamFnDataClient.java | 64 +
.../fn/harness/data/BeamFnDataGrpcClient.java | 122 +
.../harness/data/BeamFnDataGrpcMultiplexer.java | 141 +
.../harness/data/BeamFnDataInboundObserver.java | 81 +
.../beam/fn/harness/data/package-info.java | 22 +
.../fn/harness/fake/FakeAggregatorFactory.java | 52 +
.../beam/fn/harness/fake/FakeStepContext.java | 70 +
.../beam/fn/harness/fake/package-info.java | 22 +
.../harness/fn/CloseableThrowingConsumer.java | 23 +
.../beam/fn/harness/fn/ThrowingBiFunction.java | 32 +
.../beam/fn/harness/fn/ThrowingConsumer.java | 32 +
.../beam/fn/harness/fn/ThrowingFunction.java | 32 +
.../beam/fn/harness/fn/ThrowingRunnable.java | 30 +
.../apache/beam/fn/harness/fn/package-info.java | 22 +
.../fn/harness/logging/BeamFnLoggingClient.java | 310 ++
.../beam/fn/harness/logging/package-info.java | 22 +
.../apache/beam/fn/harness/package-info.java | 22 +
.../beam/fn/harness/stream/AdvancingPhaser.java | 36 +
.../harness/stream/BufferingStreamObserver.java | 166 +
.../fn/harness/stream/DirectStreamObserver.java | 71 +
.../ForwardingClientResponseObserver.java | 63 +
.../harness/stream/StreamObserverFactory.java | 91 +
.../beam/fn/harness/stream/package-info.java | 22 +
.../beam/runners/core/BeamFnDataReadRunner.java | 104 +
.../runners/core/BeamFnDataWriteRunner.java | 87 +
.../beam/runners/core/BoundedSourceRunner.java | 105 +
.../apache/beam/runners/core/package-info.java | 22 +
.../apache/beam/fn/harness/FnHarnessTest.java | 130 +
.../channel/ManagedChannelFactoryTest.java | 74 +
.../channel/SocketAddressFactoryTest.java | 56 +
.../control/BeamFnControlClientTest.java | 182 +
.../control/ProcessBundleHandlerTest.java | 675 +++
.../fn/harness/control/RegisterHandlerTest.java | 80 +
...BeamFnDataBufferingOutboundObserverTest.java | 147 +
.../harness/data/BeamFnDataGrpcClientTest.java | 318 ++
.../data/BeamFnDataGrpcMultiplexerTest.java | 98 +
.../data/BeamFnDataInboundObserverTest.java | 116 +
.../logging/BeamFnLoggingClientTest.java | 169 +
.../fn/harness/stream/AdvancingPhaserTest.java | 48 +
.../stream/BufferingStreamObserverTest.java | 146 +
.../stream/DirectStreamObserverTest.java | 139 +
.../ForwardingClientResponseObserverTest.java | 60 +
.../stream/StreamObserverFactoryTest.java | 84 +
.../beam/fn/harness/test/TestExecutors.java | 85 +
.../beam/fn/harness/test/TestExecutorsTest.java | 160 +
.../beam/fn/harness/test/TestStreams.java | 162 +
.../beam/fn/harness/test/TestStreamsTest.java | 84 +
.../runners/core/BeamFnDataReadRunnerTest.java | 187 +
.../runners/core/BeamFnDataWriteRunnerTest.java | 155 +
.../runners/core/BoundedSourceRunnerTest.java | 113 +
sdks/java/io/elasticsearch/pom.xml | 149 +
.../sdk/io/elasticsearch/ElasticsearchIO.java | 815 ++++
.../beam/sdk/io/elasticsearch/package-info.java | 20 +
.../src/test/contrib/create_elk_container.sh | 24 +
.../elasticsearch/ElasticSearchIOTestUtils.java | 129 +
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 154 +
.../io/elasticsearch/ElasticsearchIOTest.java | 358 ++
.../elasticsearch/ElasticsearchTestDataSet.java | 109 +
.../elasticsearch/ElasticsearchTestOptions.java | 46 +
sdks/java/io/google-cloud-platform/pom.xml | 16 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 450 +-
.../sdk/io/gcp/bigquery/BigQueryServices.java | 23 +-
.../io/gcp/bigquery/BigQueryServicesImpl.java | 118 +-
.../gcp/bigquery/BigQueryTableRowIterator.java | 82 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 8 -
.../io/gcp/bigtable/BigtableTestOptions.java | 37 -
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 149 +-
.../beam/sdk/io/gcp/storage/GcsFileSystem.java | 210 +-
.../io/gcp/storage/GcsFileSystemRegistrar.java | 9 +-
.../beam/sdk/io/gcp/storage/GcsResourceId.java | 116 +
.../apache/beam/sdk/io/gcp/ApiSurfaceTest.java | 134 -
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 79 +
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 335 +-
.../gcp/bigquery/BigQueryServicesImplTest.java | 141 +
.../bigquery/BigQueryTableRowIteratorTest.java | 51 +-
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 3 +-
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 24 +-
.../io/gcp/bigtable/BigtableTestOptions.java | 37 +
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 15 +-
.../sdk/io/gcp/datastore/SplitQueryFnIT.java | 2 +-
.../sdk/io/gcp/storage/GcsFileSystemTest.java | 274 ++
.../sdk/io/gcp/storage/GcsResourceIdTest.java | 147 +
sdks/java/io/hadoop-common/pom.xml | 86 +
.../io/hadoop/SerializableConfiguration.java | 96 +
.../beam/sdk/io/hadoop/WritableCoder.java | 116 +
.../apache/beam/sdk/io/hadoop/package-info.java | 22 +
.../hadoop/SerializableConfigurationTest.java | 75 +
.../beam/sdk/io/hadoop/WritableCoderTest.java | 45 +
sdks/java/io/hadoop-input-format/README.md | 167 +
sdks/java/io/hadoop-input-format/pom.xml | 136 +
.../hadoop/inputformat/HadoopInputFormatIO.java | 941 ++++
.../sdk/io/hadoop/inputformat/package-info.java | 23 +
.../ConfigurableEmployeeInputFormat.java | 131 +
.../sdk/io/hadoop/inputformat/Employee.java | 85 +
.../hadoop/inputformat/EmployeeInputFormat.java | 172 +
.../inputformat/HadoopInputFormatIOTest.java | 844 ++++
.../ReuseObjectsEmployeeInputFormat.java | 176 +
.../hadoop/inputformat/TestEmployeeDataSet.java | 76 +
sdks/java/io/hbase/pom.xml | 233 +
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 693 +++
.../beam/sdk/io/hbase/HBaseMutationCoder.java | 71 +
.../beam/sdk/io/hbase/HBaseResultCoder.java | 55 +
.../beam/sdk/io/hbase/SerializableScan.java | 55 +
.../apache/beam/sdk/io/hbase/package-info.java | 24 +
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 430 ++
.../sdk/io/hbase/HBaseMutationCoderTest.java | 52 +
.../beam/sdk/io/hbase/HBaseResultCoderTest.java | 41 +
.../beam/sdk/io/hbase/SerializableScanTest.java | 56 +
sdks/java/io/hdfs/pom.xml | 64 +-
.../beam/sdk/io/hdfs/AvroHDFSFileSource.java | 142 -
.../beam/sdk/io/hdfs/AvroWrapperCoder.java | 114 -
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 301 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 528 ++-
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 44 +-
.../beam/sdk/io/hdfs/HadoopResourceId.java | 42 +
.../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 +
.../apache/beam/sdk/io/hdfs/WritableCoder.java | 116 -
.../SimpleAuthAvroHDFSFileSource.java | 82 -
.../hdfs/simpleauth/SimpleAuthHDFSFileSink.java | 131 -
.../simpleauth/SimpleAuthHDFSFileSource.java | 117 -
.../sdk/io/hdfs/simpleauth/package-info.java | 22 -
.../beam/sdk/io/hdfs/AvroWrapperCoderTest.java | 51 -
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 173 +
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 60 +-
.../beam/sdk/io/hdfs/WritableCoderTest.java | 45 -
sdks/java/io/jdbc/pom.xml | 125 +-
sdks/java/io/jdbc/src/test/README.md | 32 +
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 178 +
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 120 +-
.../beam/sdk/io/jdbc/JdbcTestDataSet.java | 128 +
.../beam/sdk/io/jdbc/PostgresTestOptions.java | 60 +
.../kubernetes/postgres-pod-no-vol.yml | 32 +
.../kubernetes/postgres-service-public.yml | 28 +
.../jdbc/src/test/resources/kubernetes/setup.sh | 20 +
.../src/test/resources/kubernetes/teardown.sh | 20 +
sdks/java/io/jms/pom.xml | 22 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 103 +-
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 137 +-
sdks/java/io/kafka/pom.xml | 65 +-
.../apache/beam/sdk/io/kafka/ConsumerSpEL.java | 60 +
.../beam/sdk/io/kafka/KafkaCheckpointMark.java | 16 +
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 683 ++-
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 4 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 109 +-
.../beam/sdk/io/kafka/KafkaRecordCoderTest.java | 34 +
sdks/java/io/kinesis/pom.xml | 12 +-
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 4 +-
.../beam/sdk/io/kinesis/package-info.java | 2 +-
.../sdk/io/kinesis/KinesisMockReadTest.java | 7 +-
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 6 +-
sdks/java/io/mongodb/pom.xml | 20 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 5 -
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 5 -
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 9 +-
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 7 +-
sdks/java/io/mqtt/pom.xml | 122 +
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 592 +++
.../apache/beam/sdk/io/mqtt/package-info.java | 22 +
.../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 257 +
sdks/java/io/pom.xml | 76 +-
sdks/java/java8tests/pom.xml | 3 +-
.../beam/sdk/transforms/CombineJava8Test.java | 8 +-
.../beam/sdk/transforms/DistinctJava8Test.java | 5 +-
.../beam/sdk/transforms/FilterJava8Test.java | 9 +-
.../transforms/FlatMapElementsJava8Test.java | 7 +-
.../sdk/transforms/MapElementsJava8Test.java | 33 +-
.../beam/sdk/transforms/PartitionJava8Test.java | 7 +-
.../sdk/transforms/SimpleFunctionJava8Test.java | 69 +
.../beam/sdk/transforms/WithKeysJava8Test.java | 6 +-
.../sdk/transforms/WithTimestampsJava8Test.java | 14 +-
sdks/java/javadoc/ant.xml | 96 +
sdks/java/javadoc/pom.xml | 295 ++
.../maven-archetypes/examples-java8/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 78 +-
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 96 +-
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 4 +-
.../resources/projects/basic/reference/pom.xml | 4 +-
sdks/java/pom.xml | 6 +-
sdks/pom.xml | 17 +-
sdks/python/.pylintrc | 164 +
sdks/python/MANIFEST.in | 21 +
sdks/python/README.md | 298 ++
sdks/python/apache_beam/__init__.py | 82 +
sdks/python/apache_beam/coders/__init__.py | 19 +
sdks/python/apache_beam/coders/coder_impl.pxd | 143 +
sdks/python/apache_beam/coders/coder_impl.py | 734 +++
sdks/python/apache_beam/coders/coders.py | 835 ++++
sdks/python/apache_beam/coders/coders_test.py | 115 +
.../apache_beam/coders/coders_test_common.py | 389 ++
.../apache_beam/coders/fast_coders_test.py | 37 +
sdks/python/apache_beam/coders/observable.py | 38 +
.../apache_beam/coders/observable_test.py | 57 +
.../coders/proto2_coder_test_messages_pb2.py | 318 ++
.../apache_beam/coders/slow_coders_test.py | 45 +
sdks/python/apache_beam/coders/slow_stream.py | 163 +
.../apache_beam/coders/standard_coders_test.py | 156 +
sdks/python/apache_beam/coders/stream.pxd | 66 +
sdks/python/apache_beam/coders/stream.pyx | 226 +
sdks/python/apache_beam/coders/stream_test.py | 180 +
sdks/python/apache_beam/coders/typecoders.py | 182 +
.../apache_beam/coders/typecoders_test.py | 124 +
sdks/python/apache_beam/error.py | 42 +
sdks/python/apache_beam/examples/__init__.py | 16 +
.../apache_beam/examples/complete/__init__.py | 16 +
.../examples/complete/autocomplete.py | 90 +
.../examples/complete/autocomplete_test.py | 52 +
.../examples/complete/estimate_pi.py | 128 +
.../examples/complete/estimate_pi_test.py | 52 +
.../examples/complete/juliaset/__init__.py | 16 +
.../complete/juliaset/juliaset/__init__.py | 16 +
.../complete/juliaset/juliaset/juliaset.py | 124 +
.../complete/juliaset/juliaset/juliaset_test.py | 86 +
.../examples/complete/juliaset/juliaset_main.py | 58 +
.../examples/complete/juliaset/setup.py | 116 +
.../apache_beam/examples/complete/tfidf.py | 208 +
.../apache_beam/examples/complete/tfidf_test.py | 91 +
.../examples/complete/top_wikipedia_sessions.py | 182 +
.../complete/top_wikipedia_sessions_test.py | 62 +
.../apache_beam/examples/cookbook/__init__.py | 16 +
.../examples/cookbook/bigquery_schema.py | 129 +
.../examples/cookbook/bigquery_side_input.py | 121 +
.../cookbook/bigquery_side_input_test.py | 54 +
.../examples/cookbook/bigquery_tornadoes.py | 99 +
.../cookbook/bigquery_tornadoes_it_test.py | 62 +
.../cookbook/bigquery_tornadoes_test.py | 45 +
.../apache_beam/examples/cookbook/bigshuffle.py | 94 +
.../examples/cookbook/bigshuffle_test.py | 63 +
.../apache_beam/examples/cookbook/coders.py | 101 +
.../examples/cookbook/coders_test.py | 49 +
.../examples/cookbook/combiners_test.py | 74 +
.../examples/cookbook/custom_ptransform.py | 134 +
.../examples/cookbook/custom_ptransform_test.py | 53 +
.../examples/cookbook/datastore_wordcount.py | 261 ++
.../apache_beam/examples/cookbook/filters.py | 107 +
.../examples/cookbook/filters_test.py | 69 +
.../examples/cookbook/group_with_coder.py | 122 +
.../examples/cookbook/group_with_coder_test.py | 89 +
.../examples/cookbook/mergecontacts.py | 133 +
.../examples/cookbook/mergecontacts_test.py | 125 +
.../examples/cookbook/multiple_output_pardo.py | 184 +
.../cookbook/multiple_output_pardo_test.py | 72 +
.../apache_beam/examples/snippets/__init__.py | 16 +
.../apache_beam/examples/snippets/snippets.py | 1158 +++++
.../examples/snippets/snippets_test.py | 904 ++++
.../apache_beam/examples/streaming_wordcap.py | 64 +
.../apache_beam/examples/streaming_wordcount.py | 74 +
sdks/python/apache_beam/examples/wordcount.py | 116 +
.../apache_beam/examples/wordcount_debugging.py | 163 +
.../examples/wordcount_debugging_test.py | 59 +
.../apache_beam/examples/wordcount_it_test.py | 59 +
.../apache_beam/examples/wordcount_minimal.py | 121 +
.../examples/wordcount_minimal_test.py | 59 +
.../apache_beam/examples/wordcount_test.py | 58 +
sdks/python/apache_beam/internal/__init__.py | 16 +
.../python/apache_beam/internal/gcp/__init__.py | 16 +
sdks/python/apache_beam/internal/gcp/auth.py | 185 +
.../apache_beam/internal/gcp/auth_test.py | 44 +
.../apache_beam/internal/gcp/json_value.py | 147 +
.../apache_beam/internal/gcp/json_value_test.py | 93 +
sdks/python/apache_beam/internal/module_test.py | 62 +
sdks/python/apache_beam/internal/pickler.py | 230 +
.../python/apache_beam/internal/pickler_test.py | 84 +
sdks/python/apache_beam/internal/util.py | 127 +
sdks/python/apache_beam/internal/util_test.py | 61 +
sdks/python/apache_beam/io/__init__.py | 38 +
sdks/python/apache_beam/io/avroio.py | 372 ++
sdks/python/apache_beam/io/avroio_test.py | 381 ++
sdks/python/apache_beam/io/concat_source.py | 263 ++
.../python/apache_beam/io/concat_source_test.py | 231 +
sdks/python/apache_beam/io/filebasedsource.py | 329 ++
.../apache_beam/io/filebasedsource_test.py | 708 +++
sdks/python/apache_beam/io/fileio.py | 746 +++
sdks/python/apache_beam/io/fileio_test.py | 352 ++
sdks/python/apache_beam/io/gcp/__init__.py | 16 +
sdks/python/apache_beam/io/gcp/bigquery.py | 1081 +++++
sdks/python/apache_beam/io/gcp/bigquery_test.py | 828 ++++
.../apache_beam/io/gcp/datastore/__init__.py | 16 +
.../apache_beam/io/gcp/datastore/v1/__init__.py | 16 +
.../io/gcp/datastore/v1/datastoreio.py | 397 ++
.../io/gcp/datastore/v1/datastoreio_test.py | 245 +
.../io/gcp/datastore/v1/fake_datastore.py | 98 +
.../apache_beam/io/gcp/datastore/v1/helper.py | 274 ++
.../io/gcp/datastore/v1/helper_test.py | 265 ++
.../io/gcp/datastore/v1/query_splitter.py | 275 ++
.../io/gcp/datastore/v1/query_splitter_test.py | 208 +
sdks/python/apache_beam/io/gcp/gcsio.py | 871 ++++
sdks/python/apache_beam/io/gcp/gcsio_test.py | 796 ++++
.../apache_beam/io/gcp/internal/__init__.py | 16 +
.../io/gcp/internal/clients/__init__.py | 16 +
.../gcp/internal/clients/bigquery/__init__.py | 33 +
.../clients/bigquery/bigquery_v2_client.py | 660 +++
.../clients/bigquery/bigquery_v2_messages.py | 1910 ++++++++
.../io/gcp/internal/clients/storage/__init__.py | 33 +
.../clients/storage/storage_v1_client.py | 1039 +++++
.../clients/storage/storage_v1_messages.py | 1920 ++++++++
sdks/python/apache_beam/io/gcp/pubsub.py | 91 +
sdks/python/apache_beam/io/gcp/pubsub_test.py | 63 +
.../python/apache_beam/io/gcp/tests/__init__.py | 16 +
.../io/gcp/tests/bigquery_matcher.py | 108 +
.../io/gcp/tests/bigquery_matcher_test.py | 108 +
sdks/python/apache_beam/io/iobase.py | 987 ++++
sdks/python/apache_beam/io/range_trackers.py | 532 +++
.../apache_beam/io/range_trackers_test.py | 590 +++
sdks/python/apache_beam/io/source_test_utils.py | 642 +++
.../apache_beam/io/source_test_utils_test.py | 122 +
sdks/python/apache_beam/io/sources_test.py | 111 +
sdks/python/apache_beam/io/textio.py | 448 ++
sdks/python/apache_beam/io/textio_test.py | 718 +++
sdks/python/apache_beam/io/tfrecordio.py | 271 ++
sdks/python/apache_beam/io/tfrecordio_test.py | 389 ++
sdks/python/apache_beam/metrics/__init__.py | 17 +
sdks/python/apache_beam/metrics/cells.py | 315 ++
sdks/python/apache_beam/metrics/cells_test.py | 143 +
sdks/python/apache_beam/metrics/execution.pxd | 31 +
sdks/python/apache_beam/metrics/execution.py | 229 +
.../apache_beam/metrics/execution_test.py | 131 +
sdks/python/apache_beam/metrics/metric.py | 202 +
sdks/python/apache_beam/metrics/metric_test.py | 128 +
sdks/python/apache_beam/metrics/metricbase.py | 82 +
sdks/python/apache_beam/pipeline.py | 442 ++
sdks/python/apache_beam/pipeline_test.py | 444 ++
sdks/python/apache_beam/pvalue.py | 468 ++
sdks/python/apache_beam/pvalue_test.py | 68 +
sdks/python/apache_beam/runners/__init__.py | 30 +
sdks/python/apache_beam/runners/api/__init__.py | 16 +
.../runners/api/beam_runner_api_pb2.py | 2772 +++++++++++
sdks/python/apache_beam/runners/common.pxd | 77 +
sdks/python/apache_beam/runners/common.py | 436 ++
.../apache_beam/runners/dataflow/__init__.py | 16 +
.../runners/dataflow/dataflow_metrics.py | 111 +
.../runners/dataflow/dataflow_metrics_test.py | 148 +
.../runners/dataflow/dataflow_runner.py | 729 +++
.../runners/dataflow/dataflow_runner_test.py | 181 +
.../runners/dataflow/internal/__init__.py | 16 +
.../runners/dataflow/internal/apiclient.py | 739 +++
.../runners/dataflow/internal/apiclient_test.py | 96 +
.../dataflow/internal/clients/__init__.py | 16 +
.../internal/clients/dataflow/__init__.py | 33 +
.../clients/dataflow/dataflow_v1b3_client.py | 694 +++
.../clients/dataflow/dataflow_v1b3_messages.py | 4392 ++++++++++++++++++
.../clients/dataflow/message_matchers.py | 124 +
.../clients/dataflow/message_matchers_test.py | 77 +
.../runners/dataflow/internal/dependency.py | 522 +++
.../dataflow/internal/dependency_test.py | 425 ++
.../runners/dataflow/internal/names.py | 82 +
.../runners/dataflow/native_io/__init__.py | 16 +
.../runners/dataflow/native_io/iobase.py | 318 ++
.../runners/dataflow/template_runner_test.py | 97 +
.../runners/dataflow/test_dataflow_runner.py | 40 +
.../apache_beam/runners/direct/__init__.py | 19 +
.../runners/direct/bundle_factory.py | 201 +
sdks/python/apache_beam/runners/direct/clock.py | 50 +
.../consumer_tracking_pipeline_visitor.py | 59 +
.../consumer_tracking_pipeline_visitor_test.py | 127 +
.../runners/direct/direct_metrics.py | 112 +
.../runners/direct/direct_metrics_test.py | 211 +
.../apache_beam/runners/direct/direct_runner.py | 173 +
.../runners/direct/evaluation_context.py | 283 ++
.../apache_beam/runners/direct/executor.py | 578 +++
.../runners/direct/helper_transforms.py | 99 +
.../runners/direct/transform_evaluator.py | 558 +++
.../runners/direct/transform_result.py | 64 +
.../runners/direct/watermark_manager.py | 224 +
.../apache_beam/runners/pipeline_context.py | 88 +
.../runners/pipeline_context_test.py | 49 +
sdks/python/apache_beam/runners/runner.py | 368 ++
sdks/python/apache_beam/runners/runner_test.py | 123 +
.../python/apache_beam/runners/test/__init__.py | 30 +
sdks/python/apache_beam/test_pipeline.py | 163 +
sdks/python/apache_beam/test_pipeline_test.py | 112 +
sdks/python/apache_beam/tests/__init__.py | 16 +
sdks/python/apache_beam/tests/data/README.md | 20 +
.../apache_beam/tests/data/privatekey.p12 | Bin 0 -> 2452 bytes
.../apache_beam/tests/data/standard_coders.yaml | 196 +
.../apache_beam/tests/pipeline_verifiers.py | 119 +
.../tests/pipeline_verifiers_test.py | 123 +
sdks/python/apache_beam/tests/test_utils.py | 69 +
sdks/python/apache_beam/transforms/__init__.py | 25 +
sdks/python/apache_beam/transforms/combiners.py | 595 +++
.../apache_beam/transforms/combiners_test.py | 324 ++
sdks/python/apache_beam/transforms/core.py | 1389 ++++++
.../apache_beam/transforms/cy_combiners.pxd | 92 +
.../apache_beam/transforms/cy_combiners.py | 306 ++
sdks/python/apache_beam/transforms/display.py | 331 ++
.../apache_beam/transforms/display_test.py | 216 +
.../python/apache_beam/transforms/ptransform.py | 671 +++
.../apache_beam/transforms/ptransform_test.py | 1941 ++++++++
.../python/apache_beam/transforms/sideinputs.py | 214 +
.../apache_beam/transforms/sideinputs_test.py | 337 ++
sdks/python/apache_beam/transforms/timeutil.py | 133 +
sdks/python/apache_beam/transforms/trigger.py | 1109 +++++
.../apache_beam/transforms/trigger_test.py | 601 +++
.../transforms/trigger_transcripts.yaml | 224 +
sdks/python/apache_beam/transforms/util.py | 235 +
sdks/python/apache_beam/transforms/window.py | 475 ++
.../apache_beam/transforms/window_test.py | 261 ++
.../transforms/write_ptransform_test.py | 126 +
sdks/python/apache_beam/typehints/__init__.py | 22 +
sdks/python/apache_beam/typehints/decorators.py | 532 +++
sdks/python/apache_beam/typehints/opcodes.py | 334 ++
.../apache_beam/typehints/trivial_inference.py | 417 ++
.../typehints/trivial_inference_test.py | 151 +
sdks/python/apache_beam/typehints/typecheck.py | 178 +
.../typehints/typed_pipeline_test.py | 251 +
sdks/python/apache_beam/typehints/typehints.py | 1062 +++++
.../apache_beam/typehints/typehints_test.py | 1062 +++++
sdks/python/apache_beam/utils/__init__.py | 22 +
sdks/python/apache_beam/utils/annotations.py | 103 +
.../apache_beam/utils/annotations_test.py | 126 +
sdks/python/apache_beam/utils/counters.pxd | 30 +
sdks/python/apache_beam/utils/counters.py | 183 +
sdks/python/apache_beam/utils/path.py | 47 +
sdks/python/apache_beam/utils/path_test.py | 70 +
.../apache_beam/utils/pipeline_options.py | 557 +++
.../apache_beam/utils/pipeline_options_test.py | 192 +
.../utils/pipeline_options_validator.py | 199 +
.../utils/pipeline_options_validator_test.py | 342 ++
sdks/python/apache_beam/utils/processes.py | 52 +
sdks/python/apache_beam/utils/processes_test.py | 106 +
sdks/python/apache_beam/utils/profiler.py | 148 +
sdks/python/apache_beam/utils/proto_utils.py | 54 +
sdks/python/apache_beam/utils/retry.py | 207 +
sdks/python/apache_beam/utils/retry_test.py | 221 +
sdks/python/apache_beam/utils/timestamp.py | 213 +
sdks/python/apache_beam/utils/timestamp_test.py | 168 +
sdks/python/apache_beam/utils/urns.py | 24 +
.../python/apache_beam/utils/windowed_value.pxd | 38 +
sdks/python/apache_beam/utils/windowed_value.py | 122 +
.../apache_beam/utils/windowed_value_test.py | 71 +
sdks/python/apache_beam/version.py | 57 +
sdks/python/generate_pydoc.sh | 80 +
sdks/python/pom.xml | 191 +
sdks/python/run_postcommit.sh | 102 +
sdks/python/run_pylint.sh | 52 +
sdks/python/setup.cfg | 27 +
sdks/python/setup.py | 155 +
sdks/python/test_config.py | 44 +
sdks/python/tox.ini | 89 +
1310 files changed, 137874 insertions(+), 30108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/3f917987/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
----------------------------------------------------------------------
diff --cc .jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
index 847ade3,8f41c3e..c33c9b0
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
@@@ -33,9 -35,15 +35,15 @@@ mavenJob('beam_PostCommit_Java_Runnable
// Sets that this is a PostCommit job.
// 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties.
- // This job triggers only on SCM.
+ // In post-commit this job triggers only on SCM changes.
common_job_properties.setPostCommit(delegate, '0 5 31 2 *')
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Apache Gearpump Runner RunnableOnService Tests',
+ 'Run Gearpump RunnableOnService')
+
// Maven goals for this job.
- goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DrunnableOnServicePipelineOptions=\'[ "--runner=TestGearpumpRunner", "--streaming=false" ]\'')
+ goals('-B -e clean verify -am -pl runners/gearpump -Plocal-runnable-on-service-tests -Prunnable-on-service-tests')
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3f917987/.travis.yml
----------------------------------------------------------------------
diff --cc .travis.yml
index 7dcd5d1,c896431..87ade2f
--- a/.travis.yml
+++ b/.travis.yml
@@@ -55,14 -67,21 +67,22 @@@ before_install
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then export JAVA_HOME=$(/usr/libexec/java_home); fi
- if [ "$TRAVIS_OS_NAME" == "linux" ]; then jdk_switcher use "$CUSTOM_JDK"; fi
- export BEAM_SUREFIRE_ARGLINE="-Xmx512m"
+ # Python SDK environment settings.
+ - export TOX_ENV=py27
+ - if [ "$TRAVIS_OS_NAME" == "osx" ]; then export TOX_HOME=$HOME/Library/Python/2.7/bin; fi
+ - if [ "$TRAVIS_OS_NAME" == "linux" ]; then export TOX_HOME=$HOME/.local/bin; fi
install:
+ - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn -B install clean -U -DskipTests=true; fi
+ - if [ "$TEST_PYTHON" ] && pip list | grep tox; then TOX_FILE=`which tox` ; export TOX_HOME=`dirname $TOX_FILE`; fi
+ - if [ "$TEST_PYTHON" ] && ! pip list | grep tox; then travis_retry pip install tox --user; fi
# Removing this here protects from inadvertent caching
- rm -rf "$HOME/.m2/repository/org/apache/beam"
+ - rm -rf "$HOME/.m2/repository/org/apache/gearpump"
script:
- - travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates $MAVEN_OVERRIDE install && travis_retry bash -ex .travis/test_wordcount.sh
+ - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -e $TOX_ENV -c sdks/python/tox.ini; fi
+ - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .travis/test_wordcount.sh; fi
cache:
directories:
http://git-wip-us.apache.org/repos/asf/beam/blob/3f917987/runners/pom.xml
----------------------------------------------------------------------
[38/50] [abbrv] beam git commit: Merge branch 'master' of
https://github.com/apache/beam into sync-master
Posted by ke...@apache.org.
Merge branch 'master' of https://github.com/apache/beam into sync-master
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c2d3fbc3
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c2d3fbc3
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c2d3fbc3
Branch: refs/heads/master
Commit: c2d3fbc3a85a276850a2a56e040eccf2e2079339
Parents: 627ae0b 9e565f2
Author: manuzhang <ow...@gmail.com>
Authored: Fri Jul 7 16:33:03 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Fri Jul 7 16:33:03 2017 +0800
----------------------------------------------------------------------
.../jenkins/common_job_properties.groovy | 5 +-
.../job_beam_PreCommit_Website_Merge.groovy | 59 +++++++++
examples/java/pom.xml | 20 ++-
examples/java8/pom.xml | 20 ++-
.../examples/complete/game/LeaderBoardTest.java | 2 +
examples/pom.xml | 2 +-
pom.xml | 87 ++++++++++++-
runners/apex/pom.xml | 11 +-
runners/core-construction-java/pom.xml | 2 +-
runners/core-java/pom.xml | 2 +-
...eBoundedSplittableProcessElementInvoker.java | 5 +
.../beam/runners/core/ReduceFnRunner.java | 106 ++++++++++-----
.../beam/runners/core/SimpleDoFnRunner.java | 20 +++
.../beam/runners/core/ReduceFnRunnerTest.java | 81 +++++++++++-
runners/direct-java/pom.xml | 4 +-
runners/flink/pom.xml | 2 +-
runners/google-cloud-dataflow-java/pom.xml | 2 +-
.../dataflow/DataflowPipelineTranslator.java | 1 +
.../runners/dataflow/TransformTranslator.java | 4 +-
.../beam/runners/dataflow/util/TimeUtil.java | 24 ++--
.../runners/dataflow/util/TimeUtilTest.java | 6 +
runners/pom.xml | 2 +-
runners/spark/pom.xml | 16 +--
sdks/common/fn-api/pom.xml | 2 +-
sdks/common/pom.xml | 2 +-
sdks/common/runner-api/pom.xml | 2 +-
sdks/java/build-tools/pom.xml | 2 +-
sdks/java/core/pom.xml | 2 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 2 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 5 +
.../reflect/ByteBuddyDoFnInvokerFactory.java | 6 +
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 73 ++++-------
.../sdk/transforms/reflect/DoFnInvoker.java | 13 +-
.../sdk/transforms/reflect/DoFnSignature.java | 23 ++++
.../sdk/transforms/reflect/DoFnSignatures.java | 22 +++-
.../reflect/OnTimerMethodSpecifier.java | 37 ++++++
.../apache/beam/sdk/transforms/ParDoTest.java | 63 +++++++++
.../transforms/reflect/DoFnSignaturesTest.java | 14 ++
.../google-cloud-platform-core/pom.xml | 2 +-
sdks/java/extensions/jackson/pom.xml | 2 +-
sdks/java/extensions/join-library/pom.xml | 2 +-
sdks/java/extensions/pom.xml | 2 +-
sdks/java/extensions/protobuf/pom.xml | 2 +-
sdks/java/extensions/sorter/pom.xml | 8 +-
sdks/java/harness/pom.xml | 2 +-
sdks/java/io/amqp/pom.xml | 2 +-
sdks/java/io/cassandra/pom.xml | 2 +-
sdks/java/io/common/pom.xml | 2 +-
sdks/java/io/elasticsearch/pom.xml | 2 +-
sdks/java/io/google-cloud-platform/pom.xml | 2 +-
sdks/java/io/hadoop-common/pom.xml | 2 +-
sdks/java/io/hadoop-file-system/pom.xml | 33 +----
sdks/java/io/hadoop/input-format/pom.xml | 2 +-
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 4 +-
sdks/java/io/hadoop/pom.xml | 2 +-
sdks/java/io/hbase/pom.xml | 18 ++-
sdks/java/io/hcatalog/pom.xml | 20 ++-
sdks/java/io/jdbc/pom.xml | 4 +-
sdks/java/io/jms/pom.xml | 2 +-
sdks/java/io/kafka/pom.xml | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 130 +++++++++++++------
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 30 +++++
sdks/java/io/kinesis/pom.xml | 2 +-
sdks/java/io/mongodb/pom.xml | 2 +-
sdks/java/io/mqtt/pom.xml | 2 +-
sdks/java/io/pom.xml | 33 +----
sdks/java/io/xml/pom.xml | 2 +-
sdks/java/java8tests/pom.xml | 2 +-
sdks/java/javadoc/pom.xml | 19 ++-
.../maven-archetypes/examples-java8/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 1 -
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 1 -
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../resources/projects/basic/reference/pom.xml | 2 +-
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 2 +-
sdks/python/apache_beam/io/gcp/gcsio.py | 10 +-
.../runners/direct/transform_evaluator.py | 5 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/pom.xml | 2 +-
sdks/python/setup.py | 4 +-
83 files changed, 828 insertions(+), 303 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c2d3fbc3/examples/java/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c2d3fbc3/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c2d3fbc3/runners/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c2d3fbc3/sdks/java/pom.xml
----------------------------------------------------------------------
[40/50] [abbrv] beam git commit: Fix
ParDoTest#testPipelineOptionsParameter
Posted by ke...@apache.org.
Fix ParDoTest#testPipelineOptionsParameter
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/725f547f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/725f547f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/725f547f
Branch: refs/heads/master
Commit: 725f547f5e487dd3e84d5d0f95c0fa3efa853279
Parents: 2206827
Author: manuzhang <ow...@gmail.com>
Authored: Sat Jul 8 00:13:19 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Sat Jul 8 00:13:19 2017 +0800
----------------------------------------------------------------------
.../gearpump/translators/io/GearpumpSource.java | 12 ++----------
.../translators/utils/DoFnRunnerFactory.java | 5 +++--
.../translators/utils/TranslatorUtils.java | 19 +++++++++++++++++++
3 files changed, 24 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/725f547f/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
index daa8c81..2f53139 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
@@ -18,9 +18,6 @@
package org.apache.beam.runners.gearpump.translators.io;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
import java.io.IOException;
import java.time.Instant;
@@ -48,11 +45,7 @@ public abstract class GearpumpSource<T> implements DataSource {
private boolean available = false;
GearpumpSource(PipelineOptions options) {
- try {
- this.serializedOptions = new ObjectMapper().writeValueAsBytes(options);
- } catch (JsonProcessingException e) {
- throw new RuntimeException(e);
- }
+ this.serializedOptions = TranslatorUtils.serializePipelineOptions(options);
}
protected abstract Source.Reader<T> createReader(PipelineOptions options) throws IOException;
@@ -60,8 +53,7 @@ public abstract class GearpumpSource<T> implements DataSource {
@Override
public void open(TaskContext context, Instant startTime) {
try {
- PipelineOptions options = new ObjectMapper()
- .readValue(serializedOptions, PipelineOptions.class);
+ PipelineOptions options = TranslatorUtils.deserializePipelineOptions(serializedOptions);
this.reader = createReader(options);
this.available = reader.start();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/beam/blob/725f547f/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
index 35cf2b5..375b696 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
@@ -43,7 +43,7 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
private static final long serialVersionUID = -4109539010014189725L;
private final DoFn<InputT, OutputT> fn;
- private final transient PipelineOptions options;
+ private final byte[] serializedOptions;
private final Collection<PCollectionView<?>> sideInputs;
private final DoFnRunners.OutputManager outputManager;
private final TupleTag<OutputT> mainOutputTag;
@@ -61,7 +61,7 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
StepContext stepContext,
WindowingStrategy<?, ?> windowingStrategy) {
this.fn = doFn;
- this.options = pipelineOptions;
+ this.serializedOptions = TranslatorUtils.serializePipelineOptions(pipelineOptions);
this.sideInputs = sideInputs;
this.outputManager = outputManager;
this.mainOutputTag = mainOutputTag;
@@ -72,6 +72,7 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
public PushbackSideInputDoFnRunner<InputT, OutputT> createRunner(
ReadyCheckingSideInputReader sideInputReader) {
+ PipelineOptions options = TranslatorUtils.deserializePipelineOptions(serializedOptions);
DoFnRunner<InputT, OutputT> underlying = DoFnRunners.simpleRunner(
options, fn, sideInputReader, outputManager, mainOutputTag,
sideOutputTags, stepContext, windowingStrategy);
http://git-wip-us.apache.org/repos/asf/beam/blob/725f547f/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
index b1cd61c..c14298f 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
@@ -18,8 +18,11 @@
package org.apache.beam.runners.gearpump.translators.utils;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
+import java.io.IOException;
import java.time.Instant;
import java.util.Collection;
import java.util.HashMap;
@@ -27,6 +30,7 @@ import java.util.List;
import java.util.Map;
import org.apache.beam.runners.gearpump.translators.TranslationContext;
+import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
@@ -141,6 +145,21 @@ public class TranslatorUtils {
}
}
+ public static byte[] serializePipelineOptions(PipelineOptions options) {
+ try {
+ return new ObjectMapper().writeValueAsBytes(options);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static PipelineOptions deserializePipelineOptions(byte[] serializedOptions) {
+ try {
+ return new ObjectMapper().readValue(serializedOptions, PipelineOptions.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
/**
* This is copied from org.apache.beam.sdk.transforms.join.RawUnionValue.
[09/50] [abbrv] beam git commit: [BEAM-79] Fix gearpump-runner merge
conflicts and test failure
Posted by ke...@apache.org.
[BEAM-79] Fix gearpump-runner merge conflicts and test failure
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3eab6a64
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3eab6a64
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3eab6a64
Branch: refs/heads/master
Commit: 3eab6a647e4761725680c8bc40589dfa5569d75b
Parents: 3f91798
Author: manuzhang <ow...@gmail.com>
Authored: Tue Mar 14 08:09:46 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Wed Mar 15 15:21:29 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 51 ++-
.../gearpump/GearpumpPipelineResult.java | 21 +-
.../gearpump/GearpumpPipelineTranslator.java | 388 ++++++++++++++++++-
.../beam/runners/gearpump/GearpumpRunner.java | 376 +-----------------
.../runners/gearpump/TestGearpumpRunner.java | 38 +-
.../gearpump/examples/StreamingWordCount.java | 98 -----
.../gearpump/examples/UnboundedTextSource.java | 139 -------
.../runners/gearpump/examples/package-info.java | 22 --
...CreateGearpumpPCollectionViewTranslator.java | 14 +-
.../CreatePCollectionViewTranslator.java | 6 +-
.../translators/CreateValuesTranslator.java | 51 ---
.../FlattenPCollectionTranslator.java | 84 ----
.../FlattenPCollectionsTranslator.java | 83 ++++
.../translators/GroupByKeyTranslator.java | 4 +-
.../translators/ParDoBoundMultiTranslator.java | 32 +-
.../translators/ParDoBoundTranslator.java | 7 +-
.../translators/ReadBoundedTranslator.java | 4 +-
.../translators/ReadUnboundedTranslator.java | 4 +-
.../translators/TransformTranslator.java | 2 +-
.../translators/TranslationContext.java | 29 +-
.../translators/WindowAssignTranslator.java | 100 +++++
.../translators/WindowBoundTranslator.java | 100 -----
.../translators/functions/DoFnFunction.java | 12 +-
.../translators/io/UnboundedSourceWrapper.java | 1 +
.../translators/utils/DoFnRunnerFactory.java | 4 +-
.../utils/NoOpAggregatorFactory.java | 2 +-
.../translators/utils/NoOpStepContext.java | 6 +-
.../translators/utils/TranslatorUtils.java | 2 -
.../translators/utils/TranslatorUtilsTest.java | 1 -
29 files changed, 703 insertions(+), 978 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index 3efb1f6..9a6a432 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>0.5.0-incubating-SNAPSHOT</version>
+ <version>0.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -65,10 +65,12 @@
<configuration>
<groups>org.apache.beam.sdk.testing.RunnableOnService</groups>
<excludedGroups>
+ org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders,
org.apache.beam.sdk.testing.UsesStatefulParDo,
org.apache.beam.sdk.testing.UsesTimersInParDo,
org.apache.beam.sdk.testing.UsesSplittableParDo,
- org.apache.beam.sdk.testing.UsesMetrics
+ org.apache.beam.sdk.testing.UsesAttemptedMetrics,
+ org.apache.beam.sdk.testing.UsesCommittedMetrics
</excludedGroups>
<parallel>none</parallel>
<failIfNoTests>true</failIfNoTests>
@@ -136,6 +138,16 @@
<artifactId>beam-runners-core-java</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-core-construction-java</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
@@ -182,6 +194,11 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
@@ -210,8 +227,36 @@
<!-- Java compiler -->
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ <testSource>1.8</testSource>
+ <testTarget>1.8</testTarget>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <enforceBytecodeVersion>
+ <maxJdkVersion>1.8</maxJdkVersion>
+ </enforceBytecodeVersion>
+ <requireJavaVersion>
+ <version>[1.8,)</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
<!-- uber jar -->
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
index 8f90898..d833cd6 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
@@ -43,6 +43,7 @@ public class GearpumpPipelineResult implements PipelineResult {
private final ClientContext client;
private final RunningApplication app;
+ private boolean finished = false;
public GearpumpPipelineResult(ClientContext client, RunningApplication app) {
this.client = client;
@@ -51,13 +52,22 @@ public class GearpumpPipelineResult implements PipelineResult {
@Override
public State getState() {
- return getGearpumpState();
+ if (!finished) {
+ return getGearpumpState();
+ } else {
+ return State.DONE;
+ }
}
@Override
public State cancel() throws IOException {
- app.shutDown();
- return State.CANCELLED;
+ if (!finished) {
+ app.shutDown();
+ finished = true;
+ return State.CANCELLED;
+ } else {
+ return State.DONE;
+ }
}
@Override
@@ -67,7 +77,10 @@ public class GearpumpPipelineResult implements PipelineResult {
@Override
public State waitUntilFinish() {
- app.waitUntilFinish();
+ if (!finished) {
+ app.waitUntilFinish();
+ finished = true;
+ }
return State.DONE;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
index 4cc060c..1a36343 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
@@ -18,13 +18,19 @@
package org.apache.beam.runners.gearpump;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import org.apache.beam.runners.core.construction.PTransformMatchers;
+import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.runners.gearpump.translators.CreateGearpumpPCollectionViewTranslator;
import org.apache.beam.runners.gearpump.translators.CreatePCollectionViewTranslator;
-import org.apache.beam.runners.gearpump.translators.CreateValuesTranslator;
-import org.apache.beam.runners.gearpump.translators.FlattenPCollectionTranslator;
+import org.apache.beam.runners.gearpump.translators.FlattenPCollectionsTranslator;
import org.apache.beam.runners.gearpump.translators.GroupByKeyTranslator;
import org.apache.beam.runners.gearpump.translators.ParDoBoundMultiTranslator;
import org.apache.beam.runners.gearpump.translators.ParDoBoundTranslator;
@@ -32,17 +38,29 @@ import org.apache.beam.runners.gearpump.translators.ReadBoundedTranslator;
import org.apache.beam.runners.gearpump.translators.ReadUnboundedTranslator;
import org.apache.beam.runners.gearpump.translators.TransformTranslator;
import org.apache.beam.runners.gearpump.translators.TranslationContext;
-import org.apache.beam.runners.gearpump.translators.WindowBoundTranslator;
+import org.apache.beam.runners.gearpump.translators.WindowAssignTranslator;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.ListCoder;
import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.runners.PTransformMatcher;
+import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.util.InstanceBuilder;
+import org.apache.beam.sdk.util.PCollectionViews;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PValue;
import org.apache.gearpump.util.Graph;
@@ -74,14 +92,13 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
registerTransformTranslator(Read.Unbounded.class, new ReadUnboundedTranslator());
registerTransformTranslator(Read.Bounded.class, new ReadBoundedTranslator());
registerTransformTranslator(GroupByKey.class, new GroupByKeyTranslator());
- registerTransformTranslator(Flatten.FlattenPCollectionList.class,
- new FlattenPCollectionTranslator());
+ registerTransformTranslator(Flatten.PCollections.class,
+ new FlattenPCollectionsTranslator());
registerTransformTranslator(ParDo.BoundMulti.class, new ParDoBoundMultiTranslator());
- registerTransformTranslator(Window.Bound.class, new WindowBoundTranslator());
- registerTransformTranslator(Create.Values.class, new CreateValuesTranslator());
+ registerTransformTranslator(Window.Assign.class, new WindowAssignTranslator());
registerTransformTranslator(View.CreatePCollectionView.class,
new CreatePCollectionViewTranslator());
- registerTransformTranslator(GearpumpRunner.CreateGearpumpPCollectionView.class,
+ registerTransformTranslator(CreateGearpumpPCollectionView.class,
new CreateGearpumpPCollectionViewTranslator<>());
}
@@ -90,6 +107,27 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
}
public void translate(Pipeline pipeline) {
+ Map<PTransformMatcher, PTransformOverrideFactory> overrides =
+ ImmutableMap.<PTransformMatcher, PTransformOverrideFactory>builder()
+ .put(PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
+ new ReflectiveOneToOneOverrideFactory(
+ StreamingCombineGloballyAsSingletonView.class))
+ .put(PTransformMatchers.classEqualTo(View.AsMap.class),
+ new ReflectiveOneToOneOverrideFactory(StreamingViewAsMap.class))
+ .put(PTransformMatchers.classEqualTo(View.AsMultimap.class),
+ new ReflectiveOneToOneOverrideFactory(StreamingViewAsMultimap.class))
+ .put(PTransformMatchers.classEqualTo(View.AsSingleton.class),
+ new ReflectiveOneToOneOverrideFactory(StreamingViewAsSingleton.class))
+ .put(PTransformMatchers.classEqualTo(View.AsList.class),
+ new ReflectiveOneToOneOverrideFactory(StreamingViewAsList.class))
+ .put(PTransformMatchers.classEqualTo(View.AsIterable.class),
+ new ReflectiveOneToOneOverrideFactory(StreamingViewAsIterable.class))
+ .build();
+
+ for (Map.Entry<PTransformMatcher, PTransformOverrideFactory> override :
+ overrides.entrySet()) {
+ pipeline.replace(override.getKey(), override.getValue());
+ }
pipeline.traverseTopologically(this);
}
@@ -145,5 +183,337 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
return transformTranslators.get(transformClass);
}
+ // The following codes are forked from DataflowRunner for View translator
+ private static class ReflectiveOneToOneOverrideFactory<
+ InputT extends PValue,
+ OutputT extends PValue,
+ TransformT extends PTransform<InputT, OutputT>>
+ extends SingleInputOutputOverrideFactory<InputT, OutputT, TransformT> {
+ private final Class<PTransform<InputT, OutputT>> replacement;
+
+ private ReflectiveOneToOneOverrideFactory(
+ Class<PTransform<InputT, OutputT>> replacement) {
+ this.replacement = replacement;
+ }
+
+ @Override
+ public PTransform<InputT, OutputT> getReplacementTransform(TransformT transform) {
+ return InstanceBuilder.ofType(replacement)
+ .withArg((Class<PTransform<InputT, OutputT>>) transform.getClass(), transform)
+ .build();
+ }
+ }
+
+ /**
+ * Specialized implementation for
+ * {@link org.apache.beam.sdk.transforms.View.AsMap View.AsMap}
+ * for the Gearpump runner.
+ */
+ private static class StreamingViewAsMap<K, V>
+ extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, V>>> {
+
+ private static final long serialVersionUID = 4791080760092950304L;
+
+ public StreamingViewAsMap(View.AsMap<K, V> transform) {}
+
+ @Override
+ public PCollectionView<Map<K, V>> expand(PCollection<KV<K, V>> input) {
+ PCollectionView<Map<K, V>> view =
+ PCollectionViews.mapView(
+ input.getPipeline(),
+ input.getWindowingStrategy(),
+ input.getCoder());
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
+ try {
+ inputCoder.getKeyCoder().verifyDeterministic();
+ } catch (Coder.NonDeterministicException e) {
+ // throw new RuntimeException(e);
+ }
+
+ return input
+ .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
+ .apply(CreateGearpumpPCollectionView.<KV<K, V>, Map<K, V>>of(view));
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingViewAsMap";
+ }
+ }
+
+ /**
+ * Specialized expansion for {@link
+ * org.apache.beam.sdk.transforms.View.AsMultimap View.AsMultimap} for the
+ * Gearpump runner.
+ */
+ private static class StreamingViewAsMultimap<K, V>
+ extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, Iterable<V>>>> {
+
+ private static final long serialVersionUID = 5854899081751333352L;
+
+ public StreamingViewAsMultimap(View.AsMultimap<K, V> transform) {}
+
+ @Override
+ public PCollectionView<Map<K, Iterable<V>>> expand(PCollection<KV<K, V>> input) {
+ PCollectionView<Map<K, Iterable<V>>> view =
+ PCollectionViews.multimapView(
+ input.getPipeline(),
+ input.getWindowingStrategy(),
+ input.getCoder());
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
+ try {
+ inputCoder.getKeyCoder().verifyDeterministic();
+ } catch (Coder.NonDeterministicException e) {
+ // throw new RuntimeException(e);
+ }
+
+ return input
+ .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
+ .apply(CreateGearpumpPCollectionView.<KV<K, V>, Map<K, Iterable<V>>>of(view));
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingViewAsMultimap";
+ }
+ }
+
+ /**
+ * Specialized implementation for
+ * {@link org.apache.beam.sdk.transforms.View.AsIterable View.AsIterable} for the
+ * Gearpump runner.
+ */
+ private static class StreamingViewAsIterable<T>
+ extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
+
+ private static final long serialVersionUID = -3399860618995613421L;
+
+ public StreamingViewAsIterable(View.AsIterable<T> transform) {}
+
+ @Override
+ public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
+ PCollectionView<Iterable<T>> view =
+ PCollectionViews.iterableView(
+ input.getPipeline(),
+ input.getWindowingStrategy(),
+ input.getCoder());
+
+ return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
+ .apply(CreateGearpumpPCollectionView.<T, Iterable<T>>of(view));
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingViewAsIterable";
+ }
+ }
+
+ /**
+ * Specialized implementation for
+ * {@link org.apache.beam.sdk.transforms.View.AsList View.AsList} for the
+ * Gearpump runner.
+ */
+ private static class StreamingViewAsList<T>
+ extends PTransform<PCollection<T>, PCollectionView<List<T>>> {
+
+ private static final long serialVersionUID = -5018631473886330629L;
+
+ public StreamingViewAsList(View.AsList<T> transform) {}
+
+ @Override
+ public PCollectionView<List<T>> expand(PCollection<T> input) {
+ PCollectionView<List<T>> view =
+ PCollectionViews.listView(
+ input.getPipeline(),
+ input.getWindowingStrategy(),
+ input.getCoder());
+
+ return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
+ .apply(CreateGearpumpPCollectionView.<T, List<T>>of(view));
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingViewAsList";
+ }
+ }
+ private static class StreamingCombineGloballyAsSingletonView<InputT, OutputT>
+ extends PTransform<PCollection<InputT>, PCollectionView<OutputT>> {
+
+ private static final long serialVersionUID = 9064900748869035738L;
+ private final Combine.GloballyAsSingletonView<InputT, OutputT> transform;
+
+ public StreamingCombineGloballyAsSingletonView(
+ Combine.GloballyAsSingletonView<InputT, OutputT> transform) {
+ this.transform = transform;
+ }
+
+ @Override
+ public PCollectionView<OutputT> expand(PCollection<InputT> input) {
+ PCollection<OutputT> combined =
+ input.apply(Combine.globally(transform.getCombineFn())
+ .withoutDefaults()
+ .withFanout(transform.getFanout()));
+
+ PCollectionView<OutputT> view = PCollectionViews.singletonView(
+ combined.getPipeline(),
+ combined.getWindowingStrategy(),
+ transform.getInsertDefault(),
+ transform.getInsertDefault()
+ ? transform.getCombineFn().defaultValue() : null,
+ combined.getCoder());
+ return combined
+ .apply(ParDo.of(new WrapAsList<OutputT>()))
+ .apply(CreateGearpumpPCollectionView.<OutputT, OutputT>of(view));
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingCombineGloballyAsSingletonView";
+ }
+ }
+
+ private static class StreamingViewAsSingleton<T>
+ extends PTransform<PCollection<T>, PCollectionView<T>> {
+
+ private static final long serialVersionUID = 5870455965625071546L;
+ private final View.AsSingleton<T> transform;
+
+ public StreamingViewAsSingleton(View.AsSingleton<T> transform) {
+ this.transform = transform;
+ }
+
+ @Override
+ public PCollectionView<T> expand(PCollection<T> input) {
+ Combine.Globally<T, T> combine = Combine.globally(
+ new SingletonCombine<>(transform.hasDefaultValue(), transform.defaultValue()));
+ if (!transform.hasDefaultValue()) {
+ combine = combine.withoutDefaults();
+ }
+ return input.apply(combine.asSingletonView());
+ }
+
+ @Override
+ protected String getKindString() {
+ return "StreamingViewAsSingleton";
+ }
+
+ private static class SingletonCombine<T> extends Combine.BinaryCombineFn<T> {
+ private boolean hasDefaultValue;
+ private T defaultValue;
+
+ SingletonCombine(boolean hasDefaultValue, T defaultValue) {
+ this.hasDefaultValue = hasDefaultValue;
+ this.defaultValue = defaultValue;
+ }
+
+ @Override
+ public T apply(T left, T right) {
+ throw new IllegalArgumentException("PCollection with more than one element "
+ + "accessed as a singleton view. Consider using Combine.globally().asSingleton() to "
+ + "combine the PCollection into a single value");
+ }
+
+ @Override
+ public T identity() {
+ if (hasDefaultValue) {
+ return defaultValue;
+ } else {
+ throw new IllegalArgumentException(
+ "Empty PCollection accessed as a singleton view. "
+ + "Consider setting withDefault to provide a default value");
+ }
+ }
+ }
+ }
+
+ private static class WrapAsList<T> extends DoFn<T, List<T>> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(Collections.singletonList(c.element()));
+ }
+ }
+ /**
+ * Creates a primitive {@link PCollectionView}.
+ *
+ * <p>For internal use only by runner implementors.
+ *
+ * @param <ElemT> The type of the elements of the input PCollection
+ * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
+ */
+ public static class CreateGearpumpPCollectionView<ElemT, ViewT>
+ extends PTransform<PCollection<List<ElemT>>, PCollectionView<ViewT>> {
+ private static final long serialVersionUID = -2637073020800540542L;
+ private PCollectionView<ViewT> view;
+
+ private CreateGearpumpPCollectionView(PCollectionView<ViewT> view) {
+ this.view = view;
+ }
+
+ public static <ElemT, ViewT> CreateGearpumpPCollectionView<ElemT, ViewT> of(
+ PCollectionView<ViewT> view) {
+ return new CreateGearpumpPCollectionView<>(view);
+ }
+
+ public PCollectionView<ViewT> getView() {
+ return view;
+ }
+
+ @Override
+ public PCollectionView<ViewT> expand(PCollection<List<ElemT>> input) {
+ return view;
+ }
+ }
+
+ /**
+ * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
+ *
+ * <p>For internal use by {@link StreamingViewAsMap}, {@link StreamingViewAsMultimap},
+ * {@link StreamingViewAsList}, {@link StreamingViewAsIterable}.
+ * They require the input {@link PCollection} fits in memory.
+ * For a large {@link PCollection} this is expected to crash!
+ *
+ * @param <T> the type of elements to concatenate.
+ */
+ private static class Concatenate<T> extends Combine.CombineFn<T, List<T>, List<T>> {
+ @Override
+ public List<T> createAccumulator() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public List<T> addInput(List<T> accumulator, T input) {
+ accumulator.add(input);
+ return accumulator;
+ }
+
+ @Override
+ public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
+ List<T> result = createAccumulator();
+ for (List<T> accumulator : accumulators) {
+ result.addAll(accumulator);
+ }
+ return result;
+ }
+
+ @Override
+ public List<T> extractOutput(List<T> accumulator) {
+ return accumulator;
+ }
+
+ @Override
+ public Coder<List<T>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+
+ @Override
+ public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
index 72f2126..897467a 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
@@ -17,40 +17,18 @@
*/
package org.apache.beam.runners.gearpump;
-import com.google.common.collect.ImmutableMap;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigValueFactory;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import org.apache.beam.runners.gearpump.translators.TranslationContext;
+
import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.ListCoder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
import org.apache.beam.sdk.runners.PipelineRunner;
-import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.util.InstanceBuilder;
-import org.apache.beam.sdk.util.PCollectionViews;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionList;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.POutput;
import org.apache.gearpump.cluster.ClusterConfig;
import org.apache.gearpump.cluster.UserConfig;
@@ -72,21 +50,8 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
private static final String GEARPUMP_SERIALIZERS = "gearpump.serializers";
private static final String DEFAULT_APPNAME = "beam_gearpump_app";
- /** Custom transforms implementations. */
- private final Map<Class<?>, Class<?>> overrides;
-
public GearpumpRunner(GearpumpPipelineOptions options) {
this.options = options;
-
- ImmutableMap.Builder<Class<?>, Class<?>> builder = ImmutableMap.builder();
- builder.put(Combine.GloballyAsSingletonView.class,
- StreamingCombineGloballyAsSingletonView.class);
- builder.put(View.AsMap.class, StreamingViewAsMap.class);
- builder.put(View.AsMultimap.class, StreamingViewAsMultimap.class);
- builder.put(View.AsSingleton.class, StreamingViewAsSingleton.class);
- builder.put(View.AsList.class, StreamingViewAsList.class);
- builder.put(View.AsIterable.class, StreamingViewAsIterable.class);
- overrides = builder.build();
}
public static GearpumpRunner fromOptions(PipelineOptions options) {
@@ -95,31 +60,6 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
return new GearpumpRunner(pipelineOptions);
}
-
- public <OutputT extends POutput, InputT extends PInput> OutputT apply(
- PTransform<InputT, OutputT> transform, InputT input) {
- if (overrides.containsKey(transform.getClass())) {
-
- Class<PTransform<InputT, OutputT>> transformClass =
- (Class<PTransform<InputT, OutputT>>) transform.getClass();
-
- Class<PTransform<InputT, OutputT>> customTransformClass =
- (Class<PTransform<InputT, OutputT>>) overrides.get(transform.getClass());
-
- PTransform<InputT, OutputT> customTransform =
- InstanceBuilder.ofType(customTransformClass)
- .withArg(transformClass, transform)
- .build();
-
- return Pipeline.applyTransform(input, customTransform);
- } else if (Flatten.FlattenPCollectionList.class.equals(transform.getClass())
- && ((PCollectionList<?>) input).size() == 0) {
- return (OutputT) Pipeline.applyTransform(input.getPipeline().begin(), Create.of());
- } else {
- return super.apply(transform, input);
- }
- }
-
@Override
public GearpumpPipelineResult run(Pipeline pipeline) {
String appName = options.getApplicationName();
@@ -170,318 +110,4 @@ public class GearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
- // The following codes are forked from DataflowRunner for View translator
- /**
- * Specialized implementation for
- * {@link org.apache.beam.sdk.transforms.View.AsMap View.AsMap}
- * for the Gearpump runner.
- */
- private static class StreamingViewAsMap<K, V>
- extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, V>>> {
-
- private static final long serialVersionUID = 4791080760092950304L;
-
- public StreamingViewAsMap(View.AsMap<K, V> transform) {}
-
- @Override
- public PCollectionView<Map<K, V>> expand(PCollection<KV<K, V>> input) {
- PCollectionView<Map<K, V>> view =
- PCollectionViews.mapView(
- input.getPipeline(),
- input.getWindowingStrategy(),
- input.getCoder());
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
- try {
- inputCoder.getKeyCoder().verifyDeterministic();
- } catch (Coder.NonDeterministicException e) {
- // throw new RuntimeException(e);
- }
-
- return input
- .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
- .apply(CreateGearpumpPCollectionView.<KV<K, V>, Map<K, V>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsMap";
- }
- }
-
- /**
- * Specialized expansion for {@link
- * org.apache.beam.sdk.transforms.View.AsMultimap View.AsMultimap} for the
- * Gearpump runner.
- */
- private static class StreamingViewAsMultimap<K, V>
- extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, Iterable<V>>>> {
-
- private static final long serialVersionUID = 5854899081751333352L;
-
- public StreamingViewAsMultimap(View.AsMultimap<K, V> transform) {}
-
- @Override
- public PCollectionView<Map<K, Iterable<V>>> expand(PCollection<KV<K, V>> input) {
- PCollectionView<Map<K, Iterable<V>>> view =
- PCollectionViews.multimapView(
- input.getPipeline(),
- input.getWindowingStrategy(),
- input.getCoder());
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
- try {
- inputCoder.getKeyCoder().verifyDeterministic();
- } catch (Coder.NonDeterministicException e) {
- // throw new RuntimeException(e);
- }
-
- return input
- .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
- .apply(CreateGearpumpPCollectionView.<KV<K, V>, Map<K, Iterable<V>>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsMultimap";
- }
- }
-
- /**
- * Specialized implementation for
- * {@link org.apache.beam.sdk.transforms.View.AsIterable View.AsIterable} for the
- * Gearpump runner.
- */
- private static class StreamingViewAsIterable<T>
- extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
-
- private static final long serialVersionUID = -3399860618995613421L;
-
- public StreamingViewAsIterable(View.AsIterable<T> transform) {}
-
- @Override
- public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
- PCollectionView<Iterable<T>> view =
- PCollectionViews.iterableView(
- input.getPipeline(),
- input.getWindowingStrategy(),
- input.getCoder());
-
- return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
- .apply(CreateGearpumpPCollectionView.<T, Iterable<T>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsIterable";
- }
- }
-
- /**
- * Specialized implementation for
- * {@link org.apache.beam.sdk.transforms.View.AsList View.AsList} for the
- * Gearpump runner.
- */
- private static class StreamingViewAsList<T>
- extends PTransform<PCollection<T>, PCollectionView<List<T>>> {
-
- private static final long serialVersionUID = -5018631473886330629L;
-
- public StreamingViewAsList(View.AsList<T> transform) {}
-
- @Override
- public PCollectionView<List<T>> expand(PCollection<T> input) {
- PCollectionView<List<T>> view =
- PCollectionViews.listView(
- input.getPipeline(),
- input.getWindowingStrategy(),
- input.getCoder());
-
- return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
- .apply(CreateGearpumpPCollectionView.<T, List<T>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsList";
- }
- }
- private static class StreamingCombineGloballyAsSingletonView<InputT, OutputT>
- extends PTransform<PCollection<InputT>, PCollectionView<OutputT>> {
-
- private static final long serialVersionUID = 9064900748869035738L;
- private final Combine.GloballyAsSingletonView<InputT, OutputT> transform;
-
- public StreamingCombineGloballyAsSingletonView(
- Combine.GloballyAsSingletonView<InputT, OutputT> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollectionView<OutputT> expand(PCollection<InputT> input) {
- PCollection<OutputT> combined =
- input.apply(Combine.globally(transform.getCombineFn())
- .withoutDefaults()
- .withFanout(transform.getFanout()));
-
- PCollectionView<OutputT> view = PCollectionViews.singletonView(
- combined.getPipeline(),
- combined.getWindowingStrategy(),
- transform.getInsertDefault(),
- transform.getInsertDefault()
- ? transform.getCombineFn().defaultValue() : null,
- combined.getCoder());
- return combined
- .apply(ParDo.of(new WrapAsList<OutputT>()))
- .apply(CreateGearpumpPCollectionView.<OutputT, OutputT>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingCombineGloballyAsSingletonView";
- }
- }
-
- private static class StreamingViewAsSingleton<T>
- extends PTransform<PCollection<T>, PCollectionView<T>> {
-
- private static final long serialVersionUID = 5870455965625071546L;
- private final View.AsSingleton<T> transform;
-
- public StreamingViewAsSingleton(View.AsSingleton<T> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollectionView<T> expand(PCollection<T> input) {
- Combine.Globally<T, T> combine = Combine.globally(
- new SingletonCombine<>(transform.hasDefaultValue(), transform.defaultValue()));
- if (!transform.hasDefaultValue()) {
- combine = combine.withoutDefaults();
- }
- return input.apply(combine.asSingletonView());
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsSingleton";
- }
-
- private static class SingletonCombine<T> extends Combine.BinaryCombineFn<T> {
- private boolean hasDefaultValue;
- private T defaultValue;
-
- SingletonCombine(boolean hasDefaultValue, T defaultValue) {
- this.hasDefaultValue = hasDefaultValue;
- this.defaultValue = defaultValue;
- }
-
- @Override
- public T apply(T left, T right) {
- throw new IllegalArgumentException("PCollection with more than one element "
- + "accessed as a singleton view. Consider using Combine.globally().asSingleton() to "
- + "combine the PCollection into a single value");
- }
-
- @Override
- public T identity() {
- if (hasDefaultValue) {
- return defaultValue;
- } else {
- throw new IllegalArgumentException(
- "Empty PCollection accessed as a singleton view. "
- + "Consider setting withDefault to provide a default value");
- }
- }
- }
- }
-
- private static class WrapAsList<T> extends DoFn<T, List<T>> {
- @ProcessElement
- public void processElement(ProcessContext c) {
- c.output(Collections.singletonList(c.element()));
- }
- }
-
- /**
- * Creates a primitive {@link PCollectionView}.
- *
- * <p>For internal use only by runner implementors.
- *
- * @param <ElemT> The type of the elements of the input PCollection
- * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
- */
- public static class CreateGearpumpPCollectionView<ElemT, ViewT>
- extends PTransform<PCollection<List<ElemT>>, PCollectionView<ViewT>> {
- private static final long serialVersionUID = -2637073020800540542L;
- private PCollectionView<ViewT> view;
-
- private CreateGearpumpPCollectionView(PCollectionView<ViewT> view) {
- this.view = view;
- }
-
- public static <ElemT, ViewT> CreateGearpumpPCollectionView<ElemT, ViewT> of(
- PCollectionView<ViewT> view) {
- return new CreateGearpumpPCollectionView<>(view);
- }
-
- public PCollectionView<ViewT> getView() {
- return view;
- }
-
- @Override
- public PCollectionView<ViewT> expand(PCollection<List<ElemT>> input) {
- return view;
- }
- }
-
- /**
- * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
- *
- * <p>For internal use by {@link StreamingViewAsMap}, {@link StreamingViewAsMultimap},
- * {@link StreamingViewAsList}, {@link StreamingViewAsIterable}.
- * They require the input {@link PCollection} fits in memory.
- * For a large {@link PCollection} this is expected to crash!
- *
- * @param <T> the type of elements to concatenate.
- */
- private static class Concatenate<T> extends Combine.CombineFn<T, List<T>, List<T>> {
- @Override
- public List<T> createAccumulator() {
- return new ArrayList<>();
- }
-
- @Override
- public List<T> addInput(List<T> accumulator, T input) {
- accumulator.add(input);
- return accumulator;
- }
-
- @Override
- public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
- List<T> result = createAccumulator();
- for (List<T> accumulator : accumulators) {
- result.addAll(accumulator);
- }
- return result;
- }
-
- @Override
- public List<T> extractOutput(List<T> accumulator) {
- return accumulator;
- }
-
- @Override
- public Coder<List<T>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) {
- return ListCoder.of(inputCoder);
- }
-
- @Override
- public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) {
- return ListCoder.of(inputCoder);
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
index c96bcb1..ea7dd26 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
@@ -24,9 +24,6 @@ import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
import org.apache.beam.sdk.runners.PipelineRunner;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.POutput;
import org.apache.gearpump.cluster.ClusterConfig;
import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
@@ -58,36 +55,9 @@ public class TestGearpumpRunner extends PipelineRunner<GearpumpPipelineResult> {
@Override
public GearpumpPipelineResult run(Pipeline pipeline) {
- try {
- GearpumpPipelineResult result = delegate.run(pipeline);
- result.waitUntilFinish();
- cluster.stop();
- return result;
- } catch (Throwable e) {
- // copied from TestFlinkRunner to pull out AssertionError
- // which is wrapped in UserCodeException
- Throwable cause = e;
- Throwable oldCause;
- do {
- if (cause.getCause() == null) {
- break;
- }
-
- oldCause = cause;
- cause = cause.getCause();
-
- } while (!oldCause.equals(cause));
- if (cause instanceof AssertionError) {
- throw (AssertionError) cause;
- } else {
- throw e;
- }
- }
- }
-
- @Override
- public <OutputT extends POutput, InputT extends PInput>
- OutputT apply(PTransform<InputT, OutputT> transform, InputT input) {
- return delegate.apply(transform, input);
+ GearpumpPipelineResult result = delegate.run(pipeline);
+ result.waitUntilFinish();
+ cluster.stop();
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/StreamingWordCount.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/StreamingWordCount.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/StreamingWordCount.java
deleted file mode 100644
index b2d762a..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/StreamingWordCount.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump.examples;
-
-import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
-import org.apache.beam.runners.gearpump.GearpumpRunner;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-
-import org.apache.gearpump.cluster.client.ClientContext;
-import org.joda.time.Duration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * streaming word count example on Gearpump runner.
- */
-public class StreamingWordCount {
-
- static class ExtractWordsFn extends DoFn<String, String> {
-
- @ProcessElement
- public void process(ProcessContext c) {
- // Split the line into words.
- String[] words = c.element().split("[^a-zA-Z']+");
-
- // Output each word encountered into the output PCollection.
- for (String word : words) {
- if (!word.isEmpty()) {
- c.output(word);
- }
- }
- }
- }
-
- static class FormatAsStringFn extends DoFn<KV<String, Long>, String> {
- private static final Logger LOG = LoggerFactory.getLogger(FormatAsStringFn.class);
-
- @ProcessElement
- public void process(ProcessContext c) {
- String row = c.element().getKey()
- + " - " + c.element().getValue()
- + " @ " + c.timestamp().toString();
- LOG.debug("output {}", row);
- c.output(row);
- }
- }
-
-
- public static void main(String[] args) {
- GearpumpPipelineOptions options = PipelineOptionsFactory
- .fromArgs(args).as(GearpumpPipelineOptions.class);
- options.setRunner(GearpumpRunner.class);
- options.setApplicationName("StreamingWordCount");
- options.setParallelism(1);
-
- Pipeline p = Pipeline.create(options);
-
- PCollection<KV<String, Long>> wordCounts =
- p.apply(Read.from(new UnboundedTextSource()))
- .apply(ParDo.of(new ExtractWordsFn()))
- .apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(10))))
- .apply(Count.<String>perElement());
-
- wordCounts.apply(ParDo.of(new FormatAsStringFn()));
-
- p.run();
-
- ClientContext clientContext = options.getClientContext();
- clientContext.close();
-
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/UnboundedTextSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/UnboundedTextSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/UnboundedTextSource.java
deleted file mode 100644
index b014432..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/UnboundedTextSource.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump.examples;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import javax.annotation.Nullable;
-
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.UnboundedSource;
-import org.apache.beam.sdk.options.PipelineOptions;
-
-import org.joda.time.Instant;
-
-
-/**
- * unbounded source that reads from text.
- */
-public class UnboundedTextSource extends UnboundedSource<String, UnboundedSource.CheckpointMark> {
-
- @Override
- public List<? extends UnboundedSource<String, CheckpointMark>> generateInitialSplits(
- int desiredNumSplits, PipelineOptions options) throws Exception {
- return Collections.<UnboundedSource<String, CheckpointMark>>singletonList(this);
- }
-
- @Override
- public UnboundedReader<String> createReader(PipelineOptions options,
- @Nullable CheckpointMark checkpointMark) {
- return new UnboundedTextReader(this);
- }
-
- @Nullable
- @Override
- public Coder<CheckpointMark> getCheckpointMarkCoder() {
- return null;
- }
-
- @Override
- public void validate() {
- }
-
- @Override
- public Coder<String> getDefaultOutputCoder() {
- return StringUtf8Coder.of();
- }
-
- /**
- * reads from text.
- */
- public static class UnboundedTextReader extends UnboundedReader<String> implements Serializable {
-
- private static final long serialVersionUID = 7526472295622776147L;
-
- private final UnboundedTextSource source;
-
- private final String[] texts = new String[]{"foo foo foo bar bar", "foo foo bar bar bar"};
- private long index = 0;
-
- private String currentRecord;
-
- private Instant currentTimestamp;
-
- public UnboundedTextReader(UnboundedTextSource source) {
- this.source = source;
- }
-
- @Override
- public boolean start() throws IOException {
- currentRecord = texts[0];
- currentTimestamp = new Instant(0);
- return true;
- }
-
- @Override
- public boolean advance() throws IOException {
- index++;
- currentRecord = texts[(int) index % (texts.length)];
- currentTimestamp = new Instant(index * 1000);
-
- return true;
- }
-
- @Override
- public byte[] getCurrentRecordId() throws NoSuchElementException {
- return new byte[0];
- }
-
- @Override
- public String getCurrent() throws NoSuchElementException {
- return this.currentRecord;
- }
-
- @Override
- public Instant getCurrentTimestamp() throws NoSuchElementException {
- return currentTimestamp;
- }
-
- @Override
- public void close() throws IOException {
- }
-
- @Override
- public Instant getWatermark() {
- return currentTimestamp;
- }
-
- @Override
- public CheckpointMark getCheckpointMark() {
- return null;
- }
-
- @Override
- public UnboundedSource<String, ?> getCurrentSource() {
- return this.source;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/package-info.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/package-info.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/package-info.java
deleted file mode 100644
index a62a6c0..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/examples/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Examples showcase Beam application over Gearpump runner.
- */
-package org.apache.beam.runners.gearpump.examples;
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
index d05c89d..c7f24a8 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
@@ -20,25 +20,27 @@ package org.apache.beam.runners.gearpump.translators;
import java.util.List;
-import org.apache.beam.runners.gearpump.GearpumpRunner;
+import org.apache.beam.runners.gearpump.GearpumpPipelineTranslator;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
-
/**
* CreateGearpumpPCollectionView bridges input stream to down stream
* transforms.
*/
public class CreateGearpumpPCollectionViewTranslator<ElemT, ViewT> implements
- TransformTranslator<GearpumpRunner.CreateGearpumpPCollectionView<ElemT, ViewT>> {
+ TransformTranslator<GearpumpPipelineTranslator.CreateGearpumpPCollectionView<ElemT, ViewT>> {
+
+ private static final long serialVersionUID = -3955521308055056034L;
@Override
- public void translate(GearpumpRunner.CreateGearpumpPCollectionView<ElemT, ViewT> transform,
+ public void translate(
+ GearpumpPipelineTranslator.CreateGearpumpPCollectionView<ElemT, ViewT> transform,
TranslationContext context) {
JavaStream<WindowedValue<List<ElemT>>> inputStream =
- context.getInputStream(context.getInput(transform));
- PCollectionView<ViewT> view = transform.getView();
+ context.getInputStream(context.getInput());
+ PCollectionView<ViewT> view = (PCollectionView<ViewT>) context.getOutput();
context.setOutputStream(view, inputStream);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java
index e9e2e5d..da55d70 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreatePCollectionViewTranslator.java
@@ -32,12 +32,14 @@ import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
public class CreatePCollectionViewTranslator<ElemT, ViewT> implements
TransformTranslator<View.CreatePCollectionView<ElemT, ViewT>> {
+ private static final long serialVersionUID = -2394386873317515748L;
+
@Override
public void translate(View.CreatePCollectionView<ElemT, ViewT> transform,
TranslationContext context) {
JavaStream<WindowedValue<List<ElemT>>> inputStream =
- context.getInputStream(context.getInput(transform));
- PCollectionView<ViewT> view = transform.getView();
+ context.getInputStream(context.getInput());
+ PCollectionView<ViewT> view = (PCollectionView<ViewT>) context.getOutput();
context.setOutputStream(view, inputStream);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateValuesTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateValuesTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateValuesTranslator.java
deleted file mode 100644
index e5dc6dd..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateValuesTranslator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump.translators;
-
-import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
-import org.apache.beam.runners.gearpump.translators.io.ValuesSource;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
-import org.apache.beam.sdk.io.UnboundedSource;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.util.WindowedValue;
-
-import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
-
-/**
- * Wraps elements from Create.Values into an {@link UnboundedSource}.
- * mainly used for test
- */
-public class CreateValuesTranslator<T> implements TransformTranslator<Create.Values<T>> {
-
- private static final long serialVersionUID = 5411841848199229738L;
-
- @Override
- public void translate(Create.Values<T> transform, TranslationContext context) {
- try {
- UnboundedSourceWrapper<T, ?> unboundedSourceWrapper = new UnboundedSourceWrapper<>(
- new ValuesSource<>(transform.getElements(),
- transform.getDefaultOutputCoder(context.getInput(transform))),
- context.getPipelineOptions());
- JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(unboundedSourceWrapper);
- context.setOutputStream(context.getOutput(transform), sourceStream);
- } catch (CannotProvideCoderException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionTranslator.java
deleted file mode 100644
index 27e54b8..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionTranslator.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump.translators;
-
-import com.google.common.collect.Lists;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
-import org.apache.beam.runners.gearpump.translators.io.ValuesSource;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.values.PCollection;
-
-import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
-import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
-
-
-
-/**
- * Flatten.FlattenPCollectionList is translated to Gearpump merge function.
- * Note only two-way merge is working now
- */
-public class FlattenPCollectionTranslator<T> implements
- TransformTranslator<Flatten.FlattenPCollectionList<T>> {
-
- private static final long serialVersionUID = -5552148802472944759L;
-
- @Override
- public void translate(Flatten.FlattenPCollectionList<T> transform, TranslationContext context) {
- JavaStream<T> merged = null;
- Set<PCollection<T>> unique = new HashSet<>();
- for (PCollection<T> collection : context.getInput(transform).getAll()) {
- unique.add(collection);
- JavaStream<T> inputStream = context.getInputStream(collection);
- if (null == merged) {
- merged = inputStream;
- } else {
- // duplicate edges are not allowed in Gearpump graph
- // so we route through a dummy node
- if (unique.contains(collection)) {
- inputStream = inputStream.map(new DummyFunction<T>(), "dummy");
- }
-
- merged = merged.merge(inputStream, transform.getName());
- }
- }
-
- if (null == merged) {
- UnboundedSourceWrapper<String, ?> unboundedSourceWrapper = new UnboundedSourceWrapper<>(
- new ValuesSource<>(Lists.newArrayList("dummy"),
- StringUtf8Coder.of()), context.getPipelineOptions());
- merged = context.getSourceStream(unboundedSourceWrapper);
- }
- context.setOutputStream(context.getOutput(transform), merged);
- }
-
- private static class DummyFunction<T> extends MapFunction<T, T> {
-
- private static final long serialVersionUID = 5454396869997290471L;
-
- @Override
- public T map(T t) {
- return t;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
new file mode 100644
index 0000000..3a465cb
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import com.google.common.collect.Lists;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
+import org.apache.beam.runners.gearpump.translators.io.ValuesSource;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.values.PCollection;
+
+import org.apache.beam.sdk.values.TaggedPValue;
+import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+
+/**
+ * Flatten.FlattenPCollectionList is translated to Gearpump merge function.
+ */
+public class FlattenPCollectionsTranslator<T> implements
+ TransformTranslator<Flatten.PCollections<T>> {
+
+ private static final long serialVersionUID = -5552148802472944759L;
+
+ @Override
+ public void translate(Flatten.PCollections<T> transform, TranslationContext context) {
+ JavaStream<T> merged = null;
+ Set<PCollection<T>> unique = new HashSet<>();
+ for (TaggedPValue input: context.getInputs()) {
+ PCollection<T> collection = (PCollection<T>) input.getValue();
+ unique.add(collection);
+ JavaStream<T> inputStream = context.getInputStream(collection);
+ if (null == merged) {
+ merged = inputStream;
+ } else {
+ // duplicate edges are not allowed in Gearpump graph
+ // so we route through a dummy node
+ if (unique.contains(collection)) {
+ inputStream = inputStream.map(new DummyFunction<T>(), "dummy");
+ }
+
+ merged = merged.merge(inputStream, transform.getName());
+ }
+ }
+
+ if (null == merged) {
+ UnboundedSourceWrapper<String, ?> unboundedSourceWrapper = new UnboundedSourceWrapper<>(
+ new ValuesSource<>(Lists.newArrayList("dummy"),
+ StringUtf8Coder.of()), context.getPipelineOptions());
+ merged = context.getSourceStream(unboundedSourceWrapper);
+ }
+ context.setOutputStream(context.getOutput(), merged);
+ }
+
+ private static class DummyFunction<T> extends MapFunction<T, T> {
+
+ private static final long serialVersionUID = 5454396869997290471L;
+
+ @Override
+ public T map(T t) {
+ return t;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
index df8bfe9..5dfd3e9 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
@@ -61,7 +61,7 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
@Override
public void translate(GroupByKey<K, V> transform, TranslationContext context) {
- PCollection<KV<K, V>> input = context.getInput(transform);
+ PCollection<KV<K, V>> input = (PCollection<KV<K, V>>) context.getInput();
Coder<K> inputKeyCoder = ((KvCoder<K, V>) input.getCoder()).getKeyCoder();
JavaStream<WindowedValue<KV<K, V>>> inputStream =
context.getInputStream(input);
@@ -80,7 +80,7 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
.fold(new Merge<>(windowFn, outputTimeFn), "merge")
.map(new Values<K, V>(), "values");
- context.setOutputStream(context.getOutput(transform), outputStream);
+ context.setOutputStream(context.getOutput(), outputStream);
}
private static class GearpumpWindowFn<T, W extends BoundedWindow>
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
index 8c57019..e88cb73 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundMultiTranslator.java
@@ -18,17 +18,11 @@
package org.apache.beam.runners.gearpump.translators;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
-import javax.annotation.Nullable;
-
import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
import org.apache.beam.sdk.transforms.DoFn;
@@ -36,6 +30,7 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TaggedPValue;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.gearpump.streaming.dsl.api.functions.FilterFunction;
@@ -54,21 +49,21 @@ public class ParDoBoundMultiTranslator<InputT, OutputT> implements
@Override
public void translate(ParDo.BoundMulti<InputT, OutputT> transform, TranslationContext context) {
- PCollection<InputT> inputT = (PCollection<InputT>) context.getInput(transform);
+ PCollection<InputT> inputT = (PCollection<InputT>) context.getInput();
JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(inputT);
Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
Map<String, PCollectionView<?>> tagsToSideInputs =
TranslatorUtils.getTagsToSideInputs(sideInputs);
- Map<TupleTag<?>, PCollection<?>> outputs = context.getOutput(transform).getAll();
+ List<TaggedPValue> outputs = context.getOutputs();
final TupleTag<OutputT> mainOutput = transform.getMainOutputTag();
- List<TupleTag<?>> sideOutputs = Lists.newLinkedList(Sets.filter(outputs.keySet(),
- new Predicate<TupleTag<?>>() {
- @Override
- public boolean apply(@Nullable TupleTag<?> tupleTag) {
- return tupleTag != null && !tupleTag.getId().equals(mainOutput.getId());
- }
- }));
+ List<TupleTag<?>> sideOutputs = new ArrayList<>(outputs.size() - 1);
+ for (TaggedPValue output: outputs) {
+ TupleTag<?> tag = output.getTag();
+ if (tag != null && !tag.getId().equals(mainOutput.getId())) {
+ sideOutputs.add(tag);
+ }
+ }
JavaStream<TranslatorUtils.RawUnionValue> unionStream = TranslatorUtils.withSideInputStream(
context, inputStream, tagsToSideInputs);
@@ -83,10 +78,9 @@ public class ParDoBoundMultiTranslator<InputT, OutputT> implements
tagsToSideInputs,
mainOutput,
sideOutputs), transform.getName());
- for (Map.Entry<TupleTag<?>, PCollection<?>> output: outputs.entrySet()) {
- output.getValue().getCoder();
+ for (TaggedPValue output: outputs) {
JavaStream<WindowedValue<OutputT>> taggedStream = outputStream
- .filter(new FilterByOutputTag(output.getKey().getId()),
+ .filter(new FilterByOutputTag(output.getTag().getId()),
"filter_by_output_tag")
.map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
context.setOutputStream(output.getValue(), taggedStream);
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
index efae938..dc32b8c 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoBoundTranslator.java
@@ -35,7 +35,6 @@ import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
-
/**
* {@link ParDo.Bound} is translated to Gearpump flatMap function
* with {@link DoFn} wrapped in {@link DoFnFunction}.
@@ -50,14 +49,14 @@ public class ParDoBoundTranslator<InputT, OutputT> implements
@Override
public void translate(ParDo.Bound<InputT, OutputT> transform, TranslationContext context) {
DoFn<InputT, OutputT> doFn = transform.getFn();
- PCollection<OutputT> output = context.getOutput(transform);
+ PCollection<OutputT> output = (PCollection<OutputT>) context.getOutput();
WindowingStrategy<?, ?> windowingStrategy = output.getWindowingStrategy();
Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
Map<String, PCollectionView<?>> tagsToSideInputs =
TranslatorUtils.getTagsToSideInputs(sideInputs);
JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(
- context.getInput(transform));
+ context.getInput());
JavaStream<TranslatorUtils.RawUnionValue> unionStream =
TranslatorUtils.withSideInputStream(context,
inputStream, tagsToSideInputs);
@@ -71,6 +70,6 @@ public class ParDoBoundTranslator<InputT, OutputT> implements
.flatMap(doFnFunction, transform.getName())
.map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
- context.setOutputStream(context.getOutput(transform), outputStream);
+ context.setOutputStream(context.getOutput(), outputStream);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslator.java
index 478d58f..8f71a8e 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadBoundedTranslator.java
@@ -31,6 +31,8 @@ import org.apache.gearpump.streaming.source.DataSource;
*/
public class ReadBoundedTranslator <T> implements TransformTranslator<Read.Bounded<T>> {
+ private static final long serialVersionUID = -3899020490896998330L;
+
@Override
public void translate(Read.Bounded<T> transform, TranslationContext context) {
BoundedSource<T> boundedSource = transform.getSource();
@@ -38,7 +40,7 @@ public class ReadBoundedTranslator <T> implements TransformTranslator<Read.Bound
context.getPipelineOptions());
JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(sourceWrapper);
- context.setOutputStream(context.getOutput(transform), sourceStream);
+ context.setOutputStream(context.getOutput(), sourceStream);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslator.java
index 7e12a9c..0462c57 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ReadUnboundedTranslator.java
@@ -33,6 +33,8 @@ import org.apache.gearpump.streaming.source.DataSource;
public class ReadUnboundedTranslator<T> implements TransformTranslator<Read.Unbounded<T>> {
+ private static final long serialVersionUID = 3529494817859948619L;
+
@Override
public void translate(Read.Unbounded<T> transform, TranslationContext context) {
UnboundedSource<T, ?> unboundedSource = transform.getSource();
@@ -40,7 +42,7 @@ public class ReadUnboundedTranslator<T> implements TransformTranslator<Read.Unbo
unboundedSource, context.getPipelineOptions());
JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(unboundedSourceWrapper);
- context.setOutputStream(context.getOutput(transform), sourceStream);
+ context.setOutputStream(context.getOutput(), sourceStream);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TransformTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TransformTranslator.java
index c8587d3..c7becad 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TransformTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TransformTranslator.java
@@ -23,7 +23,7 @@ import java.io.Serializable;
import org.apache.beam.sdk.transforms.PTransform;
/**
- * translates {@link PTransform} to Gearpump functions.
+ * Translates {@link PTransform} to Gearpump functions.
*/
public interface TransformTranslator<T extends PTransform> extends Serializable {
void translate(T transform, TranslationContext context);
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
index b2cff8a..e88bb74 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
@@ -20,17 +20,18 @@ package org.apache.beam.runners.gearpump.translators;
import static com.google.common.base.Preconditions.checkArgument;
+import com.google.common.collect.Iterables;
+
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.POutput;
import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TaggedPValue;
import org.apache.gearpump.cluster.UserConfig;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStreamApp;
@@ -70,18 +71,26 @@ public class TranslationContext {
}
}
- public <InputT extends PInput> InputT getInput(PTransform<InputT, ?> transform) {
- return (InputT) getCurrentTransform(transform).getInput();
+ public List<TaggedPValue> getInputs() {
+ return getCurrentTransform().getInputs();
+ }
+
+ public PValue getInput() {
+ return Iterables.getOnlyElement(getInputs()).getValue();
+ }
+
+ public List<TaggedPValue> getOutputs() {
+ return getCurrentTransform().getOutputs();
}
- public <OutputT extends POutput> OutputT getOutput(PTransform<?, OutputT> transform) {
- return (OutputT) getCurrentTransform(transform).getOutput();
+ public PValue getOutput() {
+ return Iterables.getOnlyElement(getOutputs()).getValue();
}
- private AppliedPTransform<?, ?, ?> getCurrentTransform(PTransform<?, ?> transform) {
+ private AppliedPTransform<?, ?, ?> getCurrentTransform() {
checkArgument(
- currentTransform != null && currentTransform.getTransform() == transform,
- "can only be called with current transform");
+ currentTransform != null,
+ "current transform not set");
return currentTransform;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3eab6a64/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
new file mode 100644
index 0000000..fe6015a
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators;
+
+import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
+import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
+import org.joda.time.Instant;
+
+/**
+ * {@link Window.Bound} is translated to Gearpump flatMap function.
+ */
+@SuppressWarnings("unchecked")
+public class WindowAssignTranslator<T> implements TransformTranslator<Window.Assign<T>> {
+
+ private static final long serialVersionUID = -964887482120489061L;
+
+ @Override
+ public void translate(Window.Assign<T> transform, TranslationContext context) {
+ PCollection<T> input = (PCollection<T>) context.getInput();
+ PCollection<T> output = (PCollection<T>) context.getOutput();
+ JavaStream<WindowedValue<T>> inputStream = context.getInputStream(input);
+ WindowingStrategy<?, ?> outputStrategy = output.getWindowingStrategy();
+ WindowFn<T, BoundedWindow> windowFn = (WindowFn<T, BoundedWindow>) outputStrategy.getWindowFn();
+ JavaStream<WindowedValue<T>> outputStream =
+ inputStream
+ .flatMap(new AssignWindows(windowFn), "assign_windows");
+
+ context.setOutputStream(output, outputStream);
+ }
+
+ private static class AssignWindows<T> extends
+ FlatMapFunction<WindowedValue<T>, WindowedValue<T>> {
+
+ private static final long serialVersionUID = 7284565861938681360L;
+ private final WindowFn<T, BoundedWindow> windowFn;
+
+ AssignWindows(WindowFn<T, BoundedWindow> windowFn) {
+ this.windowFn = windowFn;
+ }
+
+ @Override
+ public Iterator<WindowedValue<T>> flatMap(final WindowedValue<T> value) {
+ try {
+ Collection<BoundedWindow> windows = windowFn.assignWindows(windowFn.new AssignContext() {
+ @Override
+ public T element() {
+ return value.getValue();
+ }
+
+ @Override
+ public Instant timestamp() {
+ return value.getTimestamp();
+ }
+
+ @Override
+ public BoundedWindow window() {
+ return Iterables.getOnlyElement(value.getWindows());
+ }
+ });
+ List<WindowedValue<T>> values = new ArrayList<>(windows.size());
+ for (BoundedWindow win: windows) {
+ values.add(
+ WindowedValue.of(value.getValue(), value.getTimestamp(), win, value.getPane()));
+ }
+ return values.iterator();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
[05/50] [abbrv] beam git commit: enable ParDoTest
Posted by ke...@apache.org.
enable ParDoTest
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3dc8fc81
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3dc8fc81
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3dc8fc81
Branch: refs/heads/master
Commit: 3dc8fc81d14d67318e26b99691f1583aacbd3509
Parents: 4eb50d1
Author: manuzhang <ow...@gmail.com>
Authored: Fri Mar 10 16:50:01 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Fri Mar 10 16:50:01 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/3dc8fc81/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index 6a41dc0..3efb1f6 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -75,11 +75,6 @@
<dependenciesToScan>
<dependency>org.apache.beam:beam-sdks-java-core</dependency>
</dependenciesToScan>
- <excludes>
- <exclude>
- org.apache.beam.sdk.transforms.ParDoTest
- </exclude>
- </excludes>
<systemPropertyVariables>
<beamTestPipelineOptions>
[
[30/50] [abbrv] beam git commit: Merge branch 'master' of
https://github.com/apache/beam into sync-master
Posted by ke...@apache.org.
Merge branch 'master' of https://github.com/apache/beam into sync-master
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fed98c84
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fed98c84
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fed98c84
Branch: refs/heads/master
Commit: fed98c8402f0f8c2c6b5bad544256d11cd16f540
Parents: 559e3c3 0cabdf6
Author: manuzhang <ow...@gmail.com>
Authored: Sat Jun 17 10:52:41 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Sat Jun 17 10:52:41 2017 +0800
----------------------------------------------------------------------
.gitignore | 2 +-
pom.xml | 7 +
runners/apex/pom.xml | 7 +
.../utils/ApexStateInternalsTest.java | 411 +++----------
.../core/InMemoryStateInternalsTest.java | 569 ++---------------
.../beam/runners/core/StateInternalsTest.java | 613 +++++++++++++++++++
.../beam/runners/direct/CommittedResult.java | 12 +-
.../beam/runners/direct/EvaluationContext.java | 26 +-
.../direct/ExecutorServiceParallelExecutor.java | 9 +-
.../direct/ParDoMultiOverrideFactory.java | 90 ++-
.../direct/StatefulParDoEvaluatorFactory.java | 11 +-
.../beam/runners/direct/WatermarkManager.java | 4 +-
.../runners/direct/CommittedResultTest.java | 17 +-
.../StatefulParDoEvaluatorFactoryTest.java | 65 +-
.../runners/direct/TransformExecutorTest.java | 11 +-
.../runners/direct/WatermarkManagerTest.java | 15 +-
runners/flink/pom.xml | 9 +-
.../streaming/state/FlinkStateInternals.java | 425 ++++++-------
.../FlinkBroadcastStateInternalsTest.java | 242 ++------
.../FlinkKeyGroupStateInternalsTest.java | 359 +++++------
.../streaming/FlinkSplitStateInternalsTest.java | 132 ++--
.../streaming/FlinkStateInternalsTest.java | 343 +----------
runners/spark/pom.xml | 7 +
.../apache/beam/runners/spark/SparkRunner.java | 7 +-
.../spark/stateful/SparkStateInternalsTest.java | 66 ++
.../fn-api/src/main/proto/beam_fn_api.proto | 63 +-
.../src/main/resources/beam/findbugs-filter.xml | 9 +
.../sdk/options/ProxyInvocationHandler.java | 15 +-
.../sdk/options/ProxyInvocationHandlerTest.java | 12 +
.../apache/beam/sdk/transforms/CombineTest.java | 225 ++++---
.../beam/sdk/transforms/SplittableDoFnTest.java | 145 ++++-
sdks/java/harness/pom.xml | 6 +
.../harness/control/ProcessBundleHandler.java | 293 +++------
.../beam/runners/core/BeamFnDataReadRunner.java | 70 ++-
.../runners/core/BeamFnDataWriteRunner.java | 67 +-
.../beam/runners/core/BoundedSourceRunner.java | 74 ++-
.../beam/runners/core/DoFnRunnerFactory.java | 182 ++++++
.../runners/core/PTransformRunnerFactory.java | 81 +++
.../control/ProcessBundleHandlerTest.java | 521 +++-------------
.../runners/core/BeamFnDataReadRunnerTest.java | 112 +++-
.../runners/core/BeamFnDataWriteRunnerTest.java | 120 +++-
.../runners/core/BoundedSourceRunnerTest.java | 124 +++-
.../runners/core/DoFnRunnerFactoryTest.java | 209 +++++++
.../beam/sdk/io/gcp/spanner/MutationGroup.java | 67 ++
.../io/gcp/spanner/MutationSizeEstimator.java | 9 +
.../beam/sdk/io/gcp/spanner/SpannerIO.java | 53 +-
.../gcp/spanner/MutationSizeEstimatorTest.java | 12 +
.../beam/sdk/io/gcp/spanner/SpannerIOTest.java | 76 ++-
sdks/java/io/hbase/pom.xml | 2 +-
sdks/python/apache_beam/coders/coder_impl.py | 4 +
sdks/python/apache_beam/coders/coders.py | 2 +-
.../apache_beam/coders/coders_test_common.py | 8 +
sdks/python/apache_beam/io/filesystem.py | 14 +-
.../apache_beam/options/pipeline_options.py | 26 +-
.../options/pipeline_options_test.py | 39 +-
.../apache_beam/options/value_provider_test.py | 93 +--
sdks/python/apache_beam/pipeline.py | 218 ++++++-
sdks/python/apache_beam/pipeline_test.py | 35 ++
.../apache_beam/portability/api/__init__.py | 21 +
.../apache_beam/portability/runners/__init__.py | 18 -
.../portability/runners/api/__init__.py | 21 -
sdks/python/apache_beam/pvalue.py | 2 +-
.../runners/dataflow/dataflow_runner.py | 14 +-
.../runners/dataflow/internal/apiclient_test.py | 24 +
.../runners/dataflow/native_io/iobase_test.py | 39 +-
.../runners/direct/bundle_factory.py | 2 +-
.../apache_beam/runners/direct/direct_runner.py | 39 ++
.../runners/direct/evaluation_context.py | 56 +-
.../apache_beam/runners/direct/executor.py | 64 +-
.../runners/direct/transform_evaluator.py | 157 ++++-
.../runners/direct/transform_result.py | 3 +-
.../apache_beam/runners/pipeline_context.py | 2 +-
.../runners/portability/fn_api_runner.py | 18 +-
.../runners/portability/fn_api_runner_test.py | 4 +-
.../apache_beam/runners/worker/data_plane.py | 2 +-
.../runners/worker/data_plane_test.py | 2 +-
.../apache_beam/runners/worker/log_handler.py | 2 +-
.../runners/worker/log_handler_test.py | 2 +-
.../apache_beam/runners/worker/operations.py | 1 +
.../apache_beam/runners/worker/sdk_worker.py | 4 +-
.../runners/worker/sdk_worker_main.py | 2 +-
.../runners/worker/sdk_worker_test.py | 2 +-
sdks/python/apache_beam/transforms/core.py | 91 +--
.../python/apache_beam/transforms/ptransform.py | 2 +-
sdks/python/apache_beam/transforms/trigger.py | 2 +-
sdks/python/apache_beam/transforms/window.py | 4 +-
sdks/python/apache_beam/utils/plugin.py | 42 ++
sdks/python/apache_beam/utils/urns.py | 2 +-
sdks/python/gen_protos.py | 2 +-
sdks/python/run_pylint.sh | 2 +-
90 files changed, 4049 insertions(+), 3044 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/fed98c84/pom.xml
----------------------------------------------------------------------
[26/50] [abbrv] beam git commit: Activate Gearpump
local-validates-runner-tests in precommit
Posted by ke...@apache.org.
Activate Gearpump local-validates-runner-tests in precommit
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3c7e3e6a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3c7e3e6a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3c7e3e6a
Branch: refs/heads/master
Commit: 3c7e3e6ac5d56855b55c1271d54924d9860ae1a0
Parents: c9aac96
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu May 18 13:09:36 2017 -0700
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Jun 5 19:16:34 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/3c7e3e6a/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index beb7753..7e39a48 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -49,7 +49,8 @@
<profiles>
<profile>
<id>local-validates-runner-tests</id>
- <activation><activeByDefault>false</activeByDefault></activation>
+ <!-- TODO: once on master branch, move to postcommit only -->
+ <activation><activeByDefault>true</activeByDefault></activation>
<build>
<plugins>
<plugin>
[19/50] [abbrv] beam git commit: Update gearpump-runner against
master changes
Posted by ke...@apache.org.
Update gearpump-runner against master changes
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/12b9719e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/12b9719e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/12b9719e
Branch: refs/heads/master
Commit: 12b9719e992d6fbac57efb4dc8ce7eff5e977862
Parents: 9a59ea3
Author: manuzhang <ow...@gmail.com>
Authored: Thu May 4 12:14:00 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Thu May 4 12:14:00 2017 +0800
----------------------------------------------------------------------
.../gearpump/GearpumpPipelineResult.java | 11 ----
.../translators/GroupByKeyTranslator.java | 28 ++++-----
.../translators/functions/DoFnFunction.java | 2 -
.../translators/utils/DoFnRunnerFactory.java | 6 +-
.../utils/NoOpAggregatorFactory.java | 63 --------------------
.../translators/utils/NoOpStepContext.java | 2 +-
.../translators/GroupByKeyTranslatorTest.java | 24 ++++----
7 files changed, 27 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/12b9719e/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
index d833cd6..dd7fa23 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
@@ -20,12 +20,9 @@ package org.apache.beam.runners.gearpump;
import java.io.IOException;
import java.util.List;
-import org.apache.beam.sdk.AggregatorRetrievalException;
-import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.metrics.MetricResults;
-import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.gearpump.cluster.ApplicationStatus;
import org.apache.gearpump.cluster.MasterToAppMaster.AppMasterData;
@@ -85,14 +82,6 @@ public class GearpumpPipelineResult implements PipelineResult {
}
@Override
- public <T> AggregatorValues<T> getAggregatorValues(Aggregator<?, T> aggregator)
- throws AggregatorRetrievalException {
- throw new AggregatorRetrievalException(
- "PipelineResult getAggregatorValues not supported in Gearpump pipeline",
- new UnsupportedOperationException());
- }
-
- @Override
public MetricResults metrics() {
return null;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/12b9719e/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
index 54c8737..521f665 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
@@ -33,7 +33,7 @@ import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.OutputTimeFn;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.util.WindowedValue;
@@ -66,8 +66,7 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
JavaStream<WindowedValue<KV<K, V>>> inputStream =
context.getInputStream(input);
int parallelism = context.getPipelineOptions().getParallelism();
- OutputTimeFn<? super BoundedWindow> outputTimeFn = (OutputTimeFn<? super BoundedWindow>)
- input.getWindowingStrategy().getOutputTimeFn();
+ TimestampCombiner timestampCombiner = input.getWindowingStrategy().getTimestampCombiner();
WindowFn<KV<K, V>, BoundedWindow> windowFn = (WindowFn<KV<K, V>, BoundedWindow>)
input.getWindowingStrategy().getWindowFn();
JavaStream<WindowedValue<KV<K, List<V>>>> outputStream = inputStream
@@ -75,9 +74,8 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
new GearpumpWindowFn(windowFn.isNonMerging()),
EventTimeTrigger$.MODULE$, Discarding$.MODULE$), "assign_window")
.groupBy(new GroupByFn<K, V>(inputKeyCoder), parallelism, "group_by_Key_and_Window")
- .map(new KeyedByTimestamp<K, V>((OutputTimeFn<? super BoundedWindow>)
- input.getWindowingStrategy().getOutputTimeFn()), "keyed_by_timestamp")
- .fold(new Merge<>(windowFn, outputTimeFn), "merge")
+ .map(new KeyedByTimestamp<K, V>(timestampCombiner), "keyed_by_timestamp")
+ .fold(new Merge<>(windowFn, timestampCombiner), "merge")
.map(new Values<K, V>(), "values");
context.setOutputStream(context.getOutput(), outputStream);
@@ -148,17 +146,17 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
extends MapFunction<WindowedValue<KV<K, V>>,
KV<Instant, WindowedValue<KV<K, V>>>> {
- private final OutputTimeFn<? super BoundedWindow> outputTimeFn;
+ private final TimestampCombiner timestampCombiner;
- public KeyedByTimestamp(OutputTimeFn<? super BoundedWindow> outputTimeFn) {
- this.outputTimeFn = outputTimeFn;
+ public KeyedByTimestamp(TimestampCombiner timestampCombiner) {
+ this.timestampCombiner = timestampCombiner;
}
@Override
public KV<org.joda.time.Instant, WindowedValue<KV<K, V>>> map(
WindowedValue<KV<K, V>> wv) {
- Instant timestamp = outputTimeFn.assignOutputTime(wv.getTimestamp(),
- Iterables.getOnlyElement(wv.getWindows()));
+ Instant timestamp = timestampCombiner.assign(
+ Iterables.getOnlyElement(wv.getWindows()), wv.getTimestamp());
return KV.of(timestamp, wv);
}
}
@@ -171,12 +169,12 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
KV<Instant, WindowedValue<KV<K, List<V>>>>> {
private final WindowFn<KV<K, V>, BoundedWindow> windowFn;
- private final OutputTimeFn<? super BoundedWindow> outputTimeFn;
+ private final TimestampCombiner timestampCombiner;
Merge(WindowFn<KV<K, V>, BoundedWindow> windowFn,
- OutputTimeFn<? super BoundedWindow> outputTimeFn) {
+ TimestampCombiner timestampCombiner) {
this.windowFn = windowFn;
- this.outputTimeFn = outputTimeFn;
+ this.timestampCombiner = timestampCombiner;
}
@Override
@@ -229,7 +227,7 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
mergedWindows.addAll(wv1.getWindows());
}
- Instant timestamp = outputTimeFn.combine(t1, t2);
+ Instant timestamp = timestampCombiner.combine(t1, t2);
return KV.of(timestamp,
WindowedValue.of(wv1.getValue(), timestamp,
mergedWindows, wv1.getPane()));
http://git-wip-us.apache.org/repos/asf/beam/blob/12b9719e/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index 3473f53..dfd6296 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -38,7 +38,6 @@ import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
import org.apache.beam.runners.core.SideInputHandler;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
import org.apache.beam.runners.gearpump.translators.utils.DoFnRunnerFactory;
-import org.apache.beam.runners.gearpump.translators.utils.NoOpAggregatorFactory;
import org.apache.beam.runners.gearpump.translators.utils.NoOpStepContext;
import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils.RawUnionValue;
@@ -92,7 +91,6 @@ public class DoFnFunction<InputT, OutputT> extends
mainOutput,
sideOutputs,
new NoOpStepContext(),
- new NoOpAggregatorFactory(),
windowingStrategy
);
this.sideInputs = sideInputs;
http://git-wip-us.apache.org/repos/asf/beam/blob/12b9719e/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
index 70b4271..8d55d6f 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
@@ -22,7 +22,6 @@ import java.io.Serializable;
import java.util.Collection;
import java.util.List;
-import org.apache.beam.runners.core.AggregatorFactory;
import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners;
import org.apache.beam.runners.core.ExecutionContext;
@@ -50,7 +49,6 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
private final TupleTag<OutputT> mainOutputTag;
private final List<TupleTag<?>> sideOutputTags;
private final ExecutionContext.StepContext stepContext;
- private final AggregatorFactory aggregatorFactory;
private final WindowingStrategy<?, ?> windowingStrategy;
public DoFnRunnerFactory(
@@ -61,7 +59,6 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
TupleTag<OutputT> mainOutputTag,
List<TupleTag<?>> sideOutputTags,
ExecutionContext.StepContext stepContext,
- AggregatorFactory aggregatorFactory,
WindowingStrategy<?, ?> windowingStrategy) {
this.fn = doFn;
this.options = pipelineOptions;
@@ -70,7 +67,6 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
this.mainOutputTag = mainOutputTag;
this.sideOutputTags = sideOutputTags;
this.stepContext = stepContext;
- this.aggregatorFactory = aggregatorFactory;
this.windowingStrategy = windowingStrategy;
}
@@ -78,7 +74,7 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
ReadyCheckingSideInputReader sideInputReader) {
DoFnRunner<InputT, OutputT> underlying = DoFnRunners.simpleRunner(
options, fn, sideInputReader, outputManager, mainOutputTag,
- sideOutputTags, stepContext, aggregatorFactory, windowingStrategy);
+ sideOutputTags, stepContext, windowingStrategy);
return SimplePushbackSideInputDoFnRunner.create(underlying, sideInputs, sideInputReader);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/12b9719e/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpAggregatorFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpAggregatorFactory.java
deleted file mode 100644
index 3436930..0000000
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpAggregatorFactory.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.gearpump.translators.utils;
-
-import java.io.Serializable;
-
-import org.apache.beam.runners.core.AggregatorFactory;
-import org.apache.beam.runners.core.ExecutionContext;
-import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.beam.sdk.transforms.Combine;
-
-/**
- * no-op aggregator factory.
- */
-public class NoOpAggregatorFactory implements AggregatorFactory, Serializable {
-
- @Override
- public <InputT, AccumT, OutputT> Aggregator<InputT, OutputT> createAggregatorForDoFn(
- Class<?> fnClass,
- ExecutionContext.StepContext stepContext,
- String aggregatorName,
- Combine.CombineFn<InputT, AccumT, OutputT> combine) {
- return new NoOpAggregator<>();
- }
-
- private static class NoOpAggregator<InputT, OutputT> implements Aggregator<InputT, OutputT>,
- java.io.Serializable {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void addValue(InputT value) {
- }
-
- @Override
- public String getName() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Combine.CombineFn<InputT, ?, OutputT> getCombineFn() {
- // TODO Auto-generated method stub
- return null;
- }
-
- };
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/12b9719e/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
index 4e0a74c..64fd615 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
@@ -61,7 +61,7 @@ public class NoOpStepContext implements ExecutionContext.StepContext, Serializab
}
@Override
- public StateInternals<?> stateInternals() {
+ public StateInternals stateInternals() {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/beam/blob/12b9719e/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
index 4e66ba9..86b60aa 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
@@ -33,10 +33,9 @@ import org.apache.beam.runners.gearpump.translators.GroupByKeyTranslator.Gearpum
import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.transforms.windowing.OutputTimeFn;
-import org.apache.beam.sdk.transforms.windowing.OutputTimeFns;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.transforms.windowing.Sessions;
+import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
@@ -83,15 +82,15 @@ public class GroupByKeyTranslatorTest {
}
@Parameterized.Parameters(name = "{index}: {0}")
- public static Iterable<OutputTimeFn<BoundedWindow>> data() {
+ public static Iterable<TimestampCombiner> data() {
return ImmutableList.of(
- OutputTimeFns.outputAtEarliestInputTimestamp(),
- OutputTimeFns.outputAtLatestInputTimestamp(),
- OutputTimeFns.outputAtEndOfWindow());
+ TimestampCombiner.EARLIEST,
+ TimestampCombiner.LATEST,
+ TimestampCombiner.END_OF_WINDOW);
}
@Parameterized.Parameter(0)
- public OutputTimeFn<? super BoundedWindow> outputTimeFn;
+ public TimestampCombiner timestampCombiner;
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
@@ -99,15 +98,15 @@ public class GroupByKeyTranslatorTest {
BoundedWindow window =
new IntervalWindow(new org.joda.time.Instant(0), new org.joda.time.Instant(10));
GroupByKeyTranslator.KeyedByTimestamp keyedByTimestamp =
- new GroupByKeyTranslator.KeyedByTimestamp(outputTimeFn);
+ new GroupByKeyTranslator.KeyedByTimestamp(timestampCombiner);
WindowedValue<KV<String, String>> value =
WindowedValue.of(
KV.of("key", "val"), org.joda.time.Instant.now(), window, PaneInfo.NO_FIRING);
KV<org.joda.time.Instant, WindowedValue<KV<String, String>>> result =
keyedByTimestamp.map(value);
org.joda.time.Instant time =
- outputTimeFn.assignOutputTime(
- value.getTimestamp(), Iterables.getOnlyElement(value.getWindows()));
+ timestampCombiner.assign(Iterables.getOnlyElement(value.getWindows()),
+ value.getTimestamp());
assertThat(result, equalTo(KV.of(time, value)));
}
@@ -115,7 +114,8 @@ public class GroupByKeyTranslatorTest {
@SuppressWarnings({"rawtypes", "unchecked"})
public void testMerge() {
WindowFn slidingWindows = Sessions.withGapDuration(Duration.millis(10));
- GroupByKeyTranslator.Merge merge = new GroupByKeyTranslator.Merge(slidingWindows, outputTimeFn);
+ GroupByKeyTranslator.Merge merge = new GroupByKeyTranslator.Merge(slidingWindows,
+ timestampCombiner);
org.joda.time.Instant key1 = new org.joda.time.Instant(5);
WindowedValue<KV<String, String>> value1 =
WindowedValue.of(
@@ -140,7 +140,7 @@ public class GroupByKeyTranslatorTest {
KV<org.joda.time.Instant, WindowedValue<KV<String, List<String>>>> result2 =
merge.fold(result1, KV.of(key2, value2));
- assertThat(result2.getKey(), equalTo(outputTimeFn.combine(key1, key2)));
+ assertThat(result2.getKey(), equalTo(timestampCombiner.combine(key1, key2)));
Collection<? extends BoundedWindow> resultWindows = result2.getValue().getWindows();
assertThat(resultWindows.size(), equalTo(1));
IntervalWindow expectedWindow =
[37/50] [abbrv] beam git commit: This closes #3479: [BEAM-79] Merge
master into gearpump-runner branch
Posted by ke...@apache.org.
This closes #3479: [BEAM-79] Merge master into gearpump-runner branch
Don't call .testingPipelineOptions() a second time
GCP IO ITs now all use --project option
Select SDK distribution based on the selected SDK name
[BEAM-2373] Upgrade commons-compress dependency version to 1.14
Define the projectId in the SpannerIO Read Test (utest, not itest)
Use SDK harness container for FnAPI jobs when worker_harness_container_image is not specified. Add a separate image tag to use with the SDK harness container.
Ditch apache commons
Add PubSub I/O support to Python DirectRunner
Only use ASCII 'a' through 'z' for temporary Spanner tables
ReduceFnRunner.onTrigger: add short circuit for empty pane, and move inputWM and pane after the short circuit.
WindowingStrategy: add OnTimeBehavior to control whether to emit empty ON_TIME pane.
Removed OnceTriggerStateMachine
Visit composite nodes when checking for picklability.
Upgrade beam bigtable client dependency to 0.9.7.1
Add a Combine Test for Sliding Windows without Context
[BEAM-2389] moved GcpCoreApiSurfaceTest to corresponding module, adapted exposed packagees
Add Experimental annotation to AMQP and refine Kind for the Experimental IOs
[BEAM-2488] Elasticsearch IO should read also in replica shards
Use PCollectionViews.toAdditionalInputs in Combine
Use PCollectionViews.toAdditionalInputs in ParDo
Use PCollectionViews.toAdditionalInputs in ParDoMultiOverrideFactory
Fix getAdditionalInputs for SplittableParDo transforms
Add utility to expand list of PCollectionViews
Read api with naive implementation
Pre read api refactoring. Extract `SpannerConfig` and `AbstractSpannerFn`
Bump spanner version
[BEAM-1187] Improve logging to contain the number of retries done due to IOException and unsuccessful response codes.
Add WindowFn#assignsToOneWindow
Use installed distribution name for sdk name
[BEAM-2522] upgrading jackson to 2.8.9 (mitigating #1599)
Enable grpc controller in fn_api_runner
Removed uses of proto builder clone method
[BEAM-2514] Improve error message on missing required value
[BEAM-1237] Create AmqpIO
Implement streaming GroupByKey in Python DirectRunner
Bump Dataflow worker to 0623
Reintroduces DoFn.ProcessContinuation (Dataflow worker compatibility part)
Remove old deprecated PubSub code
Fix a typo in function args
Avoid pickling the entire pipeline per-transform.
Fix python fn API data plane remote grpc port access
[BEAM-2745] Add Jenkins Suite for Python Performance Test
[BEAM-2489] Use dynamic ES port in HIFIOWithElasticTest
[BEAM-2497] Fix the reading of concat gzip files
Allow output from FinishBundle in DoFnTester
DataflowRunner: Reject merging windowing for stateful ParDo
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/627ae0bc
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/627ae0bc
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/627ae0bc
Branch: refs/heads/master
Commit: 627ae0bc3912ace4b5acf5b8e9fd779ad4461cf7
Parents: 99f4f8b f158257
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Jun 30 14:30:25 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Jun 30 14:30:25 2017 -0700
----------------------------------------------------------------------
.../jenkins/common_job_properties.groovy | 4 +-
.../job_beam_PerformanceTests_Python.groovy | 58 ++
pom.xml | 20 +-
.../apache/beam/runners/apex/ApexRunner.java | 2 +-
.../core/construction/SplittableParDo.java | 66 +-
.../WindowingStrategyTranslation.java | 26 +-
.../core/construction/SplittableParDoTest.java | 8 +-
.../beam/runners/core/ReduceFnRunner.java | 20 +-
.../core/triggers/AfterAllStateMachine.java | 25 +-
.../AfterDelayFromFirstElementStateMachine.java | 6 +-
.../core/triggers/AfterFirstStateMachine.java | 20 +-
.../core/triggers/AfterPaneStateMachine.java | 6 +-
.../triggers/AfterWatermarkStateMachine.java | 7 +-
.../triggers/ExecutableTriggerStateMachine.java | 23 +-
.../core/triggers/NeverStateMachine.java | 5 +-
.../core/triggers/TriggerStateMachine.java | 27 -
.../beam/runners/core/ReduceFnRunnerTest.java | 161 +++++
.../triggers/AfterFirstStateMachineTest.java | 5 +-
.../AfterWatermarkStateMachineTest.java | 7 +-
.../core/triggers/StubTriggerStateMachine.java | 7 +-
.../direct/ParDoMultiOverrideFactory.java | 16 +-
.../flink/FlinkStreamingPipelineTranslator.java | 2 +-
runners/google-cloud-dataflow-java/pom.xml | 2 +-
.../dataflow/BatchStatefulParDoOverrides.java | 2 +
.../dataflow/DataflowPipelineTranslator.java | 5 +-
.../beam/runners/dataflow/DataflowRunner.java | 10 +
.../dataflow/SplittableParDoOverrides.java | 2 +-
.../runners/dataflow/DataflowRunnerTest.java | 38 ++
.../src/main/proto/beam_runner_api.proto | 14 +
.../sdk/options/PipelineOptionsFactory.java | 18 +-
.../sdk/options/PipelineOptionsValidator.java | 34 +-
.../sdk/options/ProxyInvocationHandler.java | 4 +
.../apache/beam/sdk/testing/StaticWindows.java | 5 +
.../org/apache/beam/sdk/transforms/Combine.java | 13 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 3 +
.../apache/beam/sdk/transforms/DoFnTester.java | 16 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 14 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 6 +
.../sdk/transforms/reflect/DoFnInvoker.java | 4 +-
.../sdk/transforms/windowing/GlobalWindows.java | 5 +
.../windowing/PartitioningWindowFn.java | 5 +
.../transforms/windowing/SlidingWindows.java | 5 +
.../beam/sdk/transforms/windowing/Window.java | 32 +
.../beam/sdk/transforms/windowing/WindowFn.java | 11 +
.../apache/beam/sdk/util/IdentityWindowFn.java | 5 +
.../beam/sdk/values/PCollectionViews.java | 14 +
.../beam/sdk/values/WindowingStrategy.java | 46 +-
.../options/PipelineOptionsValidatorTest.java | 44 ++
.../sdk/options/ProxyInvocationHandlerTest.java | 7 +
.../apache/beam/sdk/transforms/CombineTest.java | 63 ++
.../beam/sdk/transforms/DoFnTesterTest.java | 32 +
.../windowing/SlidingWindowsTest.java | 30 +-
.../sdk/util/RetryHttpRequestInitializer.java | 148 +++--
.../extensions/gcp/GcpCoreApiSurfaceTest.java | 48 +-
.../util/RetryHttpRequestInitializerTest.java | 31 +-
sdks/java/io/amqp/pom.xml | 100 +++
.../org/apache/beam/sdk/io/amqp/AmqpIO.java | 399 ++++++++++++
.../beam/sdk/io/amqp/AmqpMessageCoder.java | 79 +++
.../amqp/AmqpMessageCoderProviderRegistrar.java | 44 ++
.../apache/beam/sdk/io/amqp/package-info.java | 22 +
.../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 148 +++++
.../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 89 +++
.../beam/sdk/io/cassandra/CassandraIO.java | 2 +-
sdks/java/io/elasticsearch/pom.xml | 8 +
.../sdk/io/elasticsearch/ElasticsearchIO.java | 13 +-
sdks/java/io/google-cloud-platform/pom.xml | 13 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 6 +-
.../io/gcp/bigtable/BigtableServiceImpl.java | 9 +-
.../sdk/io/gcp/spanner/AbstractSpannerFn.java | 58 ++
.../sdk/io/gcp/spanner/CreateTransactionFn.java | 51 ++
.../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 65 ++
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 137 +++++
.../beam/sdk/io/gcp/spanner/SpannerIO.java | 604 +++++++++++++------
.../sdk/io/gcp/spanner/SpannerWriteGroupFn.java | 125 ++++
.../beam/sdk/io/gcp/spanner/Transaction.java | 33 +
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 10 +
.../sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +-
.../io/gcp/bigtable/BigtableTestOptions.java | 5 -
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +-
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 4 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 2 +-
.../sdk/io/gcp/spanner/FakeServiceFactory.java | 82 +++
.../beam/sdk/io/gcp/spanner/RandomUtils.java | 41 ++
.../sdk/io/gcp/spanner/SpannerIOReadTest.java | 281 +++++++++
.../beam/sdk/io/gcp/spanner/SpannerIOTest.java | 317 ----------
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 258 ++++++++
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 166 +++++
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 27 +-
.../hadoop/inputformat/HadoopInputFormatIO.java | 2 +-
.../inputformat/HIFIOWithElasticTest.java | 11 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 2 +-
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 2 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 2 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 2 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 2 +-
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 2 +-
sdks/java/io/pom.xml | 1 +
.../examples/snippets/snippets_test.py | 16 +
.../apache_beam/examples/streaming_wordcount.py | 14 +-
sdks/python/apache_beam/io/filesystem.py | 8 +
sdks/python/apache_beam/io/gcp/pubsub.py | 162 +++--
sdks/python/apache_beam/io/gcp/pubsub_test.py | 89 +--
sdks/python/apache_beam/pipeline.py | 12 +-
sdks/python/apache_beam/pipeline_test.py | 18 +
.../runners/dataflow/dataflow_runner.py | 11 +-
.../runners/dataflow/internal/apiclient.py | 6 +-
.../runners/dataflow/internal/dependency.py | 69 ++-
.../apache_beam/runners/direct/direct_runner.py | 83 ++-
.../runners/direct/evaluation_context.py | 2 +-
.../runners/direct/transform_evaluator.py | 227 ++++++-
sdks/python/apache_beam/runners/direct/util.py | 25 +-
.../runners/direct/watermark_manager.py | 26 +-
.../runners/portability/fn_api_runner.py | 12 +-
.../runners/portability/fn_api_runner_test.py | 23 +-
.../apache_beam/runners/worker/data_plane.py | 10 +-
.../apache_beam/runners/worker/sdk_worker.py | 2 +-
.../apache_beam/testing/test_stream_test.py | 37 +-
sdks/python/apache_beam/transforms/trigger.py | 16 +
121 files changed, 4273 insertions(+), 1071 deletions(-)
----------------------------------------------------------------------
[20/50] [abbrv] beam git commit: This closes #2888: Merge master into
gearpump-runner branch
Posted by ke...@apache.org.
This closes #2888: Merge master into gearpump-runner branch
Update gearpump-runner against master changes
Introduces read/writePubsubMessages
Removed coder and formatFn from PubsubIO.Write
Removed coder and parseFn from PubsubIO.Read
Add Additional CloudObjectTranslators
Updating Dataflow API protos and client
Remove Coder.asCloudObject and related methods
Deprecate <pipeline>.options usage
Include LICENSE and NOTICE in python dist files
Full removal of Aggregators in Java SDK and Runners
[BEAM-59] Move GcsFileSystem to gcp-core
[BEAM-2139] Disable SplittableDoFn ValidatesRunner tests for Streaming Flink Runner
[BEAM-59] Delete old restrictions on output file paths
[BEAM-2093] Use the jackson version from the maven property in maven archetypes
[BEAM-2135] Fix pointers to sdks/java/io/hadoop-file-system
Instantiate runner briefly in Pipeline
Skip null options when converting back to argv
[BEAM-2135] Fix pointers to GCP-core
[BEAM-1316] Remove the usage of mock from ptransform tests
[BEAM-539] Fixes several issues of FileSink.
[BEAM-2020] Convert all unknown Coders into CustomCoder CloudObjects
BigQuery: swap from asSingleton to asIterable for Cleanup
[BEAM-2141] Disable JDBC tests
[BEAM-2135] Move hdfs to hadoop-file-system
Moves TextSource and TextSink to top level
Converts TextIO.Write to AutoValue
Removes TextIO.Write.Bound
Converts TextIO.Read to AutoValue
Removes TextIO.Read.Bound
Removes leftover no-op withoutValidation methods
Make DoFnSignatures robust to StateSpec subclasses
Simplify type parameters of StateSpec and related
[BEAM-2135] Move gcp-core to google-cloud-platform-core
Revert clean else-return lint changes.
Add an @Internal Annotation
Add retry logic for job creation
Update github PR template after travis removal.
[BEAM-1764] Remove aggregators from Flink Runner
Use processing time for synchronized processing time in Flink runner
Remove unneeded elements from the POM files
Remove accumulators from DoFn tester.
Removing Aggregator from core runner code
Remove Aggregators from StatefulDoFn runner
[BEAM-59] AvroIOTest: use absolute paths for display data
[BEAM-59] DataflowRunnerTests: configure FileSystems in test
[BEAM-59] TFRecordIOTest: cleanup
[BEAM-59] Core tests: stop using gs:// paths
Remove aggregators from DoFn contexts and internal SDK usage
[BEAM-2005] Fix build break, ignore test due to change in TestPipeline/FileSystems interaction
[BEAM-2005, BEAM-2030, BEAM-2031, BEAM-2032, BEAM-2033, BEAM-2070] Base implementation of HadoopFileSystem.
maptask_executor_runner_test: build fix
Scattered minor improvements per review comments
Moves AvroIO.write().withSchema into write()
Converts AvroIO.Write to AutoValue; adds writeGenericRecords()
Moves AvroIO.Read.withSchema into read()
Removes AvroIO.Write.Bound
Converts AvroIO.Read to AutoValue
Adds AvroIO.readGenericRecords()
Removes AvroIO.Read.Bound
Moves AvroSink to upper level
Fixes javadoc of TextIO to not point to AvroIO
[BEAM-1871] Remove deprecated org.apache.beam.sdk.options.GcsOptions after updating Dataflow worker
Copy CloudObject to the Dataflow Module
Rename OutputValue to TaggedOutput.
Fn API support for Python.
[BEAM-1441] remove manual join from apiclient
Add CloudObjectTranslators for Avro, Serializable
[BEAM-59] Switch mimeType from mutable protected field to constructor
[BEAM-1908] Allow setting CREATE_NEVER when using a tablespec in BigQueryIO
Remove IOChannelUtils from BigQuery TableRowWriter
[BEAM-1925] Remove deprecated context param from DoFn
[BEAM-1871] Hide CloudResourceManager client in sdks/java/extensions/gcp-core
[BEAM-1676] Internalize ClassPath from Guava 21.0 to fix the failures related to having whitespaces in resource names.
Add Cloud Object Translators for Coders
Re-enable WindowedWordCountIT on Flink runner in precommit
Add A CoderTranslator Interface
[BEAM-2130] Ensure the options id is never null.
Use a new ReflectData for each AvroCoder instance
[BEAM-2074,BEAM-2073] Fixed SourceDStream's rate control usage.
[BEAM-2072] Fixed MicrobatchSource.reader stops reading after reaching maxNumRecords for the first time.
[BEAM-2129] Fix flaky KafkaIOTest#testUnboundedSourceMetrics
[BEAM-59] DataflowRunner: Sink is always a FileBasedSink now
Do not repeat log messages in DataflowPipelineJob
Update Dataflow worker version to beam-master-20170430
Remove KeyedCombineFn
[BEAM-59] Minor style cleanups to WriteOneWindowPerFile
Do not prune branches in Jenkins
Fix hamcrest-core version in parent pom
[BEAM-2114] Tests for KafkaIO: use ExpectedException rule
[BEAM-2114] Throw instead of warning when KafkaIO cannot infer coder
[BEAM-2114] Fixed display data for Kafka read/write with coders
Remove useless continue statements
Remove redundant private on enum constructors
Remove useless return statement
Renames {id,timestamp}Label to {id,timestamp}Attribute throughout SDK
Adds PubsubIO.writeStrings(), writeProtos(), writeAvros()
Adds PubsubIO.readStrings(), readProtos(), readAvros()
Renames PubsubIO.Write builder methods to be style guide compliant
Converts PubsubIO.Write to AutoValue
Remove override of topic by subscription and vice versa
...
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/58546ace
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/58546ace
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/58546ace
Branch: refs/heads/master
Commit: 58546aceb96b9fead52b1c40574c11ba3a8e2a58
Parents: 4078c22 12b9719
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu May 4 12:52:23 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu May 4 12:52:23 2017 -0700
----------------------------------------------------------------------
.github/PULL_REQUEST_TEMPLATE.md | 3 +-
.gitignore | 2 +
.../jenkins/common_job_properties.groovy | 1 -
.../job_beam_PerformanceTests_JDBC.groovy | 3 +
.test-infra/jenkins/test_wordcount.sh | 125 +
.test-infra/travis/README.md | 23 -
.test-infra/travis/settings.xml | 33 -
.test-infra/travis/test_wordcount.sh | 125 -
.travis.yml | 94 -
README.md | 3 +-
examples/java/pom.xml | 34 +-
.../beam/examples/DebuggingWordCount.java | 22 +-
.../apache/beam/examples/MinimalWordCount.java | 7 +-
.../apache/beam/examples/WindowedWordCount.java | 2 +-
.../org/apache/beam/examples/WordCount.java | 20 +-
.../common/ExampleBigQueryTableOptions.java | 2 +-
...xamplePubsubTopicAndSubscriptionOptions.java | 2 +-
.../common/ExamplePubsubTopicOptions.java | 2 +-
.../beam/examples/common/ExampleUtils.java | 68 +-
.../examples/common/WriteOneFilePerWindow.java | 11 +-
.../beam/examples/complete/AutoComplete.java | 2 +-
.../examples/complete/StreamingWordExtract.java | 4 +-
.../apache/beam/examples/complete/TfIdf.java | 6 +-
.../examples/complete/TopWikipediaSessions.java | 4 +-
.../examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 13 +-
.../beam/examples/cookbook/DistinctExample.java | 4 +-
.../beam/examples/cookbook/JoinExamples.java | 2 +-
.../beam/examples/cookbook/TriggerExample.java | 6 +-
.../beam/examples/WindowedWordCountIT.java | 17 +-
.../org/apache/beam/examples/WordCountIT.java | 14 +-
.../examples/cookbook/BigQueryTornadoesIT.java | 4 +-
examples/java8/pom.xml | 8 +-
.../beam/examples/MinimalWordCountJava8.java | 7 +-
.../beam/examples/complete/game/GameStats.java | 18 +-
.../examples/complete/game/HourlyTeamScore.java | 2 +-
.../examples/complete/game/LeaderBoard.java | 6 +-
.../beam/examples/complete/game/UserScore.java | 10 +-
.../complete/game/utils/WriteToBigQuery.java | 2 +-
.../examples/MinimalWordCountJava8Test.java | 6 +-
pom.xml | 83 +-
runners/apex/README.md | 2 +-
runners/apex/pom.xml | 6 -
.../beam/runners/apex/ApexRunnerResult.java | 9 -
.../operators/ApexGroupByKeyOperator.java | 191 +-
.../operators/ApexParDoOperator.java | 155 +-
.../operators/ApexTimerInternals.java | 266 ++
.../translation/utils/ApexStateInternals.java | 100 +-
.../apex/translation/utils/ApexStreamTuple.java | 6 +-
.../apex/translation/utils/NoOpStepContext.java | 2 +-
.../translation/utils/StateInternalsProxy.java | 6 +-
.../runners/apex/examples/WordCountTest.java | 13 +-
.../translation/GroupByKeyTranslatorTest.java | 10 +-
.../translation/ReadUnboundTranslatorTest.java | 6 +-
.../operators/ApexTimerInternalsTest.java | 145 +
.../utils/ApexStateInternalsTest.java | 39 +-
runners/core-construction-java/pom.xml | 10 -
.../core/construction/CoderTranslator.java | 44 +
.../core/construction/CoderTranslators.java | 107 +
.../beam/runners/core/construction/Coders.java | 109 +-
.../core/construction/ForwardingPTransform.java | 5 +-
.../core/construction/PTransformMatchers.java | 6 +-
.../UnboundedReadFromBoundedSource.java | 19 +-
.../core/construction/WindowingStrategies.java | 52 +-
.../runners/core/construction/CodersTest.java | 37 +-
.../EmptyFlattenAsCreateFactoryTest.java | 10 +-
.../construction/ForwardingPTransformTest.java | 7 +-
.../core/construction/PCollectionsTest.java | 6 +-
.../construction/PTransformMatchersTest.java | 22 +-
.../PTransformReplacementsTest.java | 4 +-
.../core/construction/PTransformsTest.java | 17 +-
.../core/construction/SdkComponentsTest.java | 16 +-
.../UnboundedReadFromBoundedSourceTest.java | 14 +-
.../construction/WindowingStrategiesTest.java | 6 +-
runners/core-java/pom.xml | 15 -
.../beam/runners/core/AggregatorFactory.java | 38 -
.../beam/runners/core/BaseExecutionContext.java | 2 +-
.../apache/beam/runners/core/DoFnAdapters.java | 18 -
.../apache/beam/runners/core/DoFnRunners.java | 44 +-
.../beam/runners/core/ExecutionContext.java | 2 +-
.../runners/core/GlobalCombineFnRunner.java | 78 +
.../runners/core/GlobalCombineFnRunners.java | 193 ++
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 3 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 15 +-
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 14 +-
.../runners/core/GroupAlsoByWindowsDoFn.java | 7 -
.../runners/core/InMemoryStateInternals.java | 112 +-
.../beam/runners/core/KeyedWorkItemCoder.java | 22 +-
.../core/LateDataDroppingDoFnRunner.java | 20 +-
.../apache/beam/runners/core/LateDataUtils.java | 6 +-
.../runners/core/MergingActiveWindowSet.java | 4 +-
.../beam/runners/core/MergingStateAccessor.java | 2 +-
.../apache/beam/runners/core/NonEmptyPanes.java | 2 +-
.../org/apache/beam/runners/core/OldDoFn.java | 131 -
...eBoundedSplittableProcessElementInvoker.java | 8 -
.../beam/runners/core/PaneInfoTracker.java | 2 +-
.../runners/core/PerKeyCombineFnRunner.java | 79 -
.../runners/core/PerKeyCombineFnRunners.java | 161 -
.../runners/core/ReduceFnContextFactory.java | 37 +-
.../beam/runners/core/ReduceFnRunner.java | 27 +-
.../beam/runners/core/SideInputHandler.java | 14 +-
.../beam/runners/core/SimpleDoFnRunner.java | 37 +-
.../beam/runners/core/SimpleOldDoFnRunner.java | 24 +-
.../beam/runners/core/SplittableParDo.java | 22 +-
.../apache/beam/runners/core/StateAccessor.java | 2 +-
.../beam/runners/core/StateInternals.java | 8 +-
.../runners/core/StateInternalsFactory.java | 2 +-
.../apache/beam/runners/core/StateMerging.java | 44 +-
.../beam/runners/core/StateNamespaces.java | 13 +-
.../apache/beam/runners/core/StateTable.java | 10 +-
.../org/apache/beam/runners/core/StateTag.java | 51 +-
.../org/apache/beam/runners/core/StateTags.java | 127 +-
.../beam/runners/core/StatefulDoFnRunner.java | 18 +-
.../beam/runners/core/SystemReduceFn.java | 21 +-
.../core/TestInMemoryStateInternals.java | 8 +-
.../beam/runners/core/TimerInternals.java | 18 +-
.../apache/beam/runners/core/WatermarkHold.java | 45 +-
.../beam/runners/core/WindowingInternals.java | 2 +-
.../AfterDelayFromFirstElementStateMachine.java | 2 +-
.../core/triggers/AfterPaneStateMachine.java | 2 +-
.../TriggerStateMachineContextFactory.java | 12 +-
.../triggers/TriggerStateMachineRunner.java | 4 +-
.../core/DoFnDelegatingAggregatorTest.java | 144 -
.../core/GroupAlsoByWindowsProperties.java | 37 +-
.../core/InMemoryStateInternalsTest.java | 44 +-
.../core/LateDataDroppingDoFnRunnerTest.java | 47 +-
.../core/MergingActiveWindowSetTest.java | 2 +-
.../apache/beam/runners/core/NoOpOldDoFn.java | 7 -
.../beam/runners/core/OldDoFnContextTest.java | 72 -
.../apache/beam/runners/core/OldDoFnTest.java | 141 -
.../beam/runners/core/ReduceFnRunnerTest.java | 119 +-
.../beam/runners/core/ReduceFnTester.java | 96 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 9 -
.../runners/core/SimpleOldDoFnRunnerTest.java | 2 +-
.../beam/runners/core/SplittableParDoTest.java | 2 +-
.../apache/beam/runners/core/StateTagTest.java | 83 +-
.../runners/core/StatefulDoFnRunnerTest.java | 59 +-
runners/direct-java/pom.xml | 10 -
.../runners/direct/AggregatorContainer.java | 200 --
.../CopyOnAccessInMemoryStateInternals.java | 198 +-
.../runners/direct/DirectExecutionContext.java | 15 +-
.../beam/runners/direct/DirectRunner.java | 45 +-
.../beam/runners/direct/EvaluationContext.java | 31 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 30 +-
.../beam/runners/direct/ParDoEvaluator.java | 16 +-
.../direct/ParDoMultiOverrideFactory.java | 6 +-
...littableProcessElementsEvaluatorFactory.java | 4 +-
.../direct/StatefulParDoEvaluatorFactory.java | 6 +-
.../runners/direct/StepTransformResult.java | 12 +-
.../beam/runners/direct/TransformResult.java | 8 +-
.../direct/WriteWithShardingFactory.java | 15 +-
.../runners/direct/AggregatorContainerTest.java | 137 -
.../direct/CloningBundleFactoryTest.java | 10 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 180 +-
.../runners/direct/DirectGraphVisitorTest.java | 6 +-
.../beam/runners/direct/DirectRunnerTest.java | 15 +-
.../runners/direct/EvaluationContextTest.java | 49 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 5 -
.../StatefulParDoEvaluatorFactoryTest.java | 10 +-
.../UnboundedReadEvaluatorFactoryTest.java | 4 +-
.../direct/WriteWithShardingFactoryTest.java | 20 +-
runners/flink/pom.xml | 15 +-
.../flink/FlinkBatchTransformTranslators.java | 94 +-
.../flink/FlinkBatchTranslationContext.java | 4 +
.../flink/FlinkDetachedRunnerResult.java | 11 -
.../FlinkPipelineExecutionEnvironment.java | 8 +
.../runners/flink/FlinkPipelineOptions.java | 18 +
.../apache/beam/runners/flink/FlinkRunner.java | 3 +-
.../beam/runners/flink/FlinkRunnerResult.java | 37 +-
.../flink/FlinkStreamingPipelineTranslator.java | 3 +
.../FlinkStreamingTransformTranslators.java | 70 +-
.../flink/FlinkStreamingTranslationContext.java | 3 +
.../metrics/DoFnRunnerWithMetricsUpdate.java | 91 +
.../flink/metrics/FlinkMetricContainer.java | 315 ++
.../flink/metrics/FlinkMetricResults.java | 146 +
.../flink/metrics/ReaderInvocationUtil.java | 71 +
.../runners/flink/metrics/package-info.java | 22 +
.../functions/AbstractFlinkCombineRunner.java | 182 ++
.../functions/FlinkAggregatorFactory.java | 53 -
.../functions/FlinkDoFnFunction.java | 12 +-
.../FlinkMergingNonShuffleReduceFunction.java | 173 +-
.../FlinkMergingPartialReduceFunction.java | 201 --
.../functions/FlinkMergingReduceFunction.java | 199 --
.../functions/FlinkNoOpStepContext.java | 2 +-
.../functions/FlinkPartialReduceFunction.java | 118 +-
.../functions/FlinkReduceFunction.java | 118 +-
.../functions/FlinkStatefulDoFnFunction.java | 14 +-
.../functions/HashingFlinkCombineRunner.java | 180 ++
.../functions/SortingFlinkCombineRunner.java | 192 ++
.../SerializableFnAggregatorWrapper.java | 98 -
.../translation/wrappers/SourceInputFormat.java | 23 +-
.../wrappers/streaming/DoFnOperator.java | 87 +-
.../streaming/SingletonKeyedWorkItemCoder.java | 22 +-
.../streaming/SplittableDoFnOperator.java | 6 +-
.../wrappers/streaming/WindowDoFnOperator.java | 6 +-
.../streaming/io/BoundedSourceWrapper.java | 17 +-
.../streaming/io/UnboundedSourceWrapper.java | 18 +-
.../state/FlinkBroadcastStateInternals.java | 202 +-
.../state/FlinkKeyGroupStateInternals.java | 130 +-
.../state/FlinkSplitStateInternals.java | 130 +-
.../streaming/state/FlinkStateInternals.java | 225 +-
.../beam/runners/flink/FlinkTestPipeline.java | 9 +-
.../beam/runners/flink/PipelineOptionsTest.java | 19 +
.../beam/runners/flink/ReadSourceITCase.java | 6 +-
.../flink/ReadSourceStreamingITCase.java | 6 +-
.../beam/runners/flink/WriteSinkITCase.java | 192 --
.../flink/streaming/DoFnOperatorTest.java | 9 +-
.../FlinkBroadcastStateInternalsTest.java | 6 +-
.../FlinkKeyGroupStateInternalsTest.java | 2 +-
.../streaming/FlinkSplitStateInternalsTest.java | 2 +-
.../streaming/FlinkStateInternalsTest.java | 40 +-
.../flink/streaming/GroupByNullKeyTest.java | 13 +-
.../streaming/TopWikipediaSessionsITCase.java | 2 +-
.../streaming/UnboundedSourceWrapperTest.java | 12 +-
.../gearpump/GearpumpPipelineResult.java | 11 -
.../translators/GroupByKeyTranslator.java | 28 +-
.../translators/functions/DoFnFunction.java | 2 -
.../translators/utils/DoFnRunnerFactory.java | 6 +-
.../utils/NoOpAggregatorFactory.java | 63 -
.../translators/utils/NoOpStepContext.java | 2 +-
.../translators/GroupByKeyTranslatorTest.java | 24 +-
runners/google-cloud-dataflow-java/pom.xml | 31 +-
.../beam/runners/dataflow/AssignWindows.java | 5 +-
.../runners/dataflow/BatchViewOverrides.java | 21 +-
.../dataflow/DataflowAggregatorTransforms.java | 79 -
.../dataflow/DataflowMetricUpdateExtractor.java | 109 -
.../beam/runners/dataflow/DataflowMetrics.java | 44 +-
.../runners/dataflow/DataflowPipelineJob.java | 103 +-
.../dataflow/DataflowPipelineTranslator.java | 13 +-
.../beam/runners/dataflow/DataflowRunner.java | 164 +-
.../runners/dataflow/internal/IsmFormat.java | 92 +-
.../options/DataflowPipelineOptions.java | 8 +-
.../dataflow/testing/TestDataflowRunner.java | 181 +-
.../util/AvroCoderCloudObjectTranslator.java | 62 +
.../runners/dataflow/util/CloudKnownType.java | 138 +
.../beam/runners/dataflow/util/CloudObject.java | 185 ++
.../runners/dataflow/util/CloudObjectKinds.java | 33 +
.../dataflow/util/CloudObjectTranslator.java | 48 +
.../dataflow/util/CloudObjectTranslators.java | 575 ++++
.../runners/dataflow/util/CloudObjects.java | 92 +
.../CoderCloudObjectTranslatorRegistrar.java | 47 +
.../dataflow/util/DataflowTemplateJob.java | 2 +-
.../dataflow/util/DataflowTransport.java | 2 +-
...aultCoderCloudObjectTranslatorRegistrar.java | 136 +
.../runners/dataflow/util/MonitoringUtil.java | 4 +-
.../runners/dataflow/util/RandomAccessData.java | 9 +-
.../SerializableCoderCloudObjectTranslator.java | 65 +
.../beam/runners/dataflow/util/Serializer.java | 262 ++
.../BatchStatefulParDoOverridesTest.java | 4 +-
.../runners/dataflow/DataflowMetricsTest.java | 2 +-
.../dataflow/DataflowPipelineJobTest.java | 490 +---
.../DataflowPipelineTranslatorTest.java | 120 +-
.../runners/dataflow/DataflowRunnerTest.java | 43 +-
.../testing/TestDataflowRunnerTest.java | 374 +--
.../runners/dataflow/util/CloudObjectsTest.java | 217 ++
.../dataflow/util/MonitoringUtilTest.java | 2 +-
.../runners/dataflow/util/PackageUtilTest.java | 2 +-
runners/spark/pom.xml | 16 +-
.../spark/SparkNativePipelineVisitor.java | 2 +-
.../beam/runners/spark/SparkPipelineResult.java | 12 -
.../beam/runners/spark/TestSparkRunner.java | 35 +-
.../spark/aggregators/SparkAggregators.java | 110 -
.../beam/runners/spark/examples/WordCount.java | 17 +-
.../beam/runners/spark/io/MicrobatchSource.java | 188 +-
.../beam/runners/spark/io/SourceDStream.java | 57 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 11 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 58 +-
.../spark/stateful/SparkStateInternals.java | 111 +-
.../spark/stateful/StateSpecFunctions.java | 30 +-
.../spark/translation/MultiDoFnFunction.java | 2 -
.../translation/SparkAbstractCombineFn.java | 4 +-
.../spark/translation/SparkGlobalCombineFn.java | 37 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 13 +-
.../spark/translation/SparkKeyedCombineFn.java | 63 +-
.../spark/translation/SparkProcessContext.java | 2 +-
.../spark/translation/SparkRuntimeContext.java | 84 +-
.../spark/translation/TransformTranslator.java | 44 +-
.../spark/translation/TranslationUtils.java | 2 +-
.../streaming/StreamingTransformTranslator.java | 100 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 21 +-
.../spark/aggregators/ClearAggregatorsRule.java | 38 -
.../metrics/sink/NamedAggregatorsTest.java | 101 -
.../metrics/sink/SparkMetricsSinkTest.java | 86 +
.../beam/runners/spark/io/AvroPipelineTest.java | 4 +-
.../beam/runners/spark/io/NumShardsTest.java | 3 +-
.../translation/streaming/CreateStreamTest.java | 50 +
.../ResumeFromCheckpointStreamingTest.java | 94 +-
.../streaming/StreamingSourceMetricsTest.java | 19 +-
.../src/main/resources/beam/findbugs-filter.xml | 10 +-
sdks/java/core/pom.xml | 88 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 92 -
.../beam/sdk/AggregatorRetrievalException.java | 33 -
.../org/apache/beam/sdk/AggregatorValues.java | 51 -
.../main/java/org/apache/beam/sdk/Pipeline.java | 152 +-
.../org/apache/beam/sdk/PipelineResult.java | 12 -
.../beam/sdk/annotations/Experimental.java | 3 -
.../apache/beam/sdk/annotations/Internal.java | 42 +
.../org/apache/beam/sdk/coders/AtomicCoder.java | 52 -
.../org/apache/beam/sdk/coders/AvroCoder.java | 152 +-
.../apache/beam/sdk/coders/BigDecimalCoder.java | 26 +-
.../beam/sdk/coders/BigEndianIntegerCoder.java | 7 +-
.../beam/sdk/coders/BigEndianLongCoder.java | 5 +-
.../apache/beam/sdk/coders/BigIntegerCoder.java | 18 +-
.../org/apache/beam/sdk/coders/BitSetCoder.java | 63 +
.../apache/beam/sdk/coders/ByteArrayCoder.java | 18 +-
.../org/apache/beam/sdk/coders/ByteCoder.java | 4 +-
.../apache/beam/sdk/coders/ByteStringCoder.java | 113 -
.../java/org/apache/beam/sdk/coders/Coder.java | 42 +-
.../apache/beam/sdk/coders/CoderFactories.java | 22 +-
.../apache/beam/sdk/coders/CoderRegistrar.java | 45 +
.../apache/beam/sdk/coders/CoderRegistry.java | 128 +-
.../org/apache/beam/sdk/coders/CustomCoder.java | 80 +-
.../apache/beam/sdk/coders/DelegateCoder.java | 43 +-
.../sdk/coders/DeterministicStandardCoder.java | 39 -
.../org/apache/beam/sdk/coders/DoubleCoder.java | 4 +-
.../apache/beam/sdk/coders/DurationCoder.java | 19 +-
.../apache/beam/sdk/coders/InstantCoder.java | 20 +-
.../apache/beam/sdk/coders/IterableCoder.java | 9 -
.../beam/sdk/coders/IterableLikeCoder.java | 2 +-
.../org/apache/beam/sdk/coders/JAXBCoder.java | 201 --
.../org/apache/beam/sdk/coders/KvCoder.java | 13 +-
.../beam/sdk/coders/LengthPrefixCoder.java | 25 +-
.../org/apache/beam/sdk/coders/MapCoder.java | 15 +-
.../apache/beam/sdk/coders/NullableCoder.java | 29 +-
.../beam/sdk/coders/SerializableCoder.java | 34 +-
.../apache/beam/sdk/coders/StandardCoder.java | 265 --
.../beam/sdk/coders/StringDelegateCoder.java | 21 +-
.../apache/beam/sdk/coders/StringUtf8Coder.java | 9 +-
.../apache/beam/sdk/coders/StructuredCoder.java | 188 ++
.../beam/sdk/coders/TableRowJsonCoder.java | 88 -
.../beam/sdk/coders/TextualIntegerCoder.java | 9 +-
.../org/apache/beam/sdk/coders/VarIntCoder.java | 7 +-
.../apache/beam/sdk/coders/VarLongCoder.java | 22 +-
.../org/apache/beam/sdk/coders/VoidCoder.java | 5 +-
.../beam/sdk/coders/protobuf/ProtoCoder.java | 405 ---
.../beam/sdk/coders/protobuf/ProtobufUtil.java | 171 --
.../beam/sdk/coders/protobuf/package-info.java | 24 -
.../java/org/apache/beam/sdk/io/AvroIO.java | 1272 ++------
.../java/org/apache/beam/sdk/io/AvroSink.java | 142 +
.../java/org/apache/beam/sdk/io/AvroSource.java | 153 +-
.../apache/beam/sdk/io/BlockBasedSource.java | 10 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 9 -
.../apache/beam/sdk/io/CompressedSource.java | 26 +-
.../org/apache/beam/sdk/io/CountingInput.java | 283 --
.../org/apache/beam/sdk/io/CountingSource.java | 53 +-
.../org/apache/beam/sdk/io/FileBasedSink.java | 187 +-
.../org/apache/beam/sdk/io/FileBasedSource.java | 359 +--
.../java/org/apache/beam/sdk/io/FileSystem.java | 19 +
.../apache/beam/sdk/io/FileSystemRegistrar.java | 15 +-
.../org/apache/beam/sdk/io/FileSystems.java | 194 +-
.../apache/beam/sdk/io/GenerateSequence.java | 194 ++
.../org/apache/beam/sdk/io/LocalFileSystem.java | 39 +-
.../beam/sdk/io/LocalFileSystemRegistrar.java | 15 +-
.../org/apache/beam/sdk/io/LocalResourceId.java | 45 +-
.../org/apache/beam/sdk/io/LocalResources.java | 56 +
.../apache/beam/sdk/io/ShardNameTemplate.java | 2 +-
.../main/java/org/apache/beam/sdk/io/Sink.java | 295 --
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 42 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 1218 ++------
.../java/org/apache/beam/sdk/io/TextSink.java | 139 +
.../java/org/apache/beam/sdk/io/TextSource.java | 236 ++
.../main/java/org/apache/beam/sdk/io/Write.java | 603 ----
.../java/org/apache/beam/sdk/io/WriteFiles.java | 558 ++++
.../main/java/org/apache/beam/sdk/io/XmlIO.java | 477 ---
.../java/org/apache/beam/sdk/io/XmlSink.java | 153 -
.../java/org/apache/beam/sdk/io/XmlSource.java | 402 ---
.../org/apache/beam/sdk/io/fs/MatchResult.java | 66 +-
.../org/apache/beam/sdk/io/fs/ResourceId.java | 36 +-
.../org/apache/beam/sdk/io/package-info.java | 2 +-
.../org/apache/beam/sdk/io/range/ByteKey.java | 21 +-
.../apache/beam/sdk/metrics/CounterCell.java | 19 +-
.../beam/sdk/metrics/DistributionCell.java | 10 +-
.../org/apache/beam/sdk/metrics/GaugeCell.java | 8 +-
.../org/apache/beam/sdk/metrics/Metric.java | 8 +-
.../org/apache/beam/sdk/metrics/MetricCell.java | 8 +-
.../apache/beam/sdk/metrics/MetricResults.java | 29 +-
.../org/apache/beam/sdk/metrics/Metrics.java | 39 +
.../beam/sdk/metrics/MetricsContainer.java | 8 +-
.../apache/beam/sdk/metrics/MetricsFilter.java | 9 +-
.../apache/beam/sdk/metrics/SinkMetrics.java | 49 +
.../apache/beam/sdk/metrics/SourceMetrics.java | 116 +
.../beam/sdk/options/PipelineOptions.java | 8 +-
.../sdk/options/PipelineOptionsFactory.java | 4 +-
.../sdk/options/ProxyInvocationHandler.java | 16 +-
.../beam/sdk/testing/CoderProperties.java | 47 +-
.../org/apache/beam/sdk/testing/PAssert.java | 164 +-
.../beam/sdk/testing/SourceTestUtils.java | 5 +-
.../beam/sdk/testing/SuccessOrFailure.java | 82 +
.../apache/beam/sdk/testing/TestPipeline.java | 13 +-
.../org/apache/beam/sdk/testing/TestStream.java | 116 -
.../beam/sdk/testing/UsesCounterMetrics.java | 25 +
.../sdk/testing/UsesDistributionMetrics.java | 26 +
.../beam/sdk/testing/UsesGaugeMetrics.java | 25 +
.../beam/sdk/testing/WindowFnTestUtils.java | 53 +-
.../apache/beam/sdk/transforms/Aggregator.java | 38 +-
.../sdk/transforms/AggregatorRetriever.java | 45 -
.../sdk/transforms/ApproximateQuantiles.java | 8 +-
.../beam/sdk/transforms/ApproximateUnique.java | 3 +-
.../org/apache/beam/sdk/transforms/Combine.java | 686 ++---
.../beam/sdk/transforms/CombineFnBase.java | 136 -
.../apache/beam/sdk/transforms/CombineFns.java | 463 +--
.../beam/sdk/transforms/CombineWithContext.java | 174 +-
.../org/apache/beam/sdk/transforms/Count.java | 35 +-
.../sdk/transforms/DelegatingAggregator.java | 126 -
.../org/apache/beam/sdk/transforms/DoFn.java | 128 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 78 -
.../org/apache/beam/sdk/transforms/Filter.java | 35 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 30 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 8 +-
.../org/apache/beam/sdk/transforms/Latest.java | 5 +-
.../org/apache/beam/sdk/transforms/Mean.java | 10 +-
.../apache/beam/sdk/transforms/PTransform.java | 8 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 2 +-
.../org/apache/beam/sdk/transforms/Top.java | 6 +-
.../org/apache/beam/sdk/transforms/View.java | 27 +-
.../beam/sdk/transforms/join/CoGbkResult.java | 33 +-
.../sdk/transforms/join/CoGbkResultSchema.java | 21 +-
.../beam/sdk/transforms/join/UnionCoder.java | 18 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 27 +-
.../transforms/windowing/AfterWatermark.java | 3 -
.../sdk/transforms/windowing/GlobalWindow.java | 25 +-
.../transforms/windowing/IntervalWindow.java | 31 +-
.../sdk/transforms/windowing/OutputTimeFn.java | 314 --
.../sdk/transforms/windowing/OutputTimeFns.java | 212 --
.../beam/sdk/transforms/windowing/PaneInfo.java | 9 +-
.../transforms/windowing/TimestampCombiner.java | 186 ++
.../beam/sdk/transforms/windowing/Trigger.java | 12 +-
.../beam/sdk/transforms/windowing/Window.java | 27 +-
.../org/apache/beam/sdk/util/ApiSurface.java | 4 +-
.../apache/beam/sdk/util/AppliedCombineFn.java | 35 +-
.../org/apache/beam/sdk/util/AvroUtils.java | 137 -
.../org/apache/beam/sdk/util/BitSetCoder.java | 15 +-
.../org/apache/beam/sdk/util/ClassPath.java | 544 ++++
.../apache/beam/sdk/util/CloudKnownType.java | 9 +-
.../org/apache/beam/sdk/util/CloudObject.java | 3 +
.../org/apache/beam/sdk/util/CoderUtils.java | 15 +-
.../org/apache/beam/sdk/util/CombineFnUtil.java | 123 +-
.../beam/sdk/util/ExplicitShardedFile.java | 23 +-
.../apache/beam/sdk/util/IOChannelUtils.java | 50 +-
.../beam/sdk/util/NumberedShardedFile.java | 29 +-
.../apache/beam/sdk/util/PCollectionViews.java | 4 +-
.../org/apache/beam/sdk/util/PropertyNames.java | 4 +-
.../org/apache/beam/sdk/util/Reshuffle.java | 7 +-
.../sdk/util/RetryHttpRequestInitializer.java | 248 --
.../apache/beam/sdk/util/SerializableUtils.java | 26 +-
.../org/apache/beam/sdk/util/Serializer.java | 3 +
.../java/org/apache/beam/sdk/util/Structs.java | 74 +-
.../apache/beam/sdk/util/ValueWithRecordId.java | 16 +-
.../org/apache/beam/sdk/util/WindowedValue.java | 19 +-
.../apache/beam/sdk/util/WindowingStrategy.java | 178 +-
.../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 626 ----
.../beam/sdk/util/gcsfs/package-info.java | 20 -
.../apache/beam/sdk/util/state/StateBinder.java | 53 +-
.../apache/beam/sdk/util/state/StateSpec.java | 15 +-
.../apache/beam/sdk/util/state/StateSpecs.java | 278 +-
.../beam/sdk/util/state/WatermarkHoldState.java | 19 +-
.../org/apache/beam/sdk/values/PCollection.java | 9 +-
.../java/org/apache/beam/sdk/values/PDone.java | 33 +-
.../beam/sdk/values/POutputValueBase.java | 74 -
.../java/org/apache/beam/sdk/values/PValue.java | 2 +
.../org/apache/beam/sdk/values/PValueBase.java | 29 +-
.../beam/sdk/values/TimestampedValue.java | 22 +-
.../beam/sdk/values/ValueInSingleWindow.java | 23 +-
sdks/java/core/src/main/proto/README.md | 45 -
.../main/proto/proto2_coder_test_messages.proto | 53 -
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 3 -
.../sdk/AggregatorPipelineExtractorTest.java | 226 --
.../java/org/apache/beam/sdk/PipelineTest.java | 97 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 86 +-
.../beam/sdk/coders/AvroCoderTestPojo.java | 51 +
.../beam/sdk/coders/BigDecimalCoderTest.java | 18 -
.../sdk/coders/BigEndianIntegerCoderTest.java | 8 -
.../beam/sdk/coders/BigEndianLongCoderTest.java | 8 -
.../beam/sdk/coders/BigIntegerCoderTest.java | 18 -
.../apache/beam/sdk/coders/BitSetCoderTest.java | 103 +
.../beam/sdk/coders/ByteArrayCoderTest.java | 8 -
.../apache/beam/sdk/coders/ByteCoderTest.java | 8 -
.../beam/sdk/coders/ByteStringCoderTest.java | 128 -
.../beam/sdk/coders/CoderRegistryTest.java | 118 +-
.../beam/sdk/coders/CollectionCoderTest.java | 8 -
.../apache/beam/sdk/coders/CustomCoderTest.java | 46 -
.../beam/sdk/coders/DefaultCoderTest.java | 8 +-
.../beam/sdk/coders/DelegateCoderTest.java | 38 +-
.../apache/beam/sdk/coders/DoubleCoderTest.java | 8 -
.../beam/sdk/coders/InstantCoderTest.java | 8 -
.../beam/sdk/coders/IterableCoderTest.java | 18 -
.../apache/beam/sdk/coders/JAXBCoderTest.java | 223 --
.../org/apache/beam/sdk/coders/KvCoderTest.java | 20 -
.../beam/sdk/coders/LengthPrefixCoderTest.java | 18 +-
.../apache/beam/sdk/coders/ListCoderTest.java | 8 -
.../apache/beam/sdk/coders/MapCoderTest.java | 8 -
.../beam/sdk/coders/NullableCoderTest.java | 13 +-
.../beam/sdk/coders/SerializableCoderTest.java | 19 +-
.../apache/beam/sdk/coders/SetCoderTest.java | 8 -
.../beam/sdk/coders/StandardCoderTest.java | 238 --
.../sdk/coders/StringDelegateCoderTest.java | 6 -
.../beam/sdk/coders/StructuredCoderTest.java | 238 ++
.../beam/sdk/coders/TableRowJsonCoderTest.java | 94 -
.../sdk/coders/TextualIntegerCoderTest.java | 8 -
.../apache/beam/sdk/coders/VarIntCoderTest.java | 8 -
.../beam/sdk/coders/VarLongCoderTest.java | 8 -
.../sdk/coders/protobuf/ProtoCoderTest.java | 182 --
.../sdk/coders/protobuf/ProtobufUtilTest.java | 192 --
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 156 +-
.../apache/beam/sdk/io/AvroIOTransformTest.java | 30 +-
.../org/apache/beam/sdk/io/AvroSourceTest.java | 56 +-
.../beam/sdk/io/CompressedSourceTest.java | 12 +-
.../apache/beam/sdk/io/CountingInputTest.java | 221 --
.../apache/beam/sdk/io/CountingSourceTest.java | 4 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 102 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 192 +-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 20 +-
.../beam/sdk/io/GenerateSequenceTest.java | 194 ++
.../sdk/io/LocalFileSystemRegistrarTest.java | 6 +
.../apache/beam/sdk/io/LocalFileSystemTest.java | 91 +-
.../apache/beam/sdk/io/LocalResourceIdTest.java | 45 +
.../java/org/apache/beam/sdk/io/SimpleSink.java | 96 +
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 6 +-
.../java/org/apache/beam/sdk/io/TextIOTest.java | 106 +-
.../org/apache/beam/sdk/io/WriteFilesTest.java | 457 +++
.../java/org/apache/beam/sdk/io/WriteTest.java | 705 -----
.../org/apache/beam/sdk/io/XmlSinkTest.java | 253 --
.../org/apache/beam/sdk/io/XmlSourceTest.java | 892 ------
.../range/ByteKeyRangeInterpolateKeyTest.java | 1 -
.../beam/sdk/io/range/ByteKeyRangeTest.java | 16 +-
.../apache/beam/sdk/io/range/ByteKeyTest.java | 4 +-
.../apache/beam/sdk/metrics/MetricMatchers.java | 144 +-
.../apache/beam/sdk/metrics/MetricsTest.java | 183 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 71 +-
.../beam/sdk/options/PipelineOptionsTest.java | 14 +
.../sdk/options/ProxyInvocationHandlerTest.java | 2 +-
.../sdk/runners/TransformHierarchyTest.java | 7 +-
.../beam/sdk/runners/TransformTreeTest.java | 12 +-
.../beam/sdk/testing/GatherAllPanesTest.java | 8 +-
.../apache/beam/sdk/testing/PAssertTest.java | 71 +-
.../sdk/testing/SerializableMatchersTest.java | 4 +-
.../beam/sdk/testing/SystemNanoTimeSleeper.java | 1 -
.../apache/beam/sdk/testing/TestStreamTest.java | 24 -
.../beam/sdk/testing/WindowSupplierTest.java | 4 +-
.../beam/sdk/transforms/CombineFnsTest.java | 118 +-
.../apache/beam/sdk/transforms/CombineTest.java | 218 +-
.../apache/beam/sdk/transforms/CreateTest.java | 3 +-
.../apache/beam/sdk/transforms/DoFnTest.java | 162 --
.../beam/sdk/transforms/DoFnTesterTest.java | 47 +-
.../apache/beam/sdk/transforms/FilterTest.java | 13 +
.../apache/beam/sdk/transforms/FlattenTest.java | 8 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 19 +-
.../beam/sdk/transforms/LatestFnTest.java | 42 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 189 +-
.../org/apache/beam/sdk/transforms/SumTest.java | 2 +-
.../apache/beam/sdk/transforms/ViewTest.java | 2 +-
.../display/DisplayDataEvaluatorTest.java | 5 +-
.../sdk/transforms/join/CoGroupByKeyTest.java | 6 +-
.../sdk/transforms/join/UnionCoderTest.java | 11 +
.../transforms/reflect/DoFnInvokersTest.java | 6 +-
.../transforms/reflect/DoFnSignaturesTest.java | 26 +-
.../transforms/windowing/GlobalWindowTest.java | 7 -
.../transforms/windowing/OutputTimeFnsTest.java | 51 -
.../sdk/transforms/windowing/SessionsTest.java | 6 +-
.../sdk/transforms/windowing/WindowTest.java | 27 +-
.../sdk/transforms/windowing/WindowingTest.java | 4 +-
.../org/apache/beam/sdk/util/AvroUtilsTest.java | 147 -
.../apache/beam/sdk/util/CoderUtilsTest.java | 108 +-
.../apache/beam/sdk/util/CombineFnUtilTest.java | 18 +-
.../beam/sdk/util/IOChannelUtilsTest.java | 30 -
.../org/apache/beam/sdk/util/NameUtilsTest.java | 2 +-
.../beam/sdk/util/NumberedShardedFileTest.java | 41 +-
.../beam/sdk/util/SerializableUtilsTest.java | 56 +-
.../apache/beam/sdk/util/WindowedValueTest.java | 10 -
.../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 358 ---
.../beam/sdk/values/PCollectionListTest.java | 25 +-
.../beam/sdk/values/PCollectionTupleTest.java | 6 +-
.../org/apache/beam/sdk/values/PDoneTest.java | 2 +-
sdks/java/extensions/gcp-core/pom.xml | 222 --
.../beam/sdk/options/BigQueryOptions.java | 32 -
.../options/CloudResourceManagerOptions.java | 40 -
.../org/apache/beam/sdk/options/GcpOptions.java | 227 --
.../options/GcpPipelineOptionsRegistrar.java | 39 -
.../org/apache/beam/sdk/options/GcsOptions.java | 154 -
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 -
.../apache/beam/sdk/options/PubsubOptions.java | 36 -
.../apache/beam/sdk/options/package-info.java | 22 -
.../beam/sdk/testing/BigqueryMatcher.java | 256 --
.../apache/beam/sdk/testing/package-info.java | 21 -
.../apache/beam/sdk/util/CredentialFactory.java | 29 -
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 -
.../beam/sdk/util/GcpCredentialFactory.java | 67 -
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 -
.../beam/sdk/util/GcsIOChannelFactory.java | 111 -
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 -
.../apache/beam/sdk/util/GcsPathValidator.java | 95 -
.../java/org/apache/beam/sdk/util/GcsUtil.java | 798 -----
.../beam/sdk/util/NoopCredentialFactory.java | 68 -
.../sdk/util/NullCredentialInitializer.java | 62 -
.../apache/beam/sdk/util/TestCredential.java | 59 -
.../org/apache/beam/sdk/util/Transport.java | 178 --
.../org/apache/beam/sdk/util/package-info.java | 20 -
.../org/apache/beam/GcpCoreApiSurfaceTest.java | 62 -
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 --
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 -
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 --
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 -
.../beam/sdk/util/GcpProjectUtilTest.java | 76 -
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 -
.../beam/sdk/util/GcsPathValidatorTest.java | 87 -
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 -----
.../util/RetryHttpRequestInitializerTest.java | 290 --
.../google-cloud-platform-core/pom.xml | 185 ++
.../extensions/gcp/auth/CredentialFactory.java | 29 +
.../gcp/auth/GcpCredentialFactory.java | 67 +
.../gcp/auth/NoopCredentialFactory.java | 68 +
.../gcp/auth/NullCredentialInitializer.java | 62 +
.../sdk/extensions/gcp/auth/package-info.java | 22 +
.../options/CloudResourceManagerOptions.java | 32 +
.../sdk/extensions/gcp/options/GcpOptions.java | 389 +++
.../options/GcpPipelineOptionsRegistrar.java | 39 +
.../sdk/extensions/gcp/options/GcsOptions.java | 160 +
.../gcp/options/GoogleApiDebugOptions.java | 89 +
.../extensions/gcp/options/package-info.java | 22 +
.../extensions/gcp/storage/GcsFileSystem.java | 266 ++
.../gcp/storage/GcsFileSystemRegistrar.java | 43 +
.../extensions/gcp/storage/GcsResourceId.java | 128 +
.../extensions/gcp/storage/package-info.java | 21 +
.../beam/sdk/util/GcsIOChannelFactory.java | 111 +
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 +
.../apache/beam/sdk/util/GcsPathValidator.java | 98 +
.../java/org/apache/beam/sdk/util/GcsUtil.java | 796 +++++
.../sdk/util/RetryHttpRequestInitializer.java | 192 ++
.../org/apache/beam/sdk/util/Transport.java | 122 +
.../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 627 ++++
.../beam/sdk/util/gcsfs/package-info.java | 20 +
.../org/apache/beam/sdk/util/package-info.java | 20 +
.../extensions/gcp/GcpCoreApiSurfaceTest.java | 58 +
.../sdk/extensions/gcp/auth/TestCredential.java | 59 +
.../extensions/gcp/options/GcpOptionsTest.java | 273 ++
.../gcp/options/GoogleApiDebugOptionsTest.java | 147 +
.../gcp/storage/GcsFileSystemRegistrarTest.java | 52 +
.../gcp/storage/GcsFileSystemTest.java | 274 ++
.../gcp/storage/GcsResourceIdTest.java | 169 ++
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 +
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 +
.../beam/sdk/util/GcsPathValidatorTest.java | 106 +
.../org/apache/beam/sdk/util/GcsUtilTest.java | 799 +++++
.../util/RetryHttpRequestInitializerTest.java | 281 ++
.../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 358 +++
sdks/java/extensions/jackson/pom.xml | 99 +-
sdks/java/extensions/join-library/pom.xml | 22 -
sdks/java/extensions/pom.xml | 3 +-
sdks/java/extensions/protobuf/pom.xml | 121 +
.../extensions/protobuf/ByteStringCoder.java | 118 +
.../sdk/extensions/protobuf/ProtoCoder.java | 332 +++
.../protobuf/ProtobufCoderRegistrar.java | 39 +
.../sdk/extensions/protobuf/ProtobufUtil.java | 171 ++
.../sdk/extensions/protobuf/package-info.java | 24 +
.../protobuf/ByteStringCoderTest.java | 131 +
.../sdk/extensions/protobuf/ProtoCoderTest.java | 169 ++
.../extensions/protobuf/ProtobufUtilTest.java | 191 ++
.../test/proto/proto2_coder_test_messages.proto | 53 +
sdks/java/extensions/sorter/pom.xml | 24 +-
sdks/java/harness/pom.xml | 12 +-
.../org/apache/beam/fn/harness/FnHarness.java | 2 +-
.../harness/control/ProcessBundleHandler.java | 2 -
.../fn/harness/fake/FakeAggregatorFactory.java | 52 -
.../beam/fn/harness/fake/FakeStepContext.java | 2 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 2 +-
.../harness/stream/StreamObserverFactory.java | 2 +-
.../beam/runners/core/BeamFnDataReadRunner.java | 14 +-
.../runners/core/BeamFnDataWriteRunner.java | 15 +-
.../apache/beam/fn/harness/FnHarnessTest.java | 2 +-
.../control/ProcessBundleHandlerTest.java | 28 +-
.../runners/core/BeamFnDataReadRunnerTest.java | 12 +-
.../runners/core/BeamFnDataWriteRunnerTest.java | 12 +-
sdks/java/io/elasticsearch/pom.xml | 1 -
.../sdk/io/elasticsearch/ElasticsearchIO.java | 4 +-
.../elasticsearch/ElasticSearchIOTestUtils.java | 2 +-
sdks/java/io/google-cloud-platform/pom.xml | 19 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 28 +-
.../io/gcp/bigquery/BigQueryCoderRegistrar.java | 39 +
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 45 +-
.../sdk/io/gcp/bigquery/BigQueryOptions.java | 39 +
.../io/gcp/bigquery/BigQueryQuerySource.java | 1 -
.../sdk/io/gcp/bigquery/BigQueryServices.java | 1 -
.../io/gcp/bigquery/BigQueryServicesImpl.java | 43 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 4 +-
.../io/gcp/bigquery/BigQueryTableSource.java | 1 -
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 1 -
.../io/gcp/bigquery/PassThroughThenCleanup.java | 4 +-
.../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 1 -
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 17 +-
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 5 +-
.../io/gcp/bigquery/StreamingWriteTables.java | 1 -
.../io/gcp/bigquery/TableDestinationCoder.java | 6 +-
.../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 7 +-
.../sdk/io/gcp/bigquery/TableRowJsonCoder.java | 89 +
.../sdk/io/gcp/bigquery/TableRowWriter.java | 28 +-
.../io/gcp/bigquery/WriteBundlesToFiles.java | 9 +-
.../sdk/io/gcp/bigquery/WritePartition.java | 2 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 1 -
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 37 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 1 -
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 28 +-
.../io/gcp/bigtable/BigtableServiceImpl.java | 4 +-
.../common/GcpIoPipelineOptionsRegistrar.java | 39 +
.../beam/sdk/io/gcp/common/package-info.java | 20 +
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 6 +-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 43 +-
.../sdk/io/gcp/pubsub/PubsubGrpcClient.java | 39 +-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 644 ++--
.../sdk/io/gcp/pubsub/PubsubJsonClient.java | 37 +-
.../pubsub/PubsubMessagePayloadOnlyCoder.java | 48 +
.../PubsubMessageWithAttributesCoder.java | 57 +
.../beam/sdk/io/gcp/pubsub/PubsubOptions.java | 44 +
.../sdk/io/gcp/pubsub/PubsubTestClient.java | 7 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 166 +-
.../io/gcp/pubsub/PubsubUnboundedSource.java | 211 +-
.../beam/sdk/io/gcp/storage/GcsFileSystem.java | 242 --
.../io/gcp/storage/GcsFileSystemRegistrar.java | 49 -
.../beam/sdk/io/gcp/storage/GcsResourceId.java | 116 -
.../beam/sdk/io/gcp/storage/package-info.java | 21 -
.../beam/sdk/io/gcp/testing/package-info.java | 21 +
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 2 +
.../bigquery/BigQueryCoderRegistrarTest.java | 40 +
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 20 +-
.../io/gcp/bigquery/FakeBigQueryServices.java | 2 -
.../sdk/io/gcp/bigquery/FakeJobService.java | 1 -
.../io/gcp/bigquery/TableRowJsonCoderTest.java | 87 +
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 32 +-
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 6 +-
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 4 +-
.../beam/sdk/io/gcp/datastore/V1ReadIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 4 +-
.../beam/sdk/io/gcp/datastore/V1WriteIT.java | 6 +-
.../sdk/io/gcp/pubsub/PubsubClientTest.java | 50 +-
.../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java | 18 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 88 +-
.../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 14 +-
.../io/gcp/pubsub/PubsubUnboundedSinkTest.java | 58 +-
.../gcp/pubsub/PubsubUnboundedSourceTest.java | 112 +-
.../gcp/storage/GcsFileSystemRegistrarTest.java | 51 -
.../sdk/io/gcp/storage/GcsFileSystemTest.java | 274 --
.../sdk/io/gcp/storage/GcsResourceIdTest.java | 147 -
.../sdk/io/gcp/testing/BigqueryMatcher.java | 257 ++
.../sdk/io/gcp/testing/BigqueryMatcherTest.java | 177 ++
sdks/java/io/hadoop-common/pom.xml | 5 -
.../beam/sdk/io/hadoop/WritableCoder.java | 29 +-
sdks/java/io/hadoop-file-system/README.md | 43 +
sdks/java/io/hadoop-file-system/pom.xml | 195 ++
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 478 +++
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 625 ++++
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 240 ++
.../sdk/io/hdfs/HadoopFileSystemModule.java | 84 +
.../sdk/io/hdfs/HadoopFileSystemOptions.java | 49 +
.../hdfs/HadoopFileSystemOptionsRegistrar.java | 35 +
.../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 62 +
.../beam/sdk/io/hdfs/HadoopResourceId.java | 81 +
.../java/org/apache/beam/sdk/io/hdfs/Sink.java | 195 ++
.../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 +
.../java/org/apache/beam/sdk/io/hdfs/Write.java | 585 ++++
.../apache/beam/sdk/io/hdfs/package-info.java | 22 +
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 172 ++
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 231 ++
.../sdk/io/hdfs/HadoopFileSystemModuleTest.java | 65 +
.../HadoopFileSystemOptionsRegistrarTest.java | 49 +
.../io/hdfs/HadoopFileSystemOptionsTest.java | 48 +
.../io/hdfs/HadoopFileSystemRegistrarTest.java | 81 +
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 247 ++
sdks/java/io/hadoop/input-format/pom.xml | 15 -
.../hadoop/inputformat/HadoopInputFormatIO.java | 10 +-
.../inputformat/HadoopInputFormatIOTest.java | 29 +-
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 10 -
sdks/java/io/hadoop/pom.xml | 18 +
sdks/java/io/hbase/pom.xml | 7 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 28 +-
.../beam/sdk/io/hbase/HBaseMutationCoder.java | 5 +-
.../beam/sdk/io/hbase/HBaseResultCoder.java | 5 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 17 +-
sdks/java/io/hdfs/README.md | 43 -
sdks/java/io/hdfs/pom.xml | 141 -
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 478 ---
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 625 ----
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 71 -
.../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 42 -
.../beam/sdk/io/hdfs/HadoopResourceId.java | 42 -
.../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 -
.../apache/beam/sdk/io/hdfs/package-info.java | 22 -
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 173 --
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 231 --
.../io/hdfs/HadoopFileSystemRegistrarTest.java | 52 -
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 5 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 4 +-
sdks/java/io/kafka/pom.xml | 5 -
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 470 ++-
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 18 +-
.../CoderBasedKafkaDeserializer.java | 71 +
.../CoderBasedKafkaSerializer.java | 73 +
.../serialization/InstantDeserializer.java | 45 +
.../kafka/serialization/InstantSerializer.java | 45 +
.../io/kafka/serialization/package-info.java | 22 +
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 399 ++-
sdks/java/io/kinesis/pom.xml | 6 +
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 148 +-
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 4 +-
.../sdk/io/kinesis/KinesisMockReadTest.java | 14 +-
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 10 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 4 +-
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 4 +-
sdks/java/io/pom.xml | 3 +-
sdks/java/io/xml/pom.xml | 113 +
.../org/apache/beam/sdk/io/xml/JAXBCoder.java | 165 ++
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 543 ++++
.../org/apache/beam/sdk/io/xml/XmlSink.java | 161 +
.../org/apache/beam/sdk/io/xml/XmlSource.java | 399 +++
.../apache/beam/sdk/io/xml/package-info.java | 22 +
.../apache/beam/sdk/io/xml/JAXBCoderTest.java | 221 ++
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 276 ++
.../apache/beam/sdk/io/xml/XmlSourceTest.java | 987 +++++++
sdks/java/java8tests/pom.xml | 19 -
sdks/java/javadoc/pom.xml | 9 +-
.../main/resources/archetype-resources/pom.xml | 9 +-
.../main/resources/archetype-resources/pom.xml | 9 +-
sdks/java/maven-archetypes/pom.xml | 35 +
.../main/resources/archetype-resources/pom.xml | 2 +-
sdks/python/.pylintrc | 1 +
sdks/python/MANIFEST.in | 2 +
sdks/python/apache_beam/coders/coders.py | 16 -
sdks/python/apache_beam/coders/coders_test.py | 1 +
.../apache_beam/coders/coders_test_common.py | 7 +-
.../apache_beam/coders/fast_coders_test.py | 1 +
.../apache_beam/coders/slow_coders_test.py | 1 +
.../examples/complete/autocomplete_test.py | 1 +
.../examples/complete/game/hourly_team_score.py | 1 +
.../complete/game/hourly_team_score_test.py | 1 +
.../examples/complete/game/user_score.py | 1 +
.../examples/complete/game/user_score_test.py | 1 +
.../complete/juliaset/juliaset/juliaset_test.py | 1 +
.../examples/cookbook/bigquery_side_input.py | 2 +-
.../cookbook/bigquery_tornadoes_it_test.py | 1 +
.../apache_beam/examples/cookbook/bigshuffle.py | 94 -
.../examples/cookbook/bigshuffle_test.py | 63 -
.../apache_beam/examples/cookbook/filters.py | 2 +-
.../examples/cookbook/multiple_output_pardo.py | 6 +-
.../apache_beam/examples/snippets/snippets.py | 122 +-
.../examples/snippets/snippets_test.py | 41 +-
sdks/python/apache_beam/examples/wordcount.py | 1 +
.../examples/wordcount_debugging_test.py | 1 +
.../apache_beam/internal/gcp/json_value_test.py | 11 +
sdks/python/apache_beam/internal/module_test.py | 1 +
sdks/python/apache_beam/internal/pickler.py | 3 +
.../python/apache_beam/internal/pickler_test.py | 1 +
sdks/python/apache_beam/io/concat_source.py | 74 +-
.../python/apache_beam/io/concat_source_test.py | 3 +-
sdks/python/apache_beam/io/filebasedsource.py | 20 +-
.../apache_beam/io/filebasedsource_test.py | 19 +-
sdks/python/apache_beam/io/fileio.py | 45 +-
sdks/python/apache_beam/io/fileio_test.py | 56 +
sdks/python/apache_beam/io/filesystem.py | 35 +-
sdks/python/apache_beam/io/filesystems.py | 204 ++
sdks/python/apache_beam/io/filesystems_test.py | 224 ++
sdks/python/apache_beam/io/filesystems_util.py | 3 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 23 +-
.../io/gcp/datastore/v1/datastoreio.py | 10 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 1 +
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 48 +-
.../apache_beam/io/gcp/gcsfilesystem_test.py | 29 +
sdks/python/apache_beam/io/gcp/pubsub_test.py | 1 +
sdks/python/apache_beam/io/iobase.py | 5 +-
sdks/python/apache_beam/io/localfilesystem.py | 27 +-
.../apache_beam/io/localfilesystem_test.py | 68 +-
sdks/python/apache_beam/io/range_trackers.py | 11 +-
sdks/python/apache_beam/io/source_test_utils.py | 8 +-
.../apache_beam/io/source_test_utils_test.py | 1 +
sdks/python/apache_beam/io/sources_test.py | 2 +-
sdks/python/apache_beam/io/textio.py | 13 +-
sdks/python/apache_beam/io/tfrecordio.py | 2 +
sdks/python/apache_beam/io/tfrecordio_test.py | 9 +-
sdks/python/apache_beam/pipeline.py | 52 +-
sdks/python/apache_beam/pipeline_test.py | 14 +-
sdks/python/apache_beam/pvalue.py | 6 +-
sdks/python/apache_beam/runners/api/__init__.py | 14 +
.../apache_beam/runners/api/beam_fn_api_pb2.py | 2740 ++++++++++++++++++
.../runners/api/beam_fn_api_pb2_grpc.py | 203 ++
.../runners/api/beam_runner_api_pb2.py | 298 +-
sdks/python/apache_beam/runners/common.pxd | 62 +-
sdks/python/apache_beam/runners/common.py | 499 +++-
.../runners/dataflow/dataflow_metrics.py | 3 +
.../runners/dataflow/dataflow_metrics_test.py | 1 +
.../runners/dataflow/dataflow_runner.py | 20 +-
.../runners/dataflow/dataflow_runner_test.py | 4 +-
.../runners/dataflow/internal/apiclient.py | 17 +-
.../clients/dataflow/dataflow_v1b3_messages.py | 564 +++-
.../runners/dataflow/internal/dependency.py | 29 +-
.../dataflow/internal/dependency_test.py | 6 +-
.../runners/dataflow/native_io/iobase.py | 1 +
.../runners/dataflow/template_runner_test.py | 2 +-
.../runners/dataflow/test_dataflow_runner.py | 4 +-
.../consumer_tracking_pipeline_visitor_test.py | 2 +-
.../apache_beam/runners/direct/direct_runner.py | 12 +-
.../apache_beam/runners/portability/__init__.py | 16 +
.../runners/portability/fn_api_runner.py | 471 +++
.../runners/portability/fn_api_runner_test.py | 40 +
.../portability/maptask_executor_runner.py | 468 +++
.../portability/maptask_executor_runner_test.py | 204 ++
sdks/python/apache_beam/runners/runner.py | 34 -
.../apache_beam/runners/worker/__init__.py | 16 +
.../apache_beam/runners/worker/data_plane.py | 288 ++
.../runners/worker/data_plane_test.py | 139 +
.../apache_beam/runners/worker/log_handler.py | 100 +
.../runners/worker/log_handler_test.py | 105 +
.../apache_beam/runners/worker/logger.pxd | 25 +
.../python/apache_beam/runners/worker/logger.py | 173 ++
.../apache_beam/runners/worker/logger_test.py | 182 ++
.../apache_beam/runners/worker/opcounters.pxd | 45 +
.../apache_beam/runners/worker/opcounters.py | 162 ++
.../runners/worker/opcounters_test.py | 149 +
.../runners/worker/operation_specs.py | 368 +++
.../apache_beam/runners/worker/operations.pxd | 89 +
.../apache_beam/runners/worker/operations.py | 651 +++++
.../apache_beam/runners/worker/sdk_worker.py | 451 +++
.../runners/worker/sdk_worker_main.py | 62 +
.../runners/worker/sdk_worker_test.py | 168 ++
.../apache_beam/runners/worker/sideinputs.py | 166 ++
.../runners/worker/sideinputs_test.py | 150 +
.../apache_beam/runners/worker/statesampler.pyx | 237 ++
.../runners/worker/statesampler_fake.py | 34 +
.../runners/worker/statesampler_test.py | 102 +
.../apache_beam/tests/pipeline_verifiers.py | 7 +-
sdks/python/apache_beam/transforms/core.py | 90 +-
.../python/apache_beam/transforms/ptransform.py | 42 +-
.../apache_beam/transforms/ptransform_test.py | 195 +-
sdks/python/apache_beam/transforms/trigger.py | 10 +-
.../apache_beam/transforms/trigger_test.py | 10 +-
sdks/python/apache_beam/transforms/window.py | 114 +-
.../apache_beam/transforms/window_test.py | 16 +
sdks/python/apache_beam/typehints/decorators.py | 1 +
sdks/python/apache_beam/typehints/opcodes.py | 5 +-
sdks/python/apache_beam/typehints/typecheck.py | 4 +-
.../apache_beam/typehints/typehints_test.py | 1 +
sdks/python/apache_beam/utils/__init__.py | 4 -
sdks/python/apache_beam/utils/annotations.py | 14 +-
.../apache_beam/utils/annotations_test.py | 29 +
sdks/python/apache_beam/utils/path.py | 46 -
sdks/python/apache_beam/utils/path_test.py | 70 -
.../apache_beam/utils/pipeline_options_test.py | 2 +-
.../utils/pipeline_options_validator_test.py | 1 +
.../apache_beam/utils/test_stream_test.py | 1 +
sdks/python/apache_beam/utils/timestamp.py | 4 -
sdks/python/apache_beam/utils/urns.py | 93 +
sdks/python/apache_beam/utils/value_provider.py | 2 +-
.../apache_beam/utils/value_provider_test.py | 4 +-
sdks/python/generate_pydoc.sh | 2 +
sdks/python/pom.xml | 2 +-
sdks/python/run_postcommit.sh | 4 +-
sdks/python/run_pylint.sh | 9 +-
sdks/python/setup.py | 11 +-
sdks/python/tox.ini | 4 +-
957 files changed, 43352 insertions(+), 34953 deletions(-)
----------------------------------------------------------------------
[27/50] [abbrv] beam git commit: Respect WindowFn#getOutputTime in
gearpump-runner
Posted by ke...@apache.org.
Respect WindowFn#getOutputTime in gearpump-runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/98854d4d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/98854d4d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/98854d4d
Branch: refs/heads/master
Commit: 98854d4d01ca526ea4a44dc077d2cfb4cddf9914
Parents: 3c7e3e6
Author: manuzhang <ow...@gmail.com>
Authored: Fri May 19 09:19:42 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Jun 5 19:16:53 2017 +0800
----------------------------------------------------------------------
.../gearpump/translators/GroupByKeyTranslator.java | 12 ++++++++----
.../gearpump/translators/GroupByKeyTranslatorTest.java | 8 ++++----
2 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/98854d4d/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
index 521f665..7d944a4 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
@@ -74,7 +74,7 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
new GearpumpWindowFn(windowFn.isNonMerging()),
EventTimeTrigger$.MODULE$, Discarding$.MODULE$), "assign_window")
.groupBy(new GroupByFn<K, V>(inputKeyCoder), parallelism, "group_by_Key_and_Window")
- .map(new KeyedByTimestamp<K, V>(timestampCombiner), "keyed_by_timestamp")
+ .map(new KeyedByTimestamp<K, V>(windowFn, timestampCombiner), "keyed_by_timestamp")
.fold(new Merge<>(windowFn, timestampCombiner), "merge")
.map(new Values<K, V>(), "values");
@@ -146,17 +146,21 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
extends MapFunction<WindowedValue<KV<K, V>>,
KV<Instant, WindowedValue<KV<K, V>>>> {
+ private final WindowFn<KV<K, V>, BoundedWindow> windowFn;
private final TimestampCombiner timestampCombiner;
- public KeyedByTimestamp(TimestampCombiner timestampCombiner) {
+ public KeyedByTimestamp(WindowFn<KV<K, V>, BoundedWindow> windowFn,
+ TimestampCombiner timestampCombiner) {
+ this.windowFn = windowFn;
this.timestampCombiner = timestampCombiner;
}
@Override
public KV<org.joda.time.Instant, WindowedValue<KV<K, V>>> map(
WindowedValue<KV<K, V>> wv) {
- Instant timestamp = timestampCombiner.assign(
- Iterables.getOnlyElement(wv.getWindows()), wv.getTimestamp());
+ BoundedWindow window = Iterables.getOnlyElement(wv.getWindows());
+ Instant timestamp = timestampCombiner.assign(window
+ , windowFn.getOutputTime(wv.getTimestamp(), window));
return KV.of(timestamp, wv);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/98854d4d/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
index 86b60aa..d5b931b 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
@@ -22,7 +22,6 @@ import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import java.time.Instant;
@@ -95,18 +94,19 @@ public class GroupByKeyTranslatorTest {
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testKeyedByTimestamp() {
+ WindowFn slidingWindows = Sessions.withGapDuration(Duration.millis(10));
BoundedWindow window =
new IntervalWindow(new org.joda.time.Instant(0), new org.joda.time.Instant(10));
GroupByKeyTranslator.KeyedByTimestamp keyedByTimestamp =
- new GroupByKeyTranslator.KeyedByTimestamp(timestampCombiner);
+ new GroupByKeyTranslator.KeyedByTimestamp(slidingWindows, timestampCombiner);
WindowedValue<KV<String, String>> value =
WindowedValue.of(
KV.of("key", "val"), org.joda.time.Instant.now(), window, PaneInfo.NO_FIRING);
KV<org.joda.time.Instant, WindowedValue<KV<String, String>>> result =
keyedByTimestamp.map(value);
org.joda.time.Instant time =
- timestampCombiner.assign(Iterables.getOnlyElement(value.getWindows()),
- value.getTimestamp());
+ timestampCombiner.assign(window,
+ slidingWindows.getOutputTime(value.getTimestamp(), window));
assertThat(result, equalTo(KV.of(time, value)));
}
[41/50] [abbrv] beam git commit: This closes #3515: Sync
gearpump-runner with master and upgrade to gearpump 0.8.4
Posted by ke...@apache.org.
This closes #3515: Sync gearpump-runner with master and upgrade to gearpump 0.8.4
Fix ParDoTest#testPipelineOptionsParameter
Upgrade to gearpump 0.8.4
Fix javadoc generation for AmqpIO, CassandraIO and HCatalogIO
Simplified ByteBuddyOnTimerInvokerFactory
Fix bad merge
Made DataflowRunner TransformTranslator public
Process timer firings for a window together
Ignore processing time timers in expired windows
Add timeout to initialization of partition in KafkaIO
[BEAM-2534] Handle offset gaps in Kafka messages.
Fix PValue input in _PubSubReadEvaluator
Update SDK dependencies
Disallow Combiner Lifting for multi-window WindowFns
[BEAM-2553] Update Maven exec plugin to 1.6.0 to incorporate messaging improvements
Website Mergebot Job
Update Python SDK version
[maven-release-plugin] prepare for next development iteration
[maven-release-plugin] prepare branch release-2.1.0
For GCS operations use an http client with a default timeout value.
[BEAM-2530] Fix compilation of modules with Java 9 that depend on jdk.tools
Make modules that depend on Hadoop and Spark use the same version property
Fix DoFn javadoc: StateSpec does not require a key
Add support for PipelineOptions parameters
Properly convert milliseconds whether there's less than 3/more than 9 digits. TimeUtil did not properly convert (and returned null) when the number of digits for fractions of seconds was less than 3 digits or more than 9 digits. The solution is to pad with zeros when there is less than 3 digits and to truncate when there is more than 3.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1ce60b48
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1ce60b48
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1ce60b48
Branch: refs/heads/master
Commit: 1ce60b488e991562712937aaaf13564e3a909117
Parents: 627ae0b 725f547
Author: Kenneth Knowles <kl...@google.com>
Authored: Sun Jul 9 21:47:12 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Sun Jul 9 21:47:12 2017 -0700
----------------------------------------------------------------------
.../jenkins/common_job_properties.groovy | 5 +-
.../job_beam_PreCommit_Website_Merge.groovy | 59 +++++++++
examples/java/pom.xml | 20 ++-
examples/java8/pom.xml | 20 ++-
.../examples/complete/game/LeaderBoardTest.java | 2 +
examples/pom.xml | 2 +-
pom.xml | 87 ++++++++++++-
runners/apex/pom.xml | 11 +-
runners/core-construction-java/pom.xml | 2 +-
runners/core-java/pom.xml | 2 +-
...eBoundedSplittableProcessElementInvoker.java | 5 +
.../beam/runners/core/ReduceFnRunner.java | 106 ++++++++++-----
.../beam/runners/core/SimpleDoFnRunner.java | 20 +++
.../beam/runners/core/ReduceFnRunnerTest.java | 81 +++++++++++-
runners/direct-java/pom.xml | 4 +-
runners/flink/pom.xml | 2 +-
runners/gearpump/pom.xml | 4 +-
.../gearpump/translators/io/GearpumpSource.java | 12 +-
.../translators/utils/DoFnRunnerFactory.java | 5 +-
.../translators/utils/TranslatorUtils.java | 19 +++
runners/google-cloud-dataflow-java/pom.xml | 2 +-
.../dataflow/DataflowPipelineTranslator.java | 1 +
.../runners/dataflow/TransformTranslator.java | 4 +-
.../beam/runners/dataflow/util/TimeUtil.java | 24 ++--
.../runners/dataflow/util/TimeUtilTest.java | 6 +
runners/pom.xml | 2 +-
runners/spark/pom.xml | 16 +--
sdks/common/fn-api/pom.xml | 2 +-
sdks/common/pom.xml | 2 +-
sdks/common/runner-api/pom.xml | 2 +-
sdks/java/build-tools/pom.xml | 2 +-
sdks/java/core/pom.xml | 2 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 2 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 5 +
.../reflect/ByteBuddyDoFnInvokerFactory.java | 6 +
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 73 ++++-------
.../sdk/transforms/reflect/DoFnInvoker.java | 13 +-
.../sdk/transforms/reflect/DoFnSignature.java | 23 ++++
.../sdk/transforms/reflect/DoFnSignatures.java | 22 +++-
.../reflect/OnTimerMethodSpecifier.java | 37 ++++++
.../apache/beam/sdk/transforms/ParDoTest.java | 63 +++++++++
.../transforms/reflect/DoFnSignaturesTest.java | 14 ++
.../google-cloud-platform-core/pom.xml | 2 +-
sdks/java/extensions/jackson/pom.xml | 2 +-
sdks/java/extensions/join-library/pom.xml | 2 +-
sdks/java/extensions/pom.xml | 2 +-
sdks/java/extensions/protobuf/pom.xml | 2 +-
sdks/java/extensions/sorter/pom.xml | 8 +-
sdks/java/harness/pom.xml | 2 +-
sdks/java/io/amqp/pom.xml | 2 +-
sdks/java/io/cassandra/pom.xml | 2 +-
sdks/java/io/common/pom.xml | 2 +-
sdks/java/io/elasticsearch/pom.xml | 2 +-
sdks/java/io/google-cloud-platform/pom.xml | 2 +-
sdks/java/io/hadoop-common/pom.xml | 2 +-
sdks/java/io/hadoop-file-system/pom.xml | 33 +----
sdks/java/io/hadoop/input-format/pom.xml | 2 +-
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 4 +-
sdks/java/io/hadoop/pom.xml | 2 +-
sdks/java/io/hbase/pom.xml | 18 ++-
sdks/java/io/hcatalog/pom.xml | 20 ++-
sdks/java/io/jdbc/pom.xml | 4 +-
sdks/java/io/jms/pom.xml | 2 +-
sdks/java/io/kafka/pom.xml | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 130 +++++++++++++------
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 30 +++++
sdks/java/io/kinesis/pom.xml | 2 +-
sdks/java/io/mongodb/pom.xml | 2 +-
sdks/java/io/mqtt/pom.xml | 2 +-
sdks/java/io/pom.xml | 33 +----
sdks/java/io/xml/pom.xml | 2 +-
sdks/java/java8tests/pom.xml | 2 +-
sdks/java/javadoc/pom.xml | 19 ++-
.../maven-archetypes/examples-java8/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 1 -
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 1 -
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../resources/projects/basic/reference/pom.xml | 2 +-
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 2 +-
sdks/python/apache_beam/io/gcp/gcsio.py | 10 +-
.../runners/direct/transform_evaluator.py | 5 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/pom.xml | 2 +-
sdks/python/setup.py | 4 +-
87 files changed, 854 insertions(+), 317 deletions(-)
----------------------------------------------------------------------
[35/50] [abbrv] beam git commit: This closes #3388: Sync
gearpump-runner branch with master
Posted by ke...@apache.org.
This closes #3388: Sync gearpump-runner branch with master
Remove unused codes
Fix kryo exception
Fix PCollectionView translation
upgrade to gearpump 0.8.4-SNAPSHOT
Fix side input handling in DoFnFunction
Respect WindowFn#getOutputTime in gearpump-runner
Activate Gearpump local-validates-runner-tests in precommit
Update against master changes
Update gearpump-runner against master changes
Update gearpump-runner against master changes
Update gearpump-runner against master changes.
[BEAM-972] Add more unit test to Gearpump runner
[BEAM-972] Add unit tests to Gearpump runner
[BEAM-79] Fix gearpump-runner merge conflicts and test failure
enable ParDoTest
[BEAM-79] Add SideInput support for GearpumpRunner
[BEAM-79] Support merging windows in GearpumpRunner
[BEAM-79] Fix PostCommit test confs for Gearpump runner
note thread is interrupted on InterruptedException
Remove cache for Gearpump on travis
reduce timeout to wait for result
fix ParDo.BoundMulti translation
return encoded key for GroupByKey translation
support OutputTimeFn
update to latest gearpump dsl function interface
fix group by window
activate ROS on Gearpump by default
update ROS configurations
[BEAM-1180] Implement GearpumpPipelineResult
[BEAM-79] Upgrade to beam-0.5.0-incubating-SNAPSHOT
[BEAM-79] Update to latest Gearpump API
Fix NoOpAggregatorFactory
Remove print to stdout
Skip window assignment when windows don't change
Add Window.Bound translator
Upgrade Gearpump version
[BEAM-79] fix gearpump runner build failure
[BEAM-79] update GearpumpPipelineResult
[BEAM-79] Port Gearpump runner from OldDoFn to new DoFn
upgrade gearpump-runner to 0.4.0-incubating-SNAPSHOT
remove "pipeline" in runner name
post-merge fix
[BEAM-79] fix integration-test failure
fix import order
!fixup Minor javadoc clean-up
Added even more javadoc to TextIO#withHeader and TextIO#withFooter (2).
Added even more javadoc to TextIO#withHeader and TextIO#withFooter.
Added javadoc to TextIO#withHeader and TextIO#withFooter.
Reverted header and footer to be of type String.
Revised according to comments following a code review.
Add header/footer support to TextIO.Write
[BEAM-242] Enable and fix checkstyle in Flink runner examples
Remove timeout in JAXBCoderTest
Be more accepting in UnboundedReadDeduplicatorTest
BigQuery: limit max job polling time to 1 minute
[BEAM-242] Enable checkstyle and fix checkstyle errors in Flink runner
[BEAM-456] Add MongoDbIO
FluentBackoff: a replacement for a variety of custom backoff implementations
Remove the DataflowRunner instructions from examples
Put classes in runners-core package into runners.core namespace
Delegate populateDipslayData to wrapped combineFn's
Fixed Combine display data
Cloud Datastore naming clean-up
DatastoreIO SplitQueryFn integration test
Add Latest CombineFn and PTransforms
Remove empty unused method in TestStreamEvaluatorFactory
Test that multiple instances of TestStream are supported
Correct some accidental renames
Fix condition in FlinkStreamingPipelineTranslator
Address comments of Flink Side-Input PR
[BEAM-569] Define maxNumRecords default value to Long.MAX_VALUE in JmsIO
Add LeaderBoardTest
take advantage of setup/teardown for KafkaWriter
Returned KafkaIO getWatermark log line in debug mode
[BEAM-572] Remove Spark Reference in WordCount
Update Dataflow Container Version
[BEAM-313] Provide a context for SparkRunner
DataflowRunner: get PBegin from PInput
[BEAM-592] Fix SparkRunner Dependency Problem in WordCount
Fix javadoc in Kinesis
Organize imports in Kinesis
kinesis: a connector for Amazon Kinesis
[BEAM-589] Fixing IO.Read transformation
Query latest timestamp
travis.yml: disable updating snapshots
Added support for reporting aggregator values to Spark sinks
[BEAM-294] Rename dataflow references to beam
Modified BigtableIO to use DoFn setup/tearDown methods instead of startBundle/finishBundle
checkstyle: prohibit API client repackaged Guava
Make WriteTest more resilient to Randomness
Update DoFn javadocs to remove references to OldDoFn and Dataflow
[BEAM-545] Promote JobName to PipelineOptions
Move the samples data to gs://apache-beam-samples/
Cleanup some javadoc that referring Dataflow
BigQueryIO.Write: raise size limit to 11 TiB
Optimize imports
Update checkstyle.xml to put all imports in one group
Fix Exception Unwrapping in TestFlinkRunner
Make ParDoLifecycleTest Serializable to Fix Test with TupleTag
Use AllPanes as the PaneExtractor in IterableAssert
...
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/99f4f8b1
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/99f4f8b1
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/99f4f8b1
Branch: refs/heads/master
Commit: 99f4f8b1b8a4da2ebfec8371a1c1d9a978f001a6
Parents: 8860cce b21fa04
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Jun 22 20:03:57 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu Jun 22 20:03:57 2017 -0700
----------------------------------------------------------------------
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 2 +-
examples/java/pom.xml | 12 +
pom.xml | 6 +
runners/gearpump/README.md | 61 ++++
runners/gearpump/pom.xml | 281 +++++++++++++++++++
.../gearpump/GearpumpPipelineOptions.java | 67 +++++
.../gearpump/GearpumpPipelineResult.java | 109 +++++++
.../beam/runners/gearpump/GearpumpRunner.java | 119 ++++++++
.../gearpump/GearpumpRunnerRegistrar.java | 64 +++++
.../runners/gearpump/TestGearpumpRunner.java | 63 +++++
.../beam/runners/gearpump/package-info.java | 22 ++
...CreateGearpumpPCollectionViewTranslator.java | 45 +++
.../CreateStreamingGearpumpView.java | 157 +++++++++++
.../FlattenPCollectionsTranslator.java | 83 ++++++
.../translators/GearpumpPipelineTranslator.java | 143 ++++++++++
.../translators/GroupByKeyTranslator.java | 258 +++++++++++++++++
.../translators/ParDoMultiOutputTranslator.java | 103 +++++++
.../translators/ReadBoundedTranslator.java | 46 +++
.../translators/ReadUnboundedTranslator.java | 48 ++++
.../translators/TransformTranslator.java | 30 ++
.../translators/TranslationContext.java | 105 +++++++
.../translators/WindowAssignTranslator.java | 103 +++++++
.../translators/functions/DoFnFunction.java | 193 +++++++++++++
.../translators/functions/package-info.java | 22 ++
.../translators/io/BoundedSourceWrapper.java | 45 +++
.../gearpump/translators/io/GearpumpSource.java | 121 ++++++++
.../translators/io/UnboundedSourceWrapper.java | 46 +++
.../gearpump/translators/io/ValuesSource.java | 173 ++++++++++++
.../gearpump/translators/io/package-info.java | 22 ++
.../gearpump/translators/package-info.java | 22 ++
.../translators/utils/DoFnRunnerFactory.java | 81 ++++++
.../translators/utils/NoOpStepContext.java | 41 +++
.../translators/utils/TranslatorUtils.java | 199 +++++++++++++
.../translators/utils/package-info.java | 22 ++
.../gearpump/GearpumpRunnerRegistrarTest.java | 55 ++++
.../runners/gearpump/PipelineOptionsTest.java | 73 +++++
...teGearpumpPCollectionViewTranslatorTest.java | 56 ++++
.../FlattenPCollectionsTranslatorTest.java | 149 ++++++++++
.../translators/GroupByKeyTranslatorTest.java | 152 ++++++++++
.../translators/ReadBoundedTranslatorTest.java | 70 +++++
.../ReadUnboundedTranslatorTest.java | 70 +++++
.../translators/WindowAssignTranslatorTest.java | 110 ++++++++
.../translators/io/GearpumpSourceTest.java | 99 +++++++
.../gearpump/translators/io/ValueSoureTest.java | 79 ++++++
.../translators/utils/TranslatorUtilsTest.java | 73 +++++
runners/pom.xml | 9 +
sdks/java/pom.xml | 2 +-
47 files changed, 3909 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/99f4f8b1/pom.xml
----------------------------------------------------------------------
[43/50] [abbrv] beam git commit: Upgrade BEAM version to
2.2.0-SNAPSHOT in gearpump-runner
Posted by ke...@apache.org.
Upgrade BEAM version to 2.2.0-SNAPSHOT in gearpump-runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/daa75669
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/daa75669
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/daa75669
Branch: refs/heads/master
Commit: daa7566939e212095ef745ff004580bfe4209b38
Parents: e655f53
Author: manuzhang <ow...@gmail.com>
Authored: Fri Jul 21 14:58:13 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Fri Jul 21 19:45:14 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/daa75669/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index 54c8d5c..ee90d48 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
[45/50] [abbrv] beam git commit: Deactivate integration-tests for
gearpump-runner by default
Posted by ke...@apache.org.
Deactivate integration-tests for gearpump-runner by default
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d9bb4cb6
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d9bb4cb6
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d9bb4cb6
Branch: refs/heads/master
Commit: d9bb4cb66b6b80269795343e5be3b85aab1e7b6a
Parents: 49d4ed5
Author: manuzhang <ow...@gmail.com>
Authored: Tue Jul 25 15:14:49 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Wed Jul 26 12:29:34 2017 +0800
----------------------------------------------------------------------
examples/java/pom.xml | 12 ------------
examples/java8/pom.xml | 12 ++++++++++++
runners/gearpump/pom.xml | 3 +--
3 files changed, 13 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/d9bb4cb6/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 2d43527..ae64a79 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -89,18 +89,6 @@
</dependencies>
</profile>
- <!-- Include the Apache Gearpump (incubating) runner with -P gearpump-runner -->
- <profile>
- <id>gearpump-runner</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-runners-gearpump</artifactId>
- <scope>runtime</scope>
- </dependency>
- </dependencies>
- </profile>
-
<!-- Include the Apache Flink runner with -P flink-runner -->
<profile>
<id>flink-runner</id>
http://git-wip-us.apache.org/repos/asf/beam/blob/d9bb4cb6/examples/java8/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 6fd29a4..7beb955 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -151,6 +151,18 @@
</dependency>
</dependencies>
</profile>
+
+ <!-- Include the Apache Gearpump (incubating) runner with -P gearpump-runner -->
+ <profile>
+ <id>gearpump-runner</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gearpump</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
<build>
http://git-wip-us.apache.org/repos/asf/beam/blob/d9bb4cb6/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index ee90d48..3a4722f 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -49,8 +49,7 @@
<profiles>
<profile>
<id>local-validates-runner-tests</id>
- <!-- TODO: once on master branch, move to postcommit only -->
- <activation><activeByDefault>true</activeByDefault></activation>
+ <activation><activeByDefault>false</activeByDefault></activation>
<build>
<plugins>
<plugin>
[22/50] [abbrv] beam git commit: Update gearpump-runner against
master changes
Posted by ke...@apache.org.
Update gearpump-runner against master changes
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/6c06967c
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/6c06967c
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/6c06967c
Branch: refs/heads/master
Commit: 6c06967cf76d55fce42e87a320563ff981746f85
Parents: bc8da29
Author: manuzhang <ow...@gmail.com>
Authored: Thu May 18 11:08:11 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Thu May 18 11:08:11 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 2 +-
.../apache/beam/runners/gearpump/GearpumpPipelineTranslator.java | 4 ++--
.../java/org/apache/beam/runners/gearpump/GearpumpRunner.java | 2 +-
.../apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java | 2 +-
.../org/apache/beam/runners/gearpump/TestGearpumpRunner.java | 2 +-
.../gearpump/translators/ParDoSingleOutputTranslator.java | 2 +-
.../beam/runners/gearpump/translators/TranslationContext.java | 2 +-
.../runners/gearpump/translators/WindowAssignTranslator.java | 2 +-
.../runners/gearpump/translators/functions/DoFnFunction.java | 2 +-
.../runners/gearpump/translators/utils/DoFnRunnerFactory.java | 4 ++--
10 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/6c06967c/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index dcfa390..d4dade1 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-parent</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/beam/blob/6c06967c/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
index f5f5e70..dc4592c 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
@@ -46,9 +46,9 @@ import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.ListCoder;
import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverride;
import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
@@ -58,10 +58,10 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.util.InstanceBuilder;
-import org.apache.beam.sdk.util.PCollectionViews;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
import org.apache.beam.sdk.values.PValue;
import org.apache.gearpump.util.Graph;
http://git-wip-us.apache.org/repos/asf/beam/blob/6c06967c/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
index 897467a..6df3f2d 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunner.java
@@ -26,9 +26,9 @@ import java.util.Map;
import org.apache.beam.runners.gearpump.translators.TranslationContext;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.gearpump.cluster.ClusterConfig;
import org.apache.gearpump.cluster.UserConfig;
http://git-wip-us.apache.org/repos/asf/beam/blob/6c06967c/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
index 3183d45..5152105 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
@@ -21,9 +21,9 @@ package org.apache.beam.runners.gearpump;
import com.google.auto.service.AutoService;
import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
/**
http://git-wip-us.apache.org/repos/asf/beam/blob/6c06967c/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
index ea7dd26..0a88849 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/TestGearpumpRunner.java
@@ -21,9 +21,9 @@ package org.apache.beam.runners.gearpump;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigValueFactory;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.gearpump.cluster.ClusterConfig;
import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
http://git-wip-us.apache.org/repos/asf/beam/blob/6c06967c/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
index 86879b7..6b0e610 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
@@ -27,12 +27,12 @@ import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.WindowingStrategy;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
/**
http://git-wip-us.apache.org/repos/asf/beam/blob/6c06967c/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
index eb6bc18..c3db044 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
@@ -26,8 +26,8 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
http://git-wip-us.apache.org/repos/asf/beam/blob/6c06967c/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
index 149f80c..d144b95 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
@@ -29,8 +29,8 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.WindowingStrategy;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
import org.joda.time.Instant;
http://git-wip-us.apache.org/repos/asf/beam/blob/6c06967c/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index dfd6296..f521d7b 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -46,10 +46,10 @@ import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
/**
http://git-wip-us.apache.org/repos/asf/beam/blob/6c06967c/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
index 8d55d6f..d38f11b 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
@@ -26,15 +26,15 @@ import org.apache.beam.runners.core.DoFnRunner;
import org.apache.beam.runners.core.DoFnRunners;
import org.apache.beam.runners.core.ExecutionContext;
import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
+import org.apache.beam.runners.core.ReadyCheckingSideInputReader;
import org.apache.beam.runners.core.SimpleDoFnRunner;
import org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.WindowingStrategy;
/**
* a serializable {@link SimpleDoFnRunner}.
[10/50] [abbrv] beam git commit: This closes #2241: merge master to
gearpump-master and fixup
Posted by ke...@apache.org.
This closes #2241: merge master to gearpump-master and fixup
[BEAM-79] Fix gearpump-runner merge conflicts and test failure
Revert BigQueryIO bit of 'Make all uses of CountingOutputStream close their resources'
[BEAM-1629] Init metrics/aggregators accumulators before traversing pipeline
Generate zip distribution for pyhthon
Ignore results from the tox clean up phase
Add README to python tarball.
Remove exception suppression from PAssert.SideInputCheckerDoFn
Remove duplicated dependency from Dataflow runner pom.xml
Added assertion failure tests for `PAssert#thatSingleton`
Added a test of default PAssert failure reason
Javadoc changes
[BEAM-1551] Allow `PAssert`s to take a message
add unicode type to the typeDict attribute in Python SDK
Remove Pipeline.getRunner
[BEAM-1686] Use random MQTT clientID when not defined to avoid NPE
Properly deal with late processing-time timers
[BEAM-1661] Shade guava in the JdbcIO
[BEAM-797] A PipelineVisitor that creates a Spark-native pipeline.
Introduce Flink-specific state GC implementations
Move GC timer checking to StatefulDoFnRunner.CleanupTimer
Move pipeline context and add more tests.
Add license to new files.
Runner API translation of triggers and windowing strategies.
Runner API encoding of WindowFns.
Runner API context helper classes.
Auto-generated runner api proto bindings.
HadoopInputFormatIO with junits
Test runner to stop on EOT watermark, or timeout.
[BEAM-1184] Add integration tests to ElasticsearchIO
Jdbc k8 script: postgres data store only accessible inside test project
Jdbc k8 & data loading: add teardown and update names/docs
Bump Dataflow ROS timeout to 120 minutes
Fixup typo in WindowingStrategies
Update archetypes
[BEAM-1649] Fix unresolved references in Python SDK
Upgrade Dataflow container version to beam-master-20170307
Explicitly GBK before stateful ParDo in Dataflow batch
Add ServicesResourceTransformer to all shading configuration
Fix typo in proto: widow -> window.
[BEAM-1646] Remove duplicated bigquery dependency
Update python SDK version to the next version.
Change Json parsing from gson to jackson for ElasticsearchIO
[maven-release-plugin] prepare for next development iteration
[maven-release-plugin] prepare branch release-0.6.0
[BEAM-1633] Move .tox/ directory under target/ in Python SDK
Make all uses of CountingOutputStream close their resources
Flink: register known IOChannelFactories
[BEAM-1546] Specify exact version for Python in the SDK
[BEAM-1635] TypeError in AfterWatermark class's __repr__ method
[BEAM-1636] UnboundedDataset action() does not materialize RDD
[BEAM-1556] Make PipelineOptions a lazy-singleton and init IOs as part of it.
Add tests for serialization of BigQueryIO.TableRowInfoCoder
Fix tox warning for non-whitelisted find command
Revert "Implement Single-Output ParDo as a composite"
[BEAM-1623] Transform Reshuffle directly in Spark runner
[BEAM-1626] Remove cache of MapWithStateDStream on read.
Revert "DataflowRunner: experimental support for issuing FnAPI based jobs"
Java examples: move shade plugin into default lifecycle
Java 8 examples: add shade configuration for producing bundled/fat jar
Java 8 examples: add SNAPSHOT repository to the pom.xml file
Update Guava version from 19.0 to 20.0 in example projects
[BEAM-1625] BoundedDataset action() does not materialize RDD
Revert "[maven-release-plugin] prepare branch release-0.6.0"
Updating Dataflow API client protobufs
Implement Single-Output ParDo as a composite
Add a Test for windowed CombineGloballyAsSingletonView
Remove SingletonCombine
Only Override CreatePCollectionView in Streaming
[BEAM-1310] Add running integration tests in JdbcIO on Spark and Dataflow runners
DataflowRunner: experimental support for issuing FnAPI based jobs
Updates Python SDK source API so that sources can report limited parallelism signals.
[BEAM-1188] Python Bigquery Verifier For E2E Test
Deprecate Pipeline.getOptions
BEAM-1567 hashStream should be closed in PackageUtil#createPackageAttributes()
[BEAM-1565] Update Spark runner PostCommit Jenkins job.
[maven-release-plugin] prepare branch release-0.6.0
Do not Reassign Windows when WindowFn is null
Fix DataflowRunner message about uploaded vs cached files
Update javadoc ant to include runners/ and exclude modules with a wildcard
Adding per-stage matching to metrics filters
Upgrade dill to 0.2.6 and pin it
Remove PipelineRunner#apply
[BEAM-111] Move WritableCoder to hadoop-common
[BEAM-1297] Update maven shade plugin, fix typo and remove unneeded version
[BEAM-351] Add DisplayData to KafkaIO
Inline rather than reference FunctionSpecs.
[BEAM-1517] Garbage collect user state in Flink Runner
BEAM-1417 Count should comply with PTransform style guide
BEAM-1419 Flatten should comply with PTransform style guide
BEAM-1416 Write transform should comply with PTransform style guide
BEAM-1426 SortValues should comply with PTransform style guide
BEAM-1424 ToString should comply with PTransform style guide
BEAM-1423 Sample should comply with PTransform style guide
BEAM-1421 Latest should comply with PTransform style guide
BEAM-1420 GroupByKey should comply with PTransform style guide
Use UnsupportedSideInputReader in GroupAlsoByWindowEvaluatorFactory
Include cython tests in presubmits for linux platform
Update output stream cython declaration
Make side inputs a map, rather than embedding the name in the message.
Streaming tests, especially the ones using checkpoints, need a time buffer to finish.
...
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/555842a1
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/555842a1
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/555842a1
Branch: refs/heads/master
Commit: 555842a1a0c478b3935b3987683d2645eba770c3
Parents: 15a8ad6 3eab6a6
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Mar 17 11:15:52 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Mar 17 11:15:52 2017 -0700
----------------------------------------------------------------------
.gitignore | 10 +
.jenkins/common_job_properties.groovy | 98 +-
...job_beam_PostCommit_Java_MavenInstall.groovy | 4 +-
...ostCommit_Java_RunnableOnService_Apex.groovy | 8 +-
...ommit_Java_RunnableOnService_Dataflow.groovy | 8 +-
...stCommit_Java_RunnableOnService_Flink.groovy | 8 +-
...ommit_Java_RunnableOnService_Gearpump.groovy | 12 +-
...stCommit_Java_RunnableOnService_Spark.groovy | 10 +-
.../job_beam_PostCommit_Python_Verify.groovy | 8 +-
.../job_beam_PreCommit_Java_MavenInstall.groovy | 6 +-
.../job_beam_PreCommit_Website_Stage.groovy | 80 +
.jenkins/job_beam_PreCommit_Website_Test.groovy | 65 +
.../job_beam_Release_NightlySnapshot.groovy | 9 +-
.jenkins/job_seed.groovy | 24 +-
.travis.yml | 24 +-
.travis/README.md | 2 +-
DISCLAIMER | 10 -
NOTICE | 4 +-
README.md | 57 +-
examples/java/README.md | 16 +-
examples/java/pom.xml | 24 +-
.../beam/examples/DebuggingWordCount.java | 4 +-
.../org/apache/beam/examples/WordCount.java | 6 +-
.../beam/examples/complete/AutoComplete.java | 4 +-
.../org/apache/beam/examples/complete/README.md | 14 +-
.../apache/beam/examples/complete/TfIdf.java | 2 +-
.../examples/complete/TopWikipediaSessions.java | 27 +-
.../examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 2 +-
.../org/apache/beam/examples/cookbook/README.md | 14 +-
.../beam/examples/cookbook/TriggerExample.java | 4 +-
.../beam/examples/WindowedWordCountIT.java | 16 +-
.../org/apache/beam/examples/WordCountTest.java | 7 +-
.../examples/complete/AutoCompleteTest.java | 11 +-
.../beam/examples/complete/TfIdfTest.java | 6 +-
.../complete/TopWikipediaSessionsTest.java | 7 +-
.../examples/cookbook/DistinctExampleTest.java | 9 +-
.../examples/cookbook/JoinExamplesTest.java | 6 +-
.../examples/cookbook/TriggerExampleTest.java | 6 +-
examples/java8/pom.xml | 3 +-
.../beam/examples/complete/game/GameStats.java | 7 +-
.../examples/complete/game/LeaderBoard.java | 5 +-
.../beam/examples/complete/game/UserScore.java | 2 +-
.../examples/MinimalWordCountJava8Test.java | 6 +-
.../examples/complete/game/GameStatsTest.java | 7 +-
.../complete/game/HourlyTeamScoreTest.java | 5 +-
.../examples/complete/game/LeaderBoardTest.java | 11 +-
.../examples/complete/game/UserScoreTest.java | 10 +-
examples/pom.xml | 16 +-
pom.xml | 319 +-
runners/apex/README.md | 4 +-
runners/apex/pom.xml | 32 +-
.../beam/runners/apex/ApexPipelineOptions.java | 7 +-
.../apache/beam/runners/apex/ApexRunner.java | 144 +-
.../beam/runners/apex/ApexYarnLauncher.java | 29 +-
.../beam/runners/apex/TestApexRunner.java | 10 -
.../translation/ApexPipelineTranslator.java | 17 +-
.../translation/CreateValuesTranslator.java | 25 +-
.../FlattenPCollectionTranslator.java | 37 +-
.../apex/translation/GroupByKeyTranslator.java | 2 +-
.../translation/ParDoBoundMultiTranslator.java | 32 +-
.../apex/translation/ParDoBoundTranslator.java | 9 +-
.../apex/translation/TranslationContext.java | 43 +-
.../translation/WindowAssignTranslator.java | 78 +
.../apex/translation/WindowBoundTranslator.java | 78 -
.../operators/ApexFlattenOperator.java | 4 +-
.../operators/ApexGroupByKeyOperator.java | 22 +-
.../operators/ApexParDoOperator.java | 12 +-
.../ApexReadUnboundedInputOperator.java | 17 +-
.../translation/utils/ApexStateInternals.java | 30 +-
.../apex/translation/utils/NoOpStepContext.java | 7 +-
.../beam/runners/apex/ApexRunnerTest.java | 76 +
.../beam/runners/apex/ApexYarnLauncherTest.java | 9 +-
.../runners/apex/examples/WordCountTest.java | 2 +-
.../FlattenPCollectionTranslatorTest.java | 24 +-
.../translation/ParDoBoundTranslatorTest.java | 38 +-
.../translation/ReadUnboundTranslatorTest.java | 8 +-
.../utils/ApexStateInternalsTest.java | 14 +-
.../test/resources/beam-runners-apex.properties | 20 +
runners/core-construction-java/pom.xml | 138 +
.../EmptyFlattenAsCreateFactory.java | 71 +
.../core/construction/PTransformMatchers.java | 192 +
.../core/construction/PrimitiveCreate.java | 77 +
.../core/construction/ReplacementOutputs.java | 105 +
.../SingleInputOutputOverrideFactory.java | 50 +
.../UnsupportedOverrideFactory.java | 71 +
.../runners/core/construction/package-info.java | 22 +
.../construction/PTransformMatchersTest.java | 425 ++
.../construction/ReplacementOutputsTest.java | 254 +
.../SingleInputOutputOverrideFactoryTest.java | 114 +
.../UnsupportedOverrideFactoryTest.java | 65 +
runners/core-java/pom.xml | 24 +-
.../beam/runners/core/AggregatorFactory.java | 1 -
.../beam/runners/core/AssignWindowsDoFn.java | 3 +-
.../beam/runners/core/BaseExecutionContext.java | 174 +
.../apache/beam/runners/core/DoFnAdapters.java | 323 ++
.../apache/beam/runners/core/DoFnRunner.java | 21 -
.../apache/beam/runners/core/DoFnRunners.java | 158 +-
.../beam/runners/core/ExecutionContext.java | 100 +
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 114 +
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 17 +-
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 154 +
.../runners/core/GroupAlsoByWindowsDoFn.java | 5 +-
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 134 -
.../runners/core/InMemoryStateInternals.java | 643 +++
.../runners/core/InMemoryTimerInternals.java | 300 ++
.../apache/beam/runners/core/KeyedWorkItem.java | 2 +-
.../beam/runners/core/KeyedWorkItemCoder.java | 8 +-
.../beam/runners/core/KeyedWorkItems.java | 2 +-
.../core/LateDataDroppingDoFnRunner.java | 2 -
.../apache/beam/runners/core/LateDataUtils.java | 88 +
.../runners/core/MergingActiveWindowSet.java | 4 -
.../beam/runners/core/MergingStateAccessor.java | 41 +
.../apache/beam/runners/core/NonEmptyPanes.java | 7 +-
.../org/apache/beam/runners/core/OldDoFn.java | 471 ++
...eBoundedSplittableProcessElementInvoker.java | 285 ++
.../beam/runners/core/PaneInfoTracker.java | 4 -
.../runners/core/PerKeyCombineFnRunner.java | 70 -
.../runners/core/PerKeyCombineFnRunners.java | 101 -
.../org/apache/beam/runners/core/ReduceFn.java | 2 -
.../runners/core/ReduceFnContextFactory.java | 11 +-
.../beam/runners/core/ReduceFnRunner.java | 6 +-
.../beam/runners/core/SideInputHandler.java | 4 -
.../beam/runners/core/SimpleDoFnRunner.java | 190 +-
.../beam/runners/core/SimpleOldDoFnRunner.java | 12 +-
.../beam/runners/core/SplittableParDo.java | 233 +-
.../core/SplittableProcessElementInvoker.java | 65 +
.../apache/beam/runners/core/StateAccessor.java | 38 +
.../beam/runners/core/StateInternals.java | 59 +
.../runners/core/StateInternalsFactory.java | 35 +
.../apache/beam/runners/core/StateMerging.java | 309 ++
.../beam/runners/core/StateNamespace.java | 56 +
.../runners/core/StateNamespaceForTest.java | 65 +
.../beam/runners/core/StateNamespaces.java | 278 ++
.../apache/beam/runners/core/StateTable.java | 84 +
.../org/apache/beam/runners/core/StateTag.java | 125 +
.../org/apache/beam/runners/core/StateTags.java | 382 ++
.../beam/runners/core/StatefulDoFnRunner.java | 171 +
.../beam/runners/core/SystemReduceFn.java | 5 -
.../core/TestInMemoryStateInternals.java | 63 +
.../beam/runners/core/TimerInternals.java | 286 ++
.../runners/core/TimerInternalsFactory.java | 35 +
.../core/UnboundedReadFromBoundedSource.java | 14 +-
.../core/UnsupportedSideInputReader.java | 52 +
.../apache/beam/runners/core/WatermarkHold.java | 9 +-
.../beam/runners/core/WindowingInternals.java | 81 +
.../core/WindowingInternalsAdapters.java | 1 -
.../core/triggers/AfterAllStateMachine.java | 2 +-
.../AfterDelayFromFirstElementStateMachine.java | 16 +-
.../core/triggers/AfterFirstStateMachine.java | 2 +-
.../core/triggers/AfterPaneStateMachine.java | 12 +-
.../AfterProcessingTimeStateMachine.java | 2 +
...rSynchronizedProcessingTimeStateMachine.java | 7 +-
.../triggers/AfterWatermarkStateMachine.java | 15 +-
.../core/triggers/OrFinallyStateMachine.java | 2 +-
.../triggers/ReshuffleTriggerStateMachine.java | 6 +-
.../core/triggers/TriggerStateMachine.java | 6 +-
.../TriggerStateMachineContextFactory.java | 12 +-
.../triggers/TriggerStateMachineRunner.java | 8 +-
.../core/triggers/TriggerStateMachines.java | 250 +-
.../core/DoFnDelegatingAggregatorTest.java | 144 +
...roupAlsoByWindowViaOutputBufferDoFnTest.java | 109 +
.../core/GroupAlsoByWindowsProperties.java | 7 +-
...oupAlsoByWindowsViaOutputBufferDoFnTest.java | 110 -
.../core/InMemoryStateInternalsTest.java | 574 +++
.../core/InMemoryTimerInternalsTest.java | 195 +
.../runners/core/KeyedWorkItemCoderTest.java | 9 +-
.../core/LateDataDroppingDoFnRunnerTest.java | 3 +-
.../core/MergingActiveWindowSetTest.java | 2 -
.../apache/beam/runners/core/NoOpOldDoFn.java | 72 +
.../beam/runners/core/OldDoFnContextTest.java | 72 +
.../apache/beam/runners/core/OldDoFnTest.java | 192 +
...ndedSplittableProcessElementInvokerTest.java | 146 +
.../core/PushbackSideInputDoFnRunnerTest.java | 8 +-
.../beam/runners/core/ReduceFnRunnerTest.java | 12 +-
.../beam/runners/core/ReduceFnTester.java | 19 +-
.../beam/runners/core/SideInputHandlerTest.java | 1 -
.../beam/runners/core/SimpleDoFnRunnerTest.java | 6 +-
.../runners/core/SimpleOldDoFnRunnerTest.java | 4 +-
.../beam/runners/core/SplittableParDoTest.java | 254 +-
.../beam/runners/core/StateNamespacesTest.java | 130 +
.../apache/beam/runners/core/StateTagTest.java | 205 +
.../runners/core/StatefulDoFnRunnerTest.java | 347 ++
.../beam/runners/core/TimerInternalsTest.java | 105 +
.../UnboundedReadFromBoundedSourceTest.java | 18 +-
.../beam/runners/core/WindowMatchers.java | 3 +-
...chronizedProcessingTimeStateMachineTest.java | 3 +-
.../AfterWatermarkStateMachineTest.java | 19 +
.../core/triggers/NeverStateMachineTest.java | 1 -
.../ReshuffleTriggerStateMachineTest.java | 8 +-
.../triggers/TriggerStateMachineTester.java | 16 +-
.../core/triggers/TriggerStateMachinesTest.java | 167 +-
runners/direct-java/pom.xml | 27 +-
.../runners/direct/AggregatorContainer.java | 2 +-
.../direct/BoundedReadEvaluatorFactory.java | 10 +-
.../CopyOnAccessInMemoryStateInternals.java | 68 +-
.../runners/direct/DirectExecutionContext.java | 8 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 4 +-
.../beam/runners/direct/DirectGraphVisitor.java | 33 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 3 +-
.../beam/runners/direct/DirectMetrics.java | 31 +-
.../beam/runners/direct/DirectRunner.java | 114 +-
.../runners/direct/DirectTimerInternals.java | 14 +-
...ecycleManagerRemovingTransformEvaluator.java | 19 +-
.../beam/runners/direct/EmptyInputProvider.java | 4 +-
.../beam/runners/direct/EvaluationContext.java | 40 +-
.../direct/ExecutorServiceParallelExecutor.java | 14 +-
.../runners/direct/FlattenEvaluatorFactory.java | 8 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 38 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 10 +-
.../direct/ImmutableListBundleFactory.java | 6 +
.../direct/KeyedPValueTrackingVisitor.java | 52 +-
.../beam/runners/direct/ParDoEvaluator.java | 29 +-
.../runners/direct/ParDoEvaluatorFactory.java | 25 +-
.../direct/ParDoMultiOverrideFactory.java | 154 +-
.../ParDoSingleViaMultiOverrideFactory.java | 5 +-
.../runners/direct/RootProviderRegistry.java | 4 +-
...littableProcessElementsEvaluatorFactory.java | 84 +-
.../direct/StatefulParDoEvaluatorFactory.java | 72 +-
.../direct/TestStreamEvaluatorFactory.java | 44 +-
.../direct/TransformEvaluatorRegistry.java | 6 +-
.../direct/UnboundedReadEvaluatorFactory.java | 22 +-
.../runners/direct/ViewEvaluatorFactory.java | 16 +-
.../beam/runners/direct/WatermarkManager.java | 107 +-
.../runners/direct/WindowEvaluatorFactory.java | 17 +-
.../direct/WriteWithShardingFactory.java | 148 +-
.../runners/direct/AggregatorContainerTest.java | 18 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 18 +-
.../direct/CloningBundleFactoryTest.java | 16 +-
.../runners/direct/CommittedResultTest.java | 23 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 79 +-
.../runners/direct/DirectGraphVisitorTest.java | 55 +-
.../DirectGroupByKeyOverrideFactoryTest.java | 51 +
.../beam/runners/direct/DirectMetricsTest.java | 128 +-
.../beam/runners/direct/DirectRunnerTest.java | 8 +-
.../direct/DirectTimerInternalsTest.java | 4 +-
...leManagerRemovingTransformEvaluatorTest.java | 103 +-
.../runners/direct/EvaluationContextTest.java | 30 +-
.../direct/FlattenEvaluatorFactoryTest.java | 8 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 5 +-
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 5 +-
.../ImmutabilityCheckingBundleFactoryTest.java | 6 +-
.../ImmutabilityEnforcementFactoryTest.java | 3 +-
.../direct/ImmutableListBundleFactoryTest.java | 37 +-
.../direct/KeyedPValueTrackingVisitorTest.java | 167 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 9 +-
.../direct/ParDoMultiOverrideFactoryTest.java | 45 +
.../ParDoSingleViaMultiOverrideFactoryTest.java | 46 +
.../runners/direct/SideInputContainerTest.java | 5 +-
.../StatefulParDoEvaluatorFactoryTest.java | 129 +-
.../runners/direct/StepTransformResultTest.java | 5 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 38 +-
.../runners/direct/TransformExecutorTest.java | 4 +-
.../UnboundedReadEvaluatorFactoryTest.java | 22 +-
.../direct/ViewEvaluatorFactoryTest.java | 18 +-
.../direct/WatermarkCallbackExecutorTest.java | 5 +-
.../runners/direct/WatermarkManagerTest.java | 37 +-
.../direct/WindowEvaluatorFactoryTest.java | 87 +-
.../direct/WriteWithShardingFactoryTest.java | 154 +-
runners/flink/README.md | 250 -
runners/flink/examples/pom.xml | 2 +-
.../beam/runners/flink/examples/WordCount.java | 2 +-
.../flink/examples/streaming/AutoComplete.java | 4 +-
.../examples/streaming/KafkaIOExamples.java | 4 +-
.../KafkaWindowedWordCountExample.java | 2 +-
.../examples/streaming/WindowedWordCount.java | 2 +-
runners/flink/pom.xml | 4 +-
runners/flink/runner/pom.xml | 56 +-
.../flink/DefaultParallelismFactory.java | 3 +-
.../flink/FlinkBatchPipelineTranslator.java | 139 +
.../flink/FlinkBatchTransformTranslators.java | 797 ++++
.../flink/FlinkBatchTranslationContext.java | 154 +
.../FlinkPipelineExecutionEnvironment.java | 11 +-
.../runners/flink/FlinkPipelineOptions.java | 6 +-
.../runners/flink/FlinkPipelineTranslator.java | 53 +
.../apache/beam/runners/flink/FlinkRunner.java | 395 +-
.../flink/FlinkStreamingPipelineTranslator.java | 230 +
.../FlinkStreamingTransformTranslators.java | 1043 +++++
.../flink/FlinkStreamingTranslationContext.java | 130 +
.../flink/FlinkStreamingViewOverrides.java | 372 ++
.../flink/PipelineTranslationOptimizer.java | 72 +
.../beam/runners/flink/TestFlinkRunner.java | 24 +-
.../beam/runners/flink/TranslationMode.java | 31 +
.../FlinkBatchPipelineTranslator.java | 143 -
.../FlinkBatchTransformTranslators.java | 745 ---
.../FlinkBatchTranslationContext.java | 145 -
.../translation/FlinkPipelineTranslator.java | 53 -
.../FlinkStreamingPipelineTranslator.java | 154 -
.../FlinkStreamingTransformTranslators.java | 978 ----
.../FlinkStreamingTranslationContext.java | 110 -
.../PipelineTranslationOptimizer.java | 73 -
.../flink/translation/TranslationMode.java | 31 -
.../functions/FlinkAggregatorFactory.java | 53 +
.../functions/FlinkAssignWindows.java | 2 +-
.../functions/FlinkDoFnFunction.java | 136 +-
.../FlinkMergingNonShuffleReduceFunction.java | 57 +-
.../FlinkMergingPartialReduceFunction.java | 37 +-
.../functions/FlinkMergingReduceFunction.java | 31 +-
.../functions/FlinkMultiOutputDoFnFunction.java | 126 -
.../FlinkMultiOutputProcessContext.java | 118 -
.../FlinkMultiOutputPruningFunction.java | 2 +-
.../functions/FlinkNoElementAssignContext.java | 68 -
.../functions/FlinkNoOpStepContext.java | 73 +
.../functions/FlinkPartialReduceFunction.java | 45 +-
.../functions/FlinkProcessContextBase.java | 267 --
.../functions/FlinkReduceFunction.java | 41 +-
.../functions/FlinkSideInputReader.java | 80 +
.../FlinkSingleOutputProcessContext.java | 69 -
.../functions/FlinkStatefulDoFnFunction.java | 198 +
.../utils/SerializedPipelineOptions.java | 4 +-
.../wrappers/streaming/DoFnOperator.java | 671 ++-
.../wrappers/streaming/FlinkStateInternals.java | 1037 -----
.../streaming/KvToByteBufferKeySelector.java | 56 +
.../streaming/SingletonKeyedWorkItem.java | 2 +-
.../streaming/SingletonKeyedWorkItemCoder.java | 10 +-
.../wrappers/streaming/WindowDoFnOperator.java | 457 +-
.../streaming/io/BoundedSourceWrapper.java | 7 +-
.../streaming/io/UnboundedFlinkSink.java | 6 +
.../streaming/io/UnboundedSourceWrapper.java | 159 +-
.../state/FlinkBroadcastStateInternals.java | 865 ++++
.../state/FlinkKeyGroupStateInternals.java | 487 ++
.../state/FlinkSplitStateInternals.java | 260 ++
.../streaming/state/FlinkStateInternals.java | 1053 +++++
.../state/KeyGroupCheckpointedOperator.java | 35 +
.../state/KeyGroupRestoringOperator.java | 32 +
.../wrappers/streaming/state/package-info.java | 22 +
.../beam/runners/flink/PipelineOptionsTest.java | 42 +-
.../beam/runners/flink/WriteSinkITCase.java | 5 +-
.../flink/streaming/DoFnOperatorTest.java | 337 +-
.../FlinkBroadcastStateInternalsTest.java | 245 +
.../FlinkKeyGroupStateInternalsTest.java | 262 ++
.../streaming/FlinkSplitStateInternalsTest.java | 101 +
.../streaming/FlinkStateInternalsTest.java | 39 +-
.../streaming/UnboundedSourceWrapperTest.java | 570 ++-
runners/gearpump/pom.xml | 51 +-
.../gearpump/GearpumpPipelineResult.java | 21 +-
.../gearpump/GearpumpPipelineTranslator.java | 388 +-
.../beam/runners/gearpump/GearpumpRunner.java | 376 +-
.../runners/gearpump/TestGearpumpRunner.java | 38 +-
.../gearpump/examples/StreamingWordCount.java | 98 -
.../gearpump/examples/UnboundedTextSource.java | 139 -
.../runners/gearpump/examples/package-info.java | 22 -
...CreateGearpumpPCollectionViewTranslator.java | 14 +-
.../CreatePCollectionViewTranslator.java | 6 +-
.../translators/CreateValuesTranslator.java | 51 -
.../FlattenPCollectionTranslator.java | 84 -
.../FlattenPCollectionsTranslator.java | 83 +
.../translators/GroupByKeyTranslator.java | 4 +-
.../translators/ParDoBoundMultiTranslator.java | 32 +-
.../translators/ParDoBoundTranslator.java | 7 +-
.../translators/ReadBoundedTranslator.java | 4 +-
.../translators/ReadUnboundedTranslator.java | 4 +-
.../translators/TransformTranslator.java | 2 +-
.../translators/TranslationContext.java | 29 +-
.../translators/WindowAssignTranslator.java | 100 +
.../translators/WindowBoundTranslator.java | 100 -
.../translators/functions/DoFnFunction.java | 12 +-
.../translators/io/UnboundedSourceWrapper.java | 1 +
.../translators/utils/DoFnRunnerFactory.java | 4 +-
.../utils/NoOpAggregatorFactory.java | 2 +-
.../translators/utils/NoOpStepContext.java | 6 +-
.../translators/utils/TranslatorUtils.java | 2 -
.../translators/utils/TranslatorUtilsTest.java | 1 -
runners/google-cloud-dataflow-java/pom.xml | 40 +-
.../beam/runners/dataflow/AssignWindows.java | 89 +
.../dataflow/BatchStatefulParDoOverrides.java | 283 ++
.../runners/dataflow/BatchViewOverrides.java | 1391 ++++++
.../dataflow/DataflowAggregatorTransforms.java | 79 +
.../beam/runners/dataflow/DataflowClient.java | 44 +-
.../dataflow/DataflowMetricUpdateExtractor.java | 109 +
.../runners/dataflow/DataflowPipelineJob.java | 4 +-
.../dataflow/DataflowPipelineTranslator.java | 606 +--
.../beam/runners/dataflow/DataflowRunner.java | 2466 ++--------
.../DataflowUnboundedReadFromBoundedSource.java | 547 +++
.../beam/runners/dataflow/ReadTranslator.java | 102 +
.../dataflow/StreamingViewOverrides.java | 110 +
.../runners/dataflow/TransformTranslator.java | 131 +
.../dataflow/internal/AssignWindows.java | 89 -
.../dataflow/internal/CustomSources.java | 12 +-
.../internal/DataflowAggregatorTransforms.java | 79 -
.../internal/DataflowMetricUpdateExtractor.java | 109 -
.../DataflowUnboundedReadFromBoundedSource.java | 556 ---
.../runners/dataflow/internal/IsmFormat.java | 20 +-
.../dataflow/internal/ReadTranslator.java | 107 -
.../options/DataflowPipelineOptions.java | 36 +-
.../DataflowPipelineWorkerPoolOptions.java | 16 +-
.../dataflow/testing/TestDataflowRunner.java | 17 +-
.../beam/runners/dataflow/util/DoFnInfo.java | 75 +-
.../beam/runners/dataflow/util/GcsStager.java | 18 +-
.../beam/runners/dataflow/util/PackageUtil.java | 363 +-
.../beam/runners/dataflow/dataflow.properties | 6 +-
.../BatchStatefulParDoOverridesTest.java | 169 +
.../dataflow/BatchViewOverridesTest.java | 633 +++
.../dataflow/DataflowPipelineJobTest.java | 184 +-
.../DataflowPipelineTranslatorTest.java | 145 +-
.../runners/dataflow/DataflowRunnerTest.java | 614 +--
...aflowUnboundedReadFromBoundedSourceTest.java | 79 +
...aflowUnboundedReadFromBoundedSourceTest.java | 83 -
.../DataflowPipelineDebugOptionsTest.java | 2 +-
.../options/DataflowPipelineOptionsTest.java | 20 +-
.../options/DataflowProfilingOptionsTest.java | 4 +-
.../testing/TestDataflowRunnerTest.java | 53 +-
.../runners/dataflow/util/PackageUtilTest.java | 99 +-
runners/pom.xml | 17 +-
runners/spark/README.md | 8 +-
runners/spark/pom.xml | 70 +-
.../spark/SparkNativePipelineVisitor.java | 198 +
.../runners/spark/SparkPipelineOptions.java | 6 +-
.../beam/runners/spark/SparkPipelineResult.java | 67 +-
.../apache/beam/runners/spark/SparkRunner.java | 124 +-
.../beam/runners/spark/SparkRunnerDebugger.java | 137 +
.../runners/spark/TestSparkPipelineOptions.java | 61 +
.../beam/runners/spark/TestSparkRunner.java | 204 +-
.../spark/aggregators/AccumulatorSingleton.java | 53 -
.../aggregators/AggregatorsAccumulator.java | 131 +
.../spark/aggregators/NamedAggregators.java | 12 +-
.../spark/aggregators/SparkAggregators.java | 26 +-
.../aggregators/metrics/AggregatorMetric.java | 44 -
.../metrics/AggregatorMetricSource.java | 50 -
.../metrics/WithNamedAggregatorsSupport.java | 174 -
.../spark/aggregators/metrics/sink/CsvSink.java | 39 -
.../aggregators/metrics/sink/GraphiteSink.java | 39 -
.../aggregators/metrics/sink/package-info.java | 23 -
.../coders/BeamSparkRunnerRegistrator.java | 48 +-
.../beam/runners/spark/coders/CoderHelpers.java | 23 +
.../runners/spark/coders/NullWritableCoder.java | 76 -
.../spark/coders/StatelessJavaSerializer.java | 97 +
.../runners/spark/coders/WritableCoder.java | 122 -
.../beam/runners/spark/examples/WordCount.java | 2 +-
.../apache/beam/runners/spark/io/ConsoleIO.java | 4 +-
.../beam/runners/spark/io/CreateStream.java | 198 +-
.../beam/runners/spark/io/MicrobatchSource.java | 9 +-
.../runners/spark/io/SparkUnboundedSource.java | 166 +-
.../runners/spark/metrics/AggregatorMetric.java | 43 +
.../spark/metrics/AggregatorMetricSource.java | 51 +
.../runners/spark/metrics/CompositeSource.java | 49 +
.../spark/metrics/MetricsAccumulator.java | 132 +
.../spark/metrics/MetricsAccumulatorParam.java | 42 +
.../runners/spark/metrics/SparkBeamMetric.java | 69 +
.../spark/metrics/SparkBeamMetricSource.java | 51 +
.../spark/metrics/SparkMetricResults.java | 181 +
.../spark/metrics/SparkMetricsContainer.java | 147 +
.../spark/metrics/WithMetricsSupport.java | 209 +
.../runners/spark/metrics/package-info.java | 20 +
.../runners/spark/metrics/sink/CsvSink.java | 38 +
.../spark/metrics/sink/GraphiteSink.java | 38 +
.../spark/metrics/sink/package-info.java | 22 +
.../SparkGroupAlsoByWindowViaWindowSet.java | 431 ++
.../spark/stateful/SparkStateInternals.java | 418 ++
.../spark/stateful/SparkTimerInternals.java | 193 +
.../spark/stateful/StateSpecFunctions.java | 60 +-
.../spark/translation/BoundedDataset.java | 3 +-
.../runners/spark/translation/DoFnFunction.java | 41 +-
.../translation/DoFnRunnerWithMetrics.java | 91 +
.../spark/translation/EvaluationContext.java | 89 +-
.../translation/GroupCombineFunctions.java | 259 +-
.../spark/translation/MultiDoFnFunction.java | 39 +-
.../translation/SparkAbstractCombineFn.java | 12 +-
.../spark/translation/SparkContextFactory.java | 13 +-
.../spark/translation/SparkGlobalCombineFn.java | 13 +-
.../translation/SparkGroupAlsoByWindowFn.java | 214 -
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 179 +
.../spark/translation/SparkKeyedCombineFn.java | 13 +-
.../spark/translation/SparkPCollectionView.java | 99 +
.../spark/translation/SparkProcessContext.java | 8 +-
.../spark/translation/SparkRuntimeContext.java | 84 +-
.../spark/translation/TransformEvaluator.java | 1 +
.../spark/translation/TransformTranslator.java | 467 +-
.../spark/translation/TranslationUtils.java | 93 +-
.../spark/translation/streaming/Checkpoint.java | 137 +
.../SparkRunnerStreamingContextFactory.java | 57 +-
.../streaming/StreamingTransformTranslator.java | 492 +-
.../translation/streaming/UnboundedDataset.java | 63 +-
.../runners/spark/util/BroadcastHelper.java | 127 -
.../spark/util/GlobalWatermarkHolder.java | 200 +
.../runners/spark/util/SideInputBroadcast.java | 77 +
.../spark/util/SparkSideInputReader.java | 8 +-
.../beam/runners/spark/ClearWatermarksRule.java | 37 +
.../beam/runners/spark/ForceStreamingTest.java | 60 +-
.../spark/GlobalWatermarkHolderTest.java | 151 +
.../apache/beam/runners/spark/PipelineRule.java | 109 +
.../runners/spark/ProvidedSparkContextTest.java | 72 +-
.../runners/spark/ReuseSparkContextRule.java | 46 +
.../runners/spark/SparkPipelineStateTest.java | 27 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 180 +
.../spark/aggregators/ClearAggregatorsRule.java | 5 +-
.../metrics/sink/InMemoryMetrics.java | 10 +-
.../metrics/sink/NamedAggregatorsTest.java | 25 +-
.../coders/BeamSparkRunnerRegistratorTest.java | 57 -
.../runners/spark/coders/WritableCoderTest.java | 45 -
.../beam/runners/spark/io/AvroPipelineTest.java | 6 +-
.../beam/runners/spark/io/NumShardsTest.java | 6 +-
.../io/hadoop/HadoopFileFormatPipelineTest.java | 8 +-
.../spark/metrics/SparkBeamMetricTest.java | 60 +
.../spark/translation/StorageLevelTest.java | 8 +-
.../translation/streaming/CreateStreamTest.java | 376 ++
.../streaming/EmptyStreamAssertionTest.java | 87 -
.../streaming/FlattenStreamingTest.java | 103 -
.../streaming/KafkaStreamingTest.java | 215 -
.../ResumeFromCheckpointStreamingTest.java | 308 +-
.../streaming/SimpleStreamingWordCountTest.java | 84 -
.../streaming/TrackStreamingSourcesTest.java | 171 +
.../utils/KafkaWriteOnBatchCompleted.java | 105 -
.../streaming/utils/PAssertStreaming.java | 121 -
.../utils/SparkTestPipelineOptions.java | 42 -
.../SparkTestPipelineOptionsForStreaming.java | 37 -
.../spark/src/test/resources/log4j.properties | 30 +
.../spark/src/test/resources/metrics.properties | 10 +-
sdks/common/fn-api/pom.xml | 109 +
.../fn-api/src/main/proto/beam_fn_api.proto | 771 +++
.../org/apache/beam/fn/v1/standard_coders.yaml | 195 +
sdks/common/pom.xml | 39 +
sdks/common/runner-api/pom.xml | 89 +
.../src/main/proto/beam_runner_api.proto | 711 +++
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/findbugs-filter.xml | 95 +-
sdks/java/core/pom.xml | 28 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 75 +-
.../beam/sdk/annotations/Experimental.java | 5 +-
.../org/apache/beam/sdk/coders/AtomicCoder.java | 2 +-
.../org/apache/beam/sdk/coders/AvroCoder.java | 30 +-
.../apache/beam/sdk/coders/BigDecimalCoder.java | 6 +-
.../beam/sdk/coders/BigEndianIntegerCoder.java | 7 +
.../beam/sdk/coders/BigEndianLongCoder.java | 7 +
.../apache/beam/sdk/coders/ByteArrayCoder.java | 7 +
.../org/apache/beam/sdk/coders/ByteCoder.java | 7 +
.../apache/beam/sdk/coders/ByteStringCoder.java | 8 +
.../java/org/apache/beam/sdk/coders/Coder.java | 7 +
.../apache/beam/sdk/coders/CollectionCoder.java | 12 +-
.../org/apache/beam/sdk/coders/CustomCoder.java | 18 +-
.../apache/beam/sdk/coders/DelegateCoder.java | 29 +-
.../org/apache/beam/sdk/coders/DoubleCoder.java | 7 +
.../apache/beam/sdk/coders/DurationCoder.java | 8 +
.../apache/beam/sdk/coders/InstantCoder.java | 7 +
.../apache/beam/sdk/coders/IterableCoder.java | 12 +-
.../beam/sdk/coders/IterableLikeCoder.java | 6 +-
.../org/apache/beam/sdk/coders/JAXBCoder.java | 48 +-
.../org/apache/beam/sdk/coders/KvCoder.java | 35 +-
.../beam/sdk/coders/LengthPrefixCoder.java | 145 +
.../org/apache/beam/sdk/coders/ListCoder.java | 7 +
.../org/apache/beam/sdk/coders/MapCoder.java | 62 +-
.../apache/beam/sdk/coders/NullableCoder.java | 6 +
.../beam/sdk/coders/SerializableCoder.java | 17 +-
.../org/apache/beam/sdk/coders/SetCoder.java | 12 +-
.../apache/beam/sdk/coders/StandardCoder.java | 42 +-
.../beam/sdk/coders/StringDelegateCoder.java | 16 +-
.../apache/beam/sdk/coders/StringUtf8Coder.java | 18 +-
.../beam/sdk/coders/TableRowJsonCoder.java | 7 +
.../beam/sdk/coders/TextualIntegerCoder.java | 8 +
.../org/apache/beam/sdk/coders/VarIntCoder.java | 10 +-
.../apache/beam/sdk/coders/VarLongCoder.java | 7 +
.../org/apache/beam/sdk/coders/VoidCoder.java | 7 +
.../beam/sdk/coders/protobuf/ProtoCoder.java | 8 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 4 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 5 -
.../sdk/io/BoundedReadFromUnboundedSource.java | 79 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 8 -
.../apache/beam/sdk/io/CompressedSource.java | 40 +-
.../org/apache/beam/sdk/io/CountingSource.java | 5 -
.../org/apache/beam/sdk/io/FileBasedSink.java | 22 +
.../java/org/apache/beam/sdk/io/FileSystem.java | 115 +-
.../org/apache/beam/sdk/io/FileSystems.java | 88 +-
.../org/apache/beam/sdk/io/LocalFileSystem.java | 195 +-
.../org/apache/beam/sdk/io/LocalResourceId.java | 136 +
.../java/org/apache/beam/sdk/io/PubsubIO.java | 1142 ++---
.../apache/beam/sdk/io/PubsubUnboundedSink.java | 88 +-
.../beam/sdk/io/PubsubUnboundedSource.java | 104 +-
.../main/java/org/apache/beam/sdk/io/Read.java | 7 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 401 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 706 +--
.../java/org/apache/beam/sdk/io/XmlSource.java | 5 -
.../apache/beam/sdk/io/fs/CreateOptions.java | 60 +
.../org/apache/beam/sdk/io/fs/MatchResult.java | 125 +
.../apache/beam/sdk/io/fs/ResolveOptions.java | 41 +
.../org/apache/beam/sdk/io/fs/ResourceId.java | 85 +
.../org/apache/beam/sdk/io/fs/package-info.java | 22 +
.../beam/sdk/metrics/DistributionData.java | 3 +-
.../org/apache/beam/sdk/metrics/MetricKey.java | 3 +-
.../apache/beam/sdk/metrics/MetricUpdates.java | 3 +-
.../org/apache/beam/sdk/options/GcpOptions.java | 25 +-
.../org/apache/beam/sdk/options/GcsOptions.java | 4 +-
.../beam/sdk/options/PipelineOptions.java | 2 +-
.../sdk/options/PipelineOptionsFactory.java | 10 +-
.../apache/beam/sdk/options/ValueProvider.java | 6 +-
.../beam/sdk/runners/PTransformMatcher.java | 32 +
.../sdk/runners/PTransformOverrideFactory.java | 31 +
.../apache/beam/sdk/runners/PipelineRunner.java | 14 -
.../beam/sdk/runners/TransformHierarchy.java | 285 +-
.../apache/beam/sdk/testing/Annotations.java | 72 +
.../beam/sdk/testing/CoderProperties.java | 85 +-
.../testing/FlattenWithHeterogeneousCoders.java | 29 +
.../org/apache/beam/sdk/testing/PAssert.java | 259 +-
.../apache/beam/sdk/testing/RegexMatcher.java | 49 +
.../beam/sdk/testing/RunnableOnService.java | 14 +-
.../beam/sdk/testing/SourceTestUtils.java | 5 -
.../apache/beam/sdk/testing/TestPipeline.java | 111 +-
.../org/apache/beam/sdk/testing/TestStream.java | 18 +-
.../beam/sdk/testing/UsesAttemptedMetrics.java | 28 +
.../beam/sdk/testing/UsesCommittedMetrics.java | 28 +
.../apache/beam/sdk/testing/UsesMapState.java | 25 +
.../apache/beam/sdk/testing/UsesMetrics.java | 24 -
.../apache/beam/sdk/testing/UsesSetState.java | 25 +
.../apache/beam/sdk/testing/UsesTestStream.java | 24 +
.../sdk/testing/UsesUnboundedPCollections.java | 23 +
.../beam/sdk/testing/ValueInSingleWindow.java | 6 +-
.../apache/beam/sdk/transforms/Aggregator.java | 19 -
.../sdk/transforms/AggregatorRetriever.java | 13 +-
.../beam/sdk/transforms/AppliedPTransform.java | 31 +-
.../org/apache/beam/sdk/transforms/Combine.java | 204 +-
.../apache/beam/sdk/transforms/CombineFns.java | 14 +-
.../org/apache/beam/sdk/transforms/Count.java | 28 +-
.../org/apache/beam/sdk/transforms/Create.java | 88 +-
.../sdk/transforms/DelegatingAggregator.java | 2 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 77 +-
.../beam/sdk/transforms/DoFnAdapters.java | 504 --
.../apache/beam/sdk/transforms/DoFnTester.java | 77 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 15 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 10 +-
.../org/apache/beam/sdk/transforms/Latest.java | 80 +-
.../org/apache/beam/sdk/transforms/Max.java | 124 +-
.../org/apache/beam/sdk/transforms/Mean.java | 27 +-
.../org/apache/beam/sdk/transforms/Min.java | 122 +-
.../org/apache/beam/sdk/transforms/OldDoFn.java | 758 ---
.../apache/beam/sdk/transforms/PTransform.java | 9 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 55 +-
.../org/apache/beam/sdk/transforms/Regex.java | 589 ++-
.../org/apache/beam/sdk/transforms/Sample.java | 121 +-
.../beam/sdk/transforms/SimpleFunction.java | 44 +-
.../org/apache/beam/sdk/transforms/Sum.java | 57 +-
.../apache/beam/sdk/transforms/ToString.java | 181 +
.../org/apache/beam/sdk/transforms/Top.java | 27 +-
.../org/apache/beam/sdk/transforms/View.java | 10 +
.../sdk/transforms/display/DisplayData.java | 6 +-
.../beam/sdk/transforms/join/CoGbkResult.java | 35 +-
.../transforms/join/KeyedPCollectionTuple.java | 41 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 16 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 4 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 42 +-
.../sdk/transforms/reflect/DoFnInvokers.java | 142 +-
.../sdk/transforms/reflect/DoFnSignature.java | 71 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 82 +-
.../transforms/splittabledofn/OffsetRange.java | 71 +
.../splittabledofn/OffsetRangeTracker.java | 75 +
.../splittabledofn/RestrictionTracker.java | 2 +-
.../beam/sdk/transforms/windowing/AfterAll.java | 7 +
.../windowing/AfterDelayFromFirstElement.java | 251 -
.../sdk/transforms/windowing/AfterEach.java | 7 +
.../sdk/transforms/windowing/AfterFirst.java | 7 +
.../sdk/transforms/windowing/AfterPane.java | 10 -
.../windowing/AfterProcessingTime.java | 105 +-
.../AfterSynchronizedProcessingTime.java | 31 +-
.../sdk/transforms/windowing/BoundedWindow.java | 31 +
.../sdk/transforms/windowing/GlobalWindow.java | 6 +
.../sdk/transforms/windowing/GlobalWindows.java | 10 +
.../transforms/windowing/IntervalWindow.java | 19 +-
.../sdk/transforms/windowing/OutputTimeFns.java | 45 +
.../beam/sdk/transforms/windowing/PaneInfo.java | 2 -
.../windowing/TimestampTransform.java | 65 +
.../beam/sdk/transforms/windowing/Triggers.java | 320 ++
.../beam/sdk/transforms/windowing/Window.java | 96 +-
.../org/apache/beam/sdk/util/ApiSurface.java | 446 +-
.../beam/sdk/util/BaseExecutionContext.java | 174 -
.../org/apache/beam/sdk/util/CoderUtils.java | 30 +-
.../beam/sdk/util/CombineContextFactory.java | 18 -
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 +
.../util/EmptyOnDeserializationThreadLocal.java | 39 +
.../apache/beam/sdk/util/ExecutionContext.java | 100 -
.../apache/beam/sdk/util/GcpProjectUtil.java | 2 +-
.../apache/beam/sdk/util/GcsPathValidator.java | 3 +-
.../java/org/apache/beam/sdk/util/GcsUtil.java | 334 +-
.../org/apache/beam/sdk/util/NameUtils.java | 167 +
.../org/apache/beam/sdk/util/PropertyNames.java | 1 +
.../org/apache/beam/sdk/util/PubsubClient.java | 28 +-
.../apache/beam/sdk/util/PubsubGrpcClient.java | 6 +-
.../apache/beam/sdk/util/PubsubJsonClient.java | 4 +-
.../apache/beam/sdk/util/PubsubTestClient.java | 6 +-
.../org/apache/beam/sdk/util/StringUtils.java | 100 -
.../java/org/apache/beam/sdk/util/Timer.java | 11 +
.../apache/beam/sdk/util/TimerInternals.java | 273 --
.../org/apache/beam/sdk/util/WindowedValue.java | 23 +-
.../beam/sdk/util/WindowingInternals.java | 82 -
.../beam/sdk/util/WindowingStrategies.java | 266 ++
.../apache/beam/sdk/util/WindowingStrategy.java | 53 +-
.../beam/sdk/util/common/ReflectHelpers.java | 16 +-
.../sdk/util/state/InMemoryStateInternals.java | 430 --
.../sdk/util/state/InMemoryTimerInternals.java | 275 --
.../apache/beam/sdk/util/state/MapState.java | 93 +
.../sdk/util/state/MergingStateAccessor.java | 40 -
.../beam/sdk/util/state/ReadableState.java | 4 +-
.../apache/beam/sdk/util/state/SetState.java | 71 +
.../beam/sdk/util/state/StateAccessor.java | 37 -
.../apache/beam/sdk/util/state/StateBinder.java | 6 +
.../beam/sdk/util/state/StateContexts.java | 63 -
.../beam/sdk/util/state/StateInternals.java | 57 -
.../sdk/util/state/StateInternalsFactory.java | 35 -
.../beam/sdk/util/state/StateMerging.java | 259 --
.../beam/sdk/util/state/StateNamespace.java | 56 -
.../sdk/util/state/StateNamespaceForTest.java | 65 -
.../beam/sdk/util/state/StateNamespaces.java | 278 --
.../apache/beam/sdk/util/state/StateSpecs.java | 155 +-
.../apache/beam/sdk/util/state/StateTable.java | 82 -
.../apache/beam/sdk/util/state/StateTag.java | 111 -
.../apache/beam/sdk/util/state/StateTags.java | 290 --
.../util/state/TestInMemoryStateInternals.java | 61 -
.../sdk/util/state/TimerInternalsFactory.java | 36 -
.../java/org/apache/beam/sdk/values/PBegin.java | 9 +-
.../apache/beam/sdk/values/PCollectionList.java | 62 +-
.../beam/sdk/values/PCollectionTuple.java | 33 +-
.../java/org/apache/beam/sdk/values/PDone.java | 4 +-
.../java/org/apache/beam/sdk/values/PInput.java | 13 +-
.../org/apache/beam/sdk/values/POutput.java | 24 +-
.../beam/sdk/values/POutputValueBase.java | 4 +-
.../java/org/apache/beam/sdk/values/PValue.java | 24 +-
.../org/apache/beam/sdk/values/PValueBase.java | 18 +-
.../apache/beam/sdk/values/TaggedPValue.java | 42 +
.../beam/sdk/values/TimestampedValue.java | 10 +-
.../apache/beam/sdk/values/TupleTagList.java | 6 +
.../org/apache/beam/sdk/values/TypedPValue.java | 78 +-
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 62 +
.../sdk/AggregatorPipelineExtractorTest.java | 16 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 37 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 18 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 46 +-
.../sdk/coders/BigEndianIntegerCoderTest.java | 9 +
.../beam/sdk/coders/BigEndianLongCoderTest.java | 9 +
.../beam/sdk/coders/ByteArrayCoderTest.java | 6 +
.../apache/beam/sdk/coders/ByteCoderTest.java | 9 +
.../beam/sdk/coders/ByteStringCoderTest.java | 8 +
.../beam/sdk/coders/CoderRegistryTest.java | 12 +-
.../org/apache/beam/sdk/coders/CoderTest.java | 8 +
.../beam/sdk/coders/CollectionCoderTest.java | 16 +
.../apache/beam/sdk/coders/CommonCoderTest.java | 351 ++
.../beam/sdk/coders/DefaultCoderTest.java | 4 +-
.../beam/sdk/coders/DelegateCoderTest.java | 35 +-
.../apache/beam/sdk/coders/DoubleCoderTest.java | 9 +
.../beam/sdk/coders/DurationCoderTest.java | 10 +
.../beam/sdk/coders/InstantCoderTest.java | 9 +
.../beam/sdk/coders/IterableCoderTest.java | 27 +-
.../apache/beam/sdk/coders/JAXBCoderTest.java | 26 +-
.../org/apache/beam/sdk/coders/KvCoderTest.java | 29 +
.../beam/sdk/coders/LengthPrefixCoderTest.java | 129 +
.../apache/beam/sdk/coders/ListCoderTest.java | 16 +-
.../apache/beam/sdk/coders/MapCoderTest.java | 21 +-
.../beam/sdk/coders/NullableCoderTest.java | 12 +
.../beam/sdk/coders/SerializableCoderTest.java | 16 +-
.../apache/beam/sdk/coders/SetCoderTest.java | 16 +
.../beam/sdk/coders/StandardCoderTest.java | 40 +
.../sdk/coders/StringDelegateCoderTest.java | 11 +
.../beam/sdk/coders/StringUtf8CoderTest.java | 9 +
.../beam/sdk/coders/TableRowJsonCoderTest.java | 9 +
.../sdk/coders/TextualIntegerCoderTest.java | 9 +
.../apache/beam/sdk/coders/VarIntCoderTest.java | 9 +
.../beam/sdk/coders/VarLongCoderTest.java | 9 +
.../apache/beam/sdk/coders/VoidCoderTest.java | 40 +
.../beam/sdk/io/AvroIOGeneratedClassTest.java | 285 --
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 18 +-
.../apache/beam/sdk/io/AvroIOTransformTest.java | 324 ++
.../io/BoundedReadFromUnboundedSourceTest.java | 6 +-
.../beam/sdk/io/CompressedSourceTest.java | 89 +-
.../apache/beam/sdk/io/CountingInputTest.java | 12 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 13 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 17 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 9 +-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 61 +-
.../apache/beam/sdk/io/LocalFileSystemTest.java | 318 ++
.../apache/beam/sdk/io/LocalResourceIdTest.java | 226 +
.../beam/sdk/io/OffsetBasedSourceTest.java | 5 -
.../org/apache/beam/sdk/io/PubsubIOTest.java | 86 +-
.../beam/sdk/io/PubsubUnboundedSinkTest.java | 43 +-
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 22 +-
.../java/org/apache/beam/sdk/io/ReadTest.java | 5 -
.../java/org/apache/beam/sdk/io/TextIOTest.java | 222 +-
.../java/org/apache/beam/sdk/io/WriteTest.java | 158 +-
.../org/apache/beam/sdk/io/XmlSourceTest.java | 10 +-
.../apache/beam/sdk/metrics/MetricMatchers.java | 192 +-
.../apache/beam/sdk/metrics/MetricsTest.java | 124 +-
.../apache/beam/sdk/options/GcpOptionsTest.java | 32 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 6 +-
.../beam/sdk/options/PipelineOptionsTest.java | 3 +-
.../sdk/options/ProxyInvocationHandlerTest.java | 5 +-
.../beam/sdk/options/ValueProviderTest.java | 36 +-
.../sdk/options/ValueProviderUtilsTest.java | 2 +-
.../sdk/runners/TransformHierarchyTest.java | 320 +-
.../beam/sdk/runners/TransformTreeTest.java | 34 +-
.../beam/sdk/testing/GatherAllPanesTest.java | 7 +-
.../apache/beam/sdk/testing/PAssertTest.java | 118 +-
.../beam/sdk/testing/TestPipelineTest.java | 504 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 45 +-
.../testing/ValueInSingleWindowCoderTest.java | 7 +
.../transforms/ApproximateQuantilesTest.java | 12 +-
.../sdk/transforms/ApproximateUniqueTest.java | 487 +-
.../beam/sdk/transforms/CombineFnsTest.java | 25 +-
.../apache/beam/sdk/transforms/CombineTest.java | 174 +-
.../apache/beam/sdk/transforms/CountTest.java | 15 +-
.../apache/beam/sdk/transforms/CreateTest.java | 119 +-
.../beam/sdk/transforms/DistinctTest.java | 12 +-
.../DoFnDelegatingAggregatorTest.java | 142 -
.../apache/beam/sdk/transforms/DoFnTest.java | 19 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 12 +-
.../apache/beam/sdk/transforms/FilterTest.java | 18 +-
.../sdk/transforms/FlatMapElementsTest.java | 10 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 74 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 32 +-
.../apache/beam/sdk/transforms/KeysTest.java | 9 +-
.../apache/beam/sdk/transforms/KvSwapTest.java | 22 +-
.../apache/beam/sdk/transforms/LatestTest.java | 23 +-
.../beam/sdk/transforms/MapElementsTest.java | 14 +-
.../org/apache/beam/sdk/transforms/MaxTest.java | 20 +-
.../apache/beam/sdk/transforms/MeanTest.java | 7 +-
.../org/apache/beam/sdk/transforms/MinTest.java | 21 +-
.../apache/beam/sdk/transforms/NoOpOldDoFn.java | 71 -
.../beam/sdk/transforms/OldDoFnContextTest.java | 69 -
.../apache/beam/sdk/transforms/OldDoFnTest.java | 188 -
.../beam/sdk/transforms/ParDoLifecycleTest.java | 17 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 704 ++-
.../beam/sdk/transforms/PartitionTest.java | 8 +-
.../apache/beam/sdk/transforms/RegexTest.java | 148 +-
.../apache/beam/sdk/transforms/SampleTest.java | 419 +-
.../beam/sdk/transforms/SimpleFunctionTest.java | 43 +
.../beam/sdk/transforms/SimpleStatsFnsTest.java | 36 +-
.../beam/sdk/transforms/SplittableDoFnTest.java | 83 +-
.../org/apache/beam/sdk/transforms/SumTest.java | 24 +-
.../beam/sdk/transforms/ToStringTest.java | 125 +
.../org/apache/beam/sdk/transforms/TopTest.java | 33 +-
.../apache/beam/sdk/transforms/ValuesTest.java | 7 +-
.../apache/beam/sdk/transforms/ViewTest.java | 108 +-
.../beam/sdk/transforms/WithKeysTest.java | 8 +-
.../beam/sdk/transforms/WithTimestampsTest.java | 9 +-
.../display/DisplayDataEvaluator.java | 8 +-
.../sdk/transforms/display/DisplayDataTest.java | 15 +
.../transforms/join/CoGbkResultCoderTest.java | 10 +-
.../sdk/transforms/join/CoGroupByKeyTest.java | 11 +-
.../sdk/transforms/join/UnionCoderTest.java | 17 +-
.../transforms/reflect/DoFnInvokersTest.java | 70 +-
.../DoFnSignaturesProcessElementTest.java | 40 +-
.../DoFnSignaturesSplittableDoFnTest.java | 3 +-
.../transforms/reflect/DoFnSignaturesTest.java | 6 +-
.../splittabledofn/OffsetRangeTrackerTest.java | 111 +
.../windowing/AfterProcessingTimeTest.java | 2 +-
.../AfterSynchronizedProcessingTimeTest.java | 2 +-
.../transforms/windowing/GlobalWindowTest.java | 64 +
.../transforms/windowing/OutputTimeFnsTest.java | 51 +
.../sdk/transforms/windowing/TriggersTest.java | 100 +
.../sdk/transforms/windowing/WindowTest.java | 204 +-
.../sdk/transforms/windowing/WindowingTest.java | 15 +-
.../apache/beam/sdk/util/ApiSurfaceTest.java | 152 +-
.../apache/beam/sdk/util/CombineFnUtilTest.java | 8 +-
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 +
.../beam/sdk/util/FileIOChannelFactoryTest.java | 13 +-
.../beam/sdk/util/GcsPathValidatorTest.java | 17 +-
.../org/apache/beam/sdk/util/GcsUtilTest.java | 86 +-
.../org/apache/beam/sdk/util/NameUtilsTest.java | 177 +
.../beam/sdk/util/PubsubGrpcClientTest.java | 8 +-
.../beam/sdk/util/PubsubJsonClientTest.java | 3 +-
.../beam/sdk/util/PubsubTestClientTest.java | 4 +-
.../org/apache/beam/sdk/util/ReshuffleTest.java | 11 +-
.../beam/sdk/util/SerializableUtilsTest.java | 4 +-
.../apache/beam/sdk/util/StringUtilsTest.java | 100 -
.../beam/sdk/util/TimerInternalsTest.java | 101 -
.../beam/sdk/util/ValueWithRecordIdTest.java | 34 +
.../apache/beam/sdk/util/WindowedValueTest.java | 23 +
.../beam/sdk/util/WindowingStrategiesTest.java | 91 +
.../util/state/InMemoryStateInternalsTest.java | 348 --
.../util/state/InMemoryTimerInternalsTest.java | 153 -
.../sdk/util/state/StateNamespacesTest.java | 130 -
.../beam/sdk/util/state/StateTagTest.java | 173 -
.../beam/sdk/values/PCollectionListTest.java | 117 +
.../beam/sdk/values/PCollectionTupleTest.java | 80 +-
.../org/apache/beam/sdk/values/PDoneTest.java | 9 +-
.../beam/sdk/values/TimestampedValueTest.java | 19 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 17 +-
sdks/java/extensions/jackson/pom.xml | 125 +
.../beam/sdk/extensions/jackson/AsJsons.java | 76 +
.../beam/sdk/extensions/jackson/ParseJsons.java | 75 +
.../sdk/extensions/jackson/package-info.java | 22 +
.../jackson/JacksonTransformsTest.java | 242 +
sdks/java/extensions/join-library/README.md | 10 -
sdks/java/extensions/join-library/pom.xml | 3 +-
.../extensions/joinlibrary/InnerJoinTest.java | 23 +-
.../joinlibrary/OuterLeftJoinTest.java | 31 +-
.../joinlibrary/OuterRightJoinTest.java | 31 +-
sdks/java/extensions/pom.xml | 3 +-
sdks/java/extensions/sorter/pom.xml | 5 +-
.../sorter/BufferedExternalSorter.java | 23 +-
.../sorter/BufferedExternalSorterTest.java | 46 +-
.../sdk/extensions/sorter/SortValuesTest.java | 9 +-
sdks/java/harness/pom.xml | 177 +
.../org/apache/beam/fn/harness/FnHarness.java | 131 +
.../harness/channel/ManagedChannelFactory.java | 80 +
.../harness/channel/SocketAddressFactory.java | 64 +
.../beam/fn/harness/channel/package-info.java | 22 +
.../fn/harness/control/BeamFnControlClient.java | 166 +
.../harness/control/ProcessBundleHandler.java | 334 ++
.../fn/harness/control/RegisterHandler.java | 92 +
.../beam/fn/harness/control/package-info.java | 22 +
.../BeamFnDataBufferingOutboundObserver.java | 135 +
.../beam/fn/harness/data/BeamFnDataClient.java | 64 +
.../fn/harness/data/BeamFnDataGrpcClient.java | 122 +
.../harness/data/BeamFnDataGrpcMultiplexer.java | 141 +
.../harness/data/BeamFnDataInboundObserver.java | 81 +
.../beam/fn/harness/data/package-info.java | 22 +
.../fn/harness/fake/FakeAggregatorFactory.java | 52 +
.../beam/fn/harness/fake/FakeStepContext.java | 70 +
.../beam/fn/harness/fake/package-info.java | 22 +
.../harness/fn/CloseableThrowingConsumer.java | 23 +
.../beam/fn/harness/fn/ThrowingBiFunction.java | 32 +
.../beam/fn/harness/fn/ThrowingConsumer.java | 32 +
.../beam/fn/harness/fn/ThrowingFunction.java | 32 +
.../beam/fn/harness/fn/ThrowingRunnable.java | 30 +
.../apache/beam/fn/harness/fn/package-info.java | 22 +
.../fn/harness/logging/BeamFnLoggingClient.java | 310 ++
.../beam/fn/harness/logging/package-info.java | 22 +
.../apache/beam/fn/harness/package-info.java | 22 +
.../beam/fn/harness/stream/AdvancingPhaser.java | 36 +
.../harness/stream/BufferingStreamObserver.java | 166 +
.../fn/harness/stream/DirectStreamObserver.java | 71 +
.../ForwardingClientResponseObserver.java | 63 +
.../harness/stream/StreamObserverFactory.java | 91 +
.../beam/fn/harness/stream/package-info.java | 22 +
.../beam/runners/core/BeamFnDataReadRunner.java | 104 +
.../runners/core/BeamFnDataWriteRunner.java | 87 +
.../beam/runners/core/BoundedSourceRunner.java | 105 +
.../apache/beam/runners/core/package-info.java | 22 +
.../apache/beam/fn/harness/FnHarnessTest.java | 130 +
.../channel/ManagedChannelFactoryTest.java | 74 +
.../channel/SocketAddressFactoryTest.java | 56 +
.../control/BeamFnControlClientTest.java | 182 +
.../control/ProcessBundleHandlerTest.java | 675 +++
.../fn/harness/control/RegisterHandlerTest.java | 80 +
...BeamFnDataBufferingOutboundObserverTest.java | 147 +
.../harness/data/BeamFnDataGrpcClientTest.java | 318 ++
.../data/BeamFnDataGrpcMultiplexerTest.java | 98 +
.../data/BeamFnDataInboundObserverTest.java | 116 +
.../logging/BeamFnLoggingClientTest.java | 169 +
.../fn/harness/stream/AdvancingPhaserTest.java | 48 +
.../stream/BufferingStreamObserverTest.java | 146 +
.../stream/DirectStreamObserverTest.java | 139 +
.../ForwardingClientResponseObserverTest.java | 60 +
.../stream/StreamObserverFactoryTest.java | 84 +
.../beam/fn/harness/test/TestExecutors.java | 85 +
.../beam/fn/harness/test/TestExecutorsTest.java | 160 +
.../beam/fn/harness/test/TestStreams.java | 162 +
.../beam/fn/harness/test/TestStreamsTest.java | 84 +
.../runners/core/BeamFnDataReadRunnerTest.java | 187 +
.../runners/core/BeamFnDataWriteRunnerTest.java | 155 +
.../runners/core/BoundedSourceRunnerTest.java | 113 +
sdks/java/io/elasticsearch/pom.xml | 149 +
.../sdk/io/elasticsearch/ElasticsearchIO.java | 815 ++++
.../beam/sdk/io/elasticsearch/package-info.java | 20 +
.../src/test/contrib/create_elk_container.sh | 24 +
.../elasticsearch/ElasticSearchIOTestUtils.java | 129 +
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 154 +
.../io/elasticsearch/ElasticsearchIOTest.java | 358 ++
.../elasticsearch/ElasticsearchTestDataSet.java | 109 +
.../elasticsearch/ElasticsearchTestOptions.java | 46 +
sdks/java/io/google-cloud-platform/pom.xml | 16 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 450 +-
.../sdk/io/gcp/bigquery/BigQueryServices.java | 23 +-
.../io/gcp/bigquery/BigQueryServicesImpl.java | 118 +-
.../gcp/bigquery/BigQueryTableRowIterator.java | 82 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 8 -
.../io/gcp/bigtable/BigtableTestOptions.java | 37 -
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 149 +-
.../beam/sdk/io/gcp/storage/GcsFileSystem.java | 210 +-
.../io/gcp/storage/GcsFileSystemRegistrar.java | 9 +-
.../beam/sdk/io/gcp/storage/GcsResourceId.java | 116 +
.../apache/beam/sdk/io/gcp/ApiSurfaceTest.java | 134 -
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 79 +
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 335 +-
.../gcp/bigquery/BigQueryServicesImplTest.java | 141 +
.../bigquery/BigQueryTableRowIteratorTest.java | 51 +-
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 3 +-
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 24 +-
.../io/gcp/bigtable/BigtableTestOptions.java | 37 +
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 15 +-
.../sdk/io/gcp/datastore/SplitQueryFnIT.java | 2 +-
.../sdk/io/gcp/storage/GcsFileSystemTest.java | 274 ++
.../sdk/io/gcp/storage/GcsResourceIdTest.java | 147 +
sdks/java/io/hadoop-common/pom.xml | 86 +
.../io/hadoop/SerializableConfiguration.java | 96 +
.../beam/sdk/io/hadoop/WritableCoder.java | 116 +
.../apache/beam/sdk/io/hadoop/package-info.java | 22 +
.../hadoop/SerializableConfigurationTest.java | 75 +
.../beam/sdk/io/hadoop/WritableCoderTest.java | 45 +
sdks/java/io/hadoop-input-format/README.md | 167 +
sdks/java/io/hadoop-input-format/pom.xml | 136 +
.../hadoop/inputformat/HadoopInputFormatIO.java | 941 ++++
.../sdk/io/hadoop/inputformat/package-info.java | 23 +
.../ConfigurableEmployeeInputFormat.java | 131 +
.../sdk/io/hadoop/inputformat/Employee.java | 85 +
.../hadoop/inputformat/EmployeeInputFormat.java | 172 +
.../inputformat/HadoopInputFormatIOTest.java | 844 ++++
.../ReuseObjectsEmployeeInputFormat.java | 176 +
.../hadoop/inputformat/TestEmployeeDataSet.java | 76 +
sdks/java/io/hbase/pom.xml | 233 +
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 693 +++
.../beam/sdk/io/hbase/HBaseMutationCoder.java | 71 +
.../beam/sdk/io/hbase/HBaseResultCoder.java | 55 +
.../beam/sdk/io/hbase/SerializableScan.java | 55 +
.../apache/beam/sdk/io/hbase/package-info.java | 24 +
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 430 ++
.../sdk/io/hbase/HBaseMutationCoderTest.java | 52 +
.../beam/sdk/io/hbase/HBaseResultCoderTest.java | 41 +
.../beam/sdk/io/hbase/SerializableScanTest.java | 56 +
sdks/java/io/hdfs/pom.xml | 64 +-
.../beam/sdk/io/hdfs/AvroHDFSFileSource.java | 142 -
.../beam/sdk/io/hdfs/AvroWrapperCoder.java | 114 -
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 301 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 528 ++-
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 44 +-
.../beam/sdk/io/hdfs/HadoopResourceId.java | 42 +
.../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 +
.../apache/beam/sdk/io/hdfs/WritableCoder.java | 116 -
.../SimpleAuthAvroHDFSFileSource.java | 82 -
.../hdfs/simpleauth/SimpleAuthHDFSFileSink.java | 131 -
.../simpleauth/SimpleAuthHDFSFileSource.java | 117 -
.../sdk/io/hdfs/simpleauth/package-info.java | 22 -
.../beam/sdk/io/hdfs/AvroWrapperCoderTest.java | 51 -
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 173 +
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 60 +-
.../beam/sdk/io/hdfs/WritableCoderTest.java | 45 -
sdks/java/io/jdbc/pom.xml | 125 +-
sdks/java/io/jdbc/src/test/README.md | 32 +
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 178 +
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 120 +-
.../beam/sdk/io/jdbc/JdbcTestDataSet.java | 128 +
.../beam/sdk/io/jdbc/PostgresTestOptions.java | 60 +
.../kubernetes/postgres-pod-no-vol.yml | 32 +
.../kubernetes/postgres-service-public.yml | 28 +
.../jdbc/src/test/resources/kubernetes/setup.sh | 20 +
.../src/test/resources/kubernetes/teardown.sh | 20 +
sdks/java/io/jms/pom.xml | 22 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 103 +-
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 137 +-
sdks/java/io/kafka/pom.xml | 65 +-
.../apache/beam/sdk/io/kafka/ConsumerSpEL.java | 60 +
.../beam/sdk/io/kafka/KafkaCheckpointMark.java | 16 +
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 683 ++-
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 4 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 109 +-
.../beam/sdk/io/kafka/KafkaRecordCoderTest.java | 34 +
sdks/java/io/kinesis/pom.xml | 12 +-
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 4 +-
.../beam/sdk/io/kinesis/package-info.java | 2 +-
.../sdk/io/kinesis/KinesisMockReadTest.java | 7 +-
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 6 +-
sdks/java/io/mongodb/pom.xml | 20 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 5 -
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 5 -
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 9 +-
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 7 +-
sdks/java/io/mqtt/pom.xml | 122 +
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 592 +++
.../apache/beam/sdk/io/mqtt/package-info.java | 22 +
.../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 257 +
sdks/java/io/pom.xml | 76 +-
sdks/java/java8tests/pom.xml | 3 +-
.../beam/sdk/transforms/CombineJava8Test.java | 8 +-
.../beam/sdk/transforms/DistinctJava8Test.java | 5 +-
.../beam/sdk/transforms/FilterJava8Test.java | 9 +-
.../transforms/FlatMapElementsJava8Test.java | 7 +-
.../sdk/transforms/MapElementsJava8Test.java | 33 +-
.../beam/sdk/transforms/PartitionJava8Test.java | 7 +-
.../sdk/transforms/SimpleFunctionJava8Test.java | 69 +
.../beam/sdk/transforms/WithKeysJava8Test.java | 6 +-
.../sdk/transforms/WithTimestampsJava8Test.java | 14 +-
sdks/java/javadoc/ant.xml | 96 +
sdks/java/javadoc/pom.xml | 295 ++
.../maven-archetypes/examples-java8/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 78 +-
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 96 +-
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 4 +-
.../resources/projects/basic/reference/pom.xml | 4 +-
sdks/java/pom.xml | 6 +-
sdks/pom.xml | 17 +-
sdks/python/.pylintrc | 164 +
sdks/python/MANIFEST.in | 21 +
sdks/python/README.md | 298 ++
sdks/python/apache_beam/__init__.py | 82 +
sdks/python/apache_beam/coders/__init__.py | 19 +
sdks/python/apache_beam/coders/coder_impl.pxd | 143 +
sdks/python/apache_beam/coders/coder_impl.py | 734 +++
sdks/python/apache_beam/coders/coders.py | 835 ++++
sdks/python/apache_beam/coders/coders_test.py | 115 +
.../apache_beam/coders/coders_test_common.py | 389 ++
.../apache_beam/coders/fast_coders_test.py | 37 +
sdks/python/apache_beam/coders/observable.py | 38 +
.../apache_beam/coders/observable_test.py | 57 +
.../coders/proto2_coder_test_messages_pb2.py | 318 ++
.../apache_beam/coders/slow_coders_test.py | 45 +
sdks/python/apache_beam/coders/slow_stream.py | 163 +
.../apache_beam/coders/standard_coders_test.py | 156 +
sdks/python/apache_beam/coders/stream.pxd | 66 +
sdks/python/apache_beam/coders/stream.pyx | 226 +
sdks/python/apache_beam/coders/stream_test.py | 180 +
sdks/python/apache_beam/coders/typecoders.py | 182 +
.../apache_beam/coders/typecoders_test.py | 124 +
sdks/python/apache_beam/error.py | 42 +
sdks/python/apache_beam/examples/__init__.py | 16 +
.../apache_beam/examples/complete/__init__.py | 16 +
.../examples/complete/autocomplete.py | 90 +
.../examples/complete/autocomplete_test.py | 52 +
.../examples/complete/estimate_pi.py | 128 +
.../examples/complete/estimate_pi_test.py | 52 +
.../examples/complete/juliaset/__init__.py | 16 +
.../complete/juliaset/juliaset/__init__.py | 16 +
.../complete/juliaset/juliaset/juliaset.py | 124 +
.../complete/juliaset/juliaset/juliaset_test.py | 86 +
.../examples/complete/juliaset/juliaset_main.py | 58 +
.../examples/complete/juliaset/setup.py | 116 +
.../apache_beam/examples/complete/tfidf.py | 208 +
.../apache_beam/examples/complete/tfidf_test.py | 91 +
.../examples/complete/top_wikipedia_sessions.py | 182 +
.../complete/top_wikipedia_sessions_test.py | 62 +
.../apache_beam/examples/cookbook/__init__.py | 16 +
.../examples/cookbook/bigquery_schema.py | 129 +
.../examples/cookbook/bigquery_side_input.py | 121 +
.../cookbook/bigquery_side_input_test.py | 54 +
.../examples/cookbook/bigquery_tornadoes.py | 99 +
.../cookbook/bigquery_tornadoes_it_test.py | 62 +
.../cookbook/bigquery_tornadoes_test.py | 45 +
.../apache_beam/examples/cookbook/bigshuffle.py | 94 +
.../examples/cookbook/bigshuffle_test.py | 63 +
.../apache_beam/examples/cookbook/coders.py | 101 +
.../examples/cookbook/coders_test.py | 49 +
.../examples/cookbook/combiners_test.py | 74 +
.../examples/cookbook/custom_ptransform.py | 134 +
.../examples/cookbook/custom_ptransform_test.py | 53 +
.../examples/cookbook/datastore_wordcount.py | 261 ++
.../apache_beam/examples/cookbook/filters.py | 107 +
.../examples/cookbook/filters_test.py | 69 +
.../examples/cookbook/group_with_coder.py | 122 +
.../examples/cookbook/group_with_coder_test.py | 89 +
.../examples/cookbook/mergecontacts.py | 133 +
.../examples/cookbook/mergecontacts_test.py | 125 +
.../examples/cookbook/multiple_output_pardo.py | 184 +
.../cookbook/multiple_output_pardo_test.py | 72 +
.../apache_beam/examples/snippets/__init__.py | 16 +
.../apache_beam/examples/snippets/snippets.py | 1158 +++++
.../examples/snippets/snippets_test.py | 904 ++++
.../apache_beam/examples/streaming_wordcap.py | 64 +
.../apache_beam/examples/streaming_wordcount.py | 74 +
sdks/python/apache_beam/examples/wordcount.py | 116 +
.../apache_beam/examples/wordcount_debugging.py | 163 +
.../examples/wordcount_debugging_test.py | 59 +
.../apache_beam/examples/wordcount_it_test.py | 59 +
.../apache_beam/examples/wordcount_minimal.py | 121 +
.../examples/wordcount_minimal_test.py | 59 +
.../apache_beam/examples/wordcount_test.py | 58 +
sdks/python/apache_beam/internal/__init__.py | 16 +
.../python/apache_beam/internal/gcp/__init__.py | 16 +
sdks/python/apache_beam/internal/gcp/auth.py | 185 +
.../apache_beam/internal/gcp/auth_test.py | 44 +
.../apache_beam/internal/gcp/json_value.py | 147 +
.../apache_beam/internal/gcp/json_value_test.py | 93 +
sdks/python/apache_beam/internal/module_test.py | 62 +
sdks/python/apache_beam/internal/pickler.py | 230 +
.../python/apache_beam/internal/pickler_test.py | 84 +
sdks/python/apache_beam/internal/util.py | 127 +
sdks/python/apache_beam/internal/util_test.py | 61 +
sdks/python/apache_beam/io/__init__.py | 38 +
sdks/python/apache_beam/io/avroio.py | 372 ++
sdks/python/apache_beam/io/avroio_test.py | 381 ++
sdks/python/apache_beam/io/concat_source.py | 263 ++
.../python/apache_beam/io/concat_source_test.py | 231 +
sdks/python/apache_beam/io/filebasedsource.py | 329 ++
.../apache_beam/io/filebasedsource_test.py | 708 +++
sdks/python/apache_beam/io/fileio.py | 746 +++
sdks/python/apache_beam/io/fileio_test.py | 352 ++
sdks/python/apache_beam/io/gcp/__init__.py | 16 +
sdks/python/apache_beam/io/gcp/bigquery.py | 1081 +++++
sdks/python/apache_beam/io/gcp/bigquery_test.py | 828 ++++
.../apache_beam/io/gcp/datastore/__init__.py | 16 +
.../apache_beam/io/gcp/datastore/v1/__init__.py | 16 +
.../io/gcp/datastore/v1/datastoreio.py | 397 ++
.../io/gcp/datastore/v1/datastoreio_test.py | 245 +
.../io/gcp/datastore/v1/fake_datastore.py | 98 +
.../apache_beam/io/gcp/datastore/v1/helper.py | 274 ++
.../io/gcp/datastore/v1/helper_test.py | 265 ++
.../io/gcp/datastore/v1/query_splitter.py | 275 ++
.../io/gcp/datastore/v1/query_splitter_test.py | 208 +
sdks/python/apache_beam/io/gcp/gcsio.py | 871 ++++
sdks/python/apache_beam/io/gcp/gcsio_test.py | 796 ++++
.../apache_beam/io/gcp/internal/__init__.py | 16 +
.../io/gcp/internal/clients/__init__.py | 16 +
.../gcp/internal/clients/bigquery/__init__.py | 33 +
.../clients/bigquery/bigquery_v2_client.py | 660 +++
.../clients/bigquery/bigquery_v2_messages.py | 1910 ++++++++
.../io/gcp/internal/clients/storage/__init__.py | 33 +
.../clients/storage/storage_v1_client.py | 1039 +++++
.../clients/storage/storage_v1_messages.py | 1920 ++++++++
sdks/python/apache_beam/io/gcp/pubsub.py | 91 +
sdks/python/apache_beam/io/gcp/pubsub_test.py | 63 +
.../python/apache_beam/io/gcp/tests/__init__.py | 16 +
.../io/gcp/tests/bigquery_matcher.py | 108 +
.../io/gcp/tests/bigquery_matcher_test.py | 108 +
sdks/python/apache_beam/io/iobase.py | 987 ++++
sdks/python/apache_beam/io/range_trackers.py | 532 +++
.../apache_beam/io/range_trackers_test.py | 590 +++
sdks/python/apache_beam/io/source_test_utils.py | 642 +++
.../apache_beam/io/source_test_utils_test.py | 122 +
sdks/python/apache_beam/io/sources_test.py | 111 +
sdks/python/apache_beam/io/textio.py | 448 ++
sdks/python/apache_beam/io/textio_test.py | 718 +++
sdks/python/apache_beam/io/tfrecordio.py | 271 ++
sdks/python/apache_beam/io/tfrecordio_test.py | 389 ++
sdks/python/apache_beam/metrics/__init__.py | 17 +
sdks/python/apache_beam/metrics/cells.py | 315 ++
sdks/python/apache_beam/metrics/cells_test.py | 143 +
sdks/python/apache_beam/metrics/execution.pxd | 31 +
sdks/python/apache_beam/metrics/execution.py | 229 +
.../apache_beam/metrics/execution_test.py | 131 +
sdks/python/apache_beam/metrics/metric.py | 202 +
sdks/python/apache_beam/metrics/metric_test.py | 128 +
sdks/python/apache_beam/metrics/metricbase.py | 82 +
sdks/python/apache_beam/pipeline.py | 442 ++
sdks/python/apache_beam/pipeline_test.py | 444 ++
sdks/python/apache_beam/pvalue.py | 468 ++
sdks/python/apache_beam/pvalue_test.py | 68 +
sdks/python/apache_beam/runners/__init__.py | 30 +
sdks/python/apache_beam/runners/api/__init__.py | 16 +
.../runners/api/beam_runner_api_pb2.py | 2772 +++++++++++
sdks/python/apache_beam/runners/common.pxd | 77 +
sdks/python/apache_beam/runners/common.py | 436 ++
.../apache_beam/runners/dataflow/__init__.py | 16 +
.../runners/dataflow/dataflow_metrics.py | 111 +
.../runners/dataflow/dataflow_metrics_test.py | 148 +
.../runners/dataflow/dataflow_runner.py | 729 +++
.../runners/dataflow/dataflow_runner_test.py | 181 +
.../runners/dataflow/internal/__init__.py | 16 +
.../runners/dataflow/internal/apiclient.py | 739 +++
.../runners/dataflow/internal/apiclient_test.py | 96 +
.../dataflow/internal/clients/__init__.py | 16 +
.../internal/clients/dataflow/__init__.py | 33 +
.../clients/dataflow/dataflow_v1b3_client.py | 694 +++
.../clients/dataflow/dataflow_v1b3_messages.py | 4392 ++++++++++++++++++
.../clients/dataflow/message_matchers.py | 124 +
.../clients/dataflow/message_matchers_test.py | 77 +
.../runners/dataflow/internal/dependency.py | 522 +++
.../dataflow/internal/dependency_test.py | 425 ++
.../runners/dataflow/internal/names.py | 82 +
.../runners/dataflow/native_io/__init__.py | 16 +
.../runners/dataflow/native_io/iobase.py | 318 ++
.../runners/dataflow/template_runner_test.py | 97 +
.../runners/dataflow/test_dataflow_runner.py | 40 +
.../apache_beam/runners/direct/__init__.py | 19 +
.../runners/direct/bundle_factory.py | 201 +
sdks/python/apache_beam/runners/direct/clock.py | 50 +
.../consumer_tracking_pipeline_visitor.py | 59 +
.../consumer_tracking_pipeline_visitor_test.py | 127 +
.../runners/direct/direct_metrics.py | 112 +
.../runners/direct/direct_metrics_test.py | 211 +
.../apache_beam/runners/direct/direct_runner.py | 173 +
.../runners/direct/evaluation_context.py | 283 ++
.../apache_beam/runners/direct/executor.py | 578 +++
.../runners/direct/helper_transforms.py | 99 +
.../runners/direct/transform_evaluator.py | 558 +++
.../runners/direct/transform_result.py | 64 +
.../runners/direct/watermark_manager.py | 224 +
.../apache_beam/runners/pipeline_context.py | 88 +
.../runners/pipeline_context_test.py | 49 +
sdks/python/apache_beam/runners/runner.py | 368 ++
sdks/python/apache_beam/runners/runner_test.py | 123 +
.../python/apache_beam/runners/test/__init__.py | 30 +
sdks/python/apache_beam/test_pipeline.py | 163 +
sdks/python/apache_beam/test_pipeline_test.py | 112 +
sdks/python/apache_beam/tests/__init__.py | 16 +
sdks/python/apache_beam/tests/data/README.md | 20 +
.../apache_beam/tests/data/privatekey.p12 | Bin 0 -> 2452 bytes
.../apache_beam/tests/data/standard_coders.yaml | 196 +
.../apache_beam/tests/pipeline_verifiers.py | 119 +
.../tests/pipeline_verifiers_test.py | 123 +
sdks/python/apache_beam/tests/test_utils.py | 69 +
sdks/python/apache_beam/transforms/__init__.py | 25 +
sdks/python/apache_beam/transforms/combiners.py | 595 +++
.../apache_beam/transforms/combiners_test.py | 324 ++
sdks/python/apache_beam/transforms/core.py | 1389 ++++++
.../apache_beam/transforms/cy_combiners.pxd | 92 +
.../apache_beam/transforms/cy_combiners.py | 306 ++
sdks/python/apache_beam/transforms/display.py | 331 ++
.../apache_beam/transforms/display_test.py | 216 +
.../python/apache_beam/transforms/ptransform.py | 671 +++
.../apache_beam/transforms/ptransform_test.py | 1941 ++++++++
.../python/apache_beam/transforms/sideinputs.py | 214 +
.../apache_beam/transforms/sideinputs_test.py | 337 ++
sdks/python/apache_beam/transforms/timeutil.py | 133 +
sdks/python/apache_beam/transforms/trigger.py | 1109 +++++
.../apache_beam/transforms/trigger_test.py | 601 +++
.../transforms/trigger_transcripts.yaml | 224 +
sdks/python/apache_beam/transforms/util.py | 235 +
sdks/python/apache_beam/transforms/window.py | 475 ++
.../apache_beam/transforms/window_test.py | 261 ++
.../transforms/write_ptransform_test.py | 126 +
sdks/python/apache_beam/typehints/__init__.py | 22 +
sdks/python/apache_beam/typehints/decorators.py | 532 +++
sdks/python/apache_beam/typehints/opcodes.py | 334 ++
.../apache_beam/typehints/trivial_inference.py | 417 ++
.../typehints/trivial_inference_test.py | 151 +
sdks/python/apache_beam/typehints/typecheck.py | 178 +
.../typehints/typed_pipeline_test.py | 251 +
sdks/python/apache_beam/typehints/typehints.py | 1062 +++++
.../apache_beam/typehints/typehints_test.py | 1062 +++++
sdks/python/apache_beam/utils/__init__.py | 22 +
sdks/python/apache_beam/utils/annotations.py | 103 +
.../apache_beam/utils/annotations_test.py | 126 +
sdks/python/apache_beam/utils/counters.pxd | 30 +
sdks/python/apache_beam/utils/counters.py | 183 +
sdks/python/apache_beam/utils/path.py | 47 +
sdks/python/apache_beam/utils/path_test.py | 70 +
.../apache_beam/utils/pipeline_options.py | 557 +++
.../apache_beam/utils/pipeline_options_test.py | 192 +
.../utils/pipeline_options_validator.py | 199 +
.../utils/pipeline_options_validator_test.py | 342 ++
sdks/python/apache_beam/utils/processes.py | 52 +
sdks/python/apache_beam/utils/processes_test.py | 106 +
sdks/python/apache_beam/utils/profiler.py | 148 +
sdks/python/apache_beam/utils/proto_utils.py | 54 +
sdks/python/apache_beam/utils/retry.py | 207 +
sdks/python/apache_beam/utils/retry_test.py | 221 +
sdks/python/apache_beam/utils/timestamp.py | 213 +
sdks/python/apache_beam/utils/timestamp_test.py | 168 +
sdks/python/apache_beam/utils/urns.py | 24 +
.../python/apache_beam/utils/windowed_value.pxd | 38 +
sdks/python/apache_beam/utils/windowed_value.py | 122 +
.../apache_beam/utils/windowed_value_test.py | 71 +
sdks/python/apache_beam/version.py | 57 +
sdks/python/generate_pydoc.sh | 80 +
sdks/python/pom.xml | 191 +
sdks/python/run_postcommit.sh | 102 +
sdks/python/run_pylint.sh | 52 +
sdks/python/setup.cfg | 27 +
sdks/python/setup.py | 155 +
sdks/python/test_config.py | 44 +
sdks/python/tox.ini | 89 +
1339 files changed, 138577 insertions(+), 31086 deletions(-)
----------------------------------------------------------------------
[17/50] [abbrv] beam git commit: This closes #2610: Merge master into
gearpump-runner branch
Posted by ke...@apache.org.
This closes #2610: Merge master into gearpump-runner branch
Update gearpump-runner against master changes.
add temp dataset location for non-query BigQuerySource
added module option, use more common zero test, show module name in log
Modify types for input PCollections of Flatten transform to that of the output PCollection
[BEAM-1871] Remove another depedendency by moving TestCredential
[BEAM-2017] Fix NPE in DataflowRunner when there are no metrics
[BEAM-2013] Upgrade to Jackson 2.8.8
[BEAM-2014] Upgrade to Google Auth 0.6.1
[BEAM-2015] Remove shared profile in runners/pom.xml and fix Dataflow ValidatesRunner PostCommit
Cache result of BigQuerySourceBase.split
Ensure all Read outputs are consumed in Dataflow
[BEAM-1441] Remove deprecated ChannelFactory
[BEAM-1994] Remove Flink examples package
Pin default commons-compress version to beam-parent pom
[BEAM-1914]Â XmlIO now complies with PTransform style guide
Separate streaming writes into two pluggable components - CreateTables, and StreamingWriteTables. Also address many code review comments. Also merge with master.
Fix tests to properly fake out BigQueryService, and add tests for dynamic-table functionality.
Refactor batch loads, and add support for windowed writes.
Refactor batch load job path, and add support for data-dependent tables.
Refactor streaming write branch into separate reusable components.
Add PrepareWrite transform.
Use tableRefFunction throughout BigQueryIO. Constant table writes use ConstantTableSpecFunction.
Explodes windows before GBKIKWI
Creates ProcessFnRunner and wires it through ParDoEvaluator
Extracts interface from PushbackSideInputDoFnRunner
Minor cleanups in ParDoEvaluator
ProcessFn remembers more info about its application context
Separates side input test and side output test
Changed snappy version to 1.1.4-M3
Upgrade worker to not depend on deprecated now deleted code
Delete AppEngineEnvironment
Delete IntervalBoundedExponentialBackoff
Delete AttemptBoundedExponentialBackoff
Remove deprecated/unused code from Pipeline
Remove deprecated method in IOChannelUtils
Delete deprecated AttemptAndTimeBoundedExponentialBackoff
[BEAM-1871] Create new GCP core module package and move several GCP related classes from beam-sdks-java-core over.
[BEAM-1964] Upgrade Pylint
Remove options_id concept from templated runs.
Revert "Revert "Throw specialized exception in value providers""
Revert "Revert "Revert "Revert "Add ValueProvider class for FileBasedSource I/O Transforms""""
Removes unused validation parameter
Converts TFRecordIO.Write to AutoValue
Gets rid of TFRecordIO.Write.Bound
Converts TFRecordIO.Read to AutoValue
Gets rid of TFRecordIO.Read.Bound
runners-core-construction-java fix artifact name
Rename SideOutputValue to OutputValue
[BEAM-1990] Comment: Don't use Window.Assign
[BEAM-1272] Align the naming of "generateInitialSplits" and "splitIntoBundles" to better reflect their intention
Revert "Removes final minor usages of OldDoFn outside OldDoFn itself"
Fix Hadoop pom.xml
Making metrics usage in datastore_wordcount consistent
Remove overloading of __call__ in DirectRunner
Clean up DirectRunner Clock and TransformResult
Translate PTransforms to and from Runner API Protos
[BEAM-1993] Remove special unbounded Flink source/sink
Remove flink-annotations dependency
Fix Javadoc warnings on Flink Runner
Enable flink dependency enforcement and make dependencies explicit
[BEAM-59] Register standard FileSystems wherever we register IOChannelFactories
[BEAM-1991] Sum.SumDoubleFn => Sum.ofDoubles
clean up description for sdk_location
Set the Project of a Table Reference at Runtime
Only compile HIFIO ITs when compiling with java 8.
Update assertions of source_test_utils from camelcase to underscore-separated.
Add no-else return to pylintrc
Remove getSideInputWindow
Remove reference to the isStreaming flag
Javadoc fixups after style guide changes
Update Dataflow Worker Version
[BEAM-1922] Close datasource in JdbcIO when possible
Fix javadoc warnings
Add javadoc to getCheckpointMark in UnboundedSource
Removes final minor usages of OldDoFn outside OldDoFn itself
[BEAM-1915] Removes use of OldDoFn from Apex
Update Signature of PTransformOverrideFactory
[BEAM-1964] Fix lint issues and pylint upgrade
Rename DoFn.Context#sideOutput to output
[BEAM-1964] Fix lint issues for linter upgrade -3
[BEAM-1964] Fix lint issues for linter upgrade -2
Avoi repackaging bigtable classes in dataflow runner.
ApexRunner: register standard IOs when deserializing pipeline options
Add PCollections Utilities
Free PTransform Names if they are being Replaced
[BEAM-1347] Update protos related to State API for prototyping purposes.
Update java8 examples pom files to include maven-shade-plugin.
fix the simplest typo
[BEAM-1964] Fix lint issues for linter upgrade
Merge PR#2423: Add Kubernetes scripts for clusters for Performance and Integration tests of Cassandra and ES for Hadoop Input Format IO
Remove Triggers.java from SDK entirely
[BEAM-1708] Improve error message when GCP not installed
Improve gcloud logging message
[BEAM-1101, BEAM-1068] Remove service account name credential pipeline options
Update user_score.py
Pin versions in tox script
Improve Empty Create Default Coder Error Message
Represent a Pipeline via a list of Top-level Transforms
Test all Known Coders to ensure they Serialize via URN
[BEAM-1950] Add missing 'static' keyword to MicrobatchSource#initReaderCache
...
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4078c22f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4078c22f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4078c22f
Branch: refs/heads/master
Commit: 4078c22fde9501bc28a5119b6f59522261776106
Parents: ebbb613 44d21ac
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Apr 21 10:50:39 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Apr 21 10:50:39 2017 -0700
----------------------------------------------------------------------
.github/PULL_REQUEST_TEMPLATE.md | 2 +-
.gitignore | 3 +
.jenkins/common_job_properties.groovy | 223 --
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 -
...ostCommit_Java_RunnableOnService_Apex.groovy | 47 -
...ommit_Java_RunnableOnService_Dataflow.groovy | 45 -
...stCommit_Java_RunnableOnService_Flink.groovy | 44 -
...ommit_Java_RunnableOnService_Gearpump.groovy | 49 -
...stCommit_Java_RunnableOnService_Spark.groovy | 44 -
.../job_beam_PostCommit_Python_Verify.groovy | 43 -
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 -
.../job_beam_PreCommit_Website_Stage.groovy | 80 -
.jenkins/job_beam_PreCommit_Website_Test.groovy | 65 -
.../job_beam_Release_NightlySnapshot.groovy | 45 -
.jenkins/job_seed.groovy | 53 -
.../jenkins/common_job_properties.groovy | 261 ++
.../job_beam_PerformanceTests_Dataflow.groovy | 43 +
.../job_beam_PerformanceTests_JDBC.groovy | 60 +
.../job_beam_PerformanceTests_Spark.groovy | 44 +
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 +
..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 +
...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 +
...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 +
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 +
...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 +
.../job_beam_PostCommit_Python_Verify.groovy | 55 +
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 +
.../job_beam_PreCommit_Website_Stage.groovy | 80 +
.../job_beam_PreCommit_Website_Test.groovy | 65 +
.../job_beam_Release_NightlySnapshot.groovy | 45 +
.test-infra/jenkins/job_seed.groovy | 53 +
.../cassandra-service-for-local-dev.yaml | 28 +
.../cassandra-svc-statefulset.yaml | 114 +
.../LargeITCluster/cassandra-svc-temp.yaml | 74 +
.../cassandra/LargeITCluster/data-load.sh | 122 +
.../cassandra/LargeITCluster/show_health.sh | 47 +
.../cassandra/LargeITCluster/start-up.sh | 22 +
.../cassandra/LargeITCluster/teardown.sh | 25 +
.../cassandra-service-for-local-dev.yaml | 30 +
.../SmallITCluster/cassandra-svc-rc.yaml | 74 +
.../cassandra/SmallITCluster/data-load.sh | 86 +
.../cassandra/SmallITCluster/show_health.sh | 47 +
.../cassandra/SmallITCluster/start-up.sh | 23 +
.../cassandra/SmallITCluster/teardown.sh | 22 +
.../kubernetes/cassandra/data-load-setup.sh | 29 +
.../elasticsearch-service-for-local-dev.yaml | 33 +
.../es-services-deployments.yaml | 258 ++
.../LargeProductionCluster/start-up.sh | 22 +
.../LargeProductionCluster/teardown.sh | 21 +
.../elasticsearch-service-for-local-dev.yaml | 34 +
.../SmallITCluster/elasticsearch-svc-rc.yaml | 96 +
.../elasticsearch/SmallITCluster/start-up.sh | 23 +
.../elasticsearch/SmallITCluster/teardown.sh | 21 +
.../kubernetes/elasticsearch/data-load-setup.sh | 26 +
.../kubernetes/elasticsearch/data-load.sh | 33 +
.../kubernetes/elasticsearch/es_test_data.py | 299 ++
.../kubernetes/elasticsearch/show-health.sh | 33 +
.../postgres/postgres-service-for-local-dev.yml | 28 +
.test-infra/kubernetes/postgres/postgres.yml | 56 +
.test-infra/travis/README.md | 23 +
.test-infra/travis/settings.xml | 33 +
.test-infra/travis/test_wordcount.sh | 125 +
.travis.yml | 7 +-
.travis/README.md | 23 -
.travis/settings.xml | 33 -
.travis/test_wordcount.sh | 125 -
README.md | 3 +
examples/java/README.md | 61 +-
examples/java/pom.xml | 32 +-
.../beam/examples/DebuggingWordCount.java | 2 +-
.../apache/beam/examples/WindowedWordCount.java | 34 +-
.../examples/common/WriteOneFilePerWindow.java | 91 +
.../examples/common/WriteWindowedFilesDoFn.java | 77 -
.../beam/examples/complete/AutoComplete.java | 2 +-
.../examples/complete/StreamingWordExtract.java | 2 +-
.../apache/beam/examples/complete/TfIdf.java | 3 +-
.../examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 12 +-
.../cookbook/CombinePerKeyExamples.java | 4 +-
.../beam/examples/cookbook/FilterExamples.java | 7 +-
.../beam/examples/cookbook/JoinExamples.java | 4 +-
.../examples/cookbook/MaxPerKeyExamples.java | 4 +-
.../org/apache/beam/examples/cookbook/README.md | 2 +-
.../beam/examples/cookbook/TriggerExample.java | 4 +-
.../beam/examples/WindowedWordCountIT.java | 41 +-
.../org/apache/beam/examples/WordCountTest.java | 4 +-
.../beam/examples/complete/TfIdfTest.java | 4 +-
.../complete/TopWikipediaSessionsTest.java | 4 +-
.../examples/cookbook/DistinctExampleTest.java | 6 +-
.../examples/cookbook/JoinExamplesTest.java | 4 +-
.../examples/cookbook/TriggerExampleTest.java | 4 +-
examples/java8/pom.xml | 41 +-
.../beam/examples/MinimalWordCountJava8.java | 9 +-
.../beam/examples/complete/game/GameStats.java | 16 +-
.../examples/complete/game/LeaderBoard.java | 2 +-
.../beam/examples/complete/game/README.md | 6 +-
.../beam/examples/complete/game/UserScore.java | 5 +-
.../complete/game/injector/Injector.java | 3 +-
.../complete/game/utils/WriteToBigQuery.java | 13 +-
.../game/utils/WriteWindowedToBigQuery.java | 9 +-
.../examples/MinimalWordCountJava8Test.java | 9 +-
.../examples/complete/game/GameStatsTest.java | 6 +-
.../complete/game/HourlyTeamScoreTest.java | 11 +-
.../examples/complete/game/UserScoreTest.java | 14 +-
pom.xml | 161 +-
runners/apex/pom.xml | 12 +-
.../apache/beam/runners/apex/ApexRunner.java | 81 +-
.../beam/runners/apex/ApexYarnLauncher.java | 111 +-
.../translation/ApexPipelineTranslator.java | 5 +-
.../FlattenPCollectionTranslator.java | 13 +-
.../apex/translation/GroupByKeyTranslator.java | 4 +-
.../translation/ParDoBoundMultiTranslator.java | 185 -
.../apex/translation/ParDoBoundTranslator.java | 95 -
.../apex/translation/ParDoTranslator.java | 185 +
.../apex/translation/TranslationContext.java | 22 +-
.../translation/WindowAssignTranslator.java | 58 +-
.../operators/ApexGroupByKeyOperator.java | 273 +-
.../operators/ApexParDoOperator.java | 238 +-
.../operators/ApexProcessFnOperator.java | 184 +
.../translation/utils/ApexStateInternals.java | 101 +-
.../apex/translation/utils/NoOpStepContext.java | 2 +-
.../utils/SerializablePipelineOptions.java | 13 +-
.../translation/utils/StateInternalsProxy.java | 67 +
.../apex/translation/utils/ValuesSource.java | 2 +-
.../apex/examples/UnboundedTextSource.java | 2 +-
.../translation/ApexGroupByKeyOperatorTest.java | 2 +-
.../FlattenPCollectionTranslatorTest.java | 3 +-
.../translation/GroupByKeyTranslatorTest.java | 2 +-
.../translation/ParDoBoundTranslatorTest.java | 344 --
.../apex/translation/ParDoTranslatorTest.java | 345 ++
.../utils/ApexStateInternalsTest.java | 43 +-
.../translation/utils/CollectionSource.java | 2 +-
runners/core-construction-java/pom.xml | 97 +-
.../beam/runners/core/construction/Coders.java | 174 +
.../DeduplicatedFlattenFactory.java | 120 +
.../EmptyFlattenAsCreateFactory.java | 34 +-
.../core/construction/ForwardingPTransform.java | 62 +
.../runners/core/construction/PCollections.java | 97 +
.../core/construction/PTransformMatchers.java | 135 +-
.../construction/PTransformReplacements.java | 69 +
.../runners/core/construction/PTransforms.java | 107 +
.../core/construction/PrimitiveCreate.java | 18 +-
.../core/construction/ReplacementOutputs.java | 63 +-
.../core/construction/SdkComponents.java | 195 ++
.../SingleInputOutputOverrideFactory.java | 14 +-
.../runners/core/construction/Triggers.java | 336 ++
.../UnboundedReadFromBoundedSource.java | 543 +++
.../core/construction/UnconsumedReads.java | 72 +
.../UnsupportedOverrideFactory.java | 20 +-
.../core/construction/WindowingStrategies.java | 245 ++
.../runners/core/construction/CodersTest.java | 163 +
.../DeduplicatedFlattenFactoryTest.java | 104 +
.../EmptyFlattenAsCreateFactoryTest.java | 122 +
.../construction/ForwardingPTransformTest.java | 111 +
.../core/construction/PCollectionsTest.java | 188 +
.../construction/PTransformMatchersTest.java | 198 +-
.../PTransformReplacementsTest.java | 131 +
.../core/construction/PTransformsTest.java | 189 +
.../construction/ReplacementOutputsTest.java | 109 +-
.../core/construction/SdkComponentsTest.java | 223 ++
.../SingleInputOutputOverrideFactoryTest.java | 37 +-
.../runners/core/construction/TriggersTest.java | 111 +
.../UnboundedReadFromBoundedSourceTest.java | 373 ++
.../core/construction/UnconsumedReadsTest.java | 105 +
.../UnsupportedOverrideFactoryTest.java | 16 +-
.../construction/WindowingStrategiesTest.java | 110 +
runners/core-java/pom.xml | 58 +-
.../beam/runners/core/BaseExecutionContext.java | 13 +-
.../apache/beam/runners/core/DoFnAdapters.java | 21 +-
.../apache/beam/runners/core/DoFnRunners.java | 40 +-
.../beam/runners/core/ExecutionContext.java | 13 +-
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 2 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 2 +-
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 8 +-
.../runners/core/InMemoryStateInternals.java | 120 +-
.../apache/beam/runners/core/NonEmptyPanes.java | 4 +-
.../org/apache/beam/runners/core/OldDoFn.java | 49 +-
...eBoundedSplittableProcessElementInvoker.java | 136 +-
.../beam/runners/core/OutputWindowedValue.java | 10 +-
.../beam/runners/core/ProcessFnRunner.java | 127 +
.../core/PushbackSideInputDoFnRunner.java | 106 +-
.../runners/core/ReduceFnContextFactory.java | 3 +-
.../beam/runners/core/SideInputHandler.java | 18 +-
.../beam/runners/core/SimpleDoFnRunner.java | 76 +-
.../beam/runners/core/SimpleOldDoFnRunner.java | 65 +-
.../core/SimplePushbackSideInputDoFnRunner.java | 115 +
.../beam/runners/core/SplittableParDo.java | 154 +-
.../core/SplittableProcessElementInvoker.java | 22 +-
.../apache/beam/runners/core/StateMerging.java | 16 +-
.../org/apache/beam/runners/core/StateTag.java | 14 +-
.../org/apache/beam/runners/core/StateTags.java | 32 +-
.../beam/runners/core/StatefulDoFnRunner.java | 96 +
.../beam/runners/core/SystemReduceFn.java | 8 +-
.../core/UnboundedReadFromBoundedSource.java | 536 ---
.../beam/runners/core/WindowingInternals.java | 8 +-
.../core/WindowingInternalsAdapters.java | 8 +-
.../AfterDelayFromFirstElementStateMachine.java | 10 +-
.../core/triggers/AfterPaneStateMachine.java | 4 +-
.../triggers/AfterWatermarkStateMachine.java | 14 +-
.../core/GroupAlsoByWindowsProperties.java | 10 +-
.../core/InMemoryStateInternalsTest.java | 105 +-
.../apache/beam/runners/core/NoOpOldDoFn.java | 4 +-
.../apache/beam/runners/core/OldDoFnTest.java | 4 +-
...ndedSplittableProcessElementInvokerTest.java | 27 +-
.../core/PushbackSideInputDoFnRunnerTest.java | 282 --
.../beam/runners/core/ReduceFnRunnerTest.java | 10 +-
.../beam/runners/core/ReduceFnTester.java | 10 +-
.../beam/runners/core/SideInputHandlerTest.java | 22 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 145 +
.../runners/core/SimpleOldDoFnRunnerTest.java | 4 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 282 ++
.../beam/runners/core/SplittableParDoTest.java | 332 +-
.../runners/core/StatefulDoFnRunnerTest.java | 113 +-
.../UnboundedReadFromBoundedSourceTest.java | 373 --
runners/direct-java/pom.xml | 55 +-
.../direct/BoundedReadEvaluatorFactory.java | 4 +-
.../CopyOnAccessInMemoryStateInternals.java | 46 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 16 +-
.../beam/runners/direct/DirectGraphVisitor.java | 5 +-
.../beam/runners/direct/DirectGroupByKey.java | 1 +
.../direct/DirectGroupByKeyOverrideFactory.java | 14 +-
.../beam/runners/direct/DirectMetrics.java | 129 +-
.../beam/runners/direct/DirectOptions.java | 11 -
.../beam/runners/direct/DirectRunner.java | 132 +-
...ecycleManagerRemovingTransformEvaluator.java | 6 +-
.../beam/runners/direct/EvaluationContext.java | 34 +-
.../direct/ExecutorServiceParallelExecutor.java | 141 +-
.../runners/direct/FlattenEvaluatorFactory.java | 2 +-
.../runners/direct/ForwardingPTransform.java | 62 -
.../GroupAlsoByWindowEvaluatorFactory.java | 12 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 4 +-
.../direct/KeyedPValueTrackingVisitor.java | 18 +-
.../beam/runners/direct/ModelEnforcement.java | 13 +-
.../beam/runners/direct/ParDoEvaluator.java | 129 +-
.../runners/direct/ParDoEvaluatorFactory.java | 39 +-
.../direct/ParDoMultiOverrideFactory.java | 53 +-
.../ParDoSingleViaMultiOverrideFactory.java | 70 -
.../beam/runners/direct/PipelineExecutor.java | 23 +-
...littableProcessElementsEvaluatorFactory.java | 108 +-
.../direct/StatefulParDoEvaluatorFactory.java | 31 +-
.../direct/TestStreamEvaluatorFactory.java | 20 +-
.../direct/TransformEvaluatorFactory.java | 10 +-
.../direct/TransformEvaluatorRegistry.java | 7 +-
.../direct/TransformExecutorService.java | 6 +
.../direct/TransformExecutorServices.java | 57 +-
.../direct/UnboundedReadEvaluatorFactory.java | 39 +-
.../runners/direct/ViewEvaluatorFactory.java | 82 +-
.../runners/direct/ViewOverrideFactory.java | 114 +
.../beam/runners/direct/WatermarkManager.java | 19 +-
.../runners/direct/WindowEvaluatorFactory.java | 2 +-
.../direct/WriteWithShardingFactory.java | 26 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 4 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 34 +-
.../runners/direct/DirectGraphVisitorTest.java | 7 +-
.../DirectGroupByKeyOverrideFactoryTest.java | 12 +-
.../beam/runners/direct/DirectMetricsTest.java | 124 +-
.../beam/runners/direct/DirectRunnerTest.java | 89 +-
...leManagerRemovingTransformEvaluatorTest.java | 8 +-
.../runners/direct/EvaluationContextTest.java | 57 +-
.../direct/ForwardingPTransformTest.java | 112 -
.../beam/runners/direct/ParDoEvaluatorTest.java | 15 +-
.../direct/ParDoMultiOverrideFactoryTest.java | 45 -
.../ParDoSingleViaMultiOverrideFactoryTest.java | 46 -
.../runners/direct/SideInputContainerTest.java | 19 +-
.../StatefulParDoEvaluatorFactoryTest.java | 5 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 11 -
.../direct/TransformExecutorServicesTest.java | 48 +
.../UnboundedReadEvaluatorFactoryTest.java | 88 +-
.../direct/ViewEvaluatorFactoryTest.java | 14 +-
.../runners/direct/ViewOverrideFactoryTest.java | 138 +
.../direct/WindowEvaluatorFactoryTest.java | 12 +-
.../direct/WriteWithShardingFactoryTest.java | 28 +-
runners/flink/examples/pom.xml | 126 -
.../beam/runners/flink/examples/TFIDF.java | 456 ---
.../beam/runners/flink/examples/WordCount.java | 129 -
.../runners/flink/examples/package-info.java | 22 -
.../flink/examples/streaming/AutoComplete.java | 400 ---
.../flink/examples/streaming/JoinExamples.java | 154 -
.../examples/streaming/KafkaIOExamples.java | 338 --
.../KafkaWindowedWordCountExample.java | 164 -
.../examples/streaming/WindowedWordCount.java | 141 -
.../flink/examples/streaming/package-info.java | 22 -
runners/flink/pom.xml | 320 +-
runners/flink/runner/pom.xml | 310 --
.../flink/DefaultParallelismFactory.java | 39 -
.../flink/FlinkBatchPipelineTranslator.java | 139 -
.../flink/FlinkBatchTransformTranslators.java | 797 -----
.../flink/FlinkBatchTranslationContext.java | 154 -
.../flink/FlinkDetachedRunnerResult.java | 76 -
.../FlinkPipelineExecutionEnvironment.java | 241 --
.../runners/flink/FlinkPipelineOptions.java | 101 -
.../runners/flink/FlinkPipelineTranslator.java | 53 -
.../apache/beam/runners/flink/FlinkRunner.java | 250 --
.../runners/flink/FlinkRunnerRegistrar.java | 62 -
.../beam/runners/flink/FlinkRunnerResult.java | 98 -
.../flink/FlinkStreamingPipelineTranslator.java | 230 --
.../FlinkStreamingTransformTranslators.java | 1043 ------
.../flink/FlinkStreamingTranslationContext.java | 130 -
.../flink/FlinkStreamingViewOverrides.java | 372 --
.../flink/PipelineTranslationOptimizer.java | 72 -
.../beam/runners/flink/TestFlinkRunner.java | 97 -
.../beam/runners/flink/TranslationMode.java | 31 -
.../apache/beam/runners/flink/package-info.java | 22 -
.../functions/FlinkAggregatorFactory.java | 53 -
.../functions/FlinkAssignContext.java | 63 -
.../functions/FlinkAssignWindows.java | 49 -
.../functions/FlinkDoFnFunction.java | 161 -
.../FlinkMergingNonShuffleReduceFunction.java | 228 --
.../FlinkMergingPartialReduceFunction.java | 201 --
.../functions/FlinkMergingReduceFunction.java | 199 --
.../FlinkMultiOutputPruningFunction.java | 50 -
.../functions/FlinkNoOpStepContext.java | 73 -
.../functions/FlinkPartialReduceFunction.java | 172 -
.../functions/FlinkReduceFunction.java | 173 -
.../functions/FlinkSideInputReader.java | 80 -
.../functions/FlinkStatefulDoFnFunction.java | 198 --
.../functions/SideInputInitializer.java | 73 -
.../translation/functions/package-info.java | 22 -
.../runners/flink/translation/package-info.java | 22 -
.../translation/types/CoderTypeInformation.java | 120 -
.../translation/types/CoderTypeSerializer.java | 132 -
.../types/EncodedValueComparator.java | 195 --
.../types/EncodedValueSerializer.java | 113 -
.../types/EncodedValueTypeInformation.java | 107 -
.../flink/translation/types/FlinkCoder.java | 63 -
.../types/InspectableByteArrayOutputStream.java | 34 -
.../flink/translation/types/KvKeySelector.java | 50 -
.../flink/translation/types/package-info.java | 22 -
.../utils/SerializedPipelineOptions.java | 65 -
.../flink/translation/utils/package-info.java | 22 -
.../wrappers/DataInputViewWrapper.java | 58 -
.../wrappers/DataOutputViewWrapper.java | 51 -
.../SerializableFnAggregatorWrapper.java | 98 -
.../translation/wrappers/SourceInputFormat.java | 149 -
.../translation/wrappers/SourceInputSplit.java | 52 -
.../translation/wrappers/package-info.java | 22 -
.../wrappers/streaming/DoFnOperator.java | 861 -----
.../streaming/KvToByteBufferKeySelector.java | 56 -
.../streaming/SingletonKeyedWorkItem.java | 58 -
.../streaming/SingletonKeyedWorkItemCoder.java | 128 -
.../wrappers/streaming/WindowDoFnOperator.java | 120 -
.../wrappers/streaming/WorkItemKeySelector.java | 56 -
.../streaming/io/BoundedSourceWrapper.java | 218 --
.../streaming/io/UnboundedFlinkSink.java | 182 -
.../streaming/io/UnboundedFlinkSource.java | 120 -
.../streaming/io/UnboundedSocketSource.java | 249 --
.../streaming/io/UnboundedSourceWrapper.java | 476 ---
.../wrappers/streaming/io/package-info.java | 22 -
.../wrappers/streaming/package-info.java | 22 -
.../state/FlinkBroadcastStateInternals.java | 865 -----
.../state/FlinkKeyGroupStateInternals.java | 487 ---
.../state/FlinkSplitStateInternals.java | 260 --
.../streaming/state/FlinkStateInternals.java | 1053 ------
.../state/KeyGroupCheckpointedOperator.java | 35 -
.../state/KeyGroupRestoringOperator.java | 32 -
.../wrappers/streaming/state/package-info.java | 22 -
.../runner/src/main/resources/log4j.properties | 23 -
.../flink/EncodedValueComparatorTest.java | 70 -
.../runners/flink/FlinkRunnerRegistrarTest.java | 48 -
.../beam/runners/flink/FlinkTestPipeline.java | 72 -
.../beam/runners/flink/PipelineOptionsTest.java | 184 -
.../beam/runners/flink/ReadSourceITCase.java | 85 -
.../flink/ReadSourceStreamingITCase.java | 74 -
.../beam/runners/flink/WriteSinkITCase.java | 171 -
.../flink/streaming/DoFnOperatorTest.java | 594 ----
.../FlinkBroadcastStateInternalsTest.java | 245 --
.../FlinkKeyGroupStateInternalsTest.java | 262 --
.../streaming/FlinkSplitStateInternalsTest.java | 101 -
.../streaming/FlinkStateInternalsTest.java | 395 ---
.../flink/streaming/GroupByNullKeyTest.java | 124 -
.../flink/streaming/TestCountingSource.java | 254 --
.../streaming/TopWikipediaSessionsITCase.java | 133 -
.../streaming/UnboundedSourceWrapperTest.java | 464 ---
.../runners/flink/streaming/package-info.java | 22 -
.../src/test/resources/log4j-test.properties | 27 -
.../flink/DefaultParallelismFactory.java | 39 +
.../flink/FlinkBatchPipelineTranslator.java | 139 +
.../flink/FlinkBatchTransformTranslators.java | 723 ++++
.../flink/FlinkBatchTranslationContext.java | 153 +
.../flink/FlinkDetachedRunnerResult.java | 75 +
.../FlinkPipelineExecutionEnvironment.java | 241 ++
.../runners/flink/FlinkPipelineOptions.java | 101 +
.../runners/flink/FlinkPipelineTranslator.java | 53 +
.../apache/beam/runners/flink/FlinkRunner.java | 232 ++
.../runners/flink/FlinkRunnerRegistrar.java | 62 +
.../beam/runners/flink/FlinkRunnerResult.java | 98 +
.../flink/FlinkStreamingPipelineTranslator.java | 276 ++
.../FlinkStreamingTransformTranslators.java | 1044 ++++++
.../flink/FlinkStreamingTranslationContext.java | 130 +
.../flink/FlinkStreamingViewOverrides.java | 372 ++
.../flink/PipelineTranslationOptimizer.java | 72 +
.../beam/runners/flink/TestFlinkRunner.java | 84 +
.../beam/runners/flink/TranslationMode.java | 31 +
.../apache/beam/runners/flink/package-info.java | 22 +
.../functions/FlinkAggregatorFactory.java | 53 +
.../functions/FlinkAssignContext.java | 63 +
.../functions/FlinkAssignWindows.java | 49 +
.../functions/FlinkDoFnFunction.java | 161 +
.../FlinkMergingNonShuffleReduceFunction.java | 228 ++
.../FlinkMergingPartialReduceFunction.java | 201 ++
.../functions/FlinkMergingReduceFunction.java | 199 ++
.../FlinkMultiOutputPruningFunction.java | 50 +
.../functions/FlinkNoOpStepContext.java | 73 +
.../functions/FlinkPartialReduceFunction.java | 172 +
.../functions/FlinkReduceFunction.java | 173 +
.../functions/FlinkSideInputReader.java | 80 +
.../functions/FlinkStatefulDoFnFunction.java | 198 ++
.../functions/SideInputInitializer.java | 73 +
.../translation/functions/package-info.java | 22 +
.../runners/flink/translation/package-info.java | 22 +
.../translation/types/CoderTypeInformation.java | 120 +
.../translation/types/CoderTypeSerializer.java | 132 +
.../types/EncodedValueComparator.java | 195 ++
.../types/EncodedValueSerializer.java | 113 +
.../types/EncodedValueTypeInformation.java | 98 +
.../types/InspectableByteArrayOutputStream.java | 34 +
.../flink/translation/types/KvKeySelector.java | 50 +
.../flink/translation/types/package-info.java | 22 +
.../utils/SerializedPipelineOptions.java | 67 +
.../flink/translation/utils/package-info.java | 22 +
.../wrappers/DataInputViewWrapper.java | 58 +
.../wrappers/DataOutputViewWrapper.java | 51 +
.../SerializableFnAggregatorWrapper.java | 98 +
.../translation/wrappers/SourceInputFormat.java | 150 +
.../translation/wrappers/SourceInputSplit.java | 52 +
.../translation/wrappers/package-info.java | 22 +
.../wrappers/streaming/DoFnOperator.java | 774 ++++
.../streaming/KvToByteBufferKeySelector.java | 56 +
.../streaming/SingletonKeyedWorkItem.java | 56 +
.../streaming/SingletonKeyedWorkItemCoder.java | 126 +
.../streaming/SplittableDoFnOperator.java | 150 +
.../wrappers/streaming/WindowDoFnOperator.java | 117 +
.../wrappers/streaming/WorkItemKeySelector.java | 56 +
.../streaming/io/BoundedSourceWrapper.java | 218 ++
.../streaming/io/UnboundedSocketSource.java | 249 ++
.../streaming/io/UnboundedSourceWrapper.java | 476 +++
.../wrappers/streaming/io/package-info.java | 22 +
.../wrappers/streaming/package-info.java | 22 +
.../state/FlinkBroadcastStateInternals.java | 865 +++++
.../state/FlinkKeyGroupStateInternals.java | 487 +++
.../state/FlinkSplitStateInternals.java | 260 ++
.../streaming/state/FlinkStateInternals.java | 1053 ++++++
.../state/KeyGroupCheckpointedOperator.java | 35 +
.../state/KeyGroupRestoringOperator.java | 32 +
.../wrappers/streaming/state/package-info.java | 22 +
.../flink/src/main/resources/log4j.properties | 23 +
.../flink/EncodedValueComparatorTest.java | 70 +
.../runners/flink/FlinkRunnerRegistrarTest.java | 48 +
.../beam/runners/flink/FlinkTestPipeline.java | 72 +
.../beam/runners/flink/PipelineOptionsTest.java | 184 +
.../beam/runners/flink/ReadSourceITCase.java | 85 +
.../flink/ReadSourceStreamingITCase.java | 74 +
.../beam/runners/flink/WriteSinkITCase.java | 192 +
.../flink/streaming/DoFnOperatorTest.java | 600 ++++
.../FlinkBroadcastStateInternalsTest.java | 245 ++
.../FlinkKeyGroupStateInternalsTest.java | 262 ++
.../streaming/FlinkSplitStateInternalsTest.java | 101 +
.../streaming/FlinkStateInternalsTest.java | 395 +++
.../flink/streaming/GroupByNullKeyTest.java | 124 +
.../flink/streaming/TestCountingSource.java | 254 ++
.../streaming/TopWikipediaSessionsITCase.java | 133 +
.../streaming/UnboundedSourceWrapperTest.java | 464 +++
.../runners/flink/streaming/package-info.java | 22 +
.../src/test/resources/log4j-test.properties | 27 +
runners/gearpump/pom.xml | 11 +-
.../gearpump/GearpumpPipelineTranslator.java | 92 +-
.../FlattenPCollectionsTranslator.java | 6 +-
.../translators/ParDoBoundMultiTranslator.java | 104 -
.../translators/ParDoBoundTranslator.java | 75 -
.../translators/ParDoMultiOutputTranslator.java | 103 +
.../ParDoSingleOutputTranslator.java | 75 +
.../translators/TranslationContext.java | 11 +-
.../translators/WindowAssignTranslator.java | 2 +-
.../translators/functions/DoFnFunction.java | 2 +-
.../gearpump/translators/io/GearpumpSource.java | 3 +-
.../gearpump/translators/io/ValuesSource.java | 3 +-
.../translators/utils/DoFnRunnerFactory.java | 3 +-
.../translators/utils/NoOpStepContext.java | 3 +-
.../FlattenPCollectionsTranslatorTest.java | 48 +-
.../translators/GroupByKeyTranslatorTest.java | 3 +-
runners/google-cloud-dataflow-java/pom.xml | 88 +-
.../beam/runners/dataflow/AssignWindows.java | 4 +-
.../dataflow/BatchStatefulParDoOverrides.java | 78 +-
.../runners/dataflow/BatchViewOverrides.java | 85 +-
.../runners/dataflow/CreateDataflowView.java | 46 +
.../beam/runners/dataflow/DataflowMetrics.java | 225 ++
.../runners/dataflow/DataflowPipelineJob.java | 96 +-
.../dataflow/DataflowPipelineTranslator.java | 71 +-
.../beam/runners/dataflow/DataflowRunner.java | 387 +-
.../runners/dataflow/DataflowRunnerInfo.java | 38 +-
.../DataflowUnboundedReadFromBoundedSource.java | 547 ---
.../dataflow/PrimitiveParDoSingleFactory.java | 85 +
.../dataflow/ReshuffleOverrideFactory.java | 90 +
.../dataflow/StreamingViewOverrides.java | 17 +-
.../runners/dataflow/TransformTranslator.java | 6 +-
.../dataflow/internal/CustomSources.java | 2 +-
.../runners/dataflow/internal/IsmFormat.java | 3 +-
.../options/DataflowPipelineDebugOptions.java | 4 +-
.../DataflowPipelineWorkerPoolOptions.java | 8 +-
.../dataflow/testing/TestDataflowRunner.java | 43 +-
.../runners/dataflow/testing/package-info.java | 2 +-
.../runners/dataflow/util/MonitoringUtil.java | 4 +
.../beam/runners/dataflow/dataflow.properties | 8 +-
.../BatchStatefulParDoOverridesTest.java | 2 +-
.../dataflow/BatchViewOverridesTest.java | 4 +-
.../runners/dataflow/DataflowMetricsTest.java | 236 ++
.../dataflow/DataflowPipelineJobTest.java | 50 +-
.../DataflowPipelineTranslatorTest.java | 85 +-
.../dataflow/DataflowRunnerInfoTest.java | 23 +-
.../runners/dataflow/DataflowRunnerTest.java | 98 +-
...aflowUnboundedReadFromBoundedSourceTest.java | 79 -
.../PrimitiveParDoSingleFactoryTest.java | 161 +
.../testing/TestDataflowRunnerTest.java | 3 +-
.../dataflow/util/MonitoringUtilTest.java | 31 +-
runners/pom.xml | 40 -
runners/spark/pom.xml | 78 +-
.../spark/SparkNativePipelineVisitor.java | 1 -
.../beam/runners/spark/SparkPipelineResult.java | 81 +-
.../apache/beam/runners/spark/SparkRunner.java | 195 +-
.../beam/runners/spark/TestSparkRunner.java | 45 +-
.../spark/coders/StatelessJavaSerializer.java | 11 +
.../beam/runners/spark/io/MicrobatchSource.java | 118 +-
.../beam/runners/spark/io/SourceDStream.java | 63 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 74 +-
.../runners/spark/io/SparkUnboundedSource.java | 132 +-
.../beam/runners/spark/io/hadoop/HadoopIO.java | 216 --
.../spark/io/hadoop/ShardNameBuilder.java | 111 -
.../spark/io/hadoop/ShardNameTemplateAware.java | 31 -
.../io/hadoop/ShardNameTemplateHelper.java | 63 -
.../io/hadoop/TemplatedAvroKeyOutputFormat.java | 45 -
.../TemplatedSequenceFileOutputFormat.java | 45 -
.../io/hadoop/TemplatedTextOutputFormat.java | 45 -
.../runners/spark/io/hadoop/package-info.java | 22 -
.../runners/spark/metrics/SparkBeamMetric.java | 4 +
.../spark/metrics/SparkMetricResults.java | 67 +-
.../spark/metrics/SparkMetricsContainer.java | 31 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 48 +-
.../spark/stateful/SparkStateInternals.java | 30 +-
.../spark/stateful/StateSpecFunctions.java | 61 +-
.../spark/translation/BoundedDataset.java | 3 +-
.../runners/spark/translation/DoFnFunction.java | 130 -
.../spark/translation/EvaluationContext.java | 72 +-
.../translation/GroupCombineFunctions.java | 53 +-
.../spark/translation/MultiDoFnFunction.java | 4 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 10 +-
.../spark/translation/SparkProcessContext.java | 2 +-
.../spark/translation/SparkRuntimeContext.java | 2 +
.../spark/translation/TransformTranslator.java | 370 +-
.../spark/translation/TranslationUtils.java | 245 +-
.../SparkRunnerStreamingContextFactory.java | 45 +-
.../streaming/StreamingTransformTranslator.java | 158 +-
.../spark/util/GlobalWatermarkHolder.java | 2 +-
.../spark/util/SparkSideInputReader.java | 3 +-
.../apache/beam/runners/spark/CacheTest.java | 61 +
.../runners/spark/SparkPipelineStateTest.java | 2 +-
.../runners/spark/UsesCheckpointRecovery.java | 23 +
.../io/hadoop/HadoopFileFormatPipelineTest.java | 121 -
.../spark/io/hadoop/ShardNameBuilderTest.java | 88 -
.../spark/translation/StorageLevelTest.java | 6 +-
.../translation/streaming/CreateStreamTest.java | 4 +-
.../ResumeFromCheckpointStreamingTest.java | 17 +-
.../streaming/StreamingSourceMetricsTest.java | 71 +
.../streaming/TrackStreamingSourcesTest.java | 4 +-
runners/spark/src/test/resources/test_text.txt | 2 -
sdks/common/fn-api/pom.xml | 5 -
.../fn-api/src/main/proto/beam_fn_api.proto | 174 +-
.../src/main/proto/beam_runner_api.proto | 60 +-
sdks/java/build-tools/pom.xml | 31 +-
.../src/main/resources/beam/beam-codestyle.xml | 329 ++
.../src/main/resources/beam/findbugs-filter.xml | 43 +-
.../beam/codestyle/CodeStyleTestClass.java | 45 +
sdks/java/core/pom.xml | 155 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 8 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 123 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 170 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 4 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 4 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 20 +-
.../apache/beam/sdk/io/CompressedSource.java | 17 +-
.../org/apache/beam/sdk/io/CountingInput.java | 4 +
.../org/apache/beam/sdk/io/CountingSource.java | 10 +-
.../org/apache/beam/sdk/io/FileBasedSink.java | 429 ++-
.../org/apache/beam/sdk/io/FileBasedSource.java | 38 +-
.../org/apache/beam/sdk/io/FileSystems.java | 302 +-
.../org/apache/beam/sdk/io/LocalFileSystem.java | 57 +-
.../org/apache/beam/sdk/io/LocalResourceId.java | 2 +-
.../apache/beam/sdk/io/OffsetBasedSource.java | 4 +-
.../java/org/apache/beam/sdk/io/PubsubIO.java | 1201 -------
.../apache/beam/sdk/io/PubsubUnboundedSink.java | 500 ---
.../beam/sdk/io/PubsubUnboundedSource.java | 1435 --------
.../main/java/org/apache/beam/sdk/io/Sink.java | 57 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 694 ++++
.../java/org/apache/beam/sdk/io/TextIO.java | 104 +-
.../org/apache/beam/sdk/io/UnboundedSource.java | 19 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 377 +-
.../main/java/org/apache/beam/sdk/io/XmlIO.java | 477 +++
.../java/org/apache/beam/sdk/io/XmlSink.java | 226 +-
.../java/org/apache/beam/sdk/io/XmlSource.java | 191 +-
.../org/apache/beam/sdk/io/fs/MoveOptions.java | 34 +
.../org/apache/beam/sdk/io/fs/ResourceId.java | 8 +
.../org/apache/beam/sdk/io/package-info.java | 2 +-
.../apache/beam/sdk/io/range/ByteKeyRange.java | 4 +-
.../java/org/apache/beam/sdk/metrics/Gauge.java | 32 +
.../org/apache/beam/sdk/metrics/GaugeCell.java | 60 +
.../org/apache/beam/sdk/metrics/GaugeData.java | 81 +
.../apache/beam/sdk/metrics/GaugeResult.java | 61 +
.../beam/sdk/metrics/MetricFiltering.java | 99 +
.../beam/sdk/metrics/MetricNameFilter.java | 3 +-
.../beam/sdk/metrics/MetricQueryResults.java | 3 +
.../apache/beam/sdk/metrics/MetricUpdates.java | 11 +-
.../org/apache/beam/sdk/metrics/Metrics.java | 35 +
.../beam/sdk/metrics/MetricsContainer.java | 26 +-
.../beam/sdk/options/BigQueryOptions.java | 32 -
.../options/CloudResourceManagerOptions.java | 40 -
.../DefaultPipelineOptionsRegistrar.java | 5 -
.../org/apache/beam/sdk/options/GcpOptions.java | 227 --
.../org/apache/beam/sdk/options/GcsOptions.java | 158 -
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 -
.../beam/sdk/options/PipelineOptions.java | 16 +-
.../apache/beam/sdk/options/PubsubOptions.java | 36 -
.../beam/sdk/options/StreamingOptions.java | 7 +-
.../beam/sdk/runners/PTransformOverride.java | 44 +
.../sdk/runners/PTransformOverrideFactory.java | 31 +-
.../apache/beam/sdk/runners/PipelineRunner.java | 7 +-
.../beam/sdk/runners/TransformHierarchy.java | 80 +-
.../beam/sdk/testing/BigqueryMatcher.java | 256 --
.../apache/beam/sdk/testing/GatherAllPanes.java | 1 +
.../org/apache/beam/sdk/testing/PAssert.java | 220 +-
.../apache/beam/sdk/testing/PaneExtractors.java | 1 +
.../beam/sdk/testing/RunnableOnService.java | 38 -
.../beam/sdk/testing/SerializableMatcher.java | 2 +-
.../beam/sdk/testing/SerializableMatchers.java | 4 +-
.../beam/sdk/testing/SourceTestUtils.java | 6 +-
.../apache/beam/sdk/testing/StaticWindows.java | 17 +-
.../apache/beam/sdk/testing/StreamingIT.java | 2 +-
.../apache/beam/sdk/testing/TestPipeline.java | 89 +-
.../beam/sdk/testing/TestPipelineOptions.java | 5 +
...esSplittableParDoWithWindowedSideInputs.java | 26 +
.../beam/sdk/testing/ValidatesRunner.java | 23 +
.../beam/sdk/testing/ValueInSingleWindow.java | 134 -
.../beam/sdk/transforms/AppliedPTransform.java | 24 +-
.../org/apache/beam/sdk/transforms/Combine.java | 64 +-
.../org/apache/beam/sdk/transforms/Create.java | 86 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 138 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 51 +-
.../beam/sdk/transforms/FlatMapElements.java | 113 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 6 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 229 ++
.../apache/beam/sdk/transforms/MapElements.java | 99 +-
.../beam/sdk/transforms/Materialization.java | 36 +
.../beam/sdk/transforms/Materializations.java | 53 +
.../apache/beam/sdk/transforms/PTransform.java | 16 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 495 +--
.../apache/beam/sdk/transforms/Partition.java | 6 +-
.../org/apache/beam/sdk/transforms/Regex.java | 14 +-
.../org/apache/beam/sdk/transforms/Sample.java | 11 +-
.../org/apache/beam/sdk/transforms/View.java | 81 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 5 +
.../beam/sdk/transforms/WithTimestamps.java | 26 +-
.../sdk/transforms/display/DisplayData.java | 6 +
.../transforms/join/KeyedPCollectionTuple.java | 12 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 47 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 4 +-
.../sdk/transforms/reflect/DoFnSignature.java | 10 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 96 +-
.../splittabledofn/HasDefaultTracker.java | 30 +
.../transforms/splittabledofn/OffsetRange.java | 8 +-
.../splittabledofn/OffsetRangeTracker.java | 33 +-
.../splittabledofn/RestrictionTracker.java | 8 +
.../transforms/windowing/AfterWatermark.java | 16 +-
.../sdk/transforms/windowing/GlobalWindow.java | 10 +
.../sdk/transforms/windowing/GlobalWindows.java | 11 +-
.../transforms/windowing/InvalidWindows.java | 2 +-
.../beam/sdk/transforms/windowing/PaneInfo.java | 4 +-
.../windowing/PartitioningWindowFn.java | 17 +-
.../sdk/transforms/windowing/Repeatedly.java | 2 +-
.../beam/sdk/transforms/windowing/Sessions.java | 2 +-
.../transforms/windowing/SlidingWindows.java | 27 +-
.../beam/sdk/transforms/windowing/Triggers.java | 320 --
.../beam/sdk/transforms/windowing/Window.java | 515 ++-
.../beam/sdk/transforms/windowing/WindowFn.java | 9 +-
.../transforms/windowing/WindowMappingFn.java | 67 +
.../beam/sdk/util/AppEngineEnvironment.java | 62 -
...AttemptAndTimeBoundedExponentialBackOff.java | 170 -
.../util/AttemptBoundedExponentialBackOff.java | 86 -
.../org/apache/beam/sdk/util/CoderUtils.java | 21 +-
.../beam/sdk/util/CombineContextFactory.java | 2 +-
.../apache/beam/sdk/util/CredentialFactory.java | 29 -
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 -
.../beam/sdk/util/DirectSideInputReader.java | 74 -
.../beam/sdk/util/FileIOChannelFactory.java | 23 +-
.../beam/sdk/util/GcpCredentialFactory.java | 67 -
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 -
.../beam/sdk/util/GcsIOChannelFactory.java | 110 -
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 -
.../apache/beam/sdk/util/GcsPathValidator.java | 95 -
.../java/org/apache/beam/sdk/util/GcsUtil.java | 803 -----
.../apache/beam/sdk/util/IOChannelFactory.java | 3 +-
.../apache/beam/sdk/util/IOChannelUtils.java | 22 +-
.../apache/beam/sdk/util/IdentityWindowFn.java | 3 +-
.../util/IntervalBoundedExponentialBackOff.java | 89 -
.../apache/beam/sdk/util/MutationDetector.java | 2 +-
.../org/apache/beam/sdk/util/NameUtils.java | 5 +
.../beam/sdk/util/NoopCredentialFactory.java | 68 -
.../sdk/util/NullCredentialInitializer.java | 62 -
.../beam/sdk/util/NumberedShardedFile.java | 30 +-
.../apache/beam/sdk/util/PCollectionViews.java | 268 +-
.../java/org/apache/beam/sdk/util/PTuple.java | 160 -
.../org/apache/beam/sdk/util/PubsubClient.java | 544 ---
.../apache/beam/sdk/util/PubsubGrpcClient.java | 424 ---
.../apache/beam/sdk/util/PubsubJsonClient.java | 317 --
.../apache/beam/sdk/util/PubsubTestClient.java | 436 ---
.../apache/beam/sdk/util/ReifyTimestamps.java | 76 +
.../org/apache/beam/sdk/util/Reshuffle.java | 21 +-
.../apache/beam/sdk/util/TestCredential.java | 59 -
.../org/apache/beam/sdk/util/TimeDomain.java | 2 +-
.../org/apache/beam/sdk/util/Transport.java | 179 -
.../beam/sdk/util/WindowingStrategies.java | 266 --
.../util/state/AccumulatorCombiningState.java | 53 -
.../apache/beam/sdk/util/state/BagState.java | 2 +-
.../beam/sdk/util/state/CombiningState.java | 27 +-
.../beam/sdk/util/state/GroupingState.java | 42 +
.../apache/beam/sdk/util/state/MapState.java | 52 +-
.../beam/sdk/util/state/ReadableStates.java | 45 +
.../apache/beam/sdk/util/state/SetState.java | 36 +-
.../org/apache/beam/sdk/util/state/State.java | 2 +-
.../apache/beam/sdk/util/state/StateBinder.java | 12 +-
.../apache/beam/sdk/util/state/StateSpec.java | 15 +
.../apache/beam/sdk/util/state/StateSpecs.java | 344 +-
.../beam/sdk/util/state/WatermarkHoldState.java | 2 +-
.../java/org/apache/beam/sdk/values/PBegin.java | 6 +-
.../apache/beam/sdk/values/PCollectionList.java | 27 +-
.../beam/sdk/values/PCollectionTuple.java | 13 +-
.../apache/beam/sdk/values/PCollectionView.java | 19 +
.../java/org/apache/beam/sdk/values/PDone.java | 6 +-
.../java/org/apache/beam/sdk/values/PInput.java | 4 +-
.../org/apache/beam/sdk/values/POutput.java | 4 +-
.../beam/sdk/values/POutputValueBase.java | 41 +-
.../java/org/apache/beam/sdk/values/PValue.java | 4 +-
.../org/apache/beam/sdk/values/PValueBase.java | 6 +-
.../apache/beam/sdk/values/TaggedPValue.java | 5 +
.../org/apache/beam/sdk/values/TupleTag.java | 26 +-
.../apache/beam/sdk/values/TupleTagList.java | 2 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 25 +-
.../org/apache/beam/sdk/values/TypedPValue.java | 6 +-
.../beam/sdk/values/ValueInSingleWindow.java | 134 +
sdks/java/core/src/main/proto/README.md | 3 -
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 3 -
.../sdk/AggregatorPipelineExtractorTest.java | 56 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 218 +-
.../sdk/coders/protobuf/ProtobufUtilTest.java | 1 -
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 154 +-
.../org/apache/beam/sdk/io/AvroSourceTest.java | 10 +-
.../io/BoundedReadFromUnboundedSourceTest.java | 8 +-
.../apache/beam/sdk/io/CountingInputTest.java | 14 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 20 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 94 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 8 +-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 168 +-
.../apache/beam/sdk/io/LocalFileSystemTest.java | 12 +-
.../beam/sdk/io/OffsetBasedSourceTest.java | 8 +-
.../org/apache/beam/sdk/io/PubsubIOTest.java | 197 --
.../beam/sdk/io/PubsubUnboundedSinkTest.java | 190 -
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 398 ---
.../java/org/apache/beam/sdk/io/ReadTest.java | 10 +-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 369 ++
.../java/org/apache/beam/sdk/io/TextIOTest.java | 125 +-
.../java/org/apache/beam/sdk/io/WriteTest.java | 53 +-
.../org/apache/beam/sdk/io/XmlSinkTest.java | 89 +-
.../org/apache/beam/sdk/io/XmlSourceTest.java | 250 +-
.../apache/beam/sdk/metrics/GaugeCellTest.java | 48 +
.../beam/sdk/metrics/MetricFilteringTest.java | 145 +
.../apache/beam/sdk/metrics/MetricMatchers.java | 12 +-
.../apache/beam/sdk/metrics/MetricsTest.java | 93 +-
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 -
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 -
.../sdk/options/PipelineOptionsFactoryTest.java | 4 +-
.../beam/sdk/runners/PipelineRunnerTest.java | 46 +-
.../sdk/runners/TransformHierarchyTest.java | 86 +-
.../beam/sdk/runners/TransformTreeTest.java | 88 +-
.../runners/dataflow/TestCountingSource.java | 2 +-
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 -
.../beam/sdk/testing/GatherAllPanesTest.java | 1 +
.../apache/beam/sdk/testing/PAssertTest.java | 74 +-
.../sdk/testing/PCollectionViewTesting.java | 83 +-
.../beam/sdk/testing/PaneExtractorsTest.java | 1 +
.../beam/sdk/testing/SourceTestUtilsTest.java | 2 +-
.../beam/sdk/testing/StaticWindowsTest.java | 10 +-
.../beam/sdk/testing/TestPipelineTest.java | 47 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 6 +-
.../testing/ValueInSingleWindowCoderTest.java | 1 +
.../sdk/transforms/ApproximateUniqueTest.java | 4 +-
.../beam/sdk/transforms/CombineFnsTest.java | 8 +-
.../apache/beam/sdk/transforms/CombineTest.java | 44 +-
.../apache/beam/sdk/transforms/CountTest.java | 10 +-
.../apache/beam/sdk/transforms/CreateTest.java | 70 +-
.../beam/sdk/transforms/DistinctTest.java | 8 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 7 +-
.../apache/beam/sdk/transforms/FilterTest.java | 16 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 47 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 14 +-
.../sdk/transforms/GroupIntoBatchesTest.java | 232 ++
.../apache/beam/sdk/transforms/KeysTest.java | 6 +-
.../apache/beam/sdk/transforms/KvSwapTest.java | 6 +-
.../beam/sdk/transforms/MapElementsTest.java | 29 +-
.../beam/sdk/transforms/ParDoLifecycleTest.java | 10 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 1180 +++++--
.../beam/sdk/transforms/PartitionTest.java | 4 +-
.../apache/beam/sdk/transforms/SampleTest.java | 14 +-
.../beam/sdk/transforms/SplittableDoFnTest.java | 157 +-
.../beam/sdk/transforms/ToStringTest.java | 13 +-
.../org/apache/beam/sdk/transforms/TopTest.java | 3 +-
.../apache/beam/sdk/transforms/ValuesTest.java | 6 +-
.../apache/beam/sdk/transforms/ViewTest.java | 222 +-
.../beam/sdk/transforms/WithTimestampsTest.java | 8 +-
.../sdk/transforms/display/DisplayDataTest.java | 17 +
.../sdk/transforms/join/CoGroupByKeyTest.java | 10 +-
.../transforms/reflect/DoFnInvokersTest.java | 153 +-
.../DoFnSignaturesProcessElementTest.java | 2 +-
.../DoFnSignaturesSplittableDoFnTest.java | 117 +-
.../splittabledofn/OffsetRangeTrackerTest.java | 49 +-
.../windowing/CalendarWindowsTest.java | 33 +
.../transforms/windowing/FixedWindowsTest.java | 33 +
.../windowing/SlidingWindowsTest.java | 16 +-
.../sdk/transforms/windowing/TriggersTest.java | 100 -
.../sdk/transforms/windowing/WindowTest.java | 97 +-
.../sdk/transforms/windowing/WindowingTest.java | 10 +-
...mptAndTimeBoundedExponentialBackOffTest.java | 213 --
.../AttemptBoundedExponentialBackOffTest.java | 85 -
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 -
.../beam/sdk/util/GcpProjectUtilTest.java | 76 -
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 -
.../beam/sdk/util/GcsPathValidatorTest.java | 87 -
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 -----
.../sdk/util/IdentitySideInputWindowFn.java | 10 +-
.../IntervalBoundedExponentialBackOffTest.java | 100 -
.../org/apache/beam/sdk/util/NameUtilsTest.java | 12 +
.../org/apache/beam/sdk/util/PTupleTest.java | 40 -
.../apache/beam/sdk/util/PubsubClientTest.java | 189 -
.../beam/sdk/util/PubsubGrpcClientTest.java | 207 --
.../beam/sdk/util/PubsubJsonClientTest.java | 133 -
.../beam/sdk/util/PubsubTestClientTest.java | 114 -
.../beam/sdk/util/ReifyTimestampsTest.java | 109 +
.../org/apache/beam/sdk/util/ReshuffleTest.java | 84 +-
.../util/RetryHttpRequestInitializerTest.java | 290 --
.../beam/sdk/util/WindowingStrategiesTest.java | 91 -
.../beam/sdk/values/PCollectionListTest.java | 70 +-
.../beam/sdk/values/PCollectionTupleTest.java | 17 +-
.../org/apache/beam/sdk/values/PDoneTest.java | 4 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 50 +-
sdks/java/extensions/gcp-core/pom.xml | 222 ++
.../beam/sdk/options/BigQueryOptions.java | 32 +
.../options/CloudResourceManagerOptions.java | 40 +
.../org/apache/beam/sdk/options/GcpOptions.java | 227 ++
.../options/GcpPipelineOptionsRegistrar.java | 39 +
.../org/apache/beam/sdk/options/GcsOptions.java | 154 +
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 +
.../apache/beam/sdk/options/PubsubOptions.java | 36 +
.../apache/beam/sdk/options/package-info.java | 22 +
.../beam/sdk/testing/BigqueryMatcher.java | 256 ++
.../apache/beam/sdk/testing/package-info.java | 21 +
.../apache/beam/sdk/util/CredentialFactory.java | 29 +
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 +
.../beam/sdk/util/GcpCredentialFactory.java | 67 +
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 +
.../beam/sdk/util/GcsIOChannelFactory.java | 111 +
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 +
.../apache/beam/sdk/util/GcsPathValidator.java | 95 +
.../java/org/apache/beam/sdk/util/GcsUtil.java | 798 +++++
.../beam/sdk/util/NoopCredentialFactory.java | 68 +
.../sdk/util/NullCredentialInitializer.java | 62 +
.../apache/beam/sdk/util/TestCredential.java | 59 +
.../org/apache/beam/sdk/util/Transport.java | 178 +
.../org/apache/beam/sdk/util/package-info.java | 20 +
.../org/apache/beam/GcpCoreApiSurfaceTest.java | 62 +
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 +
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 +
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 +
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 +
.../beam/sdk/util/GcpProjectUtilTest.java | 76 +
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 +
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 +
.../beam/sdk/util/GcsPathValidatorTest.java | 87 +
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 +++++
.../util/RetryHttpRequestInitializerTest.java | 290 ++
sdks/java/extensions/jackson/pom.xml | 40 -
sdks/java/extensions/pom.xml | 1 +
sdks/java/extensions/sorter/pom.xml | 49 -
sdks/java/harness/pom.xml | 10 +-
.../beam/fn/harness/fake/FakeStepContext.java | 2 +-
.../control/ProcessBundleHandlerTest.java | 30 +-
sdks/java/io/common/pom.xml | 42 +
.../apache/beam/sdk/io/common/HashingFn.java | 109 +
.../sdk/io/common/IOTestPipelineOptions.java | 79 +
sdks/java/io/elasticsearch/pom.xml | 7 +
.../sdk/io/elasticsearch/ElasticsearchIO.java | 37 +-
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 9 +-
.../io/elasticsearch/ElasticsearchIOTest.java | 15 +-
.../elasticsearch/ElasticsearchTestDataSet.java | 16 +-
.../elasticsearch/ElasticsearchTestOptions.java | 46 -
sdks/java/io/google-cloud-platform/pom.xml | 93 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 225 ++
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 352 ++
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 3306 ++++--------------
.../io/gcp/bigquery/BigQueryQuerySource.java | 205 ++
.../io/gcp/bigquery/BigQueryServicesImpl.java | 2 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 205 ++
.../gcp/bigquery/BigQueryTableRowIterator.java | 3 +-
.../io/gcp/bigquery/BigQueryTableSource.java | 132 +
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 127 +
.../io/gcp/bigquery/GenerateShardedTable.java | 47 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 84 +
.../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 81 +
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 67 +
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 87 +
.../sdk/io/gcp/bigquery/StreamingInserts.java | 79 +
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 111 +
.../io/gcp/bigquery/StreamingWriteTables.java | 86 +
.../sdk/io/gcp/bigquery/TableDestination.java | 76 +
.../io/gcp/bigquery/TableDestinationCoder.java | 60 +
.../beam/sdk/io/gcp/bigquery/TableRowInfo.java | 34 +
.../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 68 +
.../sdk/io/gcp/bigquery/TableRowWriter.java | 91 +
.../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 62 +
.../sdk/io/gcp/bigquery/TransformingSource.java | 136 +
.../io/gcp/bigquery/WriteBundlesToFiles.java | 157 +
.../sdk/io/gcp/bigquery/WritePartition.java | 174 +
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 185 +
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 46 +
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 211 ++
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 107 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 398 ++-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 544 +++
.../sdk/io/gcp/pubsub/PubsubGrpcClient.java | 424 +++
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 1014 ++++++
.../sdk/io/gcp/pubsub/PubsubJsonClient.java | 319 ++
.../sdk/io/gcp/pubsub/PubsubTestClient.java | 436 +++
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 490 +++
.../io/gcp/pubsub/PubsubUnboundedSource.java | 1463 ++++++++
.../beam/sdk/io/gcp/pubsub/package-info.java | 24 +
.../beam/sdk/io/gcp/storage/GcsResourceId.java | 2 +-
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 8 +-
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 1812 ++++------
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 12 +-
.../io/gcp/bigquery/FakeBigQueryServices.java | 166 +
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 208 ++
.../sdk/io/gcp/bigquery/FakeJobService.java | 404 +++
.../sdk/io/gcp/bigquery/TableContainer.java | 61 +
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 20 +-
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 15 +-
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 252 +-
.../sdk/io/gcp/datastore/SplitQueryFnIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1ReadIT.java | 66 +-
.../sdk/io/gcp/datastore/V1TestOptions.java | 2 +-
.../sdk/io/gcp/pubsub/PubsubClientTest.java | 189 +
.../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java | 208 ++
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 189 +
.../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 139 +
.../sdk/io/gcp/pubsub/PubsubTestClientTest.java | 114 +
.../io/gcp/pubsub/PubsubUnboundedSinkTest.java | 188 +
.../gcp/pubsub/PubsubUnboundedSourceTest.java | 409 +++
.../io/hadoop/SerializableConfiguration.java | 3 +-
sdks/java/io/hadoop-input-format/README.md | 167 -
sdks/java/io/hadoop-input-format/pom.xml | 136 -
.../hadoop/inputformat/HadoopInputFormatIO.java | 941 -----
.../sdk/io/hadoop/inputformat/package-info.java | 23 -
.../ConfigurableEmployeeInputFormat.java | 131 -
.../sdk/io/hadoop/inputformat/Employee.java | 85 -
.../hadoop/inputformat/EmployeeInputFormat.java | 172 -
.../inputformat/HadoopInputFormatIOTest.java | 844 -----
.../ReuseObjectsEmployeeInputFormat.java | 176 -
.../hadoop/inputformat/TestEmployeeDataSet.java | 76 -
sdks/java/io/hadoop/README.md | 167 +
sdks/java/io/hadoop/input-format/pom.xml | 98 +
.../hadoop/inputformat/HadoopInputFormatIO.java | 844 +++++
.../sdk/io/hadoop/inputformat/package-info.java | 23 +
.../ConfigurableEmployeeInputFormat.java | 131 +
.../sdk/io/hadoop/inputformat/Employee.java | 85 +
.../hadoop/inputformat/EmployeeInputFormat.java | 172 +
.../inputformat/HadoopInputFormatIOTest.java | 851 +++++
.../ReuseObjectsEmployeeInputFormat.java | 176 +
.../hadoop/inputformat/TestEmployeeDataSet.java | 76 +
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 244 ++
.../inputformat/HIFIOWithElasticTest.java | 277 ++
.../HIFIOWithEmbeddedCassandraTest.java | 215 ++
.../custom/options/HIFTestOptions.java | 64 +
.../integration/tests/HIFIOCassandraIT.java | 173 +
.../integration/tests/HIFIOElasticIT.java | 215 ++
.../src/test/resources/cassandra.yaml | 1074 ++++++
sdks/java/io/hadoop/pom.xml | 64 +
sdks/java/io/hbase/pom.xml | 92 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 7 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 13 +-
.../src/test/resources/log4j-test.properties | 27 +
sdks/java/io/hdfs/README.md | 6 +-
sdks/java/io/hdfs/pom.xml | 41 -
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 29 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 16 +-
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 2 +-
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 23 +-
sdks/java/io/jdbc/pom.xml | 53 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 75 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 11 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 12 +-
.../beam/sdk/io/jdbc/JdbcTestDataSet.java | 12 +-
.../beam/sdk/io/jdbc/PostgresTestOptions.java | 60 -
.../kubernetes/postgres-pod-no-vol.yml | 32 -
.../kubernetes/postgres-service-public.yml | 28 -
.../jdbc/src/test/resources/kubernetes/setup.sh | 20 -
.../src/test/resources/kubernetes/teardown.sh | 20 -
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 4 +-
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 10 +-
sdks/java/io/kafka/pom.xml | 36 -
.../apache/beam/sdk/io/kafka/ConsumerSpEL.java | 43 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 70 +-
.../apache/beam/sdk/io/kafka/KafkaRecord.java | 15 +-
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 5 +
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 54 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 2 +
.../beam/sdk/io/kinesis/KinesisSource.java | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 14 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 4 +-
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 7 +-
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 5 -
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 4 +-
.../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 6 -
sdks/java/io/pom.xml | 3 +-
.../beam/sdk/transforms/FilterJava8Test.java | 8 +-
.../transforms/FlatMapElementsJava8Test.java | 10 +-
.../sdk/transforms/MapElementsJava8Test.java | 14 +-
.../beam/sdk/transforms/WithKeysJava8Test.java | 4 +-
.../sdk/transforms/WithTimestampsJava8Test.java | 4 +-
.../java/javadoc/dataflow-sdk-docs/package-list | 11 -
sdks/java/javadoc/overview.html | 2 +-
.../META-INF/maven/archetype-metadata.xml | 2 +-
.../META-INF/maven/archetype-metadata.xml | 2 +-
.../META-INF/maven/archetype-metadata.xml | 2 +-
sdks/java/pom.xml | 2 +-
sdks/python/.pylintrc | 6 +-
sdks/python/MANIFEST.in | 3 -
sdks/python/apache_beam/__init__.py | 7 +-
sdks/python/apache_beam/coders/coder_impl.py | 50 +-
sdks/python/apache_beam/coders/coders.py | 10 +-
sdks/python/apache_beam/coders/typecoders.py | 2 +-
sdks/python/apache_beam/error.py | 4 +
.../examples/complete/game/README.md | 69 +
.../examples/complete/game/__init__.py | 16 +
.../examples/complete/game/hourly_team_score.py | 294 ++
.../complete/game/hourly_team_score_test.py | 52 +
.../examples/complete/game/user_score.py | 217 ++
.../examples/complete/game/user_score_test.py | 49 +
.../examples/complete/top_wikipedia_sessions.py | 8 -
.../examples/cookbook/bigquery_tornadoes.py | 2 +-
.../cookbook/bigquery_tornadoes_it_test.py | 3 +
.../examples/cookbook/datastore_wordcount.py | 15 +-
.../examples/cookbook/group_with_coder.py | 6 +-
.../examples/cookbook/multiple_output_pardo.py | 47 +-
.../apache_beam/examples/snippets/snippets.py | 5 +-
.../examples/snippets/snippets_test.py | 36 +-
sdks/python/apache_beam/examples/wordcount.py | 48 +-
.../apache_beam/examples/wordcount_debugging.py | 4 -
.../apache_beam/examples/wordcount_it_test.py | 11 +-
sdks/python/apache_beam/internal/gcp/auth.py | 88 +-
.../apache_beam/internal/gcp/auth_test.py | 44 -
.../apache_beam/internal/gcp/json_value.py | 6 +
sdks/python/apache_beam/internal/pickler.py | 20 +-
sdks/python/apache_beam/io/__init__.py | 1 +
sdks/python/apache_beam/io/avroio.py | 12 +-
sdks/python/apache_beam/io/avroio_test.py | 14 +-
sdks/python/apache_beam/io/concat_source.py | 74 +-
.../python/apache_beam/io/concat_source_test.py | 12 +-
sdks/python/apache_beam/io/filebasedsource.py | 127 +-
.../apache_beam/io/filebasedsource_test.py | 43 +-
sdks/python/apache_beam/io/fileio.py | 638 +---
sdks/python/apache_beam/io/fileio_test.py | 166 +-
sdks/python/apache_beam/io/filesystem.py | 530 +++
sdks/python/apache_beam/io/filesystem_test.py | 213 ++
sdks/python/apache_beam/io/filesystems_util.py | 35 +
sdks/python/apache_beam/io/gcp/bigquery.py | 94 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 3 +-
.../io/gcp/datastore/v1/datastoreio.py | 24 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 4 +-
.../apache_beam/io/gcp/datastore/v1/helper.py | 18 +-
.../io/gcp/datastore/v1/query_splitter.py | 2 +-
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 243 ++
.../apache_beam/io/gcp/gcsfilesystem_test.py | 322 ++
sdks/python/apache_beam/io/gcp/gcsio.py | 33 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 48 +-
.../io/gcp/tests/bigquery_matcher.py | 3 +-
sdks/python/apache_beam/io/iobase.py | 20 +-
sdks/python/apache_beam/io/localfilesystem.py | 235 ++
.../apache_beam/io/localfilesystem_test.py | 189 +
sdks/python/apache_beam/io/range_trackers.py | 19 +-
sdks/python/apache_beam/io/source_test_utils.py | 79 +-
.../apache_beam/io/source_test_utils_test.py | 20 +-
sdks/python/apache_beam/io/textio.py | 23 +-
sdks/python/apache_beam/io/textio_test.py | 20 +-
sdks/python/apache_beam/io/tfrecordio.py | 26 +-
sdks/python/apache_beam/io/tfrecordio_test.py | 38 +-
sdks/python/apache_beam/metrics/cells.py | 28 +-
sdks/python/apache_beam/metrics/execution.py | 3 +-
sdks/python/apache_beam/metrics/metric.py | 9 +-
sdks/python/apache_beam/pipeline.py | 138 +-
sdks/python/apache_beam/pipeline_test.py | 31 +-
sdks/python/apache_beam/pvalue.py | 317 +-
sdks/python/apache_beam/pvalue_test.py | 33 -
.../runners/api/beam_runner_api_pb2.py | 272 +-
sdks/python/apache_beam/runners/common.pxd | 2 +-
sdks/python/apache_beam/runners/common.py | 15 +-
.../runners/dataflow/dataflow_metrics.py | 86 +-
.../runners/dataflow/dataflow_metrics_test.py | 66 +-
.../runners/dataflow/dataflow_runner.py | 164 +-
.../runners/dataflow/dataflow_runner_test.py | 67 +-
.../runners/dataflow/internal/apiclient.py | 63 +-
.../runners/dataflow/internal/apiclient_test.py | 12 +-
.../runners/dataflow/internal/dependency.py | 30 +-
.../runners/dataflow/native_io/iobase_test.py | 173 +
.../runners/dataflow/test_dataflow_runner.py | 26 +-
.../runners/direct/bundle_factory.py | 17 +-
sdks/python/apache_beam/runners/direct/clock.py | 9 +-
.../consumer_tracking_pipeline_visitor.py | 11 +-
.../consumer_tracking_pipeline_visitor_test.py | 28 +-
.../apache_beam/runners/direct/direct_runner.py | 22 +-
.../runners/direct/evaluation_context.py | 77 +-
.../apache_beam/runners/direct/executor.py | 28 +-
.../runners/direct/transform_evaluator.py | 116 +-
.../runners/direct/transform_result.py | 45 +-
.../runners/direct/watermark_manager.py | 4 +-
sdks/python/apache_beam/runners/runner.py | 16 +-
.../apache_beam/tests/pipeline_verifiers.py | 46 +-
.../tests/pipeline_verifiers_test.py | 47 +-
sdks/python/apache_beam/transforms/combiners.py | 62 +-
.../apache_beam/transforms/combiners_test.py | 4 +-
sdks/python/apache_beam/transforms/core.py | 204 +-
.../apache_beam/transforms/create_test.py | 121 +
sdks/python/apache_beam/transforms/display.py | 4 +-
.../apache_beam/transforms/display_test.py | 36 +
.../python/apache_beam/transforms/ptransform.py | 42 +-
.../apache_beam/transforms/ptransform_test.py | 41 +-
.../python/apache_beam/transforms/sideinputs.py | 143 +-
.../apache_beam/transforms/sideinputs_test.py | 99 +-
sdks/python/apache_beam/transforms/trigger.py | 40 +-
.../apache_beam/transforms/trigger_test.py | 6 +-
sdks/python/apache_beam/transforms/util.py | 20 +-
sdks/python/apache_beam/transforms/util_test.py | 50 +
sdks/python/apache_beam/transforms/window.py | 9 +-
.../apache_beam/transforms/window_test.py | 8 +-
sdks/python/apache_beam/typehints/decorators.py | 26 +-
.../apache_beam/typehints/trivial_inference.py | 30 +-
.../typehints/trivial_inference_test.py | 3 +-
sdks/python/apache_beam/typehints/typecheck.py | 11 +-
sdks/python/apache_beam/typehints/typehints.py | 66 +-
.../apache_beam/typehints/typehints_test.py | 7 +-
.../apache_beam/utils/annotations_test.py | 2 +-
sdks/python/apache_beam/utils/counters.py | 5 +-
sdks/python/apache_beam/utils/path.py | 3 +-
.../apache_beam/utils/pipeline_options.py | 106 +-
.../apache_beam/utils/pipeline_options_test.py | 52 +-
sdks/python/apache_beam/utils/proto_utils.py | 17 +-
sdks/python/apache_beam/utils/retry.py | 20 +-
sdks/python/apache_beam/utils/test_stream.py | 163 +
.../apache_beam/utils/test_stream_test.py | 82 +
sdks/python/apache_beam/utils/timestamp.py | 6 +-
sdks/python/apache_beam/utils/urns.py | 2 +
sdks/python/apache_beam/utils/value_provider.py | 103 +
.../apache_beam/utils/value_provider_test.py | 145 +
sdks/python/apache_beam/utils/windowed_value.py | 17 +-
sdks/python/apache_beam/version.py | 36 -
sdks/python/findSupportedPython.groovy | 80 +
sdks/python/generate_pydoc.sh | 2 +-
sdks/python/pom.xml | 31 +-
sdks/python/run_postcommit.sh | 35 +-
sdks/python/run_pylint.sh | 27 +-
sdks/python/setup.py | 19 +-
sdks/python/tox.ini | 18 +-
1177 files changed, 69436 insertions(+), 54855 deletions(-)
----------------------------------------------------------------------
[11/50] [abbrv] beam git commit: [BEAM-972] Add unit tests to
Gearpump runner
Posted by ke...@apache.org.
[BEAM-972] Add unit tests to Gearpump runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/eb0d333d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/eb0d333d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/eb0d333d
Branch: refs/heads/master
Commit: eb0d333df23624f54aae2abb8d7c7873f8ed2a7a
Parents: 555842a
Author: huafengw <fv...@gmail.com>
Authored: Tue Mar 21 19:45:10 2017 +0800
Committer: huafengw <fv...@gmail.com>
Committed: Thu Mar 23 19:52:11 2017 +0800
----------------------------------------------------------------------
examples/java/pom.xml | 12 +++
pom.xml | 6 ++
runners/gearpump/README.md | 41 ++++++++-
runners/gearpump/pom.xml | 2 -
.../gearpump/GearpumpRunnerRegistrar.java | 4 +-
.../translators/WindowAssignTranslator.java | 2 +-
.../gearpump/translators/io/ValuesSource.java | 2 -
.../gearpump/GearpumpRunnerRegistrarTest.java | 55 ++++++++++++
.../runners/gearpump/PipelineOptionsTest.java | 73 ++++++++++++++++
.../translators/io/GearpumpSourceTest.java | 90 ++++++++++++++++++++
.../gearpump/translators/io/ValueSoureTest.java | 82 ++++++++++++++++++
11 files changed, 362 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index ed4a1d4..0a6d8fe 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -87,6 +87,18 @@
</dependencies>
</profile>
+ <!-- Include the Apache Gearpump (incubating) runner with -P gearpump-runner -->
+ <profile>
+ <id>gearpump-runner</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gearpump</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+
<!-- Include the Apache Flink runner with -P flink-runner -->
<profile>
<id>flink-runner</id>
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c3b8476..2cdb09d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -475,6 +475,12 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gearpump</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-examples-java</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/runners/gearpump/README.md
----------------------------------------------------------------------
diff --git a/runners/gearpump/README.md b/runners/gearpump/README.md
index ad043fa..e8ce794 100644
--- a/runners/gearpump/README.md
+++ b/runners/gearpump/README.md
@@ -19,4 +19,43 @@
## Gearpump Beam Runner
-The Gearpump Beam runner allows users to execute pipelines written using the Apache Beam programming API with Apache Gearpump (incubating) as an execution engine.
\ No newline at end of file
+The Gearpump Beam runner allows users to execute pipelines written using the Apache Beam programming API with Apache Gearpump (incubating) as an execution engine.
+
+##Getting Started
+
+The following shows how to run the WordCount example that is provided with the source code on Beam.
+
+###Installing Beam
+
+To get the latest version of Beam with Gearpump-Runner, first clone the Beam repository:
+
+```
+git clone https://github.com/apache/beam
+git checkout gearpump-runner
+```
+
+Then switch to the newly created directory and run Maven to build the Apache Beam:
+
+```
+cd beam
+mvn clean install -DskipTests
+```
+
+Now Apache Beam and the Gearpump Runner are installed in your local Maven repository.
+
+###Running Wordcount Example
+
+Download something to count:
+
+```
+curl http://www.gutenberg.org/cache/epub/1128/pg1128.txt > /tmp/kinglear.txt
+```
+
+Run the pipeline, using the Gearpump runner:
+
+```
+cd examples/java
+mvn exec:java -Dexec.mainClass=org.apache.beam.examples.WordCount -Dexec.args="--inputFile=/tmp/kinglear.txt --output=/tmp/wordcounts.txt --runner=TestGearpumpRunner" -Pgearpump-runner
+```
+
+Once completed, check the output file /tmp/wordcounts.txt-00000-of-00001
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index 9a6a432..a691801 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -99,13 +99,11 @@
<groupId>org.apache.gearpump</groupId>
<artifactId>gearpump-streaming_2.11</artifactId>
<version>${gearpump.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.gearpump</groupId>
<artifactId>gearpump-core_2.11</artifactId>
<version>${gearpump.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
index b77e1e3..3183d45 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrar.java
@@ -44,7 +44,9 @@ public class GearpumpRunnerRegistrar {
@Override
public Iterable<Class<? extends PipelineRunner<?>>> getPipelineRunners() {
- return ImmutableList.<Class<? extends PipelineRunner<?>>>of(TestGearpumpRunner.class);
+ return ImmutableList.<Class<? extends PipelineRunner<?>>>of(
+ GearpumpRunner.class,
+ TestGearpumpRunner.class);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
index fe6015a..29d8f02 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
@@ -39,7 +39,7 @@ import org.joda.time.Instant;
* {@link Window.Bound} is translated to Gearpump flatMap function.
*/
@SuppressWarnings("unchecked")
-public class WindowAssignTranslator<T> implements TransformTranslator<Window.Assign<T>> {
+public class WindowAssignTranslator<T> implements TransformTranslator<Window.Assign<T>> {
private static final long serialVersionUID = -964887482120489061L;
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
index e0488cd..ccd5cdf 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
@@ -110,8 +110,6 @@ public class ValuesSource<T> extends UnboundedSource<T, UnboundedSource.Checkpoi
this.source = source;
}
-
-
@Override
public boolean start() throws IOException {
if (null == iterator) {
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrarTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrarTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrarTest.java
new file mode 100644
index 0000000..9a01d20
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/GearpumpRunnerRegistrarTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.junit.Test;
+
+/**
+ * Tests for {@link GearpumpRunnerRegistrar}.
+ */
+public class GearpumpRunnerRegistrarTest {
+
+ @Test
+ public void testFullName() {
+ String[] args =
+ new String[] {String.format("--runner=%s", GearpumpRunner.class.getName())};
+ PipelineOptions opts = PipelineOptionsFactory.fromArgs(args).create();
+ assertEquals(opts.getRunner(), GearpumpRunner.class);
+ }
+
+ @Test
+ public void testClassName() {
+ String[] args =
+ new String[] {String.format("--runner=%s", GearpumpRunner.class.getSimpleName())};
+ PipelineOptions opts = PipelineOptionsFactory.fromArgs(args).create();
+ assertEquals(opts.getRunner(), GearpumpRunner.class);
+ }
+
+ @Test
+ public void testOptions() {
+ assertEquals(
+ ImmutableList.of(GearpumpPipelineOptions.class),
+ new GearpumpRunnerRegistrar.Options().getPipelineOptions());
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/PipelineOptionsTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/PipelineOptionsTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/PipelineOptionsTest.java
new file mode 100644
index 0000000..994856b
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/PipelineOptionsTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.gearpump;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Maps;
+import com.typesafe.config.Config;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.gearpump.cluster.ClusterConfig;
+import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
+import org.junit.Test;
+
+/**
+ * Tests for {@link GearpumpPipelineOptions}.
+ */
+public class PipelineOptionsTest {
+
+ @Test
+ public void testIgnoredFieldSerialization() throws IOException {
+ String appName = "forTest";
+ Map<String, String> serializers = Maps.newHashMap();
+ serializers.put("classA", "SerializerA");
+ GearpumpPipelineOptions options = PipelineOptionsFactory.create()
+ .as(GearpumpPipelineOptions.class);
+ Config config = ClusterConfig.master(null);
+ EmbeddedCluster cluster = new EmbeddedCluster(config);
+ options.setSerializers(serializers);
+ options.setApplicationName(appName);
+ options.setEmbeddedCluster(cluster);
+ options.setParallelism(10);
+
+ byte[] serializedOptions = serialize(options);
+ GearpumpPipelineOptions deserializedOptions = new ObjectMapper()
+ .readValue(serializedOptions, PipelineOptions.class).as(GearpumpPipelineOptions.class);
+
+ assertNull(deserializedOptions.getEmbeddedCluster());
+ assertNull(deserializedOptions.getSerializers());
+ assertEquals(10, deserializedOptions.getParallelism());
+ assertEquals(appName, deserializedOptions.getApplicationName());
+ }
+
+ private byte[] serialize(Object obj) {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ new ObjectMapper().writeValue(baos, obj);
+ return baos.toByteArray();
+ } catch (Exception e) {
+ throw new RuntimeException("Couldn't serialize PipelineOptions.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
new file mode 100644
index 0000000..af5a1d2
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.gearpump.translators.io;
+
+import com.google.common.collect.Lists;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.Source;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.gearpump.Message;
+import org.apache.gearpump.streaming.source.Watermark;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for {@link GearpumpSource}.
+ */
+public class GearpumpSourceTest {
+ private static final List<TimestampedValue<String>> TEST_VALUES = Lists.newArrayList(
+ TimestampedValue.of("a", new org.joda.time.Instant(Long.MIN_VALUE)),
+ TimestampedValue.of("b", new org.joda.time.Instant(0)),
+ TimestampedValue.of("c", new org.joda.time.Instant(53)),
+ TimestampedValue.of("d", new org.joda.time.Instant(Long.MAX_VALUE - 1))
+ );
+
+ private static class SourceForTest<T> extends GearpumpSource<T> {
+ private ValuesSource<T> valuesSource;
+
+ SourceForTest(PipelineOptions options, ValuesSource<T> valuesSource) {
+ super(options);
+ this.valuesSource = valuesSource;
+ }
+
+ @Override
+ protected Source.Reader<T> createReader(PipelineOptions options) throws IOException {
+ return this.valuesSource.createReader(options, null);
+ }
+ }
+
+ @Test
+ public void testGearpumpSource() {
+ GearpumpPipelineOptions options = PipelineOptionsFactory.create()
+ .as(GearpumpPipelineOptions.class);
+ ValuesSource<TimestampedValue<String>> valuesSource = new ValuesSource<>(TEST_VALUES,
+ TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of()));
+ SourceForTest<TimestampedValue<String>> sourceForTest =
+ new SourceForTest<>(options, valuesSource);
+ sourceForTest.open(null, Instant.EPOCH);
+
+ for (TimestampedValue<String> value: TEST_VALUES) {
+ // Check the watermark first since the Source will advance when it's opened
+ Instant expectedWaterMark = TranslatorUtils.jodaTimeToJava8Time(value.getTimestamp());
+ Assert.assertEquals(expectedWaterMark, sourceForTest.getWatermark());
+
+ Message expectedMsg = Message.apply(
+ WindowedValue.timestampedValueInGlobalWindow(value, value.getTimestamp()),
+ value.getTimestamp().getMillis());
+ Message message = sourceForTest.read();
+ Assert.assertEquals(expectedMsg, message);
+ }
+
+ Assert.assertNull(sourceForTest.read());
+ Assert.assertEquals(Watermark.MAX(), sourceForTest.getWatermark());
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/eb0d333d/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java
new file mode 100644
index 0000000..8c50703
--- /dev/null
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/ValueSoureTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.gearpump.translators.io;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigValueFactory;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
+import org.apache.beam.runners.gearpump.GearpumpRunner;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.gearpump.cluster.ClusterConfig;
+import org.apache.gearpump.cluster.embedded.EmbeddedCluster;
+import org.apache.gearpump.util.Constants;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for {@link ValuesSource}.
+ */
+public class ValueSoureTest {
+
+ @Test
+ public void testValueSource() {
+ GearpumpPipelineOptions options = PipelineOptionsFactory.create()
+ .as(GearpumpPipelineOptions.class);
+ Config config = ClusterConfig.master(null);
+ config = config.withValue(Constants.APPLICATION_TOTAL_RETRIES(),
+ ConfigValueFactory.fromAnyRef(0));
+ EmbeddedCluster cluster = new EmbeddedCluster(config);
+ cluster.start();
+
+ options.setEmbeddedCluster(cluster);
+ options.setRunner(GearpumpRunner.class);
+ options.setParallelism(1);
+ Pipeline p = Pipeline.create(options);
+ List<String> values = Lists.newArrayList("1", "2", "3", "4", "5");
+ ValuesSource<String> source = new ValuesSource<>(values, StringUtf8Coder.of());
+ p.apply(Read.from(source))
+ .apply(ParDo.of(new ResultCollector()));
+
+ p.run().waitUntilFinish();
+ cluster.stop();
+
+ Assert.assertEquals(Sets.newHashSet(values), ResultCollector.RESULTS);
+ }
+
+ private static class ResultCollector extends DoFn<Object, Void> {
+ private static final Set<Object> RESULTS = Collections.synchronizedSet(new HashSet<>());
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ RESULTS.add(c.element());
+ }
+ }
+}
[39/50] [abbrv] beam git commit: Upgrade to gearpump 0.8.4
Posted by ke...@apache.org.
Upgrade to gearpump 0.8.4
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/22068274
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/22068274
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/22068274
Branch: refs/heads/master
Commit: 22068274428666da93c793a170810ddb42755704
Parents: c2d3fbc
Author: manuzhang <ow...@gmail.com>
Authored: Fri Jul 7 16:41:13 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Fri Jul 7 22:08:37 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/22068274/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index 3c98d5e..54c8d5c 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -43,7 +43,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <gearpump.version>0.8.4-SNAPSHOT</gearpump.version>
+ <gearpump.version>0.8.4</gearpump.version>
</properties>
<profiles>
@@ -58,7 +58,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<executions>
<execution>
- <id>runnable-on-service-tests</id>
+ <id>validates-runner-tests</id>
<phase>integration-test</phase>
<goals>
<goal>test</goal>
[34/50] [abbrv] beam git commit: Remove unused codes
Posted by ke...@apache.org.
Remove unused codes
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b21fa04f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b21fa04f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b21fa04f
Branch: refs/heads/master
Commit: b21fa04f28f5e7be87b41d5478c4c6decd87e433
Parents: 11caa97
Author: manuzhang <ow...@gmail.com>
Authored: Fri Jun 23 10:12:19 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Fri Jun 23 10:12:19 2017 +0800
----------------------------------------------------------------------
.../apache/beam/runners/gearpump/translators/io/GearpumpSource.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/b21fa04f/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
index 558eb0d..daa8c81 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
@@ -46,7 +46,6 @@ public abstract class GearpumpSource<T> implements DataSource {
private Source.Reader<T> reader;
private boolean available = false;
- private long count = 0L;
GearpumpSource(PipelineOptions options) {
try {
[42/50] [abbrv] beam git commit: Revert accidental changes to
sdks/java/pom.xml
Posted by ke...@apache.org.
Revert accidental changes to sdks/java/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/e655f53c
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/e655f53c
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/e655f53c
Branch: refs/heads/master
Commit: e655f53c2e732bcd082449ba8d0b551f417e8aaa
Parents: 1ce60b4
Author: manuzhang <ow...@gmail.com>
Authored: Fri Jul 21 14:27:37 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Fri Jul 21 14:29:46 2017 +0800
----------------------------------------------------------------------
sdks/java/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/e655f53c/sdks/java/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/pom.xml b/sdks/java/pom.xml
index 3c1b6ec..3144193 100644
--- a/sdks/java/pom.xml
+++ b/sdks/java/pom.xml
@@ -38,7 +38,7 @@
<module>build-tools</module> -->
<module>core</module>
<module>io</module>
- <!--<module>maven-archetypes</module>-->
+ <module>maven-archetypes</module>
<module>extensions</module>
<!-- javadoc runs directly from the root parent as the last module
in the build to be able to capture runner-specific javadoc.
[48/50] [abbrv] beam git commit: Remove gearpump-runner in precommit
tests
Posted by ke...@apache.org.
Remove gearpump-runner in precommit tests
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/bf154835
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/bf154835
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/bf154835
Branch: refs/heads/master
Commit: bf15483566ea759d2bc4af14173927402277738b
Parents: 18daed9
Author: manuzhang <ow...@gmail.com>
Authored: Thu Jul 27 12:57:28 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Thu Jul 27 12:57:28 2017 +0800
----------------------------------------------------------------------
.../job_beam_PreCommit_Java_MavenInstall.groovy | 2 +-
examples/java/pom.xml | 30 --------------------
2 files changed, 1 insertion(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/bf154835/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
index 7117f12..bc130ec 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
@@ -41,5 +41,5 @@ mavenJob('beam_PreCommit_Java_MavenInstall') {
common_job_properties.setPreCommit(delegate, 'Maven clean install')
// Maven goals for this job.
- goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner,gearpump-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report')
+ goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report')
}
http://git-wip-us.apache.org/repos/asf/beam/blob/bf154835/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 6a1ce99..ae64a79 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -155,9 +155,6 @@
<!-- Custom set up for our Jenkins precommit tests that comment on GitHub -->
<profile>
<id>jenkins-precommit</id>
- <activation>
- <jdk>[1.8,)</jdk>
- </activation>
<properties>
<skipITs>false</skipITs>
<skipDefaultIT>true</skipDefaultIT>
@@ -178,39 +175,12 @@
<version>${apex.kryo.version}</version>
<scope>runtime</scope>
</dependency>
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-runners-gearpump</artifactId>
- <scope>runtime</scope>
- </dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
- <executions>
- <execution>
- <id>enforce</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <enforceBytecodeVersion>
- <maxJdkVersion>1.8</maxJdkVersion>
- </enforceBytecodeVersion>
- <requireJavaVersion>
- <version>[1.8,)</version>
- </requireJavaVersion>
- </rules>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<useManifestOnlyJar>false</useManifestOnlyJar>
[29/50] [abbrv] beam git commit: This closes #3292: Merge master into
gearpump-runner
Posted by ke...@apache.org.
This closes #3292: Merge master into gearpump-runner
Adjustments to gearpump-runner:
Fix side input handling in DoFnFunction
Respect WindowFn#getOutputTime in gearpump-runner
Activate Gearpump local-validates-runner-tests in precommit
Update against master changes
Merged from master:
[BEAM-972] Add more unit test to Gearpump runner
[BEAM-972] Add unit tests to Gearpump runner
[BEAM-79] Fix gearpump-runner merge conflicts and test failure
enable ParDoTest
[BEAM-79] Add SideInput support for GearpumpRunner
[BEAM-79] Support merging windows in GearpumpRunner
[BEAM-79] Fix PostCommit test confs for Gearpump runner
note thread is interrupted on InterruptedException
Remove cache for Gearpump on travis
reduce timeout to wait for result
fix ParDo.BoundMulti translation
return encoded key for GroupByKey translation
support OutputTimeFn
update to latest gearpump dsl function interface
fix group by window
activate ROS on Gearpump by default
update ROS configurations
[BEAM-1180] Implement GearpumpPipelineResult
[BEAM-79] Upgrade to beam-0.5.0-incubating-SNAPSHOT
[BEAM-79] Update to latest Gearpump API
Fix NoOpAggregatorFactory
Remove print to stdout
Skip window assignment when windows don't change
Add Window.Bound translator
Upgrade Gearpump version
[BEAM-79] fix gearpump runner build failure
[BEAM-79] update GearpumpPipelineResult
[BEAM-79] Port Gearpump runner from OldDoFn to new DoFn
upgrade gearpump-runner to 0.4.0-incubating-SNAPSHOT
remove "pipeline" in runner name
post-merge fix
[BEAM-79] fix integration-test failure
fix import order
!fixup Minor javadoc clean-up
Added even more javadoc to TextIO#withHeader and TextIO#withFooter (2).
Added even more javadoc to TextIO#withHeader and TextIO#withFooter.
Added javadoc to TextIO#withHeader and TextIO#withFooter.
Reverted header and footer to be of type String.
Revised according to comments following a code review.
Add header/footer support to TextIO.Write
[BEAM-242] Enable and fix checkstyle in Flink runner examples
Remove timeout in JAXBCoderTest
Be more accepting in UnboundedReadDeduplicatorTest
BigQuery: limit max job polling time to 1 minute
[BEAM-242] Enable checkstyle and fix checkstyle errors in Flink runner
[BEAM-456] Add MongoDbIO
FluentBackoff: a replacement for a variety of custom backoff implementations
Remove the DataflowRunner instructions from examples
Put classes in runners-core package into runners.core namespace
Delegate populateDipslayData to wrapped combineFn's
Fixed Combine display data
Cloud Datastore naming clean-up
DatastoreIO SplitQueryFn integration test
Add Latest CombineFn and PTransforms
Remove empty unused method in TestStreamEvaluatorFactory
Test that multiple instances of TestStream are supported
Correct some accidental renames
Fix condition in FlinkStreamingPipelineTranslator
Address comments of Flink Side-Input PR
[BEAM-569] Define maxNumRecords default value to Long.MAX_VALUE in JmsIO
Add LeaderBoardTest
take advantage of setup/teardown for KafkaWriter
Returned KafkaIO getWatermark log line in debug mode
[BEAM-572] Remove Spark Reference in WordCount
Update Dataflow Container Version
[BEAM-313] Provide a context for SparkRunner
DataflowRunner: get PBegin from PInput
[BEAM-592] Fix SparkRunner Dependency Problem in WordCount
Fix javadoc in Kinesis
Organize imports in Kinesis
kinesis: a connector for Amazon Kinesis
[BEAM-589] Fixing IO.Read transformation
Query latest timestamp
travis.yml: disable updating snapshots
Added support for reporting aggregator values to Spark sinks
[BEAM-294] Rename dataflow references to beam
Modified BigtableIO to use DoFn setup/tearDown methods instead of startBundle/finishBundle
checkstyle: prohibit API client repackaged Guava
Make WriteTest more resilient to Randomness
Update DoFn javadocs to remove references to OldDoFn and Dataflow
[BEAM-545] Promote JobName to PipelineOptions
Move the samples data to gs://apache-beam-samples/
Cleanup some javadoc that referring Dataflow
BigQueryIO.Write: raise size limit to 11 TiB
Optimize imports
Update checkstyle.xml to put all imports in one group
Fix Exception Unwrapping in TestFlinkRunner
Make ParDoLifecycleTest Serializable to Fix Test with TupleTag
Use AllPanes as the PaneExtractor in IterableAssert
Fix combine tests with Accumulation Mode
Enable Flink Streaming Runner RunnableOnService tests
Fix Flink Runner Pom for Batch RunnableOnService tests
Don't Suppress Throwable in PAssert in Streaming Mode
...
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/559e3c34
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/559e3c34
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/559e3c34
Branch: refs/heads/master
Commit: 559e3c341f879cc06f823363ed0fa4c1eab8a6a5
Parents: 1597f3c 7653e7e
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Jun 12 09:48:29 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Jun 12 09:48:29 2017 -0700
----------------------------------------------------------------------
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 2 +-
examples/java/pom.xml | 12 +
pom.xml | 6 +
runners/gearpump/README.md | 61 +++
runners/gearpump/pom.xml | 281 ++++++++++
.../gearpump/GearpumpPipelineOptions.java | 67 +++
.../gearpump/GearpumpPipelineResult.java | 109 ++++
.../gearpump/GearpumpPipelineTranslator.java | 524 +++++++++++++++++++
.../beam/runners/gearpump/GearpumpRunner.java | 116 ++++
.../gearpump/GearpumpRunnerRegistrar.java | 64 +++
.../runners/gearpump/TestGearpumpRunner.java | 63 +++
.../beam/runners/gearpump/package-info.java | 22 +
...CreateGearpumpPCollectionViewTranslator.java | 46 ++
.../FlattenPCollectionsTranslator.java | 83 +++
.../translators/GroupByKeyTranslator.java | 253 +++++++++
.../translators/ParDoMultiOutputTranslator.java | 103 ++++
.../translators/ReadBoundedTranslator.java | 46 ++
.../translators/ReadUnboundedTranslator.java | 48 ++
.../translators/TransformTranslator.java | 30 ++
.../translators/TranslationContext.java | 104 ++++
.../translators/WindowAssignTranslator.java | 103 ++++
.../translators/functions/DoFnFunction.java | 194 +++++++
.../translators/functions/package-info.java | 22 +
.../translators/io/BoundedSourceWrapper.java | 45 ++
.../gearpump/translators/io/GearpumpSource.java | 121 +++++
.../translators/io/UnboundedSourceWrapper.java | 46 ++
.../gearpump/translators/io/ValuesSource.java | 173 ++++++
.../gearpump/translators/io/package-info.java | 22 +
.../gearpump/translators/package-info.java | 22 +
.../translators/utils/DoFnRunnerFactory.java | 81 +++
.../translators/utils/NoOpStepContext.java | 41 ++
.../translators/utils/TranslatorUtils.java | 198 +++++++
.../translators/utils/package-info.java | 22 +
.../gearpump/GearpumpRunnerRegistrarTest.java | 55 ++
.../runners/gearpump/PipelineOptionsTest.java | 73 +++
...teGearpumpPCollectionViewTranslatorTest.java | 57 ++
.../FlattenPCollectionsTranslatorTest.java | 149 ++++++
.../translators/GroupByKeyTranslatorTest.java | 152 ++++++
.../translators/ReadBoundedTranslatorTest.java | 70 +++
.../ReadUnboundedTranslatorTest.java | 70 +++
.../translators/WindowAssignTranslatorTest.java | 110 ++++
.../translators/io/GearpumpSourceTest.java | 90 ++++
.../gearpump/translators/io/ValueSoureTest.java | 79 +++
.../translators/utils/TranslatorUtilsTest.java | 74 +++
runners/pom.xml | 9 +
sdks/java/pom.xml | 2 +-
46 files changed, 4118 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[46/50] [abbrv] beam git commit: Require java 8 for precommit tests
and add gearpump-runner
Posted by ke...@apache.org.
Require java 8 for precommit tests and add gearpump-runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/a282165c
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/a282165c
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/a282165c
Branch: refs/heads/master
Commit: a282165c64ef7b5812fa9e44afd94777211a8707
Parents: d9bb4cb
Author: manuzhang <ow...@gmail.com>
Authored: Wed Jul 26 20:26:05 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Wed Jul 26 20:26:05 2017 +0800
----------------------------------------------------------------------
.../jenkins/job_beam_PreCommit_Java_MavenInstall.groovy | 2 +-
examples/java/pom.xml | 8 ++++++++
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/a282165c/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
index bc130ec..7117f12 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
@@ -41,5 +41,5 @@ mavenJob('beam_PreCommit_Java_MavenInstall') {
common_job_properties.setPreCommit(delegate, 'Maven clean install')
// Maven goals for this job.
- goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report')
+ goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner,gearpump-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report')
}
http://git-wip-us.apache.org/repos/asf/beam/blob/a282165c/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index ae64a79..bd18889 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -155,6 +155,9 @@
<!-- Custom set up for our Jenkins precommit tests that comment on GitHub -->
<profile>
<id>jenkins-precommit</id>
+ <activation>
+ <jdk>[1.8,)</jdk>
+ </activation>
<properties>
<skipITs>false</skipITs>
<skipDefaultIT>true</skipDefaultIT>
@@ -175,6 +178,11 @@
<version>${apex.kryo.version}</version>
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gearpump</artifactId>
+ <scope>runtime</scope>
+ </dependency>
</dependencies>
<build>
[15/50] [abbrv] beam git commit: Merge branch 'master' of
https://github.com/apache/incubator-beam into gearpump-runner
Posted by ke...@apache.org.
Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
# Conflicts:
# .test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
# runners/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/46c41fce
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/46c41fce
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/46c41fce
Branch: refs/heads/master
Commit: 46c41fce0e130bdd718c0d3ce7c31a564524bae6
Parents: ebbb613 104f982
Author: manuzhang <ow...@gmail.com>
Authored: Thu Apr 20 20:59:22 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Thu Apr 20 20:59:22 2017 +0800
----------------------------------------------------------------------
.github/PULL_REQUEST_TEMPLATE.md | 2 +-
.gitignore | 3 +
.jenkins/common_job_properties.groovy | 223 --
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 -
...ostCommit_Java_RunnableOnService_Apex.groovy | 47 -
...ommit_Java_RunnableOnService_Dataflow.groovy | 45 -
...stCommit_Java_RunnableOnService_Flink.groovy | 44 -
...ommit_Java_RunnableOnService_Gearpump.groovy | 49 -
...stCommit_Java_RunnableOnService_Spark.groovy | 44 -
.../job_beam_PostCommit_Python_Verify.groovy | 43 -
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 -
.../job_beam_PreCommit_Website_Stage.groovy | 80 -
.jenkins/job_beam_PreCommit_Website_Test.groovy | 65 -
.../job_beam_Release_NightlySnapshot.groovy | 45 -
.jenkins/job_seed.groovy | 53 -
.../jenkins/common_job_properties.groovy | 261 ++
.../job_beam_PerformanceTests_Dataflow.groovy | 43 +
.../job_beam_PerformanceTests_JDBC.groovy | 60 +
.../job_beam_PerformanceTests_Spark.groovy | 44 +
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 +
..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 +
...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 +
...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 +
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 +
...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 +
.../job_beam_PostCommit_Python_Verify.groovy | 55 +
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 +
.../job_beam_PreCommit_Website_Stage.groovy | 80 +
.../job_beam_PreCommit_Website_Test.groovy | 65 +
.../job_beam_Release_NightlySnapshot.groovy | 45 +
.test-infra/jenkins/job_seed.groovy | 53 +
.../cassandra-service-for-local-dev.yaml | 28 +
.../cassandra-svc-statefulset.yaml | 114 +
.../LargeITCluster/cassandra-svc-temp.yaml | 74 +
.../cassandra/LargeITCluster/data-load.sh | 122 +
.../cassandra/LargeITCluster/show_health.sh | 47 +
.../cassandra/LargeITCluster/start-up.sh | 22 +
.../cassandra/LargeITCluster/teardown.sh | 25 +
.../cassandra-service-for-local-dev.yaml | 30 +
.../SmallITCluster/cassandra-svc-rc.yaml | 74 +
.../cassandra/SmallITCluster/data-load.sh | 86 +
.../cassandra/SmallITCluster/show_health.sh | 47 +
.../cassandra/SmallITCluster/start-up.sh | 23 +
.../cassandra/SmallITCluster/teardown.sh | 22 +
.../kubernetes/cassandra/data-load-setup.sh | 29 +
.../elasticsearch-service-for-local-dev.yaml | 33 +
.../es-services-deployments.yaml | 258 ++
.../LargeProductionCluster/start-up.sh | 22 +
.../LargeProductionCluster/teardown.sh | 21 +
.../elasticsearch-service-for-local-dev.yaml | 34 +
.../SmallITCluster/elasticsearch-svc-rc.yaml | 96 +
.../elasticsearch/SmallITCluster/start-up.sh | 23 +
.../elasticsearch/SmallITCluster/teardown.sh | 21 +
.../kubernetes/elasticsearch/data-load-setup.sh | 26 +
.../kubernetes/elasticsearch/data-load.sh | 33 +
.../kubernetes/elasticsearch/es_test_data.py | 299 ++
.../kubernetes/elasticsearch/show-health.sh | 33 +
.../postgres/postgres-service-for-local-dev.yml | 28 +
.test-infra/kubernetes/postgres/postgres.yml | 56 +
.test-infra/travis/README.md | 23 +
.test-infra/travis/settings.xml | 33 +
.test-infra/travis/test_wordcount.sh | 125 +
.travis.yml | 7 +-
.travis/README.md | 23 -
.travis/settings.xml | 33 -
.travis/test_wordcount.sh | 125 -
README.md | 3 +
examples/java/README.md | 61 +-
examples/java/pom.xml | 32 +-
.../beam/examples/DebuggingWordCount.java | 2 +-
.../apache/beam/examples/WindowedWordCount.java | 34 +-
.../examples/common/WriteOneFilePerWindow.java | 91 +
.../examples/common/WriteWindowedFilesDoFn.java | 77 -
.../beam/examples/complete/AutoComplete.java | 2 +-
.../examples/complete/StreamingWordExtract.java | 2 +-
.../apache/beam/examples/complete/TfIdf.java | 3 +-
.../examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 12 +-
.../cookbook/CombinePerKeyExamples.java | 4 +-
.../beam/examples/cookbook/FilterExamples.java | 7 +-
.../beam/examples/cookbook/JoinExamples.java | 4 +-
.../examples/cookbook/MaxPerKeyExamples.java | 4 +-
.../org/apache/beam/examples/cookbook/README.md | 2 +-
.../beam/examples/cookbook/TriggerExample.java | 4 +-
.../beam/examples/WindowedWordCountIT.java | 41 +-
.../org/apache/beam/examples/WordCountTest.java | 4 +-
.../beam/examples/complete/TfIdfTest.java | 4 +-
.../complete/TopWikipediaSessionsTest.java | 4 +-
.../examples/cookbook/DistinctExampleTest.java | 6 +-
.../examples/cookbook/JoinExamplesTest.java | 4 +-
.../examples/cookbook/TriggerExampleTest.java | 4 +-
examples/java8/pom.xml | 41 +-
.../beam/examples/MinimalWordCountJava8.java | 9 +-
.../beam/examples/complete/game/GameStats.java | 16 +-
.../examples/complete/game/LeaderBoard.java | 2 +-
.../beam/examples/complete/game/README.md | 6 +-
.../beam/examples/complete/game/UserScore.java | 5 +-
.../complete/game/injector/Injector.java | 3 +-
.../complete/game/utils/WriteToBigQuery.java | 13 +-
.../game/utils/WriteWindowedToBigQuery.java | 9 +-
.../examples/MinimalWordCountJava8Test.java | 9 +-
.../examples/complete/game/GameStatsTest.java | 6 +-
.../complete/game/HourlyTeamScoreTest.java | 11 +-
.../examples/complete/game/UserScoreTest.java | 14 +-
pom.xml | 161 +-
runners/apex/pom.xml | 12 +-
.../apache/beam/runners/apex/ApexRunner.java | 81 +-
.../beam/runners/apex/ApexYarnLauncher.java | 111 +-
.../translation/ApexPipelineTranslator.java | 5 +-
.../FlattenPCollectionTranslator.java | 13 +-
.../apex/translation/GroupByKeyTranslator.java | 4 +-
.../translation/ParDoBoundMultiTranslator.java | 185 -
.../apex/translation/ParDoBoundTranslator.java | 95 -
.../apex/translation/ParDoTranslator.java | 185 +
.../apex/translation/TranslationContext.java | 22 +-
.../translation/WindowAssignTranslator.java | 58 +-
.../operators/ApexGroupByKeyOperator.java | 273 +-
.../operators/ApexParDoOperator.java | 238 +-
.../operators/ApexProcessFnOperator.java | 184 +
.../translation/utils/ApexStateInternals.java | 101 +-
.../apex/translation/utils/NoOpStepContext.java | 2 +-
.../utils/SerializablePipelineOptions.java | 13 +-
.../translation/utils/StateInternalsProxy.java | 67 +
.../apex/translation/utils/ValuesSource.java | 2 +-
.../apex/examples/UnboundedTextSource.java | 2 +-
.../translation/ApexGroupByKeyOperatorTest.java | 2 +-
.../FlattenPCollectionTranslatorTest.java | 3 +-
.../translation/GroupByKeyTranslatorTest.java | 2 +-
.../translation/ParDoBoundTranslatorTest.java | 344 --
.../apex/translation/ParDoTranslatorTest.java | 345 ++
.../utils/ApexStateInternalsTest.java | 43 +-
.../translation/utils/CollectionSource.java | 2 +-
runners/core-construction-java/pom.xml | 97 +-
.../beam/runners/core/construction/Coders.java | 174 +
.../DeduplicatedFlattenFactory.java | 120 +
.../EmptyFlattenAsCreateFactory.java | 34 +-
.../core/construction/ForwardingPTransform.java | 62 +
.../runners/core/construction/PCollections.java | 97 +
.../core/construction/PTransformMatchers.java | 135 +-
.../construction/PTransformReplacements.java | 69 +
.../runners/core/construction/PTransforms.java | 107 +
.../core/construction/PrimitiveCreate.java | 18 +-
.../core/construction/ReplacementOutputs.java | 63 +-
.../core/construction/SdkComponents.java | 195 ++
.../SingleInputOutputOverrideFactory.java | 14 +-
.../runners/core/construction/Triggers.java | 336 ++
.../UnboundedReadFromBoundedSource.java | 543 +++
.../core/construction/UnconsumedReads.java | 72 +
.../UnsupportedOverrideFactory.java | 20 +-
.../core/construction/WindowingStrategies.java | 245 ++
.../runners/core/construction/CodersTest.java | 163 +
.../DeduplicatedFlattenFactoryTest.java | 104 +
.../EmptyFlattenAsCreateFactoryTest.java | 122 +
.../construction/ForwardingPTransformTest.java | 111 +
.../core/construction/PCollectionsTest.java | 188 +
.../construction/PTransformMatchersTest.java | 198 +-
.../PTransformReplacementsTest.java | 131 +
.../core/construction/PTransformsTest.java | 189 +
.../construction/ReplacementOutputsTest.java | 109 +-
.../core/construction/SdkComponentsTest.java | 223 ++
.../SingleInputOutputOverrideFactoryTest.java | 37 +-
.../runners/core/construction/TriggersTest.java | 111 +
.../UnboundedReadFromBoundedSourceTest.java | 373 ++
.../core/construction/UnconsumedReadsTest.java | 105 +
.../UnsupportedOverrideFactoryTest.java | 16 +-
.../construction/WindowingStrategiesTest.java | 110 +
runners/core-java/pom.xml | 58 +-
.../beam/runners/core/BaseExecutionContext.java | 13 +-
.../apache/beam/runners/core/DoFnAdapters.java | 21 +-
.../apache/beam/runners/core/DoFnRunners.java | 40 +-
.../beam/runners/core/ExecutionContext.java | 13 +-
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 2 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 2 +-
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 8 +-
.../runners/core/InMemoryStateInternals.java | 120 +-
.../apache/beam/runners/core/NonEmptyPanes.java | 4 +-
.../org/apache/beam/runners/core/OldDoFn.java | 49 +-
...eBoundedSplittableProcessElementInvoker.java | 136 +-
.../beam/runners/core/OutputWindowedValue.java | 10 +-
.../beam/runners/core/ProcessFnRunner.java | 127 +
.../core/PushbackSideInputDoFnRunner.java | 106 +-
.../runners/core/ReduceFnContextFactory.java | 3 +-
.../beam/runners/core/SideInputHandler.java | 18 +-
.../beam/runners/core/SimpleDoFnRunner.java | 76 +-
.../beam/runners/core/SimpleOldDoFnRunner.java | 65 +-
.../core/SimplePushbackSideInputDoFnRunner.java | 115 +
.../beam/runners/core/SplittableParDo.java | 154 +-
.../core/SplittableProcessElementInvoker.java | 22 +-
.../apache/beam/runners/core/StateMerging.java | 16 +-
.../org/apache/beam/runners/core/StateTag.java | 14 +-
.../org/apache/beam/runners/core/StateTags.java | 32 +-
.../beam/runners/core/StatefulDoFnRunner.java | 96 +
.../beam/runners/core/SystemReduceFn.java | 8 +-
.../core/UnboundedReadFromBoundedSource.java | 536 ---
.../beam/runners/core/WindowingInternals.java | 8 +-
.../core/WindowingInternalsAdapters.java | 8 +-
.../AfterDelayFromFirstElementStateMachine.java | 10 +-
.../core/triggers/AfterPaneStateMachine.java | 4 +-
.../triggers/AfterWatermarkStateMachine.java | 14 +-
.../core/GroupAlsoByWindowsProperties.java | 10 +-
.../core/InMemoryStateInternalsTest.java | 105 +-
.../apache/beam/runners/core/NoOpOldDoFn.java | 4 +-
.../apache/beam/runners/core/OldDoFnTest.java | 4 +-
...ndedSplittableProcessElementInvokerTest.java | 27 +-
.../core/PushbackSideInputDoFnRunnerTest.java | 282 --
.../beam/runners/core/ReduceFnRunnerTest.java | 10 +-
.../beam/runners/core/ReduceFnTester.java | 10 +-
.../beam/runners/core/SideInputHandlerTest.java | 22 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 145 +
.../runners/core/SimpleOldDoFnRunnerTest.java | 4 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 282 ++
.../beam/runners/core/SplittableParDoTest.java | 332 +-
.../runners/core/StatefulDoFnRunnerTest.java | 113 +-
.../UnboundedReadFromBoundedSourceTest.java | 373 --
runners/direct-java/pom.xml | 55 +-
.../direct/BoundedReadEvaluatorFactory.java | 4 +-
.../CopyOnAccessInMemoryStateInternals.java | 46 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 16 +-
.../beam/runners/direct/DirectGraphVisitor.java | 5 +-
.../beam/runners/direct/DirectGroupByKey.java | 1 +
.../direct/DirectGroupByKeyOverrideFactory.java | 14 +-
.../beam/runners/direct/DirectMetrics.java | 129 +-
.../beam/runners/direct/DirectOptions.java | 11 -
.../beam/runners/direct/DirectRunner.java | 132 +-
...ecycleManagerRemovingTransformEvaluator.java | 6 +-
.../beam/runners/direct/EvaluationContext.java | 34 +-
.../direct/ExecutorServiceParallelExecutor.java | 141 +-
.../runners/direct/FlattenEvaluatorFactory.java | 2 +-
.../runners/direct/ForwardingPTransform.java | 62 -
.../GroupAlsoByWindowEvaluatorFactory.java | 12 +-
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 4 +-
.../direct/KeyedPValueTrackingVisitor.java | 18 +-
.../beam/runners/direct/ModelEnforcement.java | 13 +-
.../beam/runners/direct/ParDoEvaluator.java | 129 +-
.../runners/direct/ParDoEvaluatorFactory.java | 39 +-
.../direct/ParDoMultiOverrideFactory.java | 53 +-
.../ParDoSingleViaMultiOverrideFactory.java | 70 -
.../beam/runners/direct/PipelineExecutor.java | 23 +-
...littableProcessElementsEvaluatorFactory.java | 108 +-
.../direct/StatefulParDoEvaluatorFactory.java | 31 +-
.../direct/TestStreamEvaluatorFactory.java | 20 +-
.../direct/TransformEvaluatorFactory.java | 10 +-
.../direct/TransformEvaluatorRegistry.java | 7 +-
.../direct/TransformExecutorService.java | 6 +
.../direct/TransformExecutorServices.java | 57 +-
.../direct/UnboundedReadEvaluatorFactory.java | 39 +-
.../runners/direct/ViewEvaluatorFactory.java | 82 +-
.../runners/direct/ViewOverrideFactory.java | 114 +
.../beam/runners/direct/WatermarkManager.java | 19 +-
.../runners/direct/WindowEvaluatorFactory.java | 2 +-
.../direct/WriteWithShardingFactory.java | 26 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 4 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 34 +-
.../runners/direct/DirectGraphVisitorTest.java | 7 +-
.../DirectGroupByKeyOverrideFactoryTest.java | 12 +-
.../beam/runners/direct/DirectMetricsTest.java | 124 +-
.../beam/runners/direct/DirectRunnerTest.java | 89 +-
...leManagerRemovingTransformEvaluatorTest.java | 8 +-
.../runners/direct/EvaluationContextTest.java | 57 +-
.../direct/ForwardingPTransformTest.java | 112 -
.../beam/runners/direct/ParDoEvaluatorTest.java | 15 +-
.../direct/ParDoMultiOverrideFactoryTest.java | 45 -
.../ParDoSingleViaMultiOverrideFactoryTest.java | 46 -
.../runners/direct/SideInputContainerTest.java | 19 +-
.../StatefulParDoEvaluatorFactoryTest.java | 5 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 11 -
.../direct/TransformExecutorServicesTest.java | 48 +
.../UnboundedReadEvaluatorFactoryTest.java | 88 +-
.../direct/ViewEvaluatorFactoryTest.java | 14 +-
.../runners/direct/ViewOverrideFactoryTest.java | 138 +
.../direct/WindowEvaluatorFactoryTest.java | 12 +-
.../direct/WriteWithShardingFactoryTest.java | 28 +-
runners/flink/examples/pom.xml | 126 -
.../beam/runners/flink/examples/TFIDF.java | 456 ---
.../beam/runners/flink/examples/WordCount.java | 129 -
.../runners/flink/examples/package-info.java | 22 -
.../flink/examples/streaming/AutoComplete.java | 400 ---
.../flink/examples/streaming/JoinExamples.java | 154 -
.../examples/streaming/KafkaIOExamples.java | 338 --
.../KafkaWindowedWordCountExample.java | 164 -
.../examples/streaming/WindowedWordCount.java | 141 -
.../flink/examples/streaming/package-info.java | 22 -
runners/flink/pom.xml | 320 +-
runners/flink/runner/pom.xml | 310 --
.../flink/DefaultParallelismFactory.java | 39 -
.../flink/FlinkBatchPipelineTranslator.java | 139 -
.../flink/FlinkBatchTransformTranslators.java | 797 -----
.../flink/FlinkBatchTranslationContext.java | 154 -
.../flink/FlinkDetachedRunnerResult.java | 76 -
.../FlinkPipelineExecutionEnvironment.java | 241 --
.../runners/flink/FlinkPipelineOptions.java | 101 -
.../runners/flink/FlinkPipelineTranslator.java | 53 -
.../apache/beam/runners/flink/FlinkRunner.java | 250 --
.../runners/flink/FlinkRunnerRegistrar.java | 62 -
.../beam/runners/flink/FlinkRunnerResult.java | 98 -
.../flink/FlinkStreamingPipelineTranslator.java | 230 --
.../FlinkStreamingTransformTranslators.java | 1043 ------
.../flink/FlinkStreamingTranslationContext.java | 130 -
.../flink/FlinkStreamingViewOverrides.java | 372 --
.../flink/PipelineTranslationOptimizer.java | 72 -
.../beam/runners/flink/TestFlinkRunner.java | 97 -
.../beam/runners/flink/TranslationMode.java | 31 -
.../apache/beam/runners/flink/package-info.java | 22 -
.../functions/FlinkAggregatorFactory.java | 53 -
.../functions/FlinkAssignContext.java | 63 -
.../functions/FlinkAssignWindows.java | 49 -
.../functions/FlinkDoFnFunction.java | 161 -
.../FlinkMergingNonShuffleReduceFunction.java | 228 --
.../FlinkMergingPartialReduceFunction.java | 201 --
.../functions/FlinkMergingReduceFunction.java | 199 --
.../FlinkMultiOutputPruningFunction.java | 50 -
.../functions/FlinkNoOpStepContext.java | 73 -
.../functions/FlinkPartialReduceFunction.java | 172 -
.../functions/FlinkReduceFunction.java | 173 -
.../functions/FlinkSideInputReader.java | 80 -
.../functions/FlinkStatefulDoFnFunction.java | 198 --
.../functions/SideInputInitializer.java | 73 -
.../translation/functions/package-info.java | 22 -
.../runners/flink/translation/package-info.java | 22 -
.../translation/types/CoderTypeInformation.java | 120 -
.../translation/types/CoderTypeSerializer.java | 132 -
.../types/EncodedValueComparator.java | 195 --
.../types/EncodedValueSerializer.java | 113 -
.../types/EncodedValueTypeInformation.java | 107 -
.../flink/translation/types/FlinkCoder.java | 63 -
.../types/InspectableByteArrayOutputStream.java | 34 -
.../flink/translation/types/KvKeySelector.java | 50 -
.../flink/translation/types/package-info.java | 22 -
.../utils/SerializedPipelineOptions.java | 65 -
.../flink/translation/utils/package-info.java | 22 -
.../wrappers/DataInputViewWrapper.java | 58 -
.../wrappers/DataOutputViewWrapper.java | 51 -
.../SerializableFnAggregatorWrapper.java | 98 -
.../translation/wrappers/SourceInputFormat.java | 149 -
.../translation/wrappers/SourceInputSplit.java | 52 -
.../translation/wrappers/package-info.java | 22 -
.../wrappers/streaming/DoFnOperator.java | 861 -----
.../streaming/KvToByteBufferKeySelector.java | 56 -
.../streaming/SingletonKeyedWorkItem.java | 58 -
.../streaming/SingletonKeyedWorkItemCoder.java | 128 -
.../wrappers/streaming/WindowDoFnOperator.java | 120 -
.../wrappers/streaming/WorkItemKeySelector.java | 56 -
.../streaming/io/BoundedSourceWrapper.java | 218 --
.../streaming/io/UnboundedFlinkSink.java | 182 -
.../streaming/io/UnboundedFlinkSource.java | 120 -
.../streaming/io/UnboundedSocketSource.java | 249 --
.../streaming/io/UnboundedSourceWrapper.java | 476 ---
.../wrappers/streaming/io/package-info.java | 22 -
.../wrappers/streaming/package-info.java | 22 -
.../state/FlinkBroadcastStateInternals.java | 865 -----
.../state/FlinkKeyGroupStateInternals.java | 487 ---
.../state/FlinkSplitStateInternals.java | 260 --
.../streaming/state/FlinkStateInternals.java | 1053 ------
.../state/KeyGroupCheckpointedOperator.java | 35 -
.../state/KeyGroupRestoringOperator.java | 32 -
.../wrappers/streaming/state/package-info.java | 22 -
.../runner/src/main/resources/log4j.properties | 23 -
.../flink/EncodedValueComparatorTest.java | 70 -
.../runners/flink/FlinkRunnerRegistrarTest.java | 48 -
.../beam/runners/flink/FlinkTestPipeline.java | 72 -
.../beam/runners/flink/PipelineOptionsTest.java | 184 -
.../beam/runners/flink/ReadSourceITCase.java | 85 -
.../flink/ReadSourceStreamingITCase.java | 74 -
.../beam/runners/flink/WriteSinkITCase.java | 171 -
.../flink/streaming/DoFnOperatorTest.java | 594 ----
.../FlinkBroadcastStateInternalsTest.java | 245 --
.../FlinkKeyGroupStateInternalsTest.java | 262 --
.../streaming/FlinkSplitStateInternalsTest.java | 101 -
.../streaming/FlinkStateInternalsTest.java | 395 ---
.../flink/streaming/GroupByNullKeyTest.java | 124 -
.../flink/streaming/TestCountingSource.java | 254 --
.../streaming/TopWikipediaSessionsITCase.java | 133 -
.../streaming/UnboundedSourceWrapperTest.java | 464 ---
.../runners/flink/streaming/package-info.java | 22 -
.../src/test/resources/log4j-test.properties | 27 -
.../flink/DefaultParallelismFactory.java | 39 +
.../flink/FlinkBatchPipelineTranslator.java | 139 +
.../flink/FlinkBatchTransformTranslators.java | 723 ++++
.../flink/FlinkBatchTranslationContext.java | 153 +
.../flink/FlinkDetachedRunnerResult.java | 75 +
.../FlinkPipelineExecutionEnvironment.java | 241 ++
.../runners/flink/FlinkPipelineOptions.java | 101 +
.../runners/flink/FlinkPipelineTranslator.java | 53 +
.../apache/beam/runners/flink/FlinkRunner.java | 232 ++
.../runners/flink/FlinkRunnerRegistrar.java | 62 +
.../beam/runners/flink/FlinkRunnerResult.java | 98 +
.../flink/FlinkStreamingPipelineTranslator.java | 276 ++
.../FlinkStreamingTransformTranslators.java | 1044 ++++++
.../flink/FlinkStreamingTranslationContext.java | 130 +
.../flink/FlinkStreamingViewOverrides.java | 372 ++
.../flink/PipelineTranslationOptimizer.java | 72 +
.../beam/runners/flink/TestFlinkRunner.java | 84 +
.../beam/runners/flink/TranslationMode.java | 31 +
.../apache/beam/runners/flink/package-info.java | 22 +
.../functions/FlinkAggregatorFactory.java | 53 +
.../functions/FlinkAssignContext.java | 63 +
.../functions/FlinkAssignWindows.java | 49 +
.../functions/FlinkDoFnFunction.java | 161 +
.../FlinkMergingNonShuffleReduceFunction.java | 228 ++
.../FlinkMergingPartialReduceFunction.java | 201 ++
.../functions/FlinkMergingReduceFunction.java | 199 ++
.../FlinkMultiOutputPruningFunction.java | 50 +
.../functions/FlinkNoOpStepContext.java | 73 +
.../functions/FlinkPartialReduceFunction.java | 172 +
.../functions/FlinkReduceFunction.java | 173 +
.../functions/FlinkSideInputReader.java | 80 +
.../functions/FlinkStatefulDoFnFunction.java | 198 ++
.../functions/SideInputInitializer.java | 73 +
.../translation/functions/package-info.java | 22 +
.../runners/flink/translation/package-info.java | 22 +
.../translation/types/CoderTypeInformation.java | 120 +
.../translation/types/CoderTypeSerializer.java | 132 +
.../types/EncodedValueComparator.java | 195 ++
.../types/EncodedValueSerializer.java | 113 +
.../types/EncodedValueTypeInformation.java | 98 +
.../types/InspectableByteArrayOutputStream.java | 34 +
.../flink/translation/types/KvKeySelector.java | 50 +
.../flink/translation/types/package-info.java | 22 +
.../utils/SerializedPipelineOptions.java | 67 +
.../flink/translation/utils/package-info.java | 22 +
.../wrappers/DataInputViewWrapper.java | 58 +
.../wrappers/DataOutputViewWrapper.java | 51 +
.../SerializableFnAggregatorWrapper.java | 98 +
.../translation/wrappers/SourceInputFormat.java | 150 +
.../translation/wrappers/SourceInputSplit.java | 52 +
.../translation/wrappers/package-info.java | 22 +
.../wrappers/streaming/DoFnOperator.java | 774 ++++
.../streaming/KvToByteBufferKeySelector.java | 56 +
.../streaming/SingletonKeyedWorkItem.java | 56 +
.../streaming/SingletonKeyedWorkItemCoder.java | 126 +
.../streaming/SplittableDoFnOperator.java | 150 +
.../wrappers/streaming/WindowDoFnOperator.java | 117 +
.../wrappers/streaming/WorkItemKeySelector.java | 56 +
.../streaming/io/BoundedSourceWrapper.java | 218 ++
.../streaming/io/UnboundedSocketSource.java | 249 ++
.../streaming/io/UnboundedSourceWrapper.java | 476 +++
.../wrappers/streaming/io/package-info.java | 22 +
.../wrappers/streaming/package-info.java | 22 +
.../state/FlinkBroadcastStateInternals.java | 865 +++++
.../state/FlinkKeyGroupStateInternals.java | 487 +++
.../state/FlinkSplitStateInternals.java | 260 ++
.../streaming/state/FlinkStateInternals.java | 1053 ++++++
.../state/KeyGroupCheckpointedOperator.java | 35 +
.../state/KeyGroupRestoringOperator.java | 32 +
.../wrappers/streaming/state/package-info.java | 22 +
.../flink/src/main/resources/log4j.properties | 23 +
.../flink/EncodedValueComparatorTest.java | 70 +
.../runners/flink/FlinkRunnerRegistrarTest.java | 48 +
.../beam/runners/flink/FlinkTestPipeline.java | 72 +
.../beam/runners/flink/PipelineOptionsTest.java | 184 +
.../beam/runners/flink/ReadSourceITCase.java | 85 +
.../flink/ReadSourceStreamingITCase.java | 74 +
.../beam/runners/flink/WriteSinkITCase.java | 192 +
.../flink/streaming/DoFnOperatorTest.java | 600 ++++
.../FlinkBroadcastStateInternalsTest.java | 245 ++
.../FlinkKeyGroupStateInternalsTest.java | 262 ++
.../streaming/FlinkSplitStateInternalsTest.java | 101 +
.../streaming/FlinkStateInternalsTest.java | 395 +++
.../flink/streaming/GroupByNullKeyTest.java | 124 +
.../flink/streaming/TestCountingSource.java | 254 ++
.../streaming/TopWikipediaSessionsITCase.java | 133 +
.../streaming/UnboundedSourceWrapperTest.java | 464 +++
.../runners/flink/streaming/package-info.java | 22 +
.../src/test/resources/log4j-test.properties | 27 +
.../translators/ParDoBoundMultiTranslator.java | 104 -
.../translators/ParDoBoundTranslator.java | 75 -
.../translators/ParDoMultiOutputTranslator.java | 104 +
.../ParDoSingleOutputTranslator.java | 75 +
runners/google-cloud-dataflow-java/pom.xml | 88 +-
.../beam/runners/dataflow/AssignWindows.java | 4 +-
.../dataflow/BatchStatefulParDoOverrides.java | 78 +-
.../runners/dataflow/BatchViewOverrides.java | 85 +-
.../runners/dataflow/CreateDataflowView.java | 46 +
.../beam/runners/dataflow/DataflowMetrics.java | 225 ++
.../runners/dataflow/DataflowPipelineJob.java | 96 +-
.../dataflow/DataflowPipelineTranslator.java | 71 +-
.../beam/runners/dataflow/DataflowRunner.java | 387 +-
.../runners/dataflow/DataflowRunnerInfo.java | 38 +-
.../DataflowUnboundedReadFromBoundedSource.java | 547 ---
.../dataflow/PrimitiveParDoSingleFactory.java | 85 +
.../dataflow/ReshuffleOverrideFactory.java | 90 +
.../dataflow/StreamingViewOverrides.java | 17 +-
.../runners/dataflow/TransformTranslator.java | 6 +-
.../dataflow/internal/CustomSources.java | 2 +-
.../runners/dataflow/internal/IsmFormat.java | 3 +-
.../options/DataflowPipelineDebugOptions.java | 4 +-
.../DataflowPipelineWorkerPoolOptions.java | 8 +-
.../dataflow/testing/TestDataflowRunner.java | 43 +-
.../runners/dataflow/testing/package-info.java | 2 +-
.../runners/dataflow/util/MonitoringUtil.java | 4 +
.../beam/runners/dataflow/dataflow.properties | 8 +-
.../BatchStatefulParDoOverridesTest.java | 2 +-
.../dataflow/BatchViewOverridesTest.java | 4 +-
.../runners/dataflow/DataflowMetricsTest.java | 236 ++
.../dataflow/DataflowPipelineJobTest.java | 50 +-
.../DataflowPipelineTranslatorTest.java | 85 +-
.../dataflow/DataflowRunnerInfoTest.java | 23 +-
.../runners/dataflow/DataflowRunnerTest.java | 98 +-
...aflowUnboundedReadFromBoundedSourceTest.java | 79 -
.../PrimitiveParDoSingleFactoryTest.java | 161 +
.../testing/TestDataflowRunnerTest.java | 3 +-
.../dataflow/util/MonitoringUtilTest.java | 31 +-
runners/pom.xml | 40 -
runners/spark/pom.xml | 78 +-
.../spark/SparkNativePipelineVisitor.java | 1 -
.../beam/runners/spark/SparkPipelineResult.java | 81 +-
.../apache/beam/runners/spark/SparkRunner.java | 195 +-
.../beam/runners/spark/TestSparkRunner.java | 45 +-
.../spark/coders/StatelessJavaSerializer.java | 11 +
.../beam/runners/spark/io/MicrobatchSource.java | 118 +-
.../beam/runners/spark/io/SourceDStream.java | 63 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 74 +-
.../runners/spark/io/SparkUnboundedSource.java | 132 +-
.../beam/runners/spark/io/hadoop/HadoopIO.java | 216 --
.../spark/io/hadoop/ShardNameBuilder.java | 111 -
.../spark/io/hadoop/ShardNameTemplateAware.java | 31 -
.../io/hadoop/ShardNameTemplateHelper.java | 63 -
.../io/hadoop/TemplatedAvroKeyOutputFormat.java | 45 -
.../TemplatedSequenceFileOutputFormat.java | 45 -
.../io/hadoop/TemplatedTextOutputFormat.java | 45 -
.../runners/spark/io/hadoop/package-info.java | 22 -
.../runners/spark/metrics/SparkBeamMetric.java | 4 +
.../spark/metrics/SparkMetricResults.java | 67 +-
.../spark/metrics/SparkMetricsContainer.java | 31 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 48 +-
.../spark/stateful/SparkStateInternals.java | 30 +-
.../spark/stateful/StateSpecFunctions.java | 61 +-
.../spark/translation/BoundedDataset.java | 3 +-
.../runners/spark/translation/DoFnFunction.java | 130 -
.../spark/translation/EvaluationContext.java | 72 +-
.../translation/GroupCombineFunctions.java | 53 +-
.../spark/translation/MultiDoFnFunction.java | 4 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 10 +-
.../spark/translation/SparkProcessContext.java | 2 +-
.../spark/translation/SparkRuntimeContext.java | 2 +
.../spark/translation/TransformTranslator.java | 370 +-
.../spark/translation/TranslationUtils.java | 245 +-
.../SparkRunnerStreamingContextFactory.java | 45 +-
.../streaming/StreamingTransformTranslator.java | 158 +-
.../spark/util/GlobalWatermarkHolder.java | 2 +-
.../spark/util/SparkSideInputReader.java | 3 +-
.../apache/beam/runners/spark/CacheTest.java | 61 +
.../runners/spark/SparkPipelineStateTest.java | 2 +-
.../runners/spark/UsesCheckpointRecovery.java | 23 +
.../io/hadoop/HadoopFileFormatPipelineTest.java | 121 -
.../spark/io/hadoop/ShardNameBuilderTest.java | 88 -
.../spark/translation/StorageLevelTest.java | 6 +-
.../translation/streaming/CreateStreamTest.java | 4 +-
.../ResumeFromCheckpointStreamingTest.java | 17 +-
.../streaming/StreamingSourceMetricsTest.java | 71 +
.../streaming/TrackStreamingSourcesTest.java | 4 +-
runners/spark/src/test/resources/test_text.txt | 2 -
sdks/common/fn-api/pom.xml | 5 -
.../fn-api/src/main/proto/beam_fn_api.proto | 174 +-
.../src/main/proto/beam_runner_api.proto | 60 +-
sdks/java/build-tools/pom.xml | 31 +-
.../src/main/resources/beam/beam-codestyle.xml | 329 ++
.../src/main/resources/beam/findbugs-filter.xml | 43 +-
.../beam/codestyle/CodeStyleTestClass.java | 45 +
sdks/java/core/pom.xml | 155 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 8 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 123 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 170 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 4 +-
.../sdk/io/BoundedReadFromUnboundedSource.java | 4 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 20 +-
.../apache/beam/sdk/io/CompressedSource.java | 17 +-
.../org/apache/beam/sdk/io/CountingInput.java | 4 +
.../org/apache/beam/sdk/io/CountingSource.java | 10 +-
.../org/apache/beam/sdk/io/FileBasedSink.java | 429 ++-
.../org/apache/beam/sdk/io/FileBasedSource.java | 38 +-
.../org/apache/beam/sdk/io/FileSystems.java | 302 +-
.../org/apache/beam/sdk/io/LocalFileSystem.java | 57 +-
.../org/apache/beam/sdk/io/LocalResourceId.java | 2 +-
.../apache/beam/sdk/io/OffsetBasedSource.java | 4 +-
.../java/org/apache/beam/sdk/io/PubsubIO.java | 1201 -------
.../apache/beam/sdk/io/PubsubUnboundedSink.java | 500 ---
.../beam/sdk/io/PubsubUnboundedSource.java | 1435 --------
.../main/java/org/apache/beam/sdk/io/Sink.java | 57 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 694 ++++
.../java/org/apache/beam/sdk/io/TextIO.java | 104 +-
.../org/apache/beam/sdk/io/UnboundedSource.java | 19 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 377 +-
.../main/java/org/apache/beam/sdk/io/XmlIO.java | 477 +++
.../java/org/apache/beam/sdk/io/XmlSink.java | 226 +-
.../java/org/apache/beam/sdk/io/XmlSource.java | 191 +-
.../org/apache/beam/sdk/io/fs/MoveOptions.java | 34 +
.../org/apache/beam/sdk/io/fs/ResourceId.java | 8 +
.../org/apache/beam/sdk/io/package-info.java | 2 +-
.../apache/beam/sdk/io/range/ByteKeyRange.java | 4 +-
.../java/org/apache/beam/sdk/metrics/Gauge.java | 32 +
.../org/apache/beam/sdk/metrics/GaugeCell.java | 60 +
.../org/apache/beam/sdk/metrics/GaugeData.java | 81 +
.../apache/beam/sdk/metrics/GaugeResult.java | 61 +
.../beam/sdk/metrics/MetricFiltering.java | 99 +
.../beam/sdk/metrics/MetricNameFilter.java | 3 +-
.../beam/sdk/metrics/MetricQueryResults.java | 3 +
.../apache/beam/sdk/metrics/MetricUpdates.java | 11 +-
.../org/apache/beam/sdk/metrics/Metrics.java | 35 +
.../beam/sdk/metrics/MetricsContainer.java | 26 +-
.../beam/sdk/options/BigQueryOptions.java | 32 -
.../options/CloudResourceManagerOptions.java | 40 -
.../DefaultPipelineOptionsRegistrar.java | 5 -
.../org/apache/beam/sdk/options/GcpOptions.java | 227 --
.../org/apache/beam/sdk/options/GcsOptions.java | 158 -
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 -
.../beam/sdk/options/PipelineOptions.java | 16 +-
.../apache/beam/sdk/options/PubsubOptions.java | 36 -
.../beam/sdk/options/StreamingOptions.java | 7 +-
.../beam/sdk/runners/PTransformOverride.java | 44 +
.../sdk/runners/PTransformOverrideFactory.java | 31 +-
.../apache/beam/sdk/runners/PipelineRunner.java | 7 +-
.../beam/sdk/runners/TransformHierarchy.java | 80 +-
.../beam/sdk/testing/BigqueryMatcher.java | 256 --
.../apache/beam/sdk/testing/GatherAllPanes.java | 1 +
.../org/apache/beam/sdk/testing/PAssert.java | 220 +-
.../apache/beam/sdk/testing/PaneExtractors.java | 1 +
.../beam/sdk/testing/RunnableOnService.java | 38 -
.../beam/sdk/testing/SerializableMatcher.java | 2 +-
.../beam/sdk/testing/SerializableMatchers.java | 4 +-
.../beam/sdk/testing/SourceTestUtils.java | 6 +-
.../apache/beam/sdk/testing/StaticWindows.java | 17 +-
.../apache/beam/sdk/testing/StreamingIT.java | 2 +-
.../apache/beam/sdk/testing/TestPipeline.java | 89 +-
.../beam/sdk/testing/TestPipelineOptions.java | 5 +
...esSplittableParDoWithWindowedSideInputs.java | 26 +
.../beam/sdk/testing/ValidatesRunner.java | 23 +
.../beam/sdk/testing/ValueInSingleWindow.java | 134 -
.../beam/sdk/transforms/AppliedPTransform.java | 24 +-
.../org/apache/beam/sdk/transforms/Combine.java | 64 +-
.../org/apache/beam/sdk/transforms/Create.java | 86 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 138 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 51 +-
.../beam/sdk/transforms/FlatMapElements.java | 113 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 6 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 229 ++
.../apache/beam/sdk/transforms/MapElements.java | 99 +-
.../beam/sdk/transforms/Materialization.java | 36 +
.../beam/sdk/transforms/Materializations.java | 53 +
.../apache/beam/sdk/transforms/PTransform.java | 16 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 495 +--
.../apache/beam/sdk/transforms/Partition.java | 6 +-
.../org/apache/beam/sdk/transforms/Regex.java | 14 +-
.../org/apache/beam/sdk/transforms/Sample.java | 11 +-
.../org/apache/beam/sdk/transforms/View.java | 81 +-
.../org/apache/beam/sdk/transforms/ViewFn.java | 5 +
.../beam/sdk/transforms/WithTimestamps.java | 26 +-
.../sdk/transforms/display/DisplayData.java | 6 +
.../transforms/join/KeyedPCollectionTuple.java | 12 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 47 +-
.../sdk/transforms/reflect/DoFnInvoker.java | 4 +-
.../sdk/transforms/reflect/DoFnSignature.java | 10 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 96 +-
.../splittabledofn/HasDefaultTracker.java | 30 +
.../transforms/splittabledofn/OffsetRange.java | 8 +-
.../splittabledofn/OffsetRangeTracker.java | 33 +-
.../splittabledofn/RestrictionTracker.java | 8 +
.../transforms/windowing/AfterWatermark.java | 16 +-
.../sdk/transforms/windowing/GlobalWindow.java | 10 +
.../sdk/transforms/windowing/GlobalWindows.java | 11 +-
.../transforms/windowing/InvalidWindows.java | 2 +-
.../beam/sdk/transforms/windowing/PaneInfo.java | 4 +-
.../windowing/PartitioningWindowFn.java | 17 +-
.../sdk/transforms/windowing/Repeatedly.java | 2 +-
.../beam/sdk/transforms/windowing/Sessions.java | 2 +-
.../transforms/windowing/SlidingWindows.java | 27 +-
.../beam/sdk/transforms/windowing/Triggers.java | 320 --
.../beam/sdk/transforms/windowing/Window.java | 515 ++-
.../beam/sdk/transforms/windowing/WindowFn.java | 9 +-
.../transforms/windowing/WindowMappingFn.java | 67 +
.../beam/sdk/util/AppEngineEnvironment.java | 62 -
...AttemptAndTimeBoundedExponentialBackOff.java | 170 -
.../util/AttemptBoundedExponentialBackOff.java | 86 -
.../org/apache/beam/sdk/util/CoderUtils.java | 21 +-
.../beam/sdk/util/CombineContextFactory.java | 2 +-
.../apache/beam/sdk/util/CredentialFactory.java | 29 -
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 -
.../beam/sdk/util/DirectSideInputReader.java | 74 -
.../beam/sdk/util/FileIOChannelFactory.java | 23 +-
.../beam/sdk/util/GcpCredentialFactory.java | 67 -
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 -
.../beam/sdk/util/GcsIOChannelFactory.java | 110 -
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 -
.../apache/beam/sdk/util/GcsPathValidator.java | 95 -
.../java/org/apache/beam/sdk/util/GcsUtil.java | 803 -----
.../apache/beam/sdk/util/IOChannelFactory.java | 3 +-
.../apache/beam/sdk/util/IOChannelUtils.java | 22 +-
.../apache/beam/sdk/util/IdentityWindowFn.java | 3 +-
.../util/IntervalBoundedExponentialBackOff.java | 89 -
.../apache/beam/sdk/util/MutationDetector.java | 2 +-
.../org/apache/beam/sdk/util/NameUtils.java | 5 +
.../beam/sdk/util/NoopCredentialFactory.java | 68 -
.../sdk/util/NullCredentialInitializer.java | 62 -
.../beam/sdk/util/NumberedShardedFile.java | 30 +-
.../apache/beam/sdk/util/PCollectionViews.java | 268 +-
.../java/org/apache/beam/sdk/util/PTuple.java | 160 -
.../org/apache/beam/sdk/util/PubsubClient.java | 544 ---
.../apache/beam/sdk/util/PubsubGrpcClient.java | 424 ---
.../apache/beam/sdk/util/PubsubJsonClient.java | 317 --
.../apache/beam/sdk/util/PubsubTestClient.java | 436 ---
.../apache/beam/sdk/util/ReifyTimestamps.java | 76 +
.../org/apache/beam/sdk/util/Reshuffle.java | 21 +-
.../apache/beam/sdk/util/TestCredential.java | 59 -
.../org/apache/beam/sdk/util/TimeDomain.java | 2 +-
.../org/apache/beam/sdk/util/Transport.java | 179 -
.../beam/sdk/util/WindowingStrategies.java | 266 --
.../util/state/AccumulatorCombiningState.java | 53 -
.../apache/beam/sdk/util/state/BagState.java | 2 +-
.../beam/sdk/util/state/CombiningState.java | 27 +-
.../beam/sdk/util/state/GroupingState.java | 42 +
.../apache/beam/sdk/util/state/MapState.java | 52 +-
.../beam/sdk/util/state/ReadableStates.java | 45 +
.../apache/beam/sdk/util/state/SetState.java | 36 +-
.../org/apache/beam/sdk/util/state/State.java | 2 +-
.../apache/beam/sdk/util/state/StateBinder.java | 12 +-
.../apache/beam/sdk/util/state/StateSpec.java | 15 +
.../apache/beam/sdk/util/state/StateSpecs.java | 344 +-
.../beam/sdk/util/state/WatermarkHoldState.java | 2 +-
.../java/org/apache/beam/sdk/values/PBegin.java | 6 +-
.../apache/beam/sdk/values/PCollectionList.java | 27 +-
.../beam/sdk/values/PCollectionTuple.java | 13 +-
.../apache/beam/sdk/values/PCollectionView.java | 19 +
.../java/org/apache/beam/sdk/values/PDone.java | 6 +-
.../java/org/apache/beam/sdk/values/PInput.java | 4 +-
.../org/apache/beam/sdk/values/POutput.java | 4 +-
.../beam/sdk/values/POutputValueBase.java | 41 +-
.../java/org/apache/beam/sdk/values/PValue.java | 4 +-
.../org/apache/beam/sdk/values/PValueBase.java | 6 +-
.../apache/beam/sdk/values/TaggedPValue.java | 5 +
.../org/apache/beam/sdk/values/TupleTag.java | 26 +-
.../apache/beam/sdk/values/TupleTagList.java | 2 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 25 +-
.../org/apache/beam/sdk/values/TypedPValue.java | 6 +-
.../beam/sdk/values/ValueInSingleWindow.java | 134 +
sdks/java/core/src/main/proto/README.md | 3 -
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 3 -
.../sdk/AggregatorPipelineExtractorTest.java | 56 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 218 +-
.../sdk/coders/protobuf/ProtobufUtilTest.java | 1 -
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 154 +-
.../org/apache/beam/sdk/io/AvroSourceTest.java | 10 +-
.../io/BoundedReadFromUnboundedSourceTest.java | 8 +-
.../apache/beam/sdk/io/CountingInputTest.java | 14 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 20 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 94 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 8 +-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 168 +-
.../apache/beam/sdk/io/LocalFileSystemTest.java | 12 +-
.../beam/sdk/io/OffsetBasedSourceTest.java | 8 +-
.../org/apache/beam/sdk/io/PubsubIOTest.java | 197 --
.../beam/sdk/io/PubsubUnboundedSinkTest.java | 190 -
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 398 ---
.../java/org/apache/beam/sdk/io/ReadTest.java | 10 +-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 369 ++
.../java/org/apache/beam/sdk/io/TextIOTest.java | 125 +-
.../java/org/apache/beam/sdk/io/WriteTest.java | 53 +-
.../org/apache/beam/sdk/io/XmlSinkTest.java | 89 +-
.../org/apache/beam/sdk/io/XmlSourceTest.java | 250 +-
.../apache/beam/sdk/metrics/GaugeCellTest.java | 48 +
.../beam/sdk/metrics/MetricFilteringTest.java | 145 +
.../apache/beam/sdk/metrics/MetricMatchers.java | 12 +-
.../apache/beam/sdk/metrics/MetricsTest.java | 93 +-
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 -
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 -
.../sdk/options/PipelineOptionsFactoryTest.java | 4 +-
.../beam/sdk/runners/PipelineRunnerTest.java | 46 +-
.../sdk/runners/TransformHierarchyTest.java | 86 +-
.../beam/sdk/runners/TransformTreeTest.java | 88 +-
.../runners/dataflow/TestCountingSource.java | 2 +-
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 -
.../beam/sdk/testing/GatherAllPanesTest.java | 1 +
.../apache/beam/sdk/testing/PAssertTest.java | 74 +-
.../sdk/testing/PCollectionViewTesting.java | 83 +-
.../beam/sdk/testing/PaneExtractorsTest.java | 1 +
.../beam/sdk/testing/SourceTestUtilsTest.java | 2 +-
.../beam/sdk/testing/StaticWindowsTest.java | 10 +-
.../beam/sdk/testing/TestPipelineTest.java | 47 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 6 +-
.../testing/ValueInSingleWindowCoderTest.java | 1 +
.../sdk/transforms/ApproximateUniqueTest.java | 4 +-
.../beam/sdk/transforms/CombineFnsTest.java | 8 +-
.../apache/beam/sdk/transforms/CombineTest.java | 44 +-
.../apache/beam/sdk/transforms/CountTest.java | 10 +-
.../apache/beam/sdk/transforms/CreateTest.java | 70 +-
.../beam/sdk/transforms/DistinctTest.java | 8 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 7 +-
.../apache/beam/sdk/transforms/FilterTest.java | 16 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 47 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 14 +-
.../sdk/transforms/GroupIntoBatchesTest.java | 232 ++
.../apache/beam/sdk/transforms/KeysTest.java | 6 +-
.../apache/beam/sdk/transforms/KvSwapTest.java | 6 +-
.../beam/sdk/transforms/MapElementsTest.java | 29 +-
.../beam/sdk/transforms/ParDoLifecycleTest.java | 10 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 1180 +++++--
.../beam/sdk/transforms/PartitionTest.java | 4 +-
.../apache/beam/sdk/transforms/SampleTest.java | 14 +-
.../beam/sdk/transforms/SplittableDoFnTest.java | 157 +-
.../beam/sdk/transforms/ToStringTest.java | 13 +-
.../org/apache/beam/sdk/transforms/TopTest.java | 3 +-
.../apache/beam/sdk/transforms/ValuesTest.java | 6 +-
.../apache/beam/sdk/transforms/ViewTest.java | 222 +-
.../beam/sdk/transforms/WithTimestampsTest.java | 8 +-
.../sdk/transforms/display/DisplayDataTest.java | 17 +
.../sdk/transforms/join/CoGroupByKeyTest.java | 10 +-
.../transforms/reflect/DoFnInvokersTest.java | 153 +-
.../DoFnSignaturesProcessElementTest.java | 2 +-
.../DoFnSignaturesSplittableDoFnTest.java | 117 +-
.../splittabledofn/OffsetRangeTrackerTest.java | 49 +-
.../windowing/CalendarWindowsTest.java | 33 +
.../transforms/windowing/FixedWindowsTest.java | 33 +
.../windowing/SlidingWindowsTest.java | 16 +-
.../sdk/transforms/windowing/TriggersTest.java | 100 -
.../sdk/transforms/windowing/WindowTest.java | 97 +-
.../sdk/transforms/windowing/WindowingTest.java | 10 +-
...mptAndTimeBoundedExponentialBackOffTest.java | 213 --
.../AttemptBoundedExponentialBackOffTest.java | 85 -
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 -
.../beam/sdk/util/GcpProjectUtilTest.java | 76 -
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 -
.../beam/sdk/util/GcsPathValidatorTest.java | 87 -
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 -----
.../sdk/util/IdentitySideInputWindowFn.java | 10 +-
.../IntervalBoundedExponentialBackOffTest.java | 100 -
.../org/apache/beam/sdk/util/NameUtilsTest.java | 12 +
.../org/apache/beam/sdk/util/PTupleTest.java | 40 -
.../apache/beam/sdk/util/PubsubClientTest.java | 189 -
.../beam/sdk/util/PubsubGrpcClientTest.java | 207 --
.../beam/sdk/util/PubsubJsonClientTest.java | 133 -
.../beam/sdk/util/PubsubTestClientTest.java | 114 -
.../beam/sdk/util/ReifyTimestampsTest.java | 109 +
.../org/apache/beam/sdk/util/ReshuffleTest.java | 84 +-
.../util/RetryHttpRequestInitializerTest.java | 290 --
.../beam/sdk/util/WindowingStrategiesTest.java | 91 -
.../beam/sdk/values/PCollectionListTest.java | 70 +-
.../beam/sdk/values/PCollectionTupleTest.java | 17 +-
.../org/apache/beam/sdk/values/PDoneTest.java | 4 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 50 +-
sdks/java/extensions/gcp-core/pom.xml | 222 ++
.../beam/sdk/options/BigQueryOptions.java | 32 +
.../options/CloudResourceManagerOptions.java | 40 +
.../org/apache/beam/sdk/options/GcpOptions.java | 227 ++
.../options/GcpPipelineOptionsRegistrar.java | 39 +
.../org/apache/beam/sdk/options/GcsOptions.java | 154 +
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 +
.../apache/beam/sdk/options/PubsubOptions.java | 36 +
.../apache/beam/sdk/options/package-info.java | 22 +
.../beam/sdk/testing/BigqueryMatcher.java | 256 ++
.../apache/beam/sdk/testing/package-info.java | 21 +
.../apache/beam/sdk/util/CredentialFactory.java | 29 +
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 +
.../beam/sdk/util/GcpCredentialFactory.java | 67 +
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 +
.../beam/sdk/util/GcsIOChannelFactory.java | 111 +
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 +
.../apache/beam/sdk/util/GcsPathValidator.java | 95 +
.../java/org/apache/beam/sdk/util/GcsUtil.java | 798 +++++
.../beam/sdk/util/NoopCredentialFactory.java | 68 +
.../sdk/util/NullCredentialInitializer.java | 62 +
.../apache/beam/sdk/util/TestCredential.java | 59 +
.../org/apache/beam/sdk/util/Transport.java | 178 +
.../org/apache/beam/sdk/util/package-info.java | 20 +
.../org/apache/beam/GcpCoreApiSurfaceTest.java | 62 +
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 +
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 +
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 +
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 +
.../beam/sdk/util/GcpProjectUtilTest.java | 76 +
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 +
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 +
.../beam/sdk/util/GcsPathValidatorTest.java | 87 +
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 +++++
.../util/RetryHttpRequestInitializerTest.java | 290 ++
sdks/java/extensions/jackson/pom.xml | 40 -
sdks/java/extensions/pom.xml | 1 +
sdks/java/extensions/sorter/pom.xml | 49 -
sdks/java/harness/pom.xml | 10 +-
.../beam/fn/harness/fake/FakeStepContext.java | 2 +-
.../control/ProcessBundleHandlerTest.java | 30 +-
sdks/java/io/common/pom.xml | 42 +
.../apache/beam/sdk/io/common/HashingFn.java | 109 +
.../sdk/io/common/IOTestPipelineOptions.java | 79 +
sdks/java/io/elasticsearch/pom.xml | 7 +
.../sdk/io/elasticsearch/ElasticsearchIO.java | 37 +-
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 9 +-
.../io/elasticsearch/ElasticsearchIOTest.java | 15 +-
.../elasticsearch/ElasticsearchTestDataSet.java | 16 +-
.../elasticsearch/ElasticsearchTestOptions.java | 46 -
sdks/java/io/google-cloud-platform/pom.xml | 93 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 225 ++
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 352 ++
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 3306 ++++--------------
.../io/gcp/bigquery/BigQueryQuerySource.java | 205 ++
.../io/gcp/bigquery/BigQueryServicesImpl.java | 2 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 205 ++
.../gcp/bigquery/BigQueryTableRowIterator.java | 3 +-
.../io/gcp/bigquery/BigQueryTableSource.java | 132 +
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 127 +
.../io/gcp/bigquery/GenerateShardedTable.java | 47 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 84 +
.../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 81 +
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 67 +
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 87 +
.../sdk/io/gcp/bigquery/StreamingInserts.java | 79 +
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 111 +
.../io/gcp/bigquery/StreamingWriteTables.java | 86 +
.../sdk/io/gcp/bigquery/TableDestination.java | 76 +
.../io/gcp/bigquery/TableDestinationCoder.java | 60 +
.../beam/sdk/io/gcp/bigquery/TableRowInfo.java | 34 +
.../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 68 +
.../sdk/io/gcp/bigquery/TableRowWriter.java | 91 +
.../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 62 +
.../sdk/io/gcp/bigquery/TransformingSource.java | 136 +
.../io/gcp/bigquery/WriteBundlesToFiles.java | 157 +
.../sdk/io/gcp/bigquery/WritePartition.java | 174 +
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 185 +
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 46 +
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 211 ++
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 107 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 398 ++-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 544 +++
.../sdk/io/gcp/pubsub/PubsubGrpcClient.java | 424 +++
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 1014 ++++++
.../sdk/io/gcp/pubsub/PubsubJsonClient.java | 319 ++
.../sdk/io/gcp/pubsub/PubsubTestClient.java | 436 +++
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 490 +++
.../io/gcp/pubsub/PubsubUnboundedSource.java | 1463 ++++++++
.../beam/sdk/io/gcp/pubsub/package-info.java | 24 +
.../beam/sdk/io/gcp/storage/GcsResourceId.java | 2 +-
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 8 +-
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 1812 ++++------
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 12 +-
.../io/gcp/bigquery/FakeBigQueryServices.java | 166 +
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 208 ++
.../sdk/io/gcp/bigquery/FakeJobService.java | 404 +++
.../sdk/io/gcp/bigquery/TableContainer.java | 61 +
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 20 +-
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 15 +-
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 252 +-
.../sdk/io/gcp/datastore/SplitQueryFnIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1ReadIT.java | 66 +-
.../sdk/io/gcp/datastore/V1TestOptions.java | 2 +-
.../sdk/io/gcp/pubsub/PubsubClientTest.java | 189 +
.../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java | 208 ++
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 189 +
.../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 139 +
.../sdk/io/gcp/pubsub/PubsubTestClientTest.java | 114 +
.../io/gcp/pubsub/PubsubUnboundedSinkTest.java | 188 +
.../gcp/pubsub/PubsubUnboundedSourceTest.java | 409 +++
.../io/hadoop/SerializableConfiguration.java | 3 +-
sdks/java/io/hadoop-input-format/README.md | 167 -
sdks/java/io/hadoop-input-format/pom.xml | 136 -
.../hadoop/inputformat/HadoopInputFormatIO.java | 941 -----
.../sdk/io/hadoop/inputformat/package-info.java | 23 -
.../ConfigurableEmployeeInputFormat.java | 131 -
.../sdk/io/hadoop/inputformat/Employee.java | 85 -
.../hadoop/inputformat/EmployeeInputFormat.java | 172 -
.../inputformat/HadoopInputFormatIOTest.java | 844 -----
.../ReuseObjectsEmployeeInputFormat.java | 176 -
.../hadoop/inputformat/TestEmployeeDataSet.java | 76 -
sdks/java/io/hadoop/README.md | 167 +
sdks/java/io/hadoop/input-format/pom.xml | 98 +
.../hadoop/inputformat/HadoopInputFormatIO.java | 844 +++++
.../sdk/io/hadoop/inputformat/package-info.java | 23 +
.../ConfigurableEmployeeInputFormat.java | 131 +
.../sdk/io/hadoop/inputformat/Employee.java | 85 +
.../hadoop/inputformat/EmployeeInputFormat.java | 172 +
.../inputformat/HadoopInputFormatIOTest.java | 851 +++++
.../ReuseObjectsEmployeeInputFormat.java | 176 +
.../hadoop/inputformat/TestEmployeeDataSet.java | 76 +
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 244 ++
.../inputformat/HIFIOWithElasticTest.java | 277 ++
.../HIFIOWithEmbeddedCassandraTest.java | 215 ++
.../custom/options/HIFTestOptions.java | 64 +
.../integration/tests/HIFIOCassandraIT.java | 173 +
.../integration/tests/HIFIOElasticIT.java | 215 ++
.../src/test/resources/cassandra.yaml | 1074 ++++++
sdks/java/io/hadoop/pom.xml | 64 +
sdks/java/io/hbase/pom.xml | 92 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 7 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 13 +-
.../src/test/resources/log4j-test.properties | 27 +
sdks/java/io/hdfs/README.md | 6 +-
sdks/java/io/hdfs/pom.xml | 41 -
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 29 +-
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 16 +-
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 2 +-
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 23 +-
sdks/java/io/jdbc/pom.xml | 53 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 75 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 11 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 12 +-
.../beam/sdk/io/jdbc/JdbcTestDataSet.java | 12 +-
.../beam/sdk/io/jdbc/PostgresTestOptions.java | 60 -
.../kubernetes/postgres-pod-no-vol.yml | 32 -
.../kubernetes/postgres-service-public.yml | 28 -
.../jdbc/src/test/resources/kubernetes/setup.sh | 20 -
.../src/test/resources/kubernetes/teardown.sh | 20 -
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 4 +-
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 10 +-
sdks/java/io/kafka/pom.xml | 36 -
.../apache/beam/sdk/io/kafka/ConsumerSpEL.java | 43 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 70 +-
.../apache/beam/sdk/io/kafka/KafkaRecord.java | 15 +-
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 5 +
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 54 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 2 +
.../beam/sdk/io/kinesis/KinesisSource.java | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 14 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 4 +-
.../sdk/io/mongodb/MongoDBGridFSIOTest.java | 7 +-
.../beam/sdk/io/mongodb/MongoDbIOTest.java | 5 -
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 4 +-
.../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 6 -
sdks/java/io/pom.xml | 3 +-
.../beam/sdk/transforms/FilterJava8Test.java | 8 +-
.../transforms/FlatMapElementsJava8Test.java | 10 +-
.../sdk/transforms/MapElementsJava8Test.java | 14 +-
.../beam/sdk/transforms/WithKeysJava8Test.java | 4 +-
.../sdk/transforms/WithTimestampsJava8Test.java | 4 +-
.../java/javadoc/dataflow-sdk-docs/package-list | 11 -
sdks/java/javadoc/overview.html | 2 +-
.../META-INF/maven/archetype-metadata.xml | 2 +-
.../META-INF/maven/archetype-metadata.xml | 2 +-
.../META-INF/maven/archetype-metadata.xml | 2 +-
sdks/python/.pylintrc | 6 +-
sdks/python/MANIFEST.in | 3 -
sdks/python/apache_beam/__init__.py | 7 +-
sdks/python/apache_beam/coders/coder_impl.py | 50 +-
sdks/python/apache_beam/coders/coders.py | 10 +-
sdks/python/apache_beam/coders/typecoders.py | 2 +-
sdks/python/apache_beam/error.py | 4 +
.../examples/complete/game/README.md | 69 +
.../examples/complete/game/__init__.py | 16 +
.../examples/complete/game/hourly_team_score.py | 294 ++
.../complete/game/hourly_team_score_test.py | 52 +
.../examples/complete/game/user_score.py | 217 ++
.../examples/complete/game/user_score_test.py | 49 +
.../examples/complete/top_wikipedia_sessions.py | 8 -
.../examples/cookbook/bigquery_tornadoes.py | 2 +-
.../cookbook/bigquery_tornadoes_it_test.py | 3 +
.../examples/cookbook/datastore_wordcount.py | 15 +-
.../examples/cookbook/group_with_coder.py | 6 +-
.../examples/cookbook/multiple_output_pardo.py | 47 +-
.../apache_beam/examples/snippets/snippets.py | 5 +-
.../examples/snippets/snippets_test.py | 36 +-
sdks/python/apache_beam/examples/wordcount.py | 48 +-
.../apache_beam/examples/wordcount_debugging.py | 4 -
.../apache_beam/examples/wordcount_it_test.py | 11 +-
sdks/python/apache_beam/internal/gcp/auth.py | 88 +-
.../apache_beam/internal/gcp/auth_test.py | 44 -
.../apache_beam/internal/gcp/json_value.py | 6 +
sdks/python/apache_beam/internal/pickler.py | 20 +-
sdks/python/apache_beam/io/__init__.py | 1 +
sdks/python/apache_beam/io/avroio.py | 12 +-
sdks/python/apache_beam/io/avroio_test.py | 14 +-
sdks/python/apache_beam/io/concat_source.py | 74 +-
.../python/apache_beam/io/concat_source_test.py | 12 +-
sdks/python/apache_beam/io/filebasedsource.py | 127 +-
.../apache_beam/io/filebasedsource_test.py | 43 +-
sdks/python/apache_beam/io/fileio.py | 638 +---
sdks/python/apache_beam/io/fileio_test.py | 166 +-
sdks/python/apache_beam/io/filesystem.py | 530 +++
sdks/python/apache_beam/io/filesystem_test.py | 213 ++
sdks/python/apache_beam/io/filesystems_util.py | 35 +
sdks/python/apache_beam/io/gcp/bigquery.py | 94 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 3 +-
.../io/gcp/datastore/v1/datastoreio.py | 24 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 4 +-
.../apache_beam/io/gcp/datastore/v1/helper.py | 18 +-
.../io/gcp/datastore/v1/query_splitter.py | 2 +-
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 243 ++
.../apache_beam/io/gcp/gcsfilesystem_test.py | 322 ++
sdks/python/apache_beam/io/gcp/gcsio.py | 33 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 48 +-
.../io/gcp/tests/bigquery_matcher.py | 3 +-
sdks/python/apache_beam/io/iobase.py | 20 +-
sdks/python/apache_beam/io/localfilesystem.py | 235 ++
.../apache_beam/io/localfilesystem_test.py | 189 +
sdks/python/apache_beam/io/range_trackers.py | 19 +-
sdks/python/apache_beam/io/source_test_utils.py | 79 +-
.../apache_beam/io/source_test_utils_test.py | 20 +-
sdks/python/apache_beam/io/textio.py | 23 +-
sdks/python/apache_beam/io/textio_test.py | 20 +-
sdks/python/apache_beam/io/tfrecordio.py | 26 +-
sdks/python/apache_beam/io/tfrecordio_test.py | 38 +-
sdks/python/apache_beam/metrics/cells.py | 28 +-
sdks/python/apache_beam/metrics/execution.py | 3 +-
sdks/python/apache_beam/metrics/metric.py | 9 +-
sdks/python/apache_beam/pipeline.py | 138 +-
sdks/python/apache_beam/pipeline_test.py | 31 +-
sdks/python/apache_beam/pvalue.py | 317 +-
sdks/python/apache_beam/pvalue_test.py | 33 -
.../runners/api/beam_runner_api_pb2.py | 272 +-
sdks/python/apache_beam/runners/common.pxd | 2 +-
sdks/python/apache_beam/runners/common.py | 15 +-
.../runners/dataflow/dataflow_metrics.py | 86 +-
.../runners/dataflow/dataflow_metrics_test.py | 66 +-
.../runners/dataflow/dataflow_runner.py | 164 +-
.../runners/dataflow/dataflow_runner_test.py | 67 +-
.../runners/dataflow/internal/apiclient.py | 63 +-
.../runners/dataflow/internal/apiclient_test.py | 12 +-
.../runners/dataflow/internal/dependency.py | 30 +-
.../runners/dataflow/native_io/iobase_test.py | 173 +
.../runners/dataflow/test_dataflow_runner.py | 26 +-
.../runners/direct/bundle_factory.py | 17 +-
sdks/python/apache_beam/runners/direct/clock.py | 9 +-
.../consumer_tracking_pipeline_visitor.py | 11 +-
.../consumer_tracking_pipeline_visitor_test.py | 28 +-
.../apache_beam/runners/direct/direct_runner.py | 22 +-
.../runners/direct/evaluation_context.py | 77 +-
.../apache_beam/runners/direct/executor.py | 28 +-
.../runners/direct/transform_evaluator.py | 116 +-
.../runners/direct/transform_result.py | 45 +-
.../runners/direct/watermark_manager.py | 4 +-
sdks/python/apache_beam/runners/runner.py | 16 +-
.../apache_beam/tests/pipeline_verifiers.py | 46 +-
.../tests/pipeline_verifiers_test.py | 47 +-
sdks/python/apache_beam/transforms/combiners.py | 62 +-
.../apache_beam/transforms/combiners_test.py | 4 +-
sdks/python/apache_beam/transforms/core.py | 204 +-
.../apache_beam/transforms/create_test.py | 121 +
sdks/python/apache_beam/transforms/display.py | 4 +-
.../apache_beam/transforms/display_test.py | 36 +
.../python/apache_beam/transforms/ptransform.py | 42 +-
.../apache_beam/transforms/ptransform_test.py | 41 +-
.../python/apache_beam/transforms/sideinputs.py | 143 +-
.../apache_beam/transforms/sideinputs_test.py | 99 +-
sdks/python/apache_beam/transforms/trigger.py | 40 +-
.../apache_beam/transforms/trigger_test.py | 6 +-
sdks/python/apache_beam/transforms/util.py | 20 +-
sdks/python/apache_beam/transforms/util_test.py | 50 +
sdks/python/apache_beam/transforms/window.py | 9 +-
.../apache_beam/transforms/window_test.py | 8 +-
sdks/python/apache_beam/typehints/decorators.py | 26 +-
.../apache_beam/typehints/trivial_inference.py | 30 +-
.../typehints/trivial_inference_test.py | 3 +-
sdks/python/apache_beam/typehints/typecheck.py | 11 +-
sdks/python/apache_beam/typehints/typehints.py | 66 +-
.../apache_beam/typehints/typehints_test.py | 7 +-
.../apache_beam/utils/annotations_test.py | 2 +-
sdks/python/apache_beam/utils/counters.py | 5 +-
sdks/python/apache_beam/utils/path.py | 3 +-
.../apache_beam/utils/pipeline_options.py | 106 +-
.../apache_beam/utils/pipeline_options_test.py | 52 +-
sdks/python/apache_beam/utils/proto_utils.py | 17 +-
sdks/python/apache_beam/utils/retry.py | 20 +-
sdks/python/apache_beam/utils/test_stream.py | 163 +
.../apache_beam/utils/test_stream_test.py | 82 +
sdks/python/apache_beam/utils/timestamp.py | 6 +-
sdks/python/apache_beam/utils/urns.py | 2 +
sdks/python/apache_beam/utils/value_provider.py | 103 +
.../apache_beam/utils/value_provider_test.py | 145 +
sdks/python/apache_beam/utils/windowed_value.py | 17 +-
sdks/python/apache_beam/version.py | 36 -
sdks/python/findSupportedPython.groovy | 80 +
sdks/python/generate_pydoc.sh | 2 +-
sdks/python/pom.xml | 31 +-
sdks/python/run_postcommit.sh | 35 +-
sdks/python/run_pylint.sh | 27 +-
sdks/python/setup.py | 19 +-
sdks/python/tox.ini | 18 +-
1164 files changed, 69329 insertions(+), 54774 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
----------------------------------------------------------------------
diff --cc .test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
index 0000000,1348a19..e1cbafe
mode 000000,100644..100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
@@@ -1,0 -1,49 +1,49 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ import common_job_properties
+
+ // This job runs the suite of ValidatesRunner tests against the Gearpump
+ // runner.
+ mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') {
+ description('Runs the ValidatesRunner suite on the Gearpump runner.')
+
+ previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(
+ delegate,
+ 'gearpump-runner')
+
+ // Set maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PostCommit job.
+ // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties.
+ // In post-commit this job triggers only on SCM changes.
+ common_job_properties.setPostCommit(delegate, '0 5 31 2 *')
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Apache Gearpump Runner ValidatesRunner Tests',
+ 'Run Gearpump ValidatesRunner')
+
+ // Maven goals for this job.
- goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner", "--streaming=false" ]\'')
++ goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner"]\'')
+ }
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/.travis.yml
----------------------------------------------------------------------
diff --cc .travis.yml
index 87ade2f,0b5d700..a1b28f9
--- a/.travis.yml
+++ b/.travis.yml
@@@ -78,11 -77,10 +77,11 @@@ install
- if [ "$TEST_PYTHON" ] && ! pip list | grep tox; then travis_retry pip install tox --user; fi
# Removing this here protects from inadvertent caching
- rm -rf "$HOME/.m2/repository/org/apache/beam"
+ - rm -rf "$HOME/.m2/repository/org/apache/gearpump"
script:
- - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -e $TOX_ENV -c sdks/python/tox.ini; fi
- - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .travis/test_wordcount.sh; fi
+ - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -c sdks/python/tox.ini; fi
+ - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .test-infra/travis/test_wordcount.sh; fi
cache:
directories:
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/examples/java/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
----------------------------------------------------------------------
diff --cc runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
index 0000000,0000000..e78568d
new file mode 100644
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
@@@ -1,0 -1,0 +1,104 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.beam.runners.gearpump.translators;
++
++import java.util.ArrayList;
++import java.util.Collection;
++import java.util.List;
++import java.util.Map;
++
++import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
++import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
++import org.apache.beam.sdk.transforms.DoFn;
++import org.apache.beam.sdk.transforms.ParDo;
++import org.apache.beam.sdk.util.WindowedValue;
++import org.apache.beam.sdk.values.PCollection;
++import org.apache.beam.sdk.values.PCollectionView;
++import org.apache.beam.sdk.values.TaggedPValue;
++import org.apache.beam.sdk.values.TupleTag;
++
++import org.apache.gearpump.streaming.dsl.api.functions.FilterFunction;
++import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
++
++/**
++ * {@link ParDo.MultiOutput} is translated to Gearpump flatMap function
++ * with {@link DoFn} wrapped in {@link DoFnFunction}. The outputs are
++ * further filtered with Gearpump filter function by output tag
++ */
++@SuppressWarnings({"rawtypes", "unchecked"})
++public class ParDoMultiOutputTranslator<InputT, OutputT> implements
++ TransformTranslator<ParDo.MultiOutput<InputT, OutputT>> {
++
++ private static final long serialVersionUID = -6023461558200028849L;
++
++ @Override
++ public void translate(ParDo.MultiOutput<InputT, OutputT> transform, TranslationContext context) {
++ PCollection<InputT> inputT = (PCollection<InputT>) context.getInput();
++ JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(inputT);
++ Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
++ Map<String, PCollectionView<?>> tagsToSideInputs =
++ TranslatorUtils.getTagsToSideInputs(sideInputs);
++
++ List<TaggedPValue> outputs = context.getOutputs();
++ final TupleTag<OutputT> mainOutput = transform.getMainOutputTag();
++ List<TupleTag<?>> sideOutputs = new ArrayList<>(outputs.size() - 1);
++ for (TaggedPValue output: outputs) {
++ TupleTag<?> tag = output.getTag();
++ if (tag != null && !tag.getId().equals(mainOutput.getId())) {
++ sideOutputs.add(tag);
++ }
++ }
++
++ JavaStream<TranslatorUtils.RawUnionValue> unionStream = TranslatorUtils.withSideInputStream(
++ context, inputStream, tagsToSideInputs);
++
++ JavaStream<TranslatorUtils.RawUnionValue> outputStream =
++ TranslatorUtils.toList(unionStream).flatMap(
++ new DoFnFunction<>(
++ context.getPipelineOptions(),
++ transform.getFn(),
++ inputT.getWindowingStrategy(),
++ sideInputs,
++ tagsToSideInputs,
++ mainOutput,
++ sideOutputs), transform.getName());
++ for (TaggedPValue output: outputs) {
++ JavaStream<WindowedValue<OutputT>> taggedStream = outputStream
++ .filter(new FilterByOutputTag(output.getTag().getId()),
++ "filter_by_output_tag")
++ .map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
++ context.setOutputStream(output.getValue(), taggedStream);
++ }
++ }
++
++ private static class FilterByOutputTag extends FilterFunction<TranslatorUtils.RawUnionValue> {
++
++ private static final long serialVersionUID = 7276155265895637526L;
++ private final String tag;
++
++ FilterByOutputTag(String tag) {
++ this.tag = tag;
++ }
++
++ @Override
++ public boolean filter(TranslatorUtils.RawUnionValue value) {
++ return value.getUnionTag().equals(tag);
++ }
++ }
++}
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
----------------------------------------------------------------------
diff --cc runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
index 0000000,0000000..86879b7
new file mode 100644
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
@@@ -1,0 -1,0 +1,75 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.beam.runners.gearpump.translators;
++
++import java.util.Collection;
++import java.util.List;
++import java.util.Map;
++
++import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
++import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
++import org.apache.beam.sdk.transforms.DoFn;
++import org.apache.beam.sdk.transforms.ParDo;
++import org.apache.beam.sdk.util.WindowedValue;
++import org.apache.beam.sdk.util.WindowingStrategy;
++import org.apache.beam.sdk.values.PCollection;
++
++import org.apache.beam.sdk.values.PCollectionView;
++import org.apache.beam.sdk.values.TupleTag;
++import org.apache.beam.sdk.values.TupleTagList;
++import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
++
++/**
++ * {@link ParDo.SingleOutput} is translated to Gearpump flatMap function
++ * with {@link DoFn} wrapped in {@link DoFnFunction}.
++ */
++public class ParDoSingleOutputTranslator<InputT, OutputT> implements
++ TransformTranslator<ParDo.SingleOutput<InputT, OutputT>> {
++
++ private static final long serialVersionUID = -3413205558160983784L;
++ private final TupleTag<OutputT> mainOutput = new TupleTag<>();
++ private final List<TupleTag<?>> sideOutputs = TupleTagList.empty().getAll();
++
++ @Override
++ public void translate(ParDo.SingleOutput<InputT, OutputT> transform, TranslationContext context) {
++ DoFn<InputT, OutputT> doFn = transform.getFn();
++ PCollection<OutputT> output = (PCollection<OutputT>) context.getOutput();
++ WindowingStrategy<?, ?> windowingStrategy = output.getWindowingStrategy();
++
++ Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
++ Map<String, PCollectionView<?>> tagsToSideInputs =
++ TranslatorUtils.getTagsToSideInputs(sideInputs);
++ JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(
++ context.getInput());
++ JavaStream<TranslatorUtils.RawUnionValue> unionStream =
++ TranslatorUtils.withSideInputStream(context,
++ inputStream, tagsToSideInputs);
++
++ DoFnFunction<InputT, OutputT> doFnFunction = new DoFnFunction<>(context.getPipelineOptions(),
++ doFn, windowingStrategy, sideInputs, tagsToSideInputs,
++ mainOutput, sideOutputs);
++
++ JavaStream<WindowedValue<OutputT>> outputStream =
++ TranslatorUtils.toList(unionStream)
++ .flatMap(doFnFunction, transform.getName())
++ .map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
++
++ context.setOutputStream(context.getOutput(), outputStream);
++ }
++}
http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/runners/pom.xml
----------------------------------------------------------------------
diff --cc runners/pom.xml
index dd791ff,8f3cabd..a114382
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@@ -54,55 -54,6 +54,15 @@@
</plugins>
</build>
</profile>
+ <profile>
+ <id>java8</id>
+ <activation>
+ <jdk>[1.8,)</jdk>
+ </activation>
+ <modules>
+ <module>gearpump</module>
+ </modules>
+ </profile>
-
- <!-- A profile that adds an integration test phase if and only if
- the runnableOnServicePipelineOptions maven property has been set.
- It should be set to a valid PipelineOptions JSON string. -->
- <profile>
- <id>runnable-on-service-tests</id>
- <activation>
- <property><name>runnableOnServicePipelineOptions</name></property>
- </activation>
- <build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <executions>
- <execution>
- <id>runnable-on-service-tests</id>
- <phase>integration-test</phase>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <groups>org.apache.beam.sdk.testing.RunnableOnService</groups>
- <parallel>all</parallel>
- <threadCount>4</threadCount>
- <dependenciesToScan>
- <dependency>org.apache.beam:beam-sdks-java-core</dependency>
- </dependenciesToScan>
- <systemPropertyVariables>
- <beamTestPipelineOptions>${runnableOnServicePipelineOptions}</beamTestPipelineOptions>
- </systemPropertyVariables>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
- </profile>
</profiles>
<build>
[18/50] [abbrv] beam git commit: Merge remote-tracking branch
'upstream/master' into gearpump-runner
Posted by ke...@apache.org.
Merge remote-tracking branch 'upstream/master' into gearpump-runner
# Conflicts:
# .travis.yml
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9a59ea32
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9a59ea32
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9a59ea32
Branch: refs/heads/master
Commit: 9a59ea3207c1a5c606692400c837f1ca7c6c5cbb
Parents: 4078c22 7d6f8bd
Author: manuzhang <ow...@gmail.com>
Authored: Wed May 3 16:48:21 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Wed May 3 16:48:21 2017 +0800
----------------------------------------------------------------------
.github/PULL_REQUEST_TEMPLATE.md | 3 +-
.gitignore | 2 +
.../jenkins/common_job_properties.groovy | 1 -
.../job_beam_PerformanceTests_JDBC.groovy | 3 +
.test-infra/jenkins/test_wordcount.sh | 125 +
.test-infra/travis/README.md | 23 -
.test-infra/travis/settings.xml | 33 -
.test-infra/travis/test_wordcount.sh | 125 -
.travis.yml | 94 -
README.md | 3 +-
examples/java/pom.xml | 34 +-
.../beam/examples/DebuggingWordCount.java | 22 +-
.../apache/beam/examples/MinimalWordCount.java | 7 +-
.../apache/beam/examples/WindowedWordCount.java | 2 +-
.../org/apache/beam/examples/WordCount.java | 20 +-
.../common/ExampleBigQueryTableOptions.java | 2 +-
...xamplePubsubTopicAndSubscriptionOptions.java | 2 +-
.../common/ExamplePubsubTopicOptions.java | 2 +-
.../beam/examples/common/ExampleUtils.java | 68 +-
.../examples/common/WriteOneFilePerWindow.java | 11 +-
.../beam/examples/complete/AutoComplete.java | 2 +-
.../examples/complete/StreamingWordExtract.java | 4 +-
.../apache/beam/examples/complete/TfIdf.java | 6 +-
.../examples/complete/TopWikipediaSessions.java | 4 +-
.../examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 13 +-
.../beam/examples/cookbook/DistinctExample.java | 4 +-
.../beam/examples/cookbook/JoinExamples.java | 2 +-
.../beam/examples/cookbook/TriggerExample.java | 6 +-
.../beam/examples/WindowedWordCountIT.java | 17 +-
.../org/apache/beam/examples/WordCountIT.java | 14 +-
.../examples/cookbook/BigQueryTornadoesIT.java | 4 +-
examples/java8/pom.xml | 8 +-
.../beam/examples/MinimalWordCountJava8.java | 7 +-
.../beam/examples/complete/game/GameStats.java | 18 +-
.../examples/complete/game/HourlyTeamScore.java | 2 +-
.../examples/complete/game/LeaderBoard.java | 6 +-
.../beam/examples/complete/game/UserScore.java | 10 +-
.../complete/game/utils/WriteToBigQuery.java | 2 +-
.../examples/MinimalWordCountJava8Test.java | 6 +-
pom.xml | 83 +-
runners/apex/README.md | 2 +-
runners/apex/pom.xml | 6 -
.../beam/runners/apex/ApexRunnerResult.java | 9 -
.../operators/ApexGroupByKeyOperator.java | 191 +-
.../operators/ApexParDoOperator.java | 155 +-
.../operators/ApexTimerInternals.java | 266 ++
.../translation/utils/ApexStateInternals.java | 100 +-
.../apex/translation/utils/ApexStreamTuple.java | 6 +-
.../apex/translation/utils/NoOpStepContext.java | 2 +-
.../translation/utils/StateInternalsProxy.java | 6 +-
.../runners/apex/examples/WordCountTest.java | 13 +-
.../translation/GroupByKeyTranslatorTest.java | 10 +-
.../translation/ReadUnboundTranslatorTest.java | 6 +-
.../operators/ApexTimerInternalsTest.java | 145 +
.../utils/ApexStateInternalsTest.java | 39 +-
runners/core-construction-java/pom.xml | 10 -
.../core/construction/CoderTranslator.java | 44 +
.../core/construction/CoderTranslators.java | 107 +
.../beam/runners/core/construction/Coders.java | 109 +-
.../core/construction/ForwardingPTransform.java | 5 +-
.../core/construction/PTransformMatchers.java | 6 +-
.../UnboundedReadFromBoundedSource.java | 19 +-
.../core/construction/WindowingStrategies.java | 52 +-
.../runners/core/construction/CodersTest.java | 37 +-
.../EmptyFlattenAsCreateFactoryTest.java | 10 +-
.../construction/ForwardingPTransformTest.java | 7 +-
.../core/construction/PCollectionsTest.java | 6 +-
.../construction/PTransformMatchersTest.java | 22 +-
.../PTransformReplacementsTest.java | 4 +-
.../core/construction/PTransformsTest.java | 17 +-
.../core/construction/SdkComponentsTest.java | 16 +-
.../UnboundedReadFromBoundedSourceTest.java | 14 +-
.../construction/WindowingStrategiesTest.java | 6 +-
runners/core-java/pom.xml | 15 -
.../beam/runners/core/AggregatorFactory.java | 38 -
.../beam/runners/core/BaseExecutionContext.java | 2 +-
.../apache/beam/runners/core/DoFnAdapters.java | 18 -
.../apache/beam/runners/core/DoFnRunners.java | 44 +-
.../beam/runners/core/ExecutionContext.java | 2 +-
.../runners/core/GlobalCombineFnRunner.java | 78 +
.../runners/core/GlobalCombineFnRunners.java | 193 ++
.../GroupAlsoByWindowViaOutputBufferDoFn.java | 3 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 15 +-
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 14 +-
.../runners/core/GroupAlsoByWindowsDoFn.java | 7 -
.../runners/core/InMemoryStateInternals.java | 112 +-
.../beam/runners/core/KeyedWorkItemCoder.java | 22 +-
.../core/LateDataDroppingDoFnRunner.java | 20 +-
.../apache/beam/runners/core/LateDataUtils.java | 6 +-
.../runners/core/MergingActiveWindowSet.java | 4 +-
.../beam/runners/core/MergingStateAccessor.java | 2 +-
.../apache/beam/runners/core/NonEmptyPanes.java | 2 +-
.../org/apache/beam/runners/core/OldDoFn.java | 131 -
...eBoundedSplittableProcessElementInvoker.java | 8 -
.../beam/runners/core/PaneInfoTracker.java | 2 +-
.../runners/core/PerKeyCombineFnRunner.java | 79 -
.../runners/core/PerKeyCombineFnRunners.java | 161 -
.../runners/core/ReduceFnContextFactory.java | 37 +-
.../beam/runners/core/ReduceFnRunner.java | 27 +-
.../beam/runners/core/SideInputHandler.java | 14 +-
.../beam/runners/core/SimpleDoFnRunner.java | 37 +-
.../beam/runners/core/SimpleOldDoFnRunner.java | 24 +-
.../beam/runners/core/SplittableParDo.java | 22 +-
.../apache/beam/runners/core/StateAccessor.java | 2 +-
.../beam/runners/core/StateInternals.java | 8 +-
.../runners/core/StateInternalsFactory.java | 2 +-
.../apache/beam/runners/core/StateMerging.java | 44 +-
.../beam/runners/core/StateNamespaces.java | 13 +-
.../apache/beam/runners/core/StateTable.java | 10 +-
.../org/apache/beam/runners/core/StateTag.java | 51 +-
.../org/apache/beam/runners/core/StateTags.java | 127 +-
.../beam/runners/core/StatefulDoFnRunner.java | 18 +-
.../beam/runners/core/SystemReduceFn.java | 21 +-
.../core/TestInMemoryStateInternals.java | 8 +-
.../beam/runners/core/TimerInternals.java | 18 +-
.../apache/beam/runners/core/WatermarkHold.java | 45 +-
.../beam/runners/core/WindowingInternals.java | 2 +-
.../AfterDelayFromFirstElementStateMachine.java | 2 +-
.../core/triggers/AfterPaneStateMachine.java | 2 +-
.../TriggerStateMachineContextFactory.java | 12 +-
.../triggers/TriggerStateMachineRunner.java | 4 +-
.../core/DoFnDelegatingAggregatorTest.java | 144 -
.../core/GroupAlsoByWindowsProperties.java | 37 +-
.../core/InMemoryStateInternalsTest.java | 44 +-
.../core/LateDataDroppingDoFnRunnerTest.java | 47 +-
.../core/MergingActiveWindowSetTest.java | 2 +-
.../apache/beam/runners/core/NoOpOldDoFn.java | 7 -
.../beam/runners/core/OldDoFnContextTest.java | 72 -
.../apache/beam/runners/core/OldDoFnTest.java | 141 -
.../beam/runners/core/ReduceFnRunnerTest.java | 119 +-
.../beam/runners/core/ReduceFnTester.java | 96 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 9 -
.../runners/core/SimpleOldDoFnRunnerTest.java | 2 +-
.../beam/runners/core/SplittableParDoTest.java | 2 +-
.../apache/beam/runners/core/StateTagTest.java | 83 +-
.../runners/core/StatefulDoFnRunnerTest.java | 59 +-
runners/direct-java/pom.xml | 10 -
.../runners/direct/AggregatorContainer.java | 200 --
.../CopyOnAccessInMemoryStateInternals.java | 198 +-
.../runners/direct/DirectExecutionContext.java | 15 +-
.../beam/runners/direct/DirectRunner.java | 45 +-
.../beam/runners/direct/EvaluationContext.java | 31 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 30 +-
.../beam/runners/direct/ParDoEvaluator.java | 16 +-
.../direct/ParDoMultiOverrideFactory.java | 6 +-
...littableProcessElementsEvaluatorFactory.java | 4 +-
.../direct/StatefulParDoEvaluatorFactory.java | 6 +-
.../runners/direct/StepTransformResult.java | 12 +-
.../beam/runners/direct/TransformResult.java | 8 +-
.../direct/WriteWithShardingFactory.java | 15 +-
.../runners/direct/AggregatorContainerTest.java | 137 -
.../direct/CloningBundleFactoryTest.java | 10 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 180 +-
.../runners/direct/DirectGraphVisitorTest.java | 6 +-
.../beam/runners/direct/DirectRunnerTest.java | 15 +-
.../runners/direct/EvaluationContextTest.java | 49 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 5 -
.../StatefulParDoEvaluatorFactoryTest.java | 10 +-
.../UnboundedReadEvaluatorFactoryTest.java | 4 +-
.../direct/WriteWithShardingFactoryTest.java | 20 +-
runners/flink/pom.xml | 15 +-
.../flink/FlinkBatchTransformTranslators.java | 94 +-
.../flink/FlinkBatchTranslationContext.java | 4 +
.../flink/FlinkDetachedRunnerResult.java | 11 -
.../FlinkPipelineExecutionEnvironment.java | 8 +
.../runners/flink/FlinkPipelineOptions.java | 18 +
.../apache/beam/runners/flink/FlinkRunner.java | 3 +-
.../beam/runners/flink/FlinkRunnerResult.java | 37 +-
.../flink/FlinkStreamingPipelineTranslator.java | 3 +
.../FlinkStreamingTransformTranslators.java | 70 +-
.../flink/FlinkStreamingTranslationContext.java | 3 +
.../metrics/DoFnRunnerWithMetricsUpdate.java | 91 +
.../flink/metrics/FlinkMetricContainer.java | 315 ++
.../flink/metrics/FlinkMetricResults.java | 146 +
.../flink/metrics/ReaderInvocationUtil.java | 71 +
.../runners/flink/metrics/package-info.java | 22 +
.../functions/AbstractFlinkCombineRunner.java | 182 ++
.../functions/FlinkAggregatorFactory.java | 53 -
.../functions/FlinkDoFnFunction.java | 12 +-
.../FlinkMergingNonShuffleReduceFunction.java | 173 +-
.../FlinkMergingPartialReduceFunction.java | 201 --
.../functions/FlinkMergingReduceFunction.java | 199 --
.../functions/FlinkNoOpStepContext.java | 2 +-
.../functions/FlinkPartialReduceFunction.java | 118 +-
.../functions/FlinkReduceFunction.java | 118 +-
.../functions/FlinkStatefulDoFnFunction.java | 14 +-
.../functions/HashingFlinkCombineRunner.java | 180 ++
.../functions/SortingFlinkCombineRunner.java | 192 ++
.../SerializableFnAggregatorWrapper.java | 98 -
.../translation/wrappers/SourceInputFormat.java | 23 +-
.../wrappers/streaming/DoFnOperator.java | 87 +-
.../streaming/SingletonKeyedWorkItemCoder.java | 22 +-
.../streaming/SplittableDoFnOperator.java | 6 +-
.../wrappers/streaming/WindowDoFnOperator.java | 6 +-
.../streaming/io/BoundedSourceWrapper.java | 17 +-
.../streaming/io/UnboundedSourceWrapper.java | 18 +-
.../state/FlinkBroadcastStateInternals.java | 202 +-
.../state/FlinkKeyGroupStateInternals.java | 130 +-
.../state/FlinkSplitStateInternals.java | 130 +-
.../streaming/state/FlinkStateInternals.java | 225 +-
.../beam/runners/flink/FlinkTestPipeline.java | 9 +-
.../beam/runners/flink/PipelineOptionsTest.java | 19 +
.../beam/runners/flink/ReadSourceITCase.java | 6 +-
.../flink/ReadSourceStreamingITCase.java | 6 +-
.../beam/runners/flink/WriteSinkITCase.java | 192 --
.../flink/streaming/DoFnOperatorTest.java | 9 +-
.../FlinkBroadcastStateInternalsTest.java | 6 +-
.../FlinkKeyGroupStateInternalsTest.java | 2 +-
.../streaming/FlinkSplitStateInternalsTest.java | 2 +-
.../streaming/FlinkStateInternalsTest.java | 40 +-
.../flink/streaming/GroupByNullKeyTest.java | 13 +-
.../streaming/TopWikipediaSessionsITCase.java | 2 +-
.../streaming/UnboundedSourceWrapperTest.java | 12 +-
runners/google-cloud-dataflow-java/pom.xml | 31 +-
.../beam/runners/dataflow/AssignWindows.java | 5 +-
.../runners/dataflow/BatchViewOverrides.java | 21 +-
.../dataflow/DataflowAggregatorTransforms.java | 79 -
.../dataflow/DataflowMetricUpdateExtractor.java | 109 -
.../beam/runners/dataflow/DataflowMetrics.java | 44 +-
.../runners/dataflow/DataflowPipelineJob.java | 103 +-
.../dataflow/DataflowPipelineTranslator.java | 13 +-
.../beam/runners/dataflow/DataflowRunner.java | 164 +-
.../runners/dataflow/internal/IsmFormat.java | 92 +-
.../options/DataflowPipelineOptions.java | 8 +-
.../dataflow/testing/TestDataflowRunner.java | 181 +-
.../util/AvroCoderCloudObjectTranslator.java | 62 +
.../runners/dataflow/util/CloudKnownType.java | 138 +
.../beam/runners/dataflow/util/CloudObject.java | 185 ++
.../runners/dataflow/util/CloudObjectKinds.java | 33 +
.../dataflow/util/CloudObjectTranslator.java | 48 +
.../dataflow/util/CloudObjectTranslators.java | 575 ++++
.../runners/dataflow/util/CloudObjects.java | 92 +
.../CoderCloudObjectTranslatorRegistrar.java | 47 +
.../dataflow/util/DataflowTemplateJob.java | 2 +-
.../dataflow/util/DataflowTransport.java | 2 +-
...aultCoderCloudObjectTranslatorRegistrar.java | 136 +
.../runners/dataflow/util/MonitoringUtil.java | 4 +-
.../runners/dataflow/util/RandomAccessData.java | 9 +-
.../SerializableCoderCloudObjectTranslator.java | 65 +
.../beam/runners/dataflow/util/Serializer.java | 262 ++
.../BatchStatefulParDoOverridesTest.java | 4 +-
.../runners/dataflow/DataflowMetricsTest.java | 2 +-
.../dataflow/DataflowPipelineJobTest.java | 490 +---
.../DataflowPipelineTranslatorTest.java | 120 +-
.../runners/dataflow/DataflowRunnerTest.java | 43 +-
.../testing/TestDataflowRunnerTest.java | 374 +--
.../runners/dataflow/util/CloudObjectsTest.java | 217 ++
.../dataflow/util/MonitoringUtilTest.java | 2 +-
.../runners/dataflow/util/PackageUtilTest.java | 2 +-
runners/spark/pom.xml | 16 +-
.../spark/SparkNativePipelineVisitor.java | 2 +-
.../beam/runners/spark/SparkPipelineResult.java | 12 -
.../beam/runners/spark/TestSparkRunner.java | 35 +-
.../spark/aggregators/SparkAggregators.java | 110 -
.../beam/runners/spark/examples/WordCount.java | 17 +-
.../beam/runners/spark/io/MicrobatchSource.java | 188 +-
.../beam/runners/spark/io/SourceDStream.java | 57 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 11 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 58 +-
.../spark/stateful/SparkStateInternals.java | 111 +-
.../spark/stateful/StateSpecFunctions.java | 30 +-
.../spark/translation/MultiDoFnFunction.java | 2 -
.../translation/SparkAbstractCombineFn.java | 4 +-
.../spark/translation/SparkGlobalCombineFn.java | 37 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 13 +-
.../spark/translation/SparkKeyedCombineFn.java | 63 +-
.../spark/translation/SparkProcessContext.java | 2 +-
.../spark/translation/SparkRuntimeContext.java | 84 +-
.../spark/translation/TransformTranslator.java | 44 +-
.../spark/translation/TranslationUtils.java | 2 +-
.../streaming/StreamingTransformTranslator.java | 100 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 21 +-
.../spark/aggregators/ClearAggregatorsRule.java | 38 -
.../metrics/sink/NamedAggregatorsTest.java | 101 -
.../metrics/sink/SparkMetricsSinkTest.java | 86 +
.../beam/runners/spark/io/AvroPipelineTest.java | 4 +-
.../beam/runners/spark/io/NumShardsTest.java | 3 +-
.../translation/streaming/CreateStreamTest.java | 50 +
.../ResumeFromCheckpointStreamingTest.java | 94 +-
.../streaming/StreamingSourceMetricsTest.java | 19 +-
.../src/main/resources/beam/findbugs-filter.xml | 10 +-
sdks/java/core/pom.xml | 88 +-
.../beam/sdk/AggregatorPipelineExtractor.java | 92 -
.../beam/sdk/AggregatorRetrievalException.java | 33 -
.../org/apache/beam/sdk/AggregatorValues.java | 51 -
.../main/java/org/apache/beam/sdk/Pipeline.java | 152 +-
.../org/apache/beam/sdk/PipelineResult.java | 12 -
.../beam/sdk/annotations/Experimental.java | 3 -
.../apache/beam/sdk/annotations/Internal.java | 42 +
.../org/apache/beam/sdk/coders/AtomicCoder.java | 52 -
.../org/apache/beam/sdk/coders/AvroCoder.java | 152 +-
.../apache/beam/sdk/coders/BigDecimalCoder.java | 26 +-
.../beam/sdk/coders/BigEndianIntegerCoder.java | 7 +-
.../beam/sdk/coders/BigEndianLongCoder.java | 5 +-
.../apache/beam/sdk/coders/BigIntegerCoder.java | 18 +-
.../org/apache/beam/sdk/coders/BitSetCoder.java | 63 +
.../apache/beam/sdk/coders/ByteArrayCoder.java | 18 +-
.../org/apache/beam/sdk/coders/ByteCoder.java | 4 +-
.../apache/beam/sdk/coders/ByteStringCoder.java | 113 -
.../java/org/apache/beam/sdk/coders/Coder.java | 42 +-
.../apache/beam/sdk/coders/CoderFactories.java | 22 +-
.../apache/beam/sdk/coders/CoderRegistrar.java | 45 +
.../apache/beam/sdk/coders/CoderRegistry.java | 128 +-
.../org/apache/beam/sdk/coders/CustomCoder.java | 80 +-
.../apache/beam/sdk/coders/DelegateCoder.java | 43 +-
.../sdk/coders/DeterministicStandardCoder.java | 39 -
.../org/apache/beam/sdk/coders/DoubleCoder.java | 4 +-
.../apache/beam/sdk/coders/DurationCoder.java | 19 +-
.../apache/beam/sdk/coders/InstantCoder.java | 20 +-
.../apache/beam/sdk/coders/IterableCoder.java | 9 -
.../beam/sdk/coders/IterableLikeCoder.java | 2 +-
.../org/apache/beam/sdk/coders/JAXBCoder.java | 201 --
.../org/apache/beam/sdk/coders/KvCoder.java | 13 +-
.../beam/sdk/coders/LengthPrefixCoder.java | 25 +-
.../org/apache/beam/sdk/coders/MapCoder.java | 15 +-
.../apache/beam/sdk/coders/NullableCoder.java | 29 +-
.../beam/sdk/coders/SerializableCoder.java | 34 +-
.../apache/beam/sdk/coders/StandardCoder.java | 265 --
.../beam/sdk/coders/StringDelegateCoder.java | 21 +-
.../apache/beam/sdk/coders/StringUtf8Coder.java | 9 +-
.../apache/beam/sdk/coders/StructuredCoder.java | 188 ++
.../beam/sdk/coders/TableRowJsonCoder.java | 88 -
.../beam/sdk/coders/TextualIntegerCoder.java | 9 +-
.../org/apache/beam/sdk/coders/VarIntCoder.java | 7 +-
.../apache/beam/sdk/coders/VarLongCoder.java | 22 +-
.../org/apache/beam/sdk/coders/VoidCoder.java | 5 +-
.../beam/sdk/coders/protobuf/ProtoCoder.java | 405 ---
.../beam/sdk/coders/protobuf/ProtobufUtil.java | 171 --
.../beam/sdk/coders/protobuf/package-info.java | 24 -
.../java/org/apache/beam/sdk/io/AvroIO.java | 1272 ++------
.../java/org/apache/beam/sdk/io/AvroSink.java | 142 +
.../java/org/apache/beam/sdk/io/AvroSource.java | 153 +-
.../apache/beam/sdk/io/BlockBasedSource.java | 10 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 9 -
.../apache/beam/sdk/io/CompressedSource.java | 26 +-
.../org/apache/beam/sdk/io/CountingInput.java | 283 --
.../org/apache/beam/sdk/io/CountingSource.java | 53 +-
.../org/apache/beam/sdk/io/FileBasedSink.java | 187 +-
.../org/apache/beam/sdk/io/FileBasedSource.java | 359 +--
.../java/org/apache/beam/sdk/io/FileSystem.java | 19 +
.../apache/beam/sdk/io/FileSystemRegistrar.java | 15 +-
.../org/apache/beam/sdk/io/FileSystems.java | 194 +-
.../apache/beam/sdk/io/GenerateSequence.java | 194 ++
.../org/apache/beam/sdk/io/LocalFileSystem.java | 39 +-
.../beam/sdk/io/LocalFileSystemRegistrar.java | 15 +-
.../org/apache/beam/sdk/io/LocalResourceId.java | 45 +-
.../org/apache/beam/sdk/io/LocalResources.java | 56 +
.../apache/beam/sdk/io/ShardNameTemplate.java | 2 +-
.../main/java/org/apache/beam/sdk/io/Sink.java | 295 --
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 42 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 1218 ++------
.../java/org/apache/beam/sdk/io/TextSink.java | 139 +
.../java/org/apache/beam/sdk/io/TextSource.java | 236 ++
.../main/java/org/apache/beam/sdk/io/Write.java | 603 ----
.../java/org/apache/beam/sdk/io/WriteFiles.java | 558 ++++
.../main/java/org/apache/beam/sdk/io/XmlIO.java | 477 ---
.../java/org/apache/beam/sdk/io/XmlSink.java | 153 -
.../java/org/apache/beam/sdk/io/XmlSource.java | 402 ---
.../org/apache/beam/sdk/io/fs/MatchResult.java | 66 +-
.../org/apache/beam/sdk/io/fs/ResourceId.java | 36 +-
.../org/apache/beam/sdk/io/package-info.java | 2 +-
.../org/apache/beam/sdk/io/range/ByteKey.java | 21 +-
.../apache/beam/sdk/metrics/CounterCell.java | 19 +-
.../beam/sdk/metrics/DistributionCell.java | 10 +-
.../org/apache/beam/sdk/metrics/GaugeCell.java | 8 +-
.../org/apache/beam/sdk/metrics/Metric.java | 8 +-
.../org/apache/beam/sdk/metrics/MetricCell.java | 8 +-
.../apache/beam/sdk/metrics/MetricResults.java | 29 +-
.../org/apache/beam/sdk/metrics/Metrics.java | 39 +
.../beam/sdk/metrics/MetricsContainer.java | 8 +-
.../apache/beam/sdk/metrics/MetricsFilter.java | 9 +-
.../apache/beam/sdk/metrics/SinkMetrics.java | 49 +
.../apache/beam/sdk/metrics/SourceMetrics.java | 116 +
.../beam/sdk/options/PipelineOptions.java | 8 +-
.../sdk/options/PipelineOptionsFactory.java | 4 +-
.../sdk/options/ProxyInvocationHandler.java | 16 +-
.../beam/sdk/testing/CoderProperties.java | 47 +-
.../org/apache/beam/sdk/testing/PAssert.java | 164 +-
.../beam/sdk/testing/SourceTestUtils.java | 5 +-
.../beam/sdk/testing/SuccessOrFailure.java | 82 +
.../apache/beam/sdk/testing/TestPipeline.java | 13 +-
.../org/apache/beam/sdk/testing/TestStream.java | 116 -
.../beam/sdk/testing/UsesCounterMetrics.java | 25 +
.../sdk/testing/UsesDistributionMetrics.java | 26 +
.../beam/sdk/testing/UsesGaugeMetrics.java | 25 +
.../beam/sdk/testing/WindowFnTestUtils.java | 53 +-
.../apache/beam/sdk/transforms/Aggregator.java | 38 +-
.../sdk/transforms/AggregatorRetriever.java | 45 -
.../sdk/transforms/ApproximateQuantiles.java | 8 +-
.../beam/sdk/transforms/ApproximateUnique.java | 3 +-
.../org/apache/beam/sdk/transforms/Combine.java | 686 ++---
.../beam/sdk/transforms/CombineFnBase.java | 136 -
.../apache/beam/sdk/transforms/CombineFns.java | 463 +--
.../beam/sdk/transforms/CombineWithContext.java | 174 +-
.../org/apache/beam/sdk/transforms/Count.java | 35 +-
.../sdk/transforms/DelegatingAggregator.java | 126 -
.../org/apache/beam/sdk/transforms/DoFn.java | 128 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 78 -
.../org/apache/beam/sdk/transforms/Filter.java | 35 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 30 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 8 +-
.../org/apache/beam/sdk/transforms/Latest.java | 5 +-
.../org/apache/beam/sdk/transforms/Mean.java | 10 +-
.../apache/beam/sdk/transforms/PTransform.java | 8 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 2 +-
.../org/apache/beam/sdk/transforms/Top.java | 6 +-
.../org/apache/beam/sdk/transforms/View.java | 27 +-
.../beam/sdk/transforms/join/CoGbkResult.java | 33 +-
.../sdk/transforms/join/CoGbkResultSchema.java | 21 +-
.../beam/sdk/transforms/join/UnionCoder.java | 18 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 27 +-
.../transforms/windowing/AfterWatermark.java | 3 -
.../sdk/transforms/windowing/GlobalWindow.java | 25 +-
.../transforms/windowing/IntervalWindow.java | 31 +-
.../sdk/transforms/windowing/OutputTimeFn.java | 314 --
.../sdk/transforms/windowing/OutputTimeFns.java | 212 --
.../beam/sdk/transforms/windowing/PaneInfo.java | 9 +-
.../transforms/windowing/TimestampCombiner.java | 186 ++
.../beam/sdk/transforms/windowing/Trigger.java | 12 +-
.../beam/sdk/transforms/windowing/Window.java | 27 +-
.../org/apache/beam/sdk/util/ApiSurface.java | 4 +-
.../apache/beam/sdk/util/AppliedCombineFn.java | 35 +-
.../org/apache/beam/sdk/util/AvroUtils.java | 137 -
.../org/apache/beam/sdk/util/BitSetCoder.java | 15 +-
.../org/apache/beam/sdk/util/ClassPath.java | 544 ++++
.../apache/beam/sdk/util/CloudKnownType.java | 9 +-
.../org/apache/beam/sdk/util/CloudObject.java | 3 +
.../org/apache/beam/sdk/util/CoderUtils.java | 15 +-
.../org/apache/beam/sdk/util/CombineFnUtil.java | 123 +-
.../beam/sdk/util/ExplicitShardedFile.java | 23 +-
.../apache/beam/sdk/util/IOChannelUtils.java | 50 +-
.../beam/sdk/util/NumberedShardedFile.java | 29 +-
.../apache/beam/sdk/util/PCollectionViews.java | 4 +-
.../org/apache/beam/sdk/util/PropertyNames.java | 4 +-
.../org/apache/beam/sdk/util/Reshuffle.java | 7 +-
.../sdk/util/RetryHttpRequestInitializer.java | 248 --
.../apache/beam/sdk/util/SerializableUtils.java | 26 +-
.../org/apache/beam/sdk/util/Serializer.java | 3 +
.../java/org/apache/beam/sdk/util/Structs.java | 74 +-
.../apache/beam/sdk/util/ValueWithRecordId.java | 16 +-
.../org/apache/beam/sdk/util/WindowedValue.java | 19 +-
.../apache/beam/sdk/util/WindowingStrategy.java | 178 +-
.../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 626 ----
.../beam/sdk/util/gcsfs/package-info.java | 20 -
.../apache/beam/sdk/util/state/StateBinder.java | 53 +-
.../apache/beam/sdk/util/state/StateSpec.java | 15 +-
.../apache/beam/sdk/util/state/StateSpecs.java | 278 +-
.../beam/sdk/util/state/WatermarkHoldState.java | 19 +-
.../org/apache/beam/sdk/values/PCollection.java | 9 +-
.../java/org/apache/beam/sdk/values/PDone.java | 33 +-
.../beam/sdk/values/POutputValueBase.java | 74 -
.../java/org/apache/beam/sdk/values/PValue.java | 2 +
.../org/apache/beam/sdk/values/PValueBase.java | 29 +-
.../beam/sdk/values/TimestampedValue.java | 22 +-
.../beam/sdk/values/ValueInSingleWindow.java | 23 +-
sdks/java/core/src/main/proto/README.md | 45 -
.../main/proto/proto2_coder_test_messages.proto | 53 -
.../org/apache/beam/SdkCoreApiSurfaceTest.java | 3 -
.../sdk/AggregatorPipelineExtractorTest.java | 226 --
.../java/org/apache/beam/sdk/PipelineTest.java | 97 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 86 +-
.../beam/sdk/coders/AvroCoderTestPojo.java | 51 +
.../beam/sdk/coders/BigDecimalCoderTest.java | 18 -
.../sdk/coders/BigEndianIntegerCoderTest.java | 8 -
.../beam/sdk/coders/BigEndianLongCoderTest.java | 8 -
.../beam/sdk/coders/BigIntegerCoderTest.java | 18 -
.../apache/beam/sdk/coders/BitSetCoderTest.java | 103 +
.../beam/sdk/coders/ByteArrayCoderTest.java | 8 -
.../apache/beam/sdk/coders/ByteCoderTest.java | 8 -
.../beam/sdk/coders/ByteStringCoderTest.java | 128 -
.../beam/sdk/coders/CoderRegistryTest.java | 118 +-
.../beam/sdk/coders/CollectionCoderTest.java | 8 -
.../apache/beam/sdk/coders/CustomCoderTest.java | 46 -
.../beam/sdk/coders/DefaultCoderTest.java | 8 +-
.../beam/sdk/coders/DelegateCoderTest.java | 38 +-
.../apache/beam/sdk/coders/DoubleCoderTest.java | 8 -
.../beam/sdk/coders/InstantCoderTest.java | 8 -
.../beam/sdk/coders/IterableCoderTest.java | 18 -
.../apache/beam/sdk/coders/JAXBCoderTest.java | 223 --
.../org/apache/beam/sdk/coders/KvCoderTest.java | 20 -
.../beam/sdk/coders/LengthPrefixCoderTest.java | 18 +-
.../apache/beam/sdk/coders/ListCoderTest.java | 8 -
.../apache/beam/sdk/coders/MapCoderTest.java | 8 -
.../beam/sdk/coders/NullableCoderTest.java | 13 +-
.../beam/sdk/coders/SerializableCoderTest.java | 19 +-
.../apache/beam/sdk/coders/SetCoderTest.java | 8 -
.../beam/sdk/coders/StandardCoderTest.java | 238 --
.../sdk/coders/StringDelegateCoderTest.java | 6 -
.../beam/sdk/coders/StructuredCoderTest.java | 238 ++
.../beam/sdk/coders/TableRowJsonCoderTest.java | 94 -
.../sdk/coders/TextualIntegerCoderTest.java | 8 -
.../apache/beam/sdk/coders/VarIntCoderTest.java | 8 -
.../beam/sdk/coders/VarLongCoderTest.java | 8 -
.../sdk/coders/protobuf/ProtoCoderTest.java | 182 --
.../sdk/coders/protobuf/ProtobufUtilTest.java | 192 --
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 156 +-
.../apache/beam/sdk/io/AvroIOTransformTest.java | 30 +-
.../org/apache/beam/sdk/io/AvroSourceTest.java | 56 +-
.../beam/sdk/io/CompressedSourceTest.java | 12 +-
.../apache/beam/sdk/io/CountingInputTest.java | 221 --
.../apache/beam/sdk/io/CountingSourceTest.java | 4 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 102 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 192 +-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 20 +-
.../beam/sdk/io/GenerateSequenceTest.java | 194 ++
.../sdk/io/LocalFileSystemRegistrarTest.java | 6 +
.../apache/beam/sdk/io/LocalFileSystemTest.java | 91 +-
.../apache/beam/sdk/io/LocalResourceIdTest.java | 45 +
.../java/org/apache/beam/sdk/io/SimpleSink.java | 96 +
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 6 +-
.../java/org/apache/beam/sdk/io/TextIOTest.java | 106 +-
.../org/apache/beam/sdk/io/WriteFilesTest.java | 457 +++
.../java/org/apache/beam/sdk/io/WriteTest.java | 705 -----
.../org/apache/beam/sdk/io/XmlSinkTest.java | 253 --
.../org/apache/beam/sdk/io/XmlSourceTest.java | 892 ------
.../range/ByteKeyRangeInterpolateKeyTest.java | 1 -
.../beam/sdk/io/range/ByteKeyRangeTest.java | 16 +-
.../apache/beam/sdk/io/range/ByteKeyTest.java | 4 +-
.../apache/beam/sdk/metrics/MetricMatchers.java | 144 +-
.../apache/beam/sdk/metrics/MetricsTest.java | 183 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 71 +-
.../beam/sdk/options/PipelineOptionsTest.java | 14 +
.../sdk/options/ProxyInvocationHandlerTest.java | 2 +-
.../sdk/runners/TransformHierarchyTest.java | 7 +-
.../beam/sdk/runners/TransformTreeTest.java | 12 +-
.../beam/sdk/testing/GatherAllPanesTest.java | 8 +-
.../apache/beam/sdk/testing/PAssertTest.java | 71 +-
.../sdk/testing/SerializableMatchersTest.java | 4 +-
.../beam/sdk/testing/SystemNanoTimeSleeper.java | 1 -
.../apache/beam/sdk/testing/TestStreamTest.java | 24 -
.../beam/sdk/testing/WindowSupplierTest.java | 4 +-
.../beam/sdk/transforms/CombineFnsTest.java | 118 +-
.../apache/beam/sdk/transforms/CombineTest.java | 218 +-
.../apache/beam/sdk/transforms/CreateTest.java | 3 +-
.../apache/beam/sdk/transforms/DoFnTest.java | 162 --
.../beam/sdk/transforms/DoFnTesterTest.java | 47 +-
.../apache/beam/sdk/transforms/FilterTest.java | 13 +
.../apache/beam/sdk/transforms/FlattenTest.java | 8 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 19 +-
.../beam/sdk/transforms/LatestFnTest.java | 42 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 189 +-
.../org/apache/beam/sdk/transforms/SumTest.java | 2 +-
.../apache/beam/sdk/transforms/ViewTest.java | 2 +-
.../display/DisplayDataEvaluatorTest.java | 5 +-
.../sdk/transforms/join/CoGroupByKeyTest.java | 6 +-
.../sdk/transforms/join/UnionCoderTest.java | 11 +
.../transforms/reflect/DoFnInvokersTest.java | 6 +-
.../transforms/reflect/DoFnSignaturesTest.java | 26 +-
.../transforms/windowing/GlobalWindowTest.java | 7 -
.../transforms/windowing/OutputTimeFnsTest.java | 51 -
.../sdk/transforms/windowing/SessionsTest.java | 6 +-
.../sdk/transforms/windowing/WindowTest.java | 27 +-
.../sdk/transforms/windowing/WindowingTest.java | 4 +-
.../org/apache/beam/sdk/util/AvroUtilsTest.java | 147 -
.../apache/beam/sdk/util/CoderUtilsTest.java | 108 +-
.../apache/beam/sdk/util/CombineFnUtilTest.java | 18 +-
.../beam/sdk/util/IOChannelUtilsTest.java | 30 -
.../org/apache/beam/sdk/util/NameUtilsTest.java | 2 +-
.../beam/sdk/util/NumberedShardedFileTest.java | 41 +-
.../beam/sdk/util/SerializableUtilsTest.java | 56 +-
.../apache/beam/sdk/util/WindowedValueTest.java | 10 -
.../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 358 ---
.../beam/sdk/values/PCollectionListTest.java | 25 +-
.../beam/sdk/values/PCollectionTupleTest.java | 6 +-
.../org/apache/beam/sdk/values/PDoneTest.java | 2 +-
sdks/java/extensions/gcp-core/pom.xml | 222 --
.../beam/sdk/options/BigQueryOptions.java | 32 -
.../options/CloudResourceManagerOptions.java | 40 -
.../org/apache/beam/sdk/options/GcpOptions.java | 227 --
.../options/GcpPipelineOptionsRegistrar.java | 39 -
.../org/apache/beam/sdk/options/GcsOptions.java | 154 -
.../beam/sdk/options/GoogleApiDebugOptions.java | 87 -
.../apache/beam/sdk/options/PubsubOptions.java | 36 -
.../apache/beam/sdk/options/package-info.java | 22 -
.../beam/sdk/testing/BigqueryMatcher.java | 256 --
.../apache/beam/sdk/testing/package-info.java | 21 -
.../apache/beam/sdk/util/CredentialFactory.java | 29 -
.../org/apache/beam/sdk/util/DefaultBucket.java | 105 -
.../beam/sdk/util/GcpCredentialFactory.java | 67 -
.../apache/beam/sdk/util/GcpProjectUtil.java | 106 -
.../beam/sdk/util/GcsIOChannelFactory.java | 111 -
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 -
.../apache/beam/sdk/util/GcsPathValidator.java | 95 -
.../java/org/apache/beam/sdk/util/GcsUtil.java | 798 -----
.../beam/sdk/util/NoopCredentialFactory.java | 68 -
.../sdk/util/NullCredentialInitializer.java | 62 -
.../apache/beam/sdk/util/TestCredential.java | 59 -
.../org/apache/beam/sdk/util/Transport.java | 178 --
.../org/apache/beam/sdk/util/package-info.java | 20 -
.../org/apache/beam/GcpCoreApiSurfaceTest.java | 62 -
.../apache/beam/sdk/options/GcpOptionsTest.java | 171 --
.../sdk/options/GoogleApiDebugOptionsTest.java | 145 -
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 --
.../apache/beam/sdk/util/DefaultBucketTest.java | 112 -
.../beam/sdk/util/GcpProjectUtilTest.java | 76 -
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 -
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 -
.../beam/sdk/util/GcsPathValidatorTest.java | 87 -
.../org/apache/beam/sdk/util/GcsUtilTest.java | 798 -----
.../util/RetryHttpRequestInitializerTest.java | 290 --
.../google-cloud-platform-core/pom.xml | 185 ++
.../extensions/gcp/auth/CredentialFactory.java | 29 +
.../gcp/auth/GcpCredentialFactory.java | 67 +
.../gcp/auth/NoopCredentialFactory.java | 68 +
.../gcp/auth/NullCredentialInitializer.java | 62 +
.../sdk/extensions/gcp/auth/package-info.java | 22 +
.../options/CloudResourceManagerOptions.java | 32 +
.../sdk/extensions/gcp/options/GcpOptions.java | 389 +++
.../options/GcpPipelineOptionsRegistrar.java | 39 +
.../sdk/extensions/gcp/options/GcsOptions.java | 160 +
.../gcp/options/GoogleApiDebugOptions.java | 89 +
.../extensions/gcp/options/package-info.java | 22 +
.../extensions/gcp/storage/GcsFileSystem.java | 266 ++
.../gcp/storage/GcsFileSystemRegistrar.java | 43 +
.../extensions/gcp/storage/GcsResourceId.java | 128 +
.../extensions/gcp/storage/package-info.java | 21 +
.../beam/sdk/util/GcsIOChannelFactory.java | 111 +
.../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 +
.../apache/beam/sdk/util/GcsPathValidator.java | 98 +
.../java/org/apache/beam/sdk/util/GcsUtil.java | 796 +++++
.../sdk/util/RetryHttpRequestInitializer.java | 192 ++
.../org/apache/beam/sdk/util/Transport.java | 122 +
.../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 627 ++++
.../beam/sdk/util/gcsfs/package-info.java | 20 +
.../org/apache/beam/sdk/util/package-info.java | 20 +
.../extensions/gcp/GcpCoreApiSurfaceTest.java | 58 +
.../sdk/extensions/gcp/auth/TestCredential.java | 59 +
.../extensions/gcp/options/GcpOptionsTest.java | 273 ++
.../gcp/options/GoogleApiDebugOptionsTest.java | 147 +
.../gcp/storage/GcsFileSystemRegistrarTest.java | 52 +
.../gcp/storage/GcsFileSystemTest.java | 274 ++
.../gcp/storage/GcsResourceIdTest.java | 169 ++
.../util/GcsIOChannelFactoryRegistrarTest.java | 44 +
.../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 +
.../beam/sdk/util/GcsPathValidatorTest.java | 106 +
.../org/apache/beam/sdk/util/GcsUtilTest.java | 799 +++++
.../util/RetryHttpRequestInitializerTest.java | 281 ++
.../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 358 +++
sdks/java/extensions/jackson/pom.xml | 99 +-
sdks/java/extensions/join-library/pom.xml | 22 -
sdks/java/extensions/pom.xml | 3 +-
sdks/java/extensions/protobuf/pom.xml | 121 +
.../extensions/protobuf/ByteStringCoder.java | 118 +
.../sdk/extensions/protobuf/ProtoCoder.java | 332 +++
.../protobuf/ProtobufCoderRegistrar.java | 39 +
.../sdk/extensions/protobuf/ProtobufUtil.java | 171 ++
.../sdk/extensions/protobuf/package-info.java | 24 +
.../protobuf/ByteStringCoderTest.java | 131 +
.../sdk/extensions/protobuf/ProtoCoderTest.java | 169 ++
.../extensions/protobuf/ProtobufUtilTest.java | 191 ++
.../test/proto/proto2_coder_test_messages.proto | 53 +
sdks/java/extensions/sorter/pom.xml | 24 +-
sdks/java/harness/pom.xml | 12 +-
.../org/apache/beam/fn/harness/FnHarness.java | 2 +-
.../harness/control/ProcessBundleHandler.java | 2 -
.../fn/harness/fake/FakeAggregatorFactory.java | 52 -
.../beam/fn/harness/fake/FakeStepContext.java | 2 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 2 +-
.../harness/stream/StreamObserverFactory.java | 2 +-
.../beam/runners/core/BeamFnDataReadRunner.java | 14 +-
.../runners/core/BeamFnDataWriteRunner.java | 15 +-
.../apache/beam/fn/harness/FnHarnessTest.java | 2 +-
.../control/ProcessBundleHandlerTest.java | 28 +-
.../runners/core/BeamFnDataReadRunnerTest.java | 12 +-
.../runners/core/BeamFnDataWriteRunnerTest.java | 12 +-
sdks/java/io/elasticsearch/pom.xml | 1 -
.../sdk/io/elasticsearch/ElasticsearchIO.java | 4 +-
.../elasticsearch/ElasticSearchIOTestUtils.java | 2 +-
sdks/java/io/google-cloud-platform/pom.xml | 19 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 28 +-
.../io/gcp/bigquery/BigQueryCoderRegistrar.java | 39 +
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 45 +-
.../sdk/io/gcp/bigquery/BigQueryOptions.java | 39 +
.../io/gcp/bigquery/BigQueryQuerySource.java | 1 -
.../sdk/io/gcp/bigquery/BigQueryServices.java | 1 -
.../io/gcp/bigquery/BigQueryServicesImpl.java | 43 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 4 +-
.../io/gcp/bigquery/BigQueryTableSource.java | 1 -
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 1 -
.../io/gcp/bigquery/PassThroughThenCleanup.java | 4 +-
.../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 1 -
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 17 +-
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 5 +-
.../io/gcp/bigquery/StreamingWriteTables.java | 1 -
.../io/gcp/bigquery/TableDestinationCoder.java | 6 +-
.../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 7 +-
.../sdk/io/gcp/bigquery/TableRowJsonCoder.java | 89 +
.../sdk/io/gcp/bigquery/TableRowWriter.java | 28 +-
.../io/gcp/bigquery/WriteBundlesToFiles.java | 9 +-
.../sdk/io/gcp/bigquery/WritePartition.java | 2 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 1 -
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 37 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 1 -
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 28 +-
.../io/gcp/bigtable/BigtableServiceImpl.java | 4 +-
.../common/GcpIoPipelineOptionsRegistrar.java | 39 +
.../beam/sdk/io/gcp/common/package-info.java | 20 +
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 6 +-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 43 +-
.../sdk/io/gcp/pubsub/PubsubGrpcClient.java | 39 +-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 644 ++--
.../sdk/io/gcp/pubsub/PubsubJsonClient.java | 37 +-
.../pubsub/PubsubMessagePayloadOnlyCoder.java | 48 +
.../PubsubMessageWithAttributesCoder.java | 57 +
.../beam/sdk/io/gcp/pubsub/PubsubOptions.java | 44 +
.../sdk/io/gcp/pubsub/PubsubTestClient.java | 7 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 166 +-
.../io/gcp/pubsub/PubsubUnboundedSource.java | 211 +-
.../beam/sdk/io/gcp/storage/GcsFileSystem.java | 242 --
.../io/gcp/storage/GcsFileSystemRegistrar.java | 49 -
.../beam/sdk/io/gcp/storage/GcsResourceId.java | 116 -
.../beam/sdk/io/gcp/storage/package-info.java | 21 -
.../beam/sdk/io/gcp/testing/package-info.java | 21 +
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 2 +
.../bigquery/BigQueryCoderRegistrarTest.java | 40 +
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 20 +-
.../io/gcp/bigquery/FakeBigQueryServices.java | 2 -
.../sdk/io/gcp/bigquery/FakeJobService.java | 1 -
.../io/gcp/bigquery/TableRowJsonCoderTest.java | 87 +
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 32 +-
.../sdk/io/gcp/bigtable/BigtableWriteIT.java | 6 +-
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 4 +-
.../beam/sdk/io/gcp/datastore/V1ReadIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 4 +-
.../beam/sdk/io/gcp/datastore/V1WriteIT.java | 6 +-
.../sdk/io/gcp/pubsub/PubsubClientTest.java | 50 +-
.../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java | 18 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 88 +-
.../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 14 +-
.../io/gcp/pubsub/PubsubUnboundedSinkTest.java | 58 +-
.../gcp/pubsub/PubsubUnboundedSourceTest.java | 112 +-
.../gcp/storage/GcsFileSystemRegistrarTest.java | 51 -
.../sdk/io/gcp/storage/GcsFileSystemTest.java | 274 --
.../sdk/io/gcp/storage/GcsResourceIdTest.java | 147 -
.../sdk/io/gcp/testing/BigqueryMatcher.java | 257 ++
.../sdk/io/gcp/testing/BigqueryMatcherTest.java | 177 ++
sdks/java/io/hadoop-common/pom.xml | 5 -
.../beam/sdk/io/hadoop/WritableCoder.java | 29 +-
sdks/java/io/hadoop-file-system/README.md | 43 +
sdks/java/io/hadoop-file-system/pom.xml | 195 ++
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 478 +++
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 625 ++++
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 240 ++
.../sdk/io/hdfs/HadoopFileSystemModule.java | 84 +
.../sdk/io/hdfs/HadoopFileSystemOptions.java | 49 +
.../hdfs/HadoopFileSystemOptionsRegistrar.java | 35 +
.../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 62 +
.../beam/sdk/io/hdfs/HadoopResourceId.java | 81 +
.../java/org/apache/beam/sdk/io/hdfs/Sink.java | 195 ++
.../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 +
.../java/org/apache/beam/sdk/io/hdfs/Write.java | 585 ++++
.../apache/beam/sdk/io/hdfs/package-info.java | 22 +
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 172 ++
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 231 ++
.../sdk/io/hdfs/HadoopFileSystemModuleTest.java | 65 +
.../HadoopFileSystemOptionsRegistrarTest.java | 49 +
.../io/hdfs/HadoopFileSystemOptionsTest.java | 48 +
.../io/hdfs/HadoopFileSystemRegistrarTest.java | 81 +
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 247 ++
sdks/java/io/hadoop/input-format/pom.xml | 15 -
.../hadoop/inputformat/HadoopInputFormatIO.java | 10 +-
.../inputformat/HadoopInputFormatIOTest.java | 29 +-
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 10 -
sdks/java/io/hadoop/pom.xml | 18 +
sdks/java/io/hbase/pom.xml | 7 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 28 +-
.../beam/sdk/io/hbase/HBaseMutationCoder.java | 5 +-
.../beam/sdk/io/hbase/HBaseResultCoder.java | 5 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 17 +-
sdks/java/io/hdfs/README.md | 43 -
sdks/java/io/hdfs/pom.xml | 141 -
.../apache/beam/sdk/io/hdfs/HDFSFileSink.java | 478 ---
.../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 625 ----
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 71 -
.../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 42 -
.../beam/sdk/io/hdfs/HadoopResourceId.java | 42 -
.../org/apache/beam/sdk/io/hdfs/UGIHelper.java | 38 -
.../apache/beam/sdk/io/hdfs/package-info.java | 22 -
.../beam/sdk/io/hdfs/HDFSFileSinkTest.java | 173 --
.../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 231 --
.../io/hdfs/HadoopFileSystemRegistrarTest.java | 52 -
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 5 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 4 +-
sdks/java/io/kafka/pom.xml | 5 -
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 470 ++-
.../beam/sdk/io/kafka/KafkaRecordCoder.java | 18 +-
.../CoderBasedKafkaDeserializer.java | 71 +
.../CoderBasedKafkaSerializer.java | 73 +
.../serialization/InstantDeserializer.java | 45 +
.../kafka/serialization/InstantSerializer.java | 45 +
.../io/kafka/serialization/package-info.java | 22 +
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 399 ++-
sdks/java/io/kinesis/pom.xml | 6 +
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 148 +-
.../beam/sdk/io/kinesis/KinesisRecordCoder.java | 4 +-
.../sdk/io/kinesis/KinesisMockReadTest.java | 14 +-
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 10 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 4 +-
.../org/apache/beam/sdk/io/mqtt/MqttIO.java | 4 +-
sdks/java/io/pom.xml | 3 +-
sdks/java/io/xml/pom.xml | 113 +
.../org/apache/beam/sdk/io/xml/JAXBCoder.java | 165 ++
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 543 ++++
.../org/apache/beam/sdk/io/xml/XmlSink.java | 161 +
.../org/apache/beam/sdk/io/xml/XmlSource.java | 399 +++
.../apache/beam/sdk/io/xml/package-info.java | 22 +
.../apache/beam/sdk/io/xml/JAXBCoderTest.java | 221 ++
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 276 ++
.../apache/beam/sdk/io/xml/XmlSourceTest.java | 987 +++++++
sdks/java/java8tests/pom.xml | 19 -
sdks/java/javadoc/pom.xml | 9 +-
.../main/resources/archetype-resources/pom.xml | 9 +-
.../main/resources/archetype-resources/pom.xml | 9 +-
sdks/java/maven-archetypes/pom.xml | 35 +
.../main/resources/archetype-resources/pom.xml | 2 +-
sdks/python/.pylintrc | 1 +
sdks/python/MANIFEST.in | 2 +
sdks/python/apache_beam/coders/coders.py | 16 -
sdks/python/apache_beam/coders/coders_test.py | 1 +
.../apache_beam/coders/coders_test_common.py | 7 +-
.../apache_beam/coders/fast_coders_test.py | 1 +
.../apache_beam/coders/slow_coders_test.py | 1 +
.../examples/complete/autocomplete_test.py | 1 +
.../examples/complete/game/hourly_team_score.py | 1 +
.../complete/game/hourly_team_score_test.py | 1 +
.../examples/complete/game/user_score.py | 1 +
.../examples/complete/game/user_score_test.py | 1 +
.../complete/juliaset/juliaset/juliaset_test.py | 1 +
.../examples/cookbook/bigquery_side_input.py | 2 +-
.../cookbook/bigquery_tornadoes_it_test.py | 1 +
.../apache_beam/examples/cookbook/bigshuffle.py | 94 -
.../examples/cookbook/bigshuffle_test.py | 63 -
.../apache_beam/examples/cookbook/filters.py | 2 +-
.../examples/cookbook/multiple_output_pardo.py | 6 +-
.../apache_beam/examples/snippets/snippets.py | 122 +-
.../examples/snippets/snippets_test.py | 41 +-
sdks/python/apache_beam/examples/wordcount.py | 1 +
.../examples/wordcount_debugging_test.py | 1 +
.../apache_beam/internal/gcp/json_value_test.py | 11 +
sdks/python/apache_beam/internal/module_test.py | 1 +
sdks/python/apache_beam/internal/pickler.py | 3 +
.../python/apache_beam/internal/pickler_test.py | 1 +
sdks/python/apache_beam/io/concat_source.py | 74 +-
.../python/apache_beam/io/concat_source_test.py | 3 +-
sdks/python/apache_beam/io/filebasedsource.py | 20 +-
.../apache_beam/io/filebasedsource_test.py | 19 +-
sdks/python/apache_beam/io/fileio.py | 45 +-
sdks/python/apache_beam/io/fileio_test.py | 56 +
sdks/python/apache_beam/io/filesystem.py | 35 +-
sdks/python/apache_beam/io/filesystems.py | 204 ++
sdks/python/apache_beam/io/filesystems_test.py | 224 ++
sdks/python/apache_beam/io/filesystems_util.py | 3 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 23 +-
.../io/gcp/datastore/v1/datastoreio.py | 10 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 1 +
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 48 +-
.../apache_beam/io/gcp/gcsfilesystem_test.py | 29 +
sdks/python/apache_beam/io/gcp/pubsub_test.py | 1 +
sdks/python/apache_beam/io/iobase.py | 5 +-
sdks/python/apache_beam/io/localfilesystem.py | 27 +-
.../apache_beam/io/localfilesystem_test.py | 68 +-
sdks/python/apache_beam/io/range_trackers.py | 11 +-
sdks/python/apache_beam/io/source_test_utils.py | 8 +-
.../apache_beam/io/source_test_utils_test.py | 1 +
sdks/python/apache_beam/io/sources_test.py | 2 +-
sdks/python/apache_beam/io/textio.py | 13 +-
sdks/python/apache_beam/io/tfrecordio.py | 2 +
sdks/python/apache_beam/io/tfrecordio_test.py | 9 +-
sdks/python/apache_beam/pipeline.py | 52 +-
sdks/python/apache_beam/pipeline_test.py | 14 +-
sdks/python/apache_beam/pvalue.py | 6 +-
sdks/python/apache_beam/runners/api/__init__.py | 14 +
.../apache_beam/runners/api/beam_fn_api_pb2.py | 2740 ++++++++++++++++++
.../runners/api/beam_fn_api_pb2_grpc.py | 203 ++
.../runners/api/beam_runner_api_pb2.py | 298 +-
sdks/python/apache_beam/runners/common.pxd | 62 +-
sdks/python/apache_beam/runners/common.py | 499 +++-
.../runners/dataflow/dataflow_metrics.py | 3 +
.../runners/dataflow/dataflow_metrics_test.py | 1 +
.../runners/dataflow/dataflow_runner.py | 20 +-
.../runners/dataflow/dataflow_runner_test.py | 4 +-
.../runners/dataflow/internal/apiclient.py | 17 +-
.../clients/dataflow/dataflow_v1b3_messages.py | 564 +++-
.../runners/dataflow/internal/dependency.py | 29 +-
.../dataflow/internal/dependency_test.py | 6 +-
.../runners/dataflow/native_io/iobase.py | 1 +
.../runners/dataflow/template_runner_test.py | 2 +-
.../runners/dataflow/test_dataflow_runner.py | 4 +-
.../consumer_tracking_pipeline_visitor_test.py | 2 +-
.../apache_beam/runners/direct/direct_runner.py | 12 +-
.../apache_beam/runners/portability/__init__.py | 16 +
.../runners/portability/fn_api_runner.py | 471 +++
.../runners/portability/fn_api_runner_test.py | 40 +
.../portability/maptask_executor_runner.py | 468 +++
.../portability/maptask_executor_runner_test.py | 204 ++
sdks/python/apache_beam/runners/runner.py | 34 -
.../apache_beam/runners/worker/__init__.py | 16 +
.../apache_beam/runners/worker/data_plane.py | 288 ++
.../runners/worker/data_plane_test.py | 139 +
.../apache_beam/runners/worker/log_handler.py | 100 +
.../runners/worker/log_handler_test.py | 105 +
.../apache_beam/runners/worker/logger.pxd | 25 +
.../python/apache_beam/runners/worker/logger.py | 173 ++
.../apache_beam/runners/worker/logger_test.py | 182 ++
.../apache_beam/runners/worker/opcounters.pxd | 45 +
.../apache_beam/runners/worker/opcounters.py | 162 ++
.../runners/worker/opcounters_test.py | 149 +
.../runners/worker/operation_specs.py | 368 +++
.../apache_beam/runners/worker/operations.pxd | 89 +
.../apache_beam/runners/worker/operations.py | 651 +++++
.../apache_beam/runners/worker/sdk_worker.py | 451 +++
.../runners/worker/sdk_worker_main.py | 62 +
.../runners/worker/sdk_worker_test.py | 168 ++
.../apache_beam/runners/worker/sideinputs.py | 166 ++
.../runners/worker/sideinputs_test.py | 150 +
.../apache_beam/runners/worker/statesampler.pyx | 237 ++
.../runners/worker/statesampler_fake.py | 34 +
.../runners/worker/statesampler_test.py | 102 +
.../apache_beam/tests/pipeline_verifiers.py | 7 +-
sdks/python/apache_beam/transforms/core.py | 90 +-
.../python/apache_beam/transforms/ptransform.py | 42 +-
.../apache_beam/transforms/ptransform_test.py | 195 +-
sdks/python/apache_beam/transforms/trigger.py | 10 +-
.../apache_beam/transforms/trigger_test.py | 10 +-
sdks/python/apache_beam/transforms/window.py | 114 +-
.../apache_beam/transforms/window_test.py | 16 +
sdks/python/apache_beam/typehints/decorators.py | 1 +
sdks/python/apache_beam/typehints/opcodes.py | 5 +-
sdks/python/apache_beam/typehints/typecheck.py | 4 +-
.../apache_beam/typehints/typehints_test.py | 1 +
sdks/python/apache_beam/utils/__init__.py | 4 -
sdks/python/apache_beam/utils/annotations.py | 14 +-
.../apache_beam/utils/annotations_test.py | 29 +
sdks/python/apache_beam/utils/path.py | 46 -
sdks/python/apache_beam/utils/path_test.py | 70 -
.../apache_beam/utils/pipeline_options_test.py | 2 +-
.../utils/pipeline_options_validator_test.py | 1 +
.../apache_beam/utils/test_stream_test.py | 1 +
sdks/python/apache_beam/utils/timestamp.py | 4 -
sdks/python/apache_beam/utils/urns.py | 93 +
sdks/python/apache_beam/utils/value_provider.py | 2 +-
.../apache_beam/utils/value_provider_test.py | 4 +-
sdks/python/generate_pydoc.sh | 2 +
sdks/python/pom.xml | 2 +-
sdks/python/run_postcommit.sh | 4 +-
sdks/python/run_pylint.sh | 9 +-
sdks/python/setup.py | 11 +-
sdks/python/tox.ini | 4 +-
950 files changed, 43325 insertions(+), 34844 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/9a59ea32/examples/java/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/9a59ea32/pom.xml
----------------------------------------------------------------------
[16/50] [abbrv] beam git commit: Update gearpump-runner against
master changes.
Posted by ke...@apache.org.
Update gearpump-runner against master changes.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/44d21ac6
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/44d21ac6
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/44d21ac6
Branch: refs/heads/master
Commit: 44d21ac662e263c09caf2dd3b93b1c325bdfea15
Parents: 46c41fc
Author: manuzhang <ow...@gmail.com>
Authored: Thu Apr 20 20:59:47 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Fri Apr 21 23:04:18 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 11 +--
.../gearpump/GearpumpPipelineTranslator.java | 92 +++++++++++---------
.../FlattenPCollectionsTranslator.java | 6 +-
.../translators/ParDoMultiOutputTranslator.java | 11 ++-
.../translators/TranslationContext.java | 11 ++-
.../translators/WindowAssignTranslator.java | 2 +-
.../translators/functions/DoFnFunction.java | 2 +-
.../gearpump/translators/io/GearpumpSource.java | 3 +-
.../gearpump/translators/io/ValuesSource.java | 3 +-
.../translators/utils/DoFnRunnerFactory.java | 3 +-
.../translators/utils/NoOpStepContext.java | 3 +-
.../FlattenPCollectionsTranslatorTest.java | 48 ++++++----
.../translators/GroupByKeyTranslatorTest.java | 3 +-
sdks/java/pom.xml | 2 +-
14 files changed, 113 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index a691801..dcfa390 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -43,13 +43,13 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <gearpump.version>0.8.3-SNAPSHOT</gearpump.version>
+ <gearpump.version>0.8.3</gearpump.version>
</properties>
<profiles>
<profile>
- <id>local-runnable-on-service-tests</id>
- <activation><activeByDefault>true</activeByDefault></activation>
+ <id>local-validates-runner-tests</id>
+ <activation><activeByDefault>false</activeByDefault></activation>
<build>
<plugins>
<plugin>
@@ -63,14 +63,15 @@
<goal>test</goal>
</goals>
<configuration>
- <groups>org.apache.beam.sdk.testing.RunnableOnService</groups>
+ <groups>org.apache.beam.sdk.testing.ValidatesRunner</groups>
<excludedGroups>
org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders,
org.apache.beam.sdk.testing.UsesStatefulParDo,
org.apache.beam.sdk.testing.UsesTimersInParDo,
org.apache.beam.sdk.testing.UsesSplittableParDo,
org.apache.beam.sdk.testing.UsesAttemptedMetrics,
- org.apache.beam.sdk.testing.UsesCommittedMetrics
+ org.apache.beam.sdk.testing.UsesCommittedMetrics,
+ org.apache.beam.sdk.testing.UsesTestStream
</excludedGroups>
<parallel>none</parallel>
<failIfNoTests>true</failIfNoTests>
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
index 1a36343..f5f5e70 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
@@ -18,7 +18,7 @@
package org.apache.beam.runners.gearpump;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.Collections;
@@ -27,13 +27,14 @@ import java.util.List;
import java.util.Map;
import org.apache.beam.runners.core.construction.PTransformMatchers;
+import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.runners.gearpump.translators.CreateGearpumpPCollectionViewTranslator;
import org.apache.beam.runners.gearpump.translators.CreatePCollectionViewTranslator;
import org.apache.beam.runners.gearpump.translators.FlattenPCollectionsTranslator;
import org.apache.beam.runners.gearpump.translators.GroupByKeyTranslator;
-import org.apache.beam.runners.gearpump.translators.ParDoBoundMultiTranslator;
-import org.apache.beam.runners.gearpump.translators.ParDoBoundTranslator;
+import org.apache.beam.runners.gearpump.translators.ParDoMultiOutputTranslator;
+import org.apache.beam.runners.gearpump.translators.ParDoSingleOutputTranslator;
import org.apache.beam.runners.gearpump.translators.ReadBoundedTranslator;
import org.apache.beam.runners.gearpump.translators.ReadUnboundedTranslator;
import org.apache.beam.runners.gearpump.translators.TransformTranslator;
@@ -45,9 +46,9 @@ import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.ListCoder;
import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.runners.PTransformMatcher;
-import org.apache.beam.sdk.runners.PTransformOverrideFactory;
+import org.apache.beam.sdk.runners.PTransformOverride;
import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
@@ -72,7 +73,7 @@ import org.slf4j.LoggerFactory;
* into Gearpump {@link Graph}.
*/
@SuppressWarnings({"rawtypes", "unchecked"})
-public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
+public class GearpumpPipelineTranslator implements Pipeline.PipelineVisitor {
private static final Logger LOG = LoggerFactory.getLogger(
GearpumpPipelineTranslator.class);
@@ -88,13 +89,13 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
static {
// register TransformTranslators
- registerTransformTranslator(ParDo.Bound.class, new ParDoBoundTranslator());
+ registerTransformTranslator(ParDo.SingleOutput.class, new ParDoSingleOutputTranslator());
registerTransformTranslator(Read.Unbounded.class, new ReadUnboundedTranslator());
registerTransformTranslator(Read.Bounded.class, new ReadBoundedTranslator());
registerTransformTranslator(GroupByKey.class, new GroupByKeyTranslator());
registerTransformTranslator(Flatten.PCollections.class,
new FlattenPCollectionsTranslator());
- registerTransformTranslator(ParDo.BoundMulti.class, new ParDoBoundMultiTranslator());
+ registerTransformTranslator(ParDo.MultiOutput.class, new ParDoMultiOutputTranslator());
registerTransformTranslator(Window.Assign.class, new WindowAssignTranslator());
registerTransformTranslator(View.CreatePCollectionView.class,
new CreatePCollectionViewTranslator());
@@ -107,27 +108,30 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
}
public void translate(Pipeline pipeline) {
- Map<PTransformMatcher, PTransformOverrideFactory> overrides =
- ImmutableMap.<PTransformMatcher, PTransformOverrideFactory>builder()
- .put(PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
+ List<PTransformOverride> overrides =
+ ImmutableList.<PTransformOverride>builder()
+ .add(PTransformOverride.of(
+ PTransformMatchers.classEqualTo(View.AsMap.class),
+ new ReflectiveOneToOneOverrideFactory(StreamingViewAsMap.class)))
+ .add(PTransformOverride.of(
+ PTransformMatchers.classEqualTo(View.AsMultimap.class),
+ new ReflectiveOneToOneOverrideFactory(StreamingViewAsMultimap.class)))
+ .add(PTransformOverride.of(
+ PTransformMatchers.classEqualTo(View.AsSingleton.class),
+ new ReflectiveOneToOneOverrideFactory(StreamingViewAsSingleton.class)))
+ .add(PTransformOverride.of(
+ PTransformMatchers.classEqualTo(View.AsList.class),
+ new ReflectiveOneToOneOverrideFactory(StreamingViewAsList.class)))
+ .add(PTransformOverride.of(
+ PTransformMatchers.classEqualTo(View.AsIterable.class),
+ new ReflectiveOneToOneOverrideFactory(StreamingViewAsIterable.class)))
+ .add(PTransformOverride.of(
+ PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
new ReflectiveOneToOneOverrideFactory(
- StreamingCombineGloballyAsSingletonView.class))
- .put(PTransformMatchers.classEqualTo(View.AsMap.class),
- new ReflectiveOneToOneOverrideFactory(StreamingViewAsMap.class))
- .put(PTransformMatchers.classEqualTo(View.AsMultimap.class),
- new ReflectiveOneToOneOverrideFactory(StreamingViewAsMultimap.class))
- .put(PTransformMatchers.classEqualTo(View.AsSingleton.class),
- new ReflectiveOneToOneOverrideFactory(StreamingViewAsSingleton.class))
- .put(PTransformMatchers.classEqualTo(View.AsList.class),
- new ReflectiveOneToOneOverrideFactory(StreamingViewAsList.class))
- .put(PTransformMatchers.classEqualTo(View.AsIterable.class),
- new ReflectiveOneToOneOverrideFactory(StreamingViewAsIterable.class))
+ StreamingCombineGloballyAsSingletonView.class)))
.build();
- for (Map.Entry<PTransformMatcher, PTransformOverrideFactory> override :
- overrides.entrySet()) {
- pipeline.replace(override.getKey(), override.getValue());
- }
+ pipeline.replaceAll(overrides);
pipeline.traverseTopologically(this);
}
@@ -185,22 +189,27 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
// The following codes are forked from DataflowRunner for View translator
private static class ReflectiveOneToOneOverrideFactory<
- InputT extends PValue,
- OutputT extends PValue,
- TransformT extends PTransform<InputT, OutputT>>
- extends SingleInputOutputOverrideFactory<InputT, OutputT, TransformT> {
- private final Class<PTransform<InputT, OutputT>> replacement;
+ InputT, OutputT, TransformT extends PTransform<PCollection<InputT>, PCollection<OutputT>>>
+ extends SingleInputOutputOverrideFactory<
+ PCollection<InputT>, PCollection<OutputT>, TransformT> {
+ private final Class<PTransform<PCollection<InputT>, PCollection<OutputT>>> replacement;
private ReflectiveOneToOneOverrideFactory(
- Class<PTransform<InputT, OutputT>> replacement) {
+ Class<PTransform<PCollection<InputT>, PCollection<OutputT>>> replacement) {
this.replacement = replacement;
}
@Override
- public PTransform<InputT, OutputT> getReplacementTransform(TransformT transform) {
- return InstanceBuilder.ofType(replacement)
- .withArg((Class<PTransform<InputT, OutputT>>) transform.getClass(), transform)
- .build();
+ public PTransformReplacement<PCollection<InputT>, PCollection<OutputT>> getReplacementTransform(
+ AppliedPTransform<PCollection<InputT>, PCollection<OutputT>, TransformT> transform) {
+ return PTransformReplacement.of(
+ PTransformReplacements.getSingletonMainInput(transform),
+ InstanceBuilder.ofType(replacement)
+ .withArg(
+ (Class<PTransform<PCollection<InputT>, PCollection<OutputT>>>)
+ transform.getTransform().getClass(),
+ transform.getTransform())
+ .build());
}
}
@@ -220,7 +229,7 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
public PCollectionView<Map<K, V>> expand(PCollection<KV<K, V>> input) {
PCollectionView<Map<K, V>> view =
PCollectionViews.mapView(
- input.getPipeline(),
+ input,
input.getWindowingStrategy(),
input.getCoder());
@@ -259,7 +268,7 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
public PCollectionView<Map<K, Iterable<V>>> expand(PCollection<KV<K, V>> input) {
PCollectionView<Map<K, Iterable<V>>> view =
PCollectionViews.multimapView(
- input.getPipeline(),
+ input,
input.getWindowingStrategy(),
input.getCoder());
@@ -298,7 +307,7 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
PCollectionView<Iterable<T>> view =
PCollectionViews.iterableView(
- input.getPipeline(),
+ input,
input.getWindowingStrategy(),
input.getCoder());
@@ -328,7 +337,7 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
public PCollectionView<List<T>> expand(PCollection<T> input) {
PCollectionView<List<T>> view =
PCollectionViews.listView(
- input.getPipeline(),
+ input,
input.getWindowingStrategy(),
input.getCoder());
@@ -341,6 +350,7 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
return "StreamingViewAsList";
}
}
+
private static class StreamingCombineGloballyAsSingletonView<InputT, OutputT>
extends PTransform<PCollection<InputT>, PCollectionView<OutputT>> {
@@ -360,7 +370,7 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
.withFanout(transform.getFanout()));
PCollectionView<OutputT> view = PCollectionViews.singletonView(
- combined.getPipeline(),
+ combined,
combined.getWindowingStrategy(),
transform.getInsertDefault(),
transform.getInsertDefault()
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
index 56f7d1a..5ca05d8 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
@@ -29,7 +29,7 @@ import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.TaggedPValue;
+import org.apache.beam.sdk.values.PValue;
import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
@@ -45,8 +45,8 @@ public class FlattenPCollectionsTranslator<T> implements
public void translate(Flatten.PCollections<T> transform, TranslationContext context) {
JavaStream<T> merged = null;
Set<PCollection<T>> unique = new HashSet<>();
- for (TaggedPValue input: context.getInputs()) {
- PCollection<T> collection = (PCollection<T>) input.getValue();
+ for (PValue input: context.getInputs().values()) {
+ PCollection<T> collection = (PCollection<T>) input;
JavaStream<T> inputStream = context.getInputStream(collection);
if (null == merged) {
merged = inputStream;
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
index e78568d..d92979b 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
@@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TaggedPValue;
+import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.gearpump.streaming.dsl.api.functions.FilterFunction;
@@ -55,11 +55,10 @@ public class ParDoMultiOutputTranslator<InputT, OutputT> implements
Map<String, PCollectionView<?>> tagsToSideInputs =
TranslatorUtils.getTagsToSideInputs(sideInputs);
- List<TaggedPValue> outputs = context.getOutputs();
+ Map<TupleTag<?>, PValue> outputs = context.getOutputs();
final TupleTag<OutputT> mainOutput = transform.getMainOutputTag();
List<TupleTag<?>> sideOutputs = new ArrayList<>(outputs.size() - 1);
- for (TaggedPValue output: outputs) {
- TupleTag<?> tag = output.getTag();
+ for (TupleTag<?> tag: outputs.keySet()) {
if (tag != null && !tag.getId().equals(mainOutput.getId())) {
sideOutputs.add(tag);
}
@@ -78,9 +77,9 @@ public class ParDoMultiOutputTranslator<InputT, OutputT> implements
tagsToSideInputs,
mainOutput,
sideOutputs), transform.getName());
- for (TaggedPValue output: outputs) {
+ for (Map.Entry<TupleTag<?>, PValue> output: outputs.entrySet()) {
JavaStream<WindowedValue<OutputT>> taggedStream = outputStream
- .filter(new FilterByOutputTag(output.getTag().getId()),
+ .filter(new FilterByOutputTag(output.getKey().getId()),
"filter_by_output_tag")
.map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
context.setOutputStream(output.getValue(), taggedStream);
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
index e88bb74..eb6bc18 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
@@ -23,7 +23,6 @@ import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.collect.Iterables;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
@@ -31,7 +30,7 @@ import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TaggedPValue;
+import org.apache.beam.sdk.values.TupleTag;
import org.apache.gearpump.cluster.UserConfig;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStreamApp;
@@ -71,20 +70,20 @@ public class TranslationContext {
}
}
- public List<TaggedPValue> getInputs() {
+ public Map<TupleTag<?>, PValue> getInputs() {
return getCurrentTransform().getInputs();
}
public PValue getInput() {
- return Iterables.getOnlyElement(getInputs()).getValue();
+ return Iterables.getOnlyElement(getInputs().values());
}
- public List<TaggedPValue> getOutputs() {
+ public Map<TupleTag<?>, PValue> getOutputs() {
return getCurrentTransform().getOutputs();
}
public PValue getOutput() {
- return Iterables.getOnlyElement(getOutputs()).getValue();
+ return Iterables.getOnlyElement(getOutputs().values());
}
private AppliedPTransform<?, ?, ?> getCurrentTransform() {
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
index 2d70b63..149f80c 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/WindowAssignTranslator.java
@@ -36,7 +36,7 @@ import org.apache.gearpump.streaming.dsl.javaapi.functions.FlatMapFunction;
import org.joda.time.Instant;
/**
- * {@link Window.Bound} is translated to Gearpump flatMap function.
+ * {@link Window.Assign} is translated to Gearpump flatMap function.
*/
@SuppressWarnings("unchecked")
public class WindowAssignTranslator<T> implements TransformTranslator<Window.Assign<T>> {
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index 9941e71..3473f53 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -156,7 +156,7 @@ public class DoFnFunction<InputT, OutputT> extends
for (WindowedValue<InputT> value : pushedBackValues) {
for (BoundedWindow win: value.getWindows()) {
BoundedWindow sideInputWindow =
- sideInput.getWindowingStrategyInternal().getWindowFn().getSideInputWindow(win);
+ sideInput.getWindowMappingFn().getSideInputWindow(win);
if (!sideInputReader.isReady(sideInput, sideInputWindow)) {
Object emptyValue = WindowedValue.of(
Lists.newArrayList(), value.getTimestamp(), sideInputWindow, value.getPane());
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
index c079603..5e79151 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
@@ -46,6 +46,7 @@ public abstract class GearpumpSource<T> implements DataSource {
private Source.Reader<T> reader;
private boolean available = false;
+ private long count = 0L;
GearpumpSource(PipelineOptions options) {
try {
@@ -112,7 +113,7 @@ public abstract class GearpumpSource<T> implements DataSource {
}
} else {
if (available) {
- return TranslatorUtils.jodaTimeToJava8Time(reader.getCurrentTimestamp());
+ return Watermark.MIN();
} else {
return Watermark.MAX();
}
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
index ccd5cdf..b62da19 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/ValuesSource.java
@@ -23,6 +23,7 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.NoSuchElementException;
import javax.annotation.Nullable;
@@ -68,7 +69,7 @@ public class ValuesSource<T> extends UnboundedSource<T, UnboundedSource.Checkpoi
}
@Override
- public java.util.List<? extends UnboundedSource<T, CheckpointMark>> generateInitialSplits(
+ public java.util.List<? extends UnboundedSource<T, CheckpointMark>> split(
int desiredNumSplits, PipelineOptions options) throws Exception {
return Collections.singletonList(this);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
index bdfc336..70b4271 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/DoFnRunnerFactory.java
@@ -28,6 +28,7 @@ import org.apache.beam.runners.core.DoFnRunners;
import org.apache.beam.runners.core.ExecutionContext;
import org.apache.beam.runners.core.PushbackSideInputDoFnRunner;
import org.apache.beam.runners.core.SimpleDoFnRunner;
+import org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner;
import org.apache.beam.runners.gearpump.GearpumpPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
@@ -78,7 +79,7 @@ public class DoFnRunnerFactory<InputT, OutputT> implements Serializable {
DoFnRunner<InputT, OutputT> underlying = DoFnRunners.simpleRunner(
options, fn, sideInputReader, outputManager, mainOutputTag,
sideOutputTags, stepContext, aggregatorFactory, windowingStrategy);
- return PushbackSideInputDoFnRunner.create(underlying, sideInputs, sideInputReader);
+ return SimplePushbackSideInputDoFnRunner.create(underlying, sideInputs, sideInputReader);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
index 140df2a..4e0a74c 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/NoOpStepContext.java
@@ -49,7 +49,8 @@ public class NoOpStepContext implements ExecutionContext.StepContext, Serializab
}
@Override
- public void noteSideOutput(TupleTag<?> tag, WindowedValue<?> output) {
+ public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {
+
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
index fa89d4a..ac12fa4 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
@@ -26,13 +26,15 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import com.google.common.collect.Lists;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.beam.runners.gearpump.translators.io.UnboundedSourceWrapper;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TaggedPValue;
+import org.apache.beam.sdk.values.TupleTag;
import org.apache.gearpump.streaming.dsl.api.functions.MapFunction;
import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
import org.apache.gearpump.streaming.source.DataSource;
@@ -55,10 +57,10 @@ public class FlattenPCollectionsTranslatorTest {
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testTranslateWithEmptyCollection() {
- PValue mockOutput = mock(PValue.class);
+ PCollection mockOutput = mock(PCollection.class);
TranslationContext translationContext = mock(TranslationContext.class);
- when(translationContext.getInputs()).thenReturn(Collections.EMPTY_LIST);
+ when(translationContext.getInputs()).thenReturn(Collections.EMPTY_MAP);
when(translationContext.getOutput()).thenReturn(mockOutput);
translator.translate(transform, translationContext);
@@ -71,11 +73,12 @@ public class FlattenPCollectionsTranslatorTest {
JavaStream javaStream = mock(JavaStream.class);
TranslationContext translationContext = mock(TranslationContext.class);
- TaggedPValue mockInput = mock(TaggedPValue.class);
+ Map<TupleTag<?>, PValue> inputs = new HashMap<>();
+ TupleTag tag = mock(TupleTag.class);
PCollection mockCollection = mock(PCollection.class);
- when(mockInput.getValue()).thenReturn(mockCollection);
+ inputs.put(tag, mockCollection);
- when(translationContext.getInputs()).thenReturn(Lists.newArrayList(mockInput));
+ when(translationContext.getInputs()).thenReturn(inputs);
when(translationContext.getInputStream(mockCollection)).thenReturn(javaStream);
PValue mockOutput = mock(PValue.class);
@@ -93,22 +96,30 @@ public class FlattenPCollectionsTranslatorTest {
JavaStream javaStream1 = mock(JavaStream.class);
JavaStream javaStream2 = mock(JavaStream.class);
+ JavaStream mergedStream = mock(JavaStream.class);
TranslationContext translationContext = mock(TranslationContext.class);
- TaggedPValue mockInput1 = mock(TaggedPValue.class);
+ Map<TupleTag<?>, PValue> inputs = new HashMap<>();
+ TupleTag tag1 = mock(TupleTag.class);
PCollection mockCollection1 = mock(PCollection.class);
- when(mockInput1.getValue()).thenReturn(mockCollection1);
+ inputs.put(tag1, mockCollection1);
- TaggedPValue mockInput2 = mock(TaggedPValue.class);
+ TupleTag tag2 = mock(TupleTag.class);
PCollection mockCollection2 = mock(PCollection.class);
- when(mockInput2.getValue()).thenReturn(mockCollection2);
+ inputs.put(tag2, mockCollection2);
- when(translationContext.getInputs()).thenReturn(Lists.newArrayList(mockInput1, mockInput2));
+ PCollection output = mock(PCollection.class);
+
+ when(translationContext.getInputs()).thenReturn(inputs);
when(translationContext.getInputStream(mockCollection1)).thenReturn(javaStream1);
when(translationContext.getInputStream(mockCollection2)).thenReturn(javaStream2);
+ when(javaStream1.merge(javaStream2, transformName)).thenReturn(mergedStream);
+ when(javaStream2.merge(javaStream1, transformName)).thenReturn(mergedStream);
+
+ when(translationContext.getOutput()).thenReturn(output);
translator.translate(transform, translationContext);
- verify(javaStream1).merge(javaStream2, transformName);
+ verify(translationContext).setOutputStream(output, mergedStream);
}
@Test
@@ -120,14 +131,15 @@ public class FlattenPCollectionsTranslatorTest {
JavaStream javaStream1 = mock(JavaStream.class);
TranslationContext translationContext = mock(TranslationContext.class);
+ Map<TupleTag<?>, PValue> inputs = new HashMap<>();
+ TupleTag tag1 = mock(TupleTag.class);
PCollection mockCollection1 = mock(PCollection.class);
- TaggedPValue mockInput1 = mock(TaggedPValue.class);
- when(mockInput1.getValue()).thenReturn(mockCollection1);
+ inputs.put(tag1, mockCollection1);
- TaggedPValue mockInput2 = mock(TaggedPValue.class);
- when(mockInput2.getValue()).thenReturn(mockCollection1);
+ TupleTag tag2 = mock(TupleTag.class);
+ inputs.put(tag2, mockCollection1);
- when(translationContext.getInputs()).thenReturn(Lists.newArrayList(mockInput1, mockInput2));
+ when(translationContext.getInputs()).thenReturn(inputs);
when(translationContext.getInputStream(mockCollection1)).thenReturn(javaStream1);
translator.translate(transform, translationContext);
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
index 9135022..4e66ba9 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslatorTest.java
@@ -133,7 +133,8 @@ public class GroupByKeyTranslatorTest {
PaneInfo.NO_FIRING);
KV<org.joda.time.Instant, WindowedValue<KV<String, List<String>>>> result1 =
- merge.fold(KV.of(null, null), KV.of(key1, value1));
+ merge.fold(KV.<org.joda.time.Instant, WindowedValue<KV<String, List<String>>>>of(
+ null, null), KV.of(key1, value1));
assertThat(result1.getKey(), equalTo(key1));
assertThat(result1.getValue().getValue().getValue(), equalTo(Lists.newArrayList("value1")));
http://git-wip-us.apache.org/repos/asf/beam/blob/44d21ac6/sdks/java/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/pom.xml b/sdks/java/pom.xml
index 7ca6109..21b5841 100644
--- a/sdks/java/pom.xml
+++ b/sdks/java/pom.xml
@@ -38,7 +38,7 @@
<module>build-tools</module> -->
<module>core</module>
<module>io</module>
- <module>maven-archetypes</module>
+ <!--<module>maven-archetypes</module>-->
<module>extensions</module>
<!-- javadoc runs directly from the root parent as the last module
in the build to be able to capture runner-specific javadoc.
[31/50] [abbrv] beam git commit: upgrade to gearpump 0.8.4-SNAPSHOT
Posted by ke...@apache.org.
upgrade to gearpump 0.8.4-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f61822d4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f61822d4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f61822d4
Branch: refs/heads/master
Commit: f61822d41653def8332cb3cc55140685c3dd75a2
Parents: fed98c8
Author: manuzhang <ow...@gmail.com>
Authored: Wed Jun 7 14:06:43 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Sat Jun 17 11:00:29 2017 +0800
----------------------------------------------------------------------
runners/gearpump/pom.xml | 2 +-
.../runners/gearpump/GearpumpPipelineTranslator.java | 4 +++-
.../CreateGearpumpPCollectionViewTranslator.java | 2 +-
.../translators/FlattenPCollectionsTranslator.java | 2 +-
.../gearpump/translators/GroupByKeyTranslator.java | 13 +++++++++----
.../gearpump/translators/functions/DoFnFunction.java | 1 +
.../gearpump/translators/io/GearpumpSource.java | 5 +++--
.../gearpump/translators/utils/TranslatorUtils.java | 2 +-
.../translators/FlattenPCollectionsTranslatorTest.java | 6 +++---
.../gearpump/translators/io/GearpumpSourceTest.java | 3 ++-
10 files changed, 25 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/f61822d4/runners/gearpump/pom.xml
----------------------------------------------------------------------
diff --git a/runners/gearpump/pom.xml b/runners/gearpump/pom.xml
index 7e39a48..3c98d5e 100644
--- a/runners/gearpump/pom.xml
+++ b/runners/gearpump/pom.xml
@@ -43,7 +43,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <gearpump.version>0.8.3</gearpump.version>
+ <gearpump.version>0.8.4-SNAPSHOT</gearpump.version>
</properties>
<profiles>
http://git-wip-us.apache.org/repos/asf/beam/blob/f61822d4/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
index daf65d9..58b44a3 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineTranslator.java
@@ -54,6 +54,7 @@ import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.util.InstanceBuilder;
import org.apache.beam.sdk.values.KV;
@@ -143,7 +144,7 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
@Override
public void visitPrimitiveTransform(TransformHierarchy.Node node) {
- LOG.debug("visiting transform {}", node.getTransform());
+ LOG.info("visiting transform {}", node.getTransform());
PTransform transform = node.getTransform();
TransformTranslator translator = getTransformTranslator(transform.getClass());
if (null == translator) {
@@ -346,6 +347,7 @@ public class GearpumpPipelineTranslator extends Pipeline.PipelineVisitor.Default
}
}
+
private static class StreamingCombineGloballyAsSingletonView<InputT, OutputT>
extends PTransform<PCollection<InputT>, PCollectionView<OutputT>> {
http://git-wip-us.apache.org/repos/asf/beam/blob/f61822d4/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
index c7f24a8..d7588c2 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/CreateGearpumpPCollectionViewTranslator.java
@@ -41,6 +41,6 @@ public class CreateGearpumpPCollectionViewTranslator<ElemT, ViewT> implements
JavaStream<WindowedValue<List<ElemT>>> inputStream =
context.getInputStream(context.getInput());
PCollectionView<ViewT> view = (PCollectionView<ViewT>) context.getOutput();
- context.setOutputStream(view, inputStream);
+ context.setOutputStream(view.getPCollection(), inputStream);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/f61822d4/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
index 5ca05d8..8cc0058 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslator.java
@@ -57,7 +57,7 @@ public class FlattenPCollectionsTranslator<T> implements
inputStream = inputStream.map(new DummyFunction<T>(), "dummy");
}
- merged = merged.merge(inputStream, transform.getName());
+ merged = merged.merge(inputStream, 1, transform.getName());
}
unique.add(collection);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/f61822d4/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
index 7d944a4..8409beb 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GroupByKeyTranslator.java
@@ -72,7 +72,7 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
JavaStream<WindowedValue<KV<K, List<V>>>> outputStream = inputStream
.window(Windows.apply(
new GearpumpWindowFn(windowFn.isNonMerging()),
- EventTimeTrigger$.MODULE$, Discarding$.MODULE$), "assign_window")
+ EventTimeTrigger$.MODULE$, Discarding$.MODULE$, windowFn.toString()))
.groupBy(new GroupByFn<K, V>(inputKeyCoder), parallelism, "group_by_Key_and_Window")
.map(new KeyedByTimestamp<K, V>(windowFn, timestampCombiner), "keyed_by_timestamp")
.fold(new Merge<>(windowFn, timestampCombiner), "merge")
@@ -85,7 +85,7 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
* A transform used internally to translate Beam's Window to Gearpump's Window.
*/
protected static class GearpumpWindowFn<T, W extends BoundedWindow>
- implements WindowFunction<WindowedValue<T>>, Serializable {
+ implements WindowFunction, Serializable {
private final boolean isNonMerging;
@@ -94,9 +94,14 @@ public class GroupByKeyTranslator<K, V> implements TransformTranslator<GroupByKe
}
@Override
- public Window[] apply(Context<WindowedValue<T>> context) {
+ public <T> Window[] apply(Context<T> context) {
try {
- return toGearpumpWindows(context.element().getWindows().toArray(new BoundedWindow[0]));
+ Object element = context.element();
+ if (element instanceof TranslatorUtils.RawUnionValue) {
+ element = ((TranslatorUtils.RawUnionValue) element).getValue();
+ }
+ return toGearpumpWindows(((WindowedValue<T>) element).getWindows()
+ .toArray(new BoundedWindow[0]));
} catch (Exception e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/f61822d4/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index 6e4fbeb..e2777df 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -145,6 +145,7 @@ public class DoFnFunction<InputT, OutputT> extends
Object emptyValue = WindowedValue.of(
Lists.newArrayList(), value.getTimestamp(), sideInputWindow, value.getPane());
sideInputReader.addSideInputValue(sideInput, (WindowedValue<Iterable<?>>) emptyValue);
+ System.out.println(sideInput + " in " + sideInputWindow.toString() + " not ready");
}
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/f61822d4/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
index 60f319d..6637a9b 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
@@ -31,6 +31,7 @@ import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.gearpump.DefaultMessage;
import org.apache.gearpump.Message;
import org.apache.gearpump.streaming.source.DataSource;
import org.apache.gearpump.streaming.source.Watermark;
@@ -77,9 +78,9 @@ public abstract class GearpumpSource<T> implements DataSource {
if (available) {
T data = reader.getCurrent();
org.joda.time.Instant timestamp = reader.getCurrentTimestamp();
- message = Message.apply(
+ message = new DefaultMessage(
WindowedValue.timestampedValueInGlobalWindow(data, timestamp),
- timestamp.getMillis());
+ TranslatorUtils.jodaTimeToJava8Time(timestamp));
}
available = reader.advance();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/beam/blob/f61822d4/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
index 282f261..83fc6e6 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
@@ -76,7 +76,7 @@ public class TranslatorUtils {
JavaStream<WindowedValue<List<?>>> sideInputStream = context.getInputStream(
tagToSideInput.getValue());
mainStream = mainStream.merge(sideInputStream.map(new ToRawUnionValue<>(
- tagToSideInput.getKey()), "map_to_RawUnionValue"), "merge_to_MainStream");
+ tagToSideInput.getKey()), "map_to_RawUnionValue"), 1, "merge_to_MainStream");
}
return mainStream;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/f61822d4/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
index ac12fa4..1262177 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/FlattenPCollectionsTranslatorTest.java
@@ -113,8 +113,8 @@ public class FlattenPCollectionsTranslatorTest {
when(translationContext.getInputs()).thenReturn(inputs);
when(translationContext.getInputStream(mockCollection1)).thenReturn(javaStream1);
when(translationContext.getInputStream(mockCollection2)).thenReturn(javaStream2);
- when(javaStream1.merge(javaStream2, transformName)).thenReturn(mergedStream);
- when(javaStream2.merge(javaStream1, transformName)).thenReturn(mergedStream);
+ when(javaStream1.merge(javaStream2, 1, transformName)).thenReturn(mergedStream);
+ when(javaStream2.merge(javaStream1, 1, transformName)).thenReturn(mergedStream);
when(translationContext.getOutput()).thenReturn(output);
@@ -144,6 +144,6 @@ public class FlattenPCollectionsTranslatorTest {
translator.translate(transform, translationContext);
verify(javaStream1).map(any(MapFunction.class), eq("dummy"));
- verify(javaStream1).merge(any(JavaStream.class), eq(transformName));
+ verify(javaStream1).merge(any(JavaStream.class), eq(1), eq(transformName));
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/f61822d4/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
index b244484..4490737 100644
--- a/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
+++ b/runners/gearpump/src/test/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSourceTest.java
@@ -32,6 +32,7 @@ import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.gearpump.DefaultMessage;
import org.apache.gearpump.Message;
import org.apache.gearpump.streaming.source.Watermark;
import org.junit.Assert;
@@ -77,7 +78,7 @@ public class GearpumpSourceTest {
Assert.assertEquals(expectedWaterMark, sourceForTest.getWatermark());
Message expectedMsg =
- Message.apply(
+ new DefaultMessage(
WindowedValue.timestampedValueInGlobalWindow(value, value.getTimestamp()),
value.getTimestamp().getMillis());
Message message = sourceForTest.read();
[49/50] [abbrv] beam git commit: This closes #3637: [BEAM-79] Prepare
for merging gearpump-runner
Posted by ke...@apache.org.
This closes #3637: [BEAM-79] Prepare for merging gearpump-runner
Remove gearpump-runner in precommit tests
Override maven-enforcer-plugin for precommit tests
Require java 8 for precommit tests and add gearpump-runner
Deactivate integration-tests for gearpump-runner by default
Add beam-runners-gearpump dependency to javadoc
Upgrade BEAM version to 2.2.0-SNAPSHOT in gearpump-runner
Revert accidental changes to sdks/java/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c35febf1
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c35febf1
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c35febf1
Branch: refs/heads/master
Commit: c35febf1ac60433096bf9ca4b9d43170f3480842
Parents: 1ce60b4 bf15483
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Jul 28 09:25:55 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Jul 28 09:25:55 2017 -0700
----------------------------------------------------------------------
examples/java/pom.xml | 12 ------------
examples/java8/pom.xml | 12 ++++++++++++
runners/gearpump/pom.xml | 5 ++---
sdks/java/javadoc/pom.xml | 5 +++++
sdks/java/pom.xml | 2 +-
5 files changed, 20 insertions(+), 16 deletions(-)
----------------------------------------------------------------------