You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by am...@apache.org on 2017/02/20 09:57:57 UTC
[1/9] beam git commit: A global Watermark holder to update and
broadcast to workers.
Repository: beam
Updated Branches:
refs/heads/master 92190ba5d -> aa45ccb08
A global Watermark holder to update and broadcast to workers.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/a6206535
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/a6206535
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/a6206535
Branch: refs/heads/master
Commit: a6206535ec84d8c01695368c8973a6577ae6d953
Parents: 92190ba
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:28:19 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:13 2017 +0200
----------------------------------------------------------------------
.../spark/util/GlobalWatermarkHolder.java | 200 +++++++++++++++++++
1 file changed, 200 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/a6206535/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
new file mode 100644
index 0000000..b215b5f
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.util;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.spark.streaming.api.java.JavaStreamingContext;
+import org.apache.spark.streaming.api.java.JavaStreamingListener;
+import org.apache.spark.streaming.api.java.JavaStreamingListenerBatchCompleted;
+import org.joda.time.Instant;
+
+
+/**
+ * A {@link Broadcast} variable to hold the global watermarks for a micro-batch.
+ *
+ * <p>For each source, holds a queue for the watermarks of each micro-batch that was read,
+ * and advances the watermarks according to the queue (first-in-first-out).
+ */
+public class GlobalWatermarkHolder {
+ // the broadcast is broadcasted to the workers.
+ private static volatile Broadcast<Map<Integer, SparkWatermarks>> broadcast = null;
+ // this should only live in the driver so transient.
+ private static final transient Map<Integer, Queue<SparkWatermarks>> sourceTimes = new HashMap<>();
+
+ public static void add(int sourceId, SparkWatermarks sparkWatermarks) {
+ Queue<SparkWatermarks> timesQueue = sourceTimes.get(sourceId);
+ if (timesQueue == null) {
+ timesQueue = new ConcurrentLinkedQueue<>();
+ }
+ timesQueue.offer(sparkWatermarks);
+ sourceTimes.put(sourceId, timesQueue);
+ }
+
+ @VisibleForTesting
+ public static void addAll(Map<Integer, Queue<SparkWatermarks>> sourceTimes) {
+ for (Map.Entry<Integer, Queue<SparkWatermarks>> en: sourceTimes.entrySet()) {
+ int sourceId = en.getKey();
+ Queue<SparkWatermarks> timesQueue = en.getValue();
+ while (!timesQueue.isEmpty()) {
+ add(sourceId, timesQueue.poll());
+ }
+ }
+ }
+
+ /**
+ * Returns the {@link Broadcast} containing the {@link SparkWatermarks} mapped
+ * to their sources.
+ */
+ public static Broadcast<Map<Integer, SparkWatermarks>> get() {
+ return broadcast;
+ }
+
+ /**
+ * Advances the watermarks to the next-in-line watermarks.
+ * SparkWatermarks are monotonically increasing.
+ */
+ public static void advance(JavaSparkContext jsc) {
+ synchronized (GlobalWatermarkHolder.class){
+ if (sourceTimes.isEmpty()) {
+ return;
+ }
+
+ // update all sources' watermarks into the new broadcast.
+ Map<Integer, SparkWatermarks> newBroadcast = new HashMap<>();
+
+ for (Map.Entry<Integer, Queue<SparkWatermarks>> en: sourceTimes.entrySet()) {
+ if (en.getValue().isEmpty()) {
+ continue;
+ }
+ Integer sourceId = en.getKey();
+ Queue<SparkWatermarks> timesQueue = en.getValue();
+
+ // current state, if exists.
+ Instant currentLowWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ Instant currentHighWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ Instant currentSynchronizedProcessingTime = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ if (broadcast != null && broadcast.getValue().containsKey(sourceId)) {
+ SparkWatermarks currentTimes = broadcast.getValue().get(sourceId);
+ currentLowWatermark = currentTimes.getLowWatermark();
+ currentHighWatermark = currentTimes.getHighWatermark();
+ currentSynchronizedProcessingTime = currentTimes.getSynchronizedProcessingTime();
+ }
+
+ SparkWatermarks next = timesQueue.poll();
+ // advance watermarks monotonically.
+ Instant nextLowWatermark = next.getLowWatermark().isAfter(currentLowWatermark)
+ ? next.getLowWatermark() : currentLowWatermark;
+ Instant nextHighWatermark = next.getHighWatermark().isAfter(currentHighWatermark)
+ ? next.getHighWatermark() : currentHighWatermark;
+ Instant nextSynchronizedProcessingTime = next.getSynchronizedProcessingTime();
+ checkState(!nextLowWatermark.isAfter(nextHighWatermark),
+ String.format(
+ "Low watermark %s cannot be later then high watermark %s",
+ nextLowWatermark, nextHighWatermark));
+ checkState(nextSynchronizedProcessingTime.isAfter(currentSynchronizedProcessingTime),
+ "Synchronized processing time must advance.");
+ newBroadcast.put(
+ sourceId,
+ new SparkWatermarks(
+ nextLowWatermark, nextHighWatermark, nextSynchronizedProcessingTime));
+ }
+
+ // update the watermarks broadcast only if something has changed.
+ if (!newBroadcast.isEmpty()) {
+ if (broadcast != null) {
+ // for now this is blocking, we could make this asynchronous
+ // but it could slow down WM propagation.
+ broadcast.unpersist(true);
+ }
+ broadcast = jsc.broadcast(newBroadcast);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public static synchronized void clear() {
+ sourceTimes.clear();
+ broadcast = null;
+ }
+
+ /**
+ * A {@link SparkWatermarks} holds the watermarks and batch time
+ * relevant to a micro-batch input from a specific source.
+ */
+ public static class SparkWatermarks implements Serializable {
+ private final Instant lowWatermark;
+ private final Instant highWatermark;
+ private final Instant synchronizedProcessingTime;
+
+ @VisibleForTesting
+ public SparkWatermarks(
+ Instant lowWatermark,
+ Instant highWatermark,
+ Instant synchronizedProcessingTime) {
+ this.lowWatermark = lowWatermark;
+ this.highWatermark = highWatermark;
+ this.synchronizedProcessingTime = synchronizedProcessingTime;
+ }
+
+ public Instant getLowWatermark() {
+ return lowWatermark;
+ }
+
+ public Instant getHighWatermark() {
+ return highWatermark;
+ }
+
+ public Instant getSynchronizedProcessingTime() {
+ return synchronizedProcessingTime;
+ }
+
+ @Override
+ public String toString() {
+ return "SparkWatermarks{"
+ + "lowWatermark=" + lowWatermark
+ + ", highWatermark=" + highWatermark
+ + ", synchronizedProcessingTime=" + synchronizedProcessingTime + '}';
+ }
+ }
+
+ /** Advance the WMs onBatchCompleted event. */
+ public static class WatermarksListener extends JavaStreamingListener {
+ private final JavaStreamingContext jssc;
+
+ public WatermarksListener(JavaStreamingContext jssc) {
+ this.jssc = jssc;
+ }
+
+ @Override
+ public void onBatchCompleted(JavaStreamingListenerBatchCompleted batchCompleted) {
+ GlobalWatermarkHolder.advance(jssc.sparkContext());
+ }
+ }
+}
[7/9] beam git commit: Streaming sources tracking test.
Posted by am...@apache.org.
Streaming sources tracking test.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/add87166
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/add87166
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/add87166
Branch: refs/heads/master
Commit: add87166fd445d634f6faf0f838f234c3f908c2e
Parents: 9784f20
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 19:23:28 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:16 2017 +0200
----------------------------------------------------------------------
.../translation/streaming/UnboundedDataset.java | 6 +
.../streaming/TrackStreamingSourcesTest.java | 152 +++++++++++++++++++
2 files changed, 158 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/add87166/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
index 80c0515..08d1ab6 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
@@ -18,6 +18,7 @@
package org.apache.beam.runners.spark.translation.streaming;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
import java.util.ArrayList;
@@ -72,6 +73,11 @@ public class UnboundedDataset<T> implements Dataset {
this.streamingSources.add(queuedStreamIds.decrementAndGet());
}
+ @VisibleForTesting
+ public static void resetQueuedStreamIds() {
+ queuedStreamIds.set(0);
+ }
+
@SuppressWarnings("ConstantConditions")
JavaDStream<WindowedValue<T>> getDStream() {
if (dStream == null) {
http://git-wip-us.apache.org/repos/asf/beam/blob/add87166/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
new file mode 100644
index 0000000..f102ac8
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -0,0 +1,152 @@
+package org.apache.beam.runners.spark.translation.streaming;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.runners.spark.ReuseSparkContext;
+import org.apache.beam.runners.spark.SparkPipelineOptions;
+import org.apache.beam.runners.spark.SparkRunner;
+import org.apache.beam.runners.spark.io.CreateStream;
+import org.apache.beam.runners.spark.translation.Dataset;
+import org.apache.beam.runners.spark.translation.EvaluationContext;
+import org.apache.beam.runners.spark.translation.SparkContextFactory;
+import org.apache.beam.runners.spark.translation.TransformTranslator;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.spark.SparkStatusTracker;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.streaming.Duration;
+import org.apache.spark.streaming.api.java.JavaStreamingContext;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+
+/**
+ * A test suite that tests tracking of the streaming sources created an
+ * {@link org.apache.beam.runners.spark.translation.streaming.UnboundedDataset}.
+ */
+public class TrackStreamingSourcesTest {
+
+ @Rule
+ public ReuseSparkContext reuseContext = ReuseSparkContext.yes();
+
+ private static final transient SparkPipelineOptions options =
+ PipelineOptionsFactory.create().as(SparkPipelineOptions.class);
+
+ @Before
+ public void before() {
+ UnboundedDataset.resetQueuedStreamIds();
+ StreamingSourceTracker.numAssertions = 0;
+ }
+
+ @Test
+ public void testTrackSingle() {
+ options.setRunner(SparkRunner.class);
+ JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+ JavaStreamingContext jssc = new JavaStreamingContext(jsc,
+ new Duration(options.getBatchIntervalMillis()));
+
+ Pipeline p = Pipeline.create(options);
+
+ CreateStream.QueuedValues<Integer> queueStream =
+ CreateStream.fromQueue(Collections.<Iterable<Integer>>emptyList());
+
+ p.apply(queueStream).setCoder(VarIntCoder.of())
+ .apply(ParDo.of(new PassthroughFn<>()));
+
+ p.traverseTopologically(new StreamingSourceTracker(jssc, p, ParDo.Bound.class, -1));
+ assertThat(StreamingSourceTracker.numAssertions, equalTo(1));
+ }
+
+ @Test
+ public void testTrackFlattened() {
+ options.setRunner(SparkRunner.class);
+ JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+ JavaStreamingContext jssc = new JavaStreamingContext(jsc,
+ new Duration(options.getBatchIntervalMillis()));
+
+ Pipeline p = Pipeline.create(options);
+
+ CreateStream.QueuedValues<Integer> queueStream1 =
+ CreateStream.fromQueue(Collections.<Iterable<Integer>>emptyList());
+ CreateStream.QueuedValues<Integer> queueStream2 =
+ CreateStream.fromQueue(Collections.<Iterable<Integer>>emptyList());
+
+ PCollection<Integer> pcol1 = p.apply(queueStream1).setCoder(VarIntCoder.of());
+ PCollection<Integer> pcol2 = p.apply(queueStream2).setCoder(VarIntCoder.of());
+ PCollection<Integer> flattened =
+ PCollectionList.of(pcol1).and(pcol2).apply(Flatten.<Integer>pCollections());
+ flattened.apply(ParDo.of(new PassthroughFn<>()));
+
+ p.traverseTopologically(new StreamingSourceTracker(jssc, p, ParDo.Bound.class, -1, -2));
+ assertThat(StreamingSourceTracker.numAssertions, equalTo(1));
+ }
+
+ private static class PassthroughFn<T> extends DoFn<T, T> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element());
+ }
+ }
+
+ private static class StreamingSourceTracker extends Pipeline.PipelineVisitor.Defaults {
+ private final EvaluationContext ctxt;
+ private final SparkRunner.Evaluator evaluator;
+ private final Class<? extends PTransform> transformClassToAssert;
+ private final Integer[] expected;
+
+ private static int numAssertions = 0;
+
+ private StreamingSourceTracker(
+ JavaStreamingContext jssc,
+ Pipeline pipeline,
+ Class<? extends PTransform> transformClassToAssert,
+ Integer... expected) {
+ this.ctxt = new EvaluationContext(jssc.sparkContext(), pipeline, jssc);
+ this.evaluator = new SparkRunner.Evaluator(
+ new StreamingTransformTranslator.Translator(new TransformTranslator.Translator()), ctxt);
+ this.transformClassToAssert = transformClassToAssert;
+ this.expected = expected;
+ }
+
+ private void assertSourceIds(List<Integer> streamingSources) {
+ numAssertions++;
+ assertThat(streamingSources, containsInAnyOrder(expected));
+ }
+
+ @Override
+ public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
+ return evaluator.enterCompositeTransform(node);
+ }
+
+ @Override
+ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
+ PTransform transform = node.getTransform();
+ if (transform.getClass() == transformClassToAssert) {
+ AppliedPTransform<?, ?, ?> appliedTransform = node.toAppliedPTransform();
+ ctxt.setCurrentTransform(appliedTransform);
+ //noinspection unchecked
+ Dataset dataset = ctxt.borrowDataset((PTransform<? extends PValue, ?>) transform);
+ assertSourceIds(((UnboundedDataset<?>) dataset).getStreamingSources());
+ ctxt.setCurrentTransform(null);
+ } else {
+ evaluator.visitPrimitiveTransform(node);
+ }
+ }
+ }
+
+}
[8/9] beam git commit: This relied on a wrong functionality as
described in BEAM-1444 and should be revisited there.
Posted by am...@apache.org.
This relied on a wrong functionality as described in BEAM-1444 and should be revisited there.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3d25b9cc
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3d25b9cc
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3d25b9cc
Branch: refs/heads/master
Commit: 3d25b9cc794ce39a0ac3051eacf28127da138449
Parents: add8716
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 19:47:00 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:33 2017 +0200
----------------------------------------------------------------------
.../beam/runners/spark/TestSparkRunner.java | 11 +++++-----
.../translation/streaming/UnboundedDataset.java | 1 -
.../beam/runners/spark/WatermarkTest.java | 19 +++++++++++++++--
.../streaming/FlattenStreamingTest.java | 22 --------------------
.../streaming/TrackStreamingSourcesTest.java | 18 +++++++++++++++-
.../apache/beam/sdk/testing/RegexMatcher.java | 17 +++++++++++++++
6 files changed, 57 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index a634dd4..8b8f9ba 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -23,8 +23,8 @@ import static org.hamcrest.Matchers.is;
import org.apache.beam.runners.core.UnboundedReadFromBoundedSource;
import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
-import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineResult.State;
import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
@@ -108,14 +108,15 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
@Override
public SparkPipelineResult run(Pipeline pipeline) {
+ // clear state of Aggregators, Metrics and Watermarks if exists.
+ AggregatorsAccumulator.clear();
+ SparkMetricsContainer.clear();
+ GlobalWatermarkHolder.clear();
+
TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
SparkPipelineResult result = delegate.run(pipeline);
result.waitUntilFinish();
- // clear state of Aggregators, Metrics and Watermarks.
- AggregatorsAccumulator.clear();
- SparkMetricsContainer.clear();
- GlobalWatermarkHolder.clear();
// make sure the test pipeline finished successfully.
State resultState = result.getState();
http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
index 08d1ab6..8b65dca 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
@@ -20,7 +20,6 @@ package org.apache.beam.runners.spark.translation.streaming;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
-
import java.util.ArrayList;
import java.util.List;
import java.util.Queue;
http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
index 0b56403..e7a5481 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark;
import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -172,8 +189,6 @@ public class WatermarkTest {
p.run().waitUntilFinish(Duration.millis(options.getBatchIntervalMillis()).multipliedBy(3));
- System.out.println(WatermarksDoFn.strings);
-
// this is a hacky way to assert but it will do until triggers are supported.
assertThat(
WatermarksDoFn.strings,
http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
index 530721b..fc40bbd 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.runners.spark.translation.streaming;
-import com.google.common.collect.ImmutableList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -27,7 +26,6 @@ import org.apache.beam.runners.spark.translation.streaming.utils.PAssertStreamin
import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
@@ -81,24 +79,4 @@ public class FlattenStreamingTest {
PAssertStreaming.runAndAssertContents(p, union, EXPECTED_UNION, Duration.standardSeconds(1L));
}
- @Test
- public void testFlattenBoundedUnbounded() throws Exception {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- options.setStreaming(true);
-
- Pipeline p = Pipeline.create(options);
- PCollection<String> w1 =
- p.apply(CreateStream.fromQueue(WORDS_QUEUE_1)).setCoder(StringUtf8Coder.of());
- PCollection<String> windowedW1 =
- w1.apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(1))));
- PCollection<String> w2 =
- p.apply(Create.of(ImmutableList.copyOf(WORDS_ARRAY_2)).withCoder(StringUtf8Coder.of()));
- PCollection<String> windowedW2 =
- w2.apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(1))));
- PCollectionList<String> list = PCollectionList.of(windowedW1).and(windowedW2);
- PCollection<String> union = list.apply(Flatten.<String>pCollections());
-
- PAssertStreaming.runAndAssertContents(p, union, EXPECTED_UNION, Duration.standardSeconds(1L));
- }
-
}
http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
index f102ac8..fbe5777 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.translation.streaming;
import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -26,7 +43,6 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionList;
import org.apache.beam.sdk.values.PValue;
-import org.apache.spark.SparkStatusTracker;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
index a4b14fe..aab4a09 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.sdk.testing;
import java.util.regex.Pattern;
[9/9] beam git commit: This closes #1987
Posted by am...@apache.org.
This closes #1987
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/aa45ccb0
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/aa45ccb0
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/aa45ccb0
Branch: refs/heads/master
Commit: aa45ccb0800741ddf5ee7ecc7965c85c1914acf7
Parents: 92190ba 3d25b9c
Author: Sela <an...@paypal.com>
Authored: Mon Feb 20 11:40:26 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:40:26 2017 +0200
----------------------------------------------------------------------
.../runners/dataflow/util/PackageUtilTest.java | 30 +--
.../apache/beam/runners/spark/SparkRunner.java | 6 +-
.../beam/runners/spark/TestSparkRunner.java | 10 +-
.../aggregators/AggregatorsAccumulator.java | 2 +-
.../runners/spark/io/SparkUnboundedSource.java | 69 ++++--
.../spark/stateful/StateSpecFunctions.java | 31 ++-
.../spark/translation/SparkContextFactory.java | 2 +-
.../streaming/StreamingTransformTranslator.java | 92 +++++---
.../translation/streaming/UnboundedDataset.java | 34 ++-
.../spark/util/GlobalWatermarkHolder.java | 200 ++++++++++++++++
.../beam/runners/spark/ClearWatermarksRule.java | 37 +++
.../beam/runners/spark/ReuseSparkContext.java | 46 ++++
.../beam/runners/spark/WatermarkTest.java | 227 +++++++++++++++++++
.../streaming/FlattenStreamingTest.java | 22 --
.../streaming/TrackStreamingSourcesTest.java | 168 ++++++++++++++
.../apache/beam/sdk/testing/RegexMatcher.java | 49 ++++
16 files changed, 898 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
[4/9] beam git commit: Ingest the input watermarks into the
GlobalWatermarkHolder.
Posted by am...@apache.org.
Ingest the input watermarks into the GlobalWatermarkHolder.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/bbf3744d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/bbf3744d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/bbf3744d
Branch: refs/heads/master
Commit: bbf3744d4cc1a6a58712b4c54c421b0009c5bb5e
Parents: a620653
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:30:47 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:14 2017 +0200
----------------------------------------------------------------------
.../runners/spark/io/SparkUnboundedSource.java | 69 ++++++++++++++------
.../spark/stateful/StateSpecFunctions.java | 31 ++++++---
2 files changed, 72 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/bbf3744d/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
index f03dc8c..354461f 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
@@ -24,8 +24,13 @@ import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.stateful.StateSpecFunctions;
import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
+import org.apache.beam.runners.spark.translation.streaming.UnboundedDataset;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.spark.api.java.JavaRDD;
@@ -55,7 +60,7 @@ import scala.runtime.BoxedUnit;
* <p>This read is a composite of the following steps:
* <ul>
* <li>Create a single-element (per-partition) stream, that contains the (partitioned)
- * {@link Source} and an optional {@link UnboundedSource.CheckpointMark} to start from.</li>
+ * {@link Source} and an optional {@link CheckpointMark} to start from.</li>
* <li>Read from within a stateful operation {@link JavaPairInputDStream#mapWithState(StateSpec)}
* using the {@link StateSpecFunctions#mapSourceFunction(SparkRuntimeContext)} mapping function,
* which manages the state of the CheckpointMark per partition.</li>
@@ -65,10 +70,11 @@ import scala.runtime.BoxedUnit;
*/
public class SparkUnboundedSource {
- public static <T, CheckpointMarkT extends UnboundedSource.CheckpointMark>
- JavaDStream<WindowedValue<T>> read(JavaStreamingContext jssc,
- SparkRuntimeContext rc,
- UnboundedSource<T, CheckpointMarkT> source) {
+ public static <T, CheckpointMarkT extends CheckpointMark> UnboundedDataset<T> read(
+ JavaStreamingContext jssc,
+ SparkRuntimeContext rc,
+ UnboundedSource<T, CheckpointMarkT> source) {
+
SparkPipelineOptions options = rc.getPipelineOptions().as(SparkPipelineOptions.class);
Long maxRecordsPerBatch = options.getMaxRecordsPerBatch();
SourceDStream<T, CheckpointMarkT> sourceDStream = new SourceDStream<>(jssc.ssc(), source, rc);
@@ -82,7 +88,7 @@ public class SparkUnboundedSource {
JavaSparkContext$.MODULE$.<CheckpointMarkT>fakeClassTag());
// call mapWithState to read from a checkpointable sources.
- JavaMapWithStateDStream<Source<T>, CheckpointMarkT, byte[],
+ JavaMapWithStateDStream<Source<T>, CheckpointMarkT, Tuple2<byte[], Instant>,
Tuple2<Iterable<byte[]>, Metadata>> mapWithStateDStream = inputDStream.mapWithState(
StateSpec.function(StateSpecFunctions.<T, CheckpointMarkT>mapSourceFunction(rc)));
@@ -109,13 +115,14 @@ public class SparkUnboundedSource {
WindowedValue.FullWindowedValueCoder.of(
source.getDefaultOutputCoder(),
GlobalWindow.Coder.INSTANCE);
- return mapWithStateDStream.flatMap(
+ JavaDStream<WindowedValue<T>> readUnboundedStream = mapWithStateDStream.flatMap(
new FlatMapFunction<Tuple2<Iterable<byte[]>, Metadata>, byte[]>() {
@Override
public Iterable<byte[]> call(Tuple2<Iterable<byte[]>, Metadata> t2) throws Exception {
return t2._1();
}
}).map(CoderHelpers.fromByteFunction(coder));
+ return new UnboundedDataset<>(readUnboundedStream, Collections.singletonList(id));
}
private static <T> String getSourceName(Source<T> source, int id) {
@@ -173,30 +180,46 @@ public class SparkUnboundedSource {
// compute parent.
scala.Option<RDD<Metadata>> parentRDDOpt = parent.getOrCompute(validTime);
long count = 0;
- Instant globalWatermark = new Instant(Long.MIN_VALUE);
+ SparkWatermarks sparkWatermark = null;
+ Instant globalLowWatermarkForBatch = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ Instant globalHighWatermarkForBatch = BoundedWindow.TIMESTAMP_MIN_VALUE;
if (parentRDDOpt.isDefined()) {
JavaRDD<Metadata> parentRDD = parentRDDOpt.get().toJavaRDD();
for (Metadata metadata: parentRDD.collect()) {
count += metadata.getNumRecords();
- // a monotonically increasing watermark.
- globalWatermark = globalWatermark.isBefore(metadata.getWatermark())
- ? metadata.getWatermark() : globalWatermark;
+ // compute the global input watermark - advance to latest of all partitions.
+ Instant partitionLowWatermark = metadata.getLowWatermark();
+ globalLowWatermarkForBatch =
+ globalLowWatermarkForBatch.isBefore(partitionLowWatermark)
+ ? partitionLowWatermark : globalLowWatermarkForBatch;
+ Instant partitionHighWatermark = metadata.getHighWatermark();
+ globalHighWatermarkForBatch =
+ globalHighWatermarkForBatch.isBefore(partitionHighWatermark)
+ ? partitionHighWatermark : globalHighWatermarkForBatch;
}
+
+ sparkWatermark =
+ new SparkWatermarks(
+ globalLowWatermarkForBatch,
+ globalHighWatermarkForBatch,
+ new Instant(validTime.milliseconds()));
+ // add to watermark queue.
+ GlobalWatermarkHolder.add(inputDStreamId, sparkWatermark);
}
// report - for RateEstimator and visibility.
- report(validTime, count, globalWatermark);
+ report(validTime, count, sparkWatermark);
return scala.Option.empty();
}
- private void report(Time batchTime, long count, Instant watermark) {
+ private void report(Time batchTime, long count, SparkWatermarks sparkWatermark) {
// metadata - #records read and a description.
scala.collection.immutable.Map<String, Object> metadata =
new scala.collection.immutable.Map.Map1<String, Object>(
StreamInputInfo.METADATA_KEY_DESCRIPTION(),
String.format(
- "Read %d records with observed watermark %s, from %s for batch time: %s",
+ "Read %d records with observed watermarks %s, from %s for batch time: %s",
count,
- watermark,
+ sparkWatermark == null ? "N/A" : sparkWatermark,
sourceName,
batchTime));
StreamInputInfo streamInputInfo = new StreamInputInfo(inputDStreamId, count, metadata);
@@ -209,19 +232,25 @@ public class SparkUnboundedSource {
*/
public static class Metadata implements Serializable {
private final long numRecords;
- private final Instant watermark;
+ private final Instant lowWatermark;
+ private final Instant highWatermark;
- public Metadata(long numRecords, Instant watermark) {
+ public Metadata(long numRecords, Instant lowWatermark, Instant highWatermark) {
this.numRecords = numRecords;
- this.watermark = watermark;
+ this.lowWatermark = lowWatermark;
+ this.highWatermark = highWatermark;
}
public long getNumRecords() {
return numRecords;
}
- public Instant getWatermark() {
- return watermark;
+ public Instant getLowWatermark() {
+ return lowWatermark;
+ }
+
+ public Instant getHighWatermark() {
+ return highWatermark;
}
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/bbf3744d/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
index ffe0ddd..ae5a746 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
@@ -35,6 +35,7 @@ import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
@@ -94,29 +95,36 @@ public class StateSpecFunctions {
* @return The appropriate {@link org.apache.spark.streaming.StateSpec} function.
*/
public static <T, CheckpointMarkT extends UnboundedSource.CheckpointMark>
- scala.Function3<Source<T>, scala.Option<CheckpointMarkT>, /* CheckpointMarkT */State<byte[]>,
+ scala.Function3<Source<T>, scala.Option<CheckpointMarkT>, State<Tuple2<byte[], Instant>>,
Tuple2<Iterable<byte[]>, Metadata>> mapSourceFunction(
final SparkRuntimeContext runtimeContext) {
- return new SerializableFunction3<Source<T>, Option<CheckpointMarkT>, State<byte[]>,
- Tuple2<Iterable<byte[]>, Metadata>>() {
+ return new SerializableFunction3<Source<T>, Option<CheckpointMarkT>,
+ State<Tuple2<byte[], Instant>>, Tuple2<Iterable<byte[]>, Metadata>>() {
@Override
public Tuple2<Iterable<byte[]>, Metadata> apply(
Source<T> source,
scala.Option<CheckpointMarkT> startCheckpointMark,
- State<byte[]> state) {
+ State<Tuple2<byte[], Instant>> state) {
+
// source as MicrobatchSource
MicrobatchSource<T, CheckpointMarkT> microbatchSource =
(MicrobatchSource<T, CheckpointMarkT>) source;
+ // Initial high/low watermarks.
+ Instant lowWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ Instant highWatermark;
+
// if state exists, use it, otherwise it's first time so use the startCheckpointMark.
// startCheckpointMark may be EmptyCheckpointMark (the Spark Java API tries to apply
// Optional(null)), which is handled by the UnboundedSource implementation.
Coder<CheckpointMarkT> checkpointCoder = microbatchSource.getCheckpointMarkCoder();
CheckpointMarkT checkpointMark;
if (state.exists()) {
- checkpointMark = CoderHelpers.fromByteArray(state.get(), checkpointCoder);
+ // previous (output) watermark is now the low watermark.
+ lowWatermark = state.get()._2();
+ checkpointMark = CoderHelpers.fromByteArray(state.get()._1(), checkpointCoder);
LOG.info("Continue reading from an existing CheckpointMark.");
} else if (startCheckpointMark.isDefined()
&& !startCheckpointMark.get().equals(EmptyCheckpointMark.get())) {
@@ -154,7 +162,10 @@ public class StateSpecFunctions {
finished = !reader.advance();
}
- watermark = ((MicrobatchSource.Reader) reader).getWatermark();
+ // end-of-read watermark is the high watermark, but don't allow decrease.
+ Instant sourceWatermark = ((MicrobatchSource.Reader) reader).getWatermark();
+ highWatermark = sourceWatermark.isAfter(lowWatermark) ? sourceWatermark : lowWatermark;
+
// close and checkpoint reader.
reader.close();
LOG.info("Source id {} spent {} msec on reading.", microbatchSource.getId(),
@@ -164,11 +175,15 @@ public class StateSpecFunctions {
@SuppressWarnings("unchecked")
CheckpointMarkT finishedReadCheckpointMark =
(CheckpointMarkT) ((MicrobatchSource.Reader) reader).getCheckpointMark();
+ byte[] codedCheckpoint = new byte[0];
if (finishedReadCheckpointMark != null) {
- state.update(CoderHelpers.toByteArray(finishedReadCheckpointMark, checkpointCoder));
+ codedCheckpoint = CoderHelpers.toByteArray(finishedReadCheckpointMark, checkpointCoder);
} else {
LOG.info("Skipping checkpoint marking because the reader failed to supply one.");
}
+ // persist the end-of-read (high) watermark for following read, where it will become
+ // the next low watermark.
+ state.update(new Tuple2<>(codedCheckpoint, highWatermark));
} catch (IOException e) {
throw new RuntimeException("Failed to read from reader.", e);
}
@@ -179,7 +194,7 @@ public class StateSpecFunctions {
return Iterators.unmodifiableIterator(readValues.iterator());
}
};
- return new Tuple2<>(iterable, new Metadata(readValues.size(), watermark));
+ return new Tuple2<>(iterable, new Metadata(readValues.size(), lowWatermark, highWatermark));
}
};
}
[5/9] beam git commit: Watermark tests.
Posted by am...@apache.org.
Watermark tests.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9784f204
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9784f204
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9784f204
Branch: refs/heads/master
Commit: 9784f204793ab8e8ec3ec84e3c7c8d2ca4ddaf6a
Parents: c18f8a2
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:34:30 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:15 2017 +0200
----------------------------------------------------------------------
.../beam/runners/spark/ClearWatermarksRule.java | 37 ++++
.../beam/runners/spark/ReuseSparkContext.java | 46 ++++
.../beam/runners/spark/WatermarkTest.java | 212 +++++++++++++++++++
3 files changed, 295 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/9784f204/runners/spark/src/test/java/org/apache/beam/runners/spark/ClearWatermarksRule.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ClearWatermarksRule.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ClearWatermarksRule.java
new file mode 100644
index 0000000..4c0c99a
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ClearWatermarksRule.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark;
+
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
+import org.junit.rules.ExternalResource;
+
+/**
+ * A rule that clears the {@link GlobalWatermarkHolder}.
+ */
+public class ClearWatermarksRule extends ExternalResource {
+
+ @Override
+ protected void before() throws Throwable {
+ clearWatermarks();
+ }
+
+ public void clearWatermarks() {
+ GlobalWatermarkHolder.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/9784f204/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java
new file mode 100644
index 0000000..027f9fd
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark;
+
+import org.apache.beam.runners.spark.translation.SparkContextFactory;
+import org.junit.rules.ExternalResource;
+
+/**
+ * Explicitly set {@link org.apache.spark.SparkContext} to be reused (or not) in tests.
+ */
+public class ReuseSparkContext extends ExternalResource {
+
+ private final boolean reuse;
+
+ private ReuseSparkContext(boolean reuse) {
+ this.reuse = reuse;
+ }
+
+ public static ReuseSparkContext no() {
+ return new ReuseSparkContext(false);
+ }
+
+ public static ReuseSparkContext yes() {
+ return new ReuseSparkContext(true);
+ }
+
+ @Override
+ protected void before() throws Throwable {
+ System.setProperty(SparkContextFactory.TEST_REUSE_SPARK_CONTEXT, Boolean.toString(reuse));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/9784f204/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
new file mode 100644
index 0000000..0b56403
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
@@ -0,0 +1,212 @@
+package org.apache.beam.runners.spark;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.runners.spark.io.CreateStream;
+import org.apache.beam.runners.spark.translation.SparkContextFactory;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.RegexMatcher;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+
+/**
+ * A test suite for the propagation of watermarks in the Spark runner.
+ */
+public class WatermarkTest {
+
+ @Rule
+ public ClearWatermarksRule clearWatermarksRule = new ClearWatermarksRule();
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Rule
+ public ReuseSparkContext reuseContext = ReuseSparkContext.yes();
+
+ private static final SparkPipelineOptions options =
+ PipelineOptionsFactory.create().as(SparkPipelineOptions.class);
+
+ private static final String INSTANT_PATTERN =
+ "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z";
+
+ @Test
+ public void testLowHighWatermarksAdvance() {
+ JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+
+ Instant instant = new Instant(0);
+ // low == high.
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(5)),
+ instant.plus(Duration.millis(5)),
+ instant));
+ GlobalWatermarkHolder.advance(jsc);
+ // low < high.
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(10)),
+ instant.plus(Duration.millis(15)),
+ instant.plus(Duration.millis(100))));
+ GlobalWatermarkHolder.advance(jsc);
+
+ // assert watermarks in Broadcast.
+ SparkWatermarks currentWatermarks = GlobalWatermarkHolder.get().getValue().get(1);
+ assertThat(currentWatermarks.getLowWatermark(), equalTo(instant.plus(Duration.millis(10))));
+ assertThat(currentWatermarks.getHighWatermark(), equalTo(instant.plus(Duration.millis(15))));
+ assertThat(currentWatermarks.getSynchronizedProcessingTime(),
+ equalTo(instant.plus(Duration.millis(100))));
+
+ // assert illegal watermark advance.
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage(
+ RegexMatcher.matches(
+ "Low watermark " + INSTANT_PATTERN + " cannot be later then high watermark "
+ + INSTANT_PATTERN));
+ // low > high -> not allowed!
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(25)),
+ instant.plus(Duration.millis(20)),
+ instant.plus(Duration.millis(200))));
+ GlobalWatermarkHolder.advance(jsc);
+ }
+
+ @Test
+ public void testSynchronizedTimeMonotonic() {
+ JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+
+ Instant instant = new Instant(0);
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(5)),
+ instant.plus(Duration.millis(10)),
+ instant));
+ GlobalWatermarkHolder.advance(jsc);
+
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Synchronized processing time must advance.");
+ // no actual advancement of watermarks - fine by Watermarks
+ // but not by synchronized processing time.
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(5)),
+ instant.plus(Duration.millis(10)),
+ instant));
+ GlobalWatermarkHolder.advance(jsc);
+ }
+
+ @Test
+ public void testMultiSource() {
+ JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+
+ Instant instant = new Instant(0);
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(5)),
+ instant.plus(Duration.millis(10)),
+ instant));
+ GlobalWatermarkHolder.add(2,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(3)),
+ instant.plus(Duration.millis(6)),
+ instant));
+
+ GlobalWatermarkHolder.advance(jsc);
+
+ // assert watermarks for source 1.
+ SparkWatermarks watermarksForSource1 = GlobalWatermarkHolder.get().getValue().get(1);
+ assertThat(watermarksForSource1.getLowWatermark(), equalTo(instant.plus(Duration.millis(5))));
+ assertThat(watermarksForSource1.getHighWatermark(), equalTo(instant.plus(Duration.millis(10))));
+
+ // assert watermarks for source 2.
+ SparkWatermarks watermarksForSource2 = GlobalWatermarkHolder.get().getValue().get(2);
+ assertThat(watermarksForSource2.getLowWatermark(), equalTo(instant.plus(Duration.millis(3))));
+ assertThat(watermarksForSource2.getHighWatermark(), equalTo(instant.plus(Duration.millis(6))));
+ }
+
+ @Test
+ public void testInDoFn() {
+ // because watermark advances onBatchCompleted.
+ Iterable<Integer> zeroBatch = Collections.emptyList();
+ Iterable<Integer> firstBatch = Collections.singletonList(1);
+ Iterable<Integer> secondBatch = Collections.singletonList(2);
+
+ Instant instant = new Instant(0);
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(5)),
+ instant.plus(Duration.millis(10)),
+ instant));
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(10)),
+ instant.plus(Duration.millis(15)),
+ instant.plus(options.getBatchIntervalMillis())));
+
+ options.setRunner(SparkRunner.class);
+ options.setStreaming(true);
+ options.setBatchIntervalMillis(500L);
+ Pipeline p = Pipeline.create(options);
+
+ CreateStream.QueuedValues<Integer> queueStream =
+ CreateStream.fromQueue(Arrays.asList(zeroBatch, firstBatch, secondBatch));
+
+ p.apply(queueStream).setCoder(VarIntCoder.of()).apply(ParDo.of(new WatermarksDoFn(1)));
+
+ p.run().waitUntilFinish(Duration.millis(options.getBatchIntervalMillis()).multipliedBy(3));
+
+ System.out.println(WatermarksDoFn.strings);
+
+ // this is a hacky way to assert but it will do until triggers are supported.
+ assertThat(
+ WatermarksDoFn.strings,
+ containsInAnyOrder(
+ "element: 1 lowWatermark: 5 highWatermark: 10 processingTime: 0",
+ "element: 2 lowWatermark: 10 highWatermark: 15 processingTime: 1000"));
+ }
+
+ private static class WatermarksDoFn extends DoFn<Integer, String> {
+ private final int sourceId;
+
+ static List<String> strings = new ArrayList<>();
+
+ private WatermarksDoFn(int sourceId) {
+ this.sourceId = sourceId;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ if (GlobalWatermarkHolder.get() == null
+ || GlobalWatermarkHolder.get().getValue().get(sourceId) == null) {
+ // watermark not yet updated.
+ return;
+ }
+ SparkWatermarks sparkWatermarks = GlobalWatermarkHolder.get().getValue().get(sourceId);
+ Integer element = c.element();
+ String output =
+ "element: " + element
+ + " lowWatermark: " + sparkWatermarks.getLowWatermark().getMillis()
+ + " highWatermark: " + sparkWatermarks.getHighWatermark().getMillis()
+ + " processingTime: " + sparkWatermarks.getSynchronizedProcessingTime().getMillis();
+ strings.add(output);
+ }
+ }
+
+}
[2/9] beam git commit: Advance watermarks onBatchCompleted hook.
Posted by am...@apache.org.
Advance watermarks onBatchCompleted hook.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fa31f18e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fa31f18e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fa31f18e
Branch: refs/heads/master
Commit: fa31f18e489d4cbe44fe4a9be7ba3d7dbee7c354
Parents: bbf3744
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:31:14 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:14 2017 +0200
----------------------------------------------------------------------
.../main/java/org/apache/beam/runners/spark/SparkRunner.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/fa31f18e/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index ebac375..52a080b 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -39,6 +39,7 @@ import org.apache.beam.runners.spark.translation.TransformEvaluator;
import org.apache.beam.runners.spark.translation.TransformTranslator;
import org.apache.beam.runners.spark.translation.streaming.Checkpoint.CheckpointDir;
import org.apache.beam.runners.spark.translation.streaming.SparkRunnerStreamingContextFactory;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.WatermarksListener;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.metrics.MetricsEnvironment;
@@ -191,12 +192,15 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
new JavaStreamingListenerWrapper(
new MetricsAccumulator.AccumulatorCheckpointingSparkListener()));
- // register listeners.
+ // register user-defined listeners.
for (JavaStreamingListener listener: mOptions.as(SparkContextOptions.class).getListeners()) {
LOG.info("Registered listener {}." + listener.getClass().getSimpleName());
jssc.addStreamingListener(new JavaStreamingListenerWrapper(listener));
}
+ // register Watermarks listener to broadcast the advanced WMs.
+ jssc.addStreamingListener(new JavaStreamingListenerWrapper(new WatermarksListener(jssc)));
+
startPipeline = executorService.submit(new Runnable() {
@Override
[3/9] beam git commit: Handle QueuedStream (for testing) and track
sources upstream.
Posted by am...@apache.org.
Handle QueuedStream (for testing) and track sources upstream.
Refactor according to changes.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/705695eb
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/705695eb
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/705695eb
Branch: refs/heads/master
Commit: 705695eb726acf086915e610cb2304bd968e3682
Parents: fa31f18
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:32:06 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:14 2017 +0200
----------------------------------------------------------------------
.../beam/runners/spark/TestSparkRunner.java | 11 ++-
.../aggregators/AggregatorsAccumulator.java | 2 +-
.../spark/translation/SparkContextFactory.java | 2 +-
.../streaming/StreamingTransformTranslator.java | 92 +++++++++++++-------
.../translation/streaming/UnboundedDataset.java | 29 ++++--
5 files changed, 88 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/705695eb/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index e770164..a634dd4 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -22,6 +22,8 @@ import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import org.apache.beam.runners.core.UnboundedReadFromBoundedSource;
+import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineResult.State;
@@ -107,13 +109,14 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
@Override
public SparkPipelineResult run(Pipeline pipeline) {
TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
-
- // clear metrics singleton
- SparkMetricsContainer.clear();
-
SparkPipelineResult result = delegate.run(pipeline);
result.waitUntilFinish();
+ // clear state of Aggregators, Metrics and Watermarks.
+ AggregatorsAccumulator.clear();
+ SparkMetricsContainer.clear();
+ GlobalWatermarkHolder.clear();
+
// make sure the test pipeline finished successfully.
State resultState = result.getState();
assertThat(
http://git-wip-us.apache.org/repos/asf/beam/blob/705695eb/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
index 1b49e91..a4dfda6 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
@@ -89,7 +89,7 @@ public class AggregatorsAccumulator {
}
@VisibleForTesting
- static void clear() {
+ public static void clear() {
synchronized (AggregatorsAccumulator.class) {
instance = null;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/705695eb/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
index 326838a..cdeddad 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
@@ -38,7 +38,7 @@ public final class SparkContextFactory {
* {@code true} then the Spark context will be reused for beam pipelines.
* This property should only be enabled for tests.
*/
- static final String TEST_REUSE_SPARK_CONTEXT = "beam.spark.test.reuseSparkContext";
+ public static final String TEST_REUSE_SPARK_CONTEXT = "beam.spark.test.reuseSparkContext";
// Spark allows only one context for JVM so this can be static.
private static JavaSparkContext sparkContext;
http://git-wip-us.apache.org/repos/asf/beam/blob/705695eb/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index a49b959..9451df7 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -105,9 +105,12 @@ final class StreamingTransformTranslator {
return new TransformEvaluator<Read.Unbounded<T>>() {
@Override
public void evaluate(Read.Unbounded<T> transform, EvaluationContext context) {
- context.putDataset(transform,
- new UnboundedDataset<>(SparkUnboundedSource.read(context.getStreamingContext(),
- context.getRuntimeContext(), transform.getSource())));
+ context.putDataset(
+ transform,
+ SparkUnboundedSource.read(
+ context.getStreamingContext(),
+ context.getRuntimeContext(),
+ transform.getSource()));
}
};
}
@@ -134,6 +137,7 @@ final class StreamingTransformTranslator {
// So we could end up with an unbounded unified DStream.
final List<JavaRDD<WindowedValue<T>>> rdds = new ArrayList<>();
final List<JavaDStream<WindowedValue<T>>> dStreams = new ArrayList<>();
+ final List<Integer> streamingSources = new ArrayList<>();
for (TaggedPValue pv : pcs) {
checkArgument(
pv.getValue() instanceof PCollection,
@@ -143,7 +147,9 @@ final class StreamingTransformTranslator {
PCollection<T> pcol = (PCollection<T>) pv.getValue();
Dataset dataset = context.borrowDataset(pcol);
if (dataset instanceof UnboundedDataset) {
- dStreams.add(((UnboundedDataset<T>) dataset).getDStream());
+ UnboundedDataset<T> unboundedDataset = (UnboundedDataset<T>) dataset;
+ streamingSources.addAll(unboundedDataset.getStreamingSources());
+ dStreams.add(unboundedDataset.getDStream());
} else {
rdds.add(((BoundedDataset<T>) dataset).getRDD());
}
@@ -162,9 +168,9 @@ final class StreamingTransformTranslator {
return new JavaSparkContext(streamRdd.context()).union(streamRdd, rdds);
}
});
- context.putDataset(transform, new UnboundedDataset<>(joined));
+ context.putDataset(transform, new UnboundedDataset<>(joined, streamingSources));
} else {
- context.putDataset(transform, new UnboundedDataset<>(unifiedStreams));
+ context.putDataset(transform, new UnboundedDataset<>(unifiedStreams, streamingSources));
}
}
};
@@ -177,8 +183,9 @@ final class StreamingTransformTranslator {
@SuppressWarnings("unchecked")
WindowFn<? super T, W> windowFn = (WindowFn<? super T, W>) transform.getWindowFn();
@SuppressWarnings("unchecked")
- JavaDStream<WindowedValue<T>> dStream =
- ((UnboundedDataset<T>) context.borrowDataset(transform)).getDStream();
+ UnboundedDataset<T> unboundedDataset =
+ ((UnboundedDataset<T>) context.borrowDataset(transform));
+ JavaDStream<WindowedValue<T>> dStream = unboundedDataset.getDStream();
// get the right window durations.
Duration windowDuration;
Duration slideDuration;
@@ -197,7 +204,8 @@ final class StreamingTransformTranslator {
dStream.window(windowDuration, slideDuration);
//--- then we apply windowing to the elements
if (TranslationUtils.skipAssignWindows(transform, context)) {
- context.putDataset(transform, new UnboundedDataset<>(windowedDStream));
+ context.putDataset(transform,
+ new UnboundedDataset<>(windowedDStream, unboundedDataset.getStreamingSources()));
} else {
JavaDStream<WindowedValue<T>> outStream = windowedDStream.transform(
new Function<JavaRDD<WindowedValue<T>>, JavaRDD<WindowedValue<T>>>() {
@@ -206,7 +214,8 @@ final class StreamingTransformTranslator {
return rdd.map(new SparkAssignWindowFn<>(transform.getWindowFn()));
}
});
- context.putDataset(transform, new UnboundedDataset<>(outStream));
+ context.putDataset(transform,
+ new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
}
}
};
@@ -217,8 +226,9 @@ final class StreamingTransformTranslator {
@Override
public void evaluate(GroupByKey<K, V> transform, EvaluationContext context) {
@SuppressWarnings("unchecked")
- JavaDStream<WindowedValue<KV<K, V>>> dStream =
- ((UnboundedDataset<KV<K, V>>) context.borrowDataset(transform)).getDStream();
+ UnboundedDataset<KV<K, V>> unboundedDataset =
+ ((UnboundedDataset<KV<K, V>>) context.borrowDataset(transform));
+ JavaDStream<WindowedValue<KV<K, V>>> dStream = unboundedDataset.getDStream();
@SuppressWarnings("unchecked")
final KvCoder<K, V> coder = (KvCoder<K, V>) context.getInput(transform).getCoder();
@@ -239,7 +249,8 @@ final class StreamingTransformTranslator {
windowingStrategy);
}
});
- context.putDataset(transform, new UnboundedDataset<>(outStream));
+ context.putDataset(transform,
+ new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
}
};
}
@@ -247,7 +258,6 @@ final class StreamingTransformTranslator {
private static <K, InputT, OutputT> TransformEvaluator<Combine.GroupedValues<K, InputT, OutputT>>
combineGrouped() {
return new TransformEvaluator<Combine.GroupedValues<K, InputT, OutputT>>() {
- @SuppressWarnings("unchecked")
@Override
public void evaluate(final Combine.GroupedValues<K, InputT, OutputT> transform,
EvaluationContext context) {
@@ -255,13 +265,15 @@ final class StreamingTransformTranslator {
PCollection<? extends KV<K, ? extends Iterable<InputT>>> input =
context.getInput(transform);
final WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
+ @SuppressWarnings("unchecked")
final CombineWithContext.KeyedCombineFnWithContext<K, InputT, ?, OutputT> fn =
(CombineWithContext.KeyedCombineFnWithContext<K, InputT, ?, OutputT>)
CombineFnUtil.toFnWithContext(transform.getFn());
- JavaDStream<WindowedValue<KV<K, Iterable<InputT>>>> dStream =
- ((UnboundedDataset<KV<K, Iterable<InputT>>>) context.borrowDataset(transform))
- .getDStream();
+ @SuppressWarnings("unchecked")
+ UnboundedDataset<KV<K, Iterable<InputT>>> unboundedDataset =
+ ((UnboundedDataset<KV<K, Iterable<InputT>>>) context.borrowDataset(transform));
+ JavaDStream<WindowedValue<KV<K, Iterable<InputT>>>> dStream = unboundedDataset.getDStream();
final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
final SparkPCollectionView pviews = context.getPViews();
@@ -283,7 +295,8 @@ final class StreamingTransformTranslator {
}
});
- context.putDataset(transform, new UnboundedDataset<>(outStream));
+ context.putDataset(transform,
+ new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
}
};
}
@@ -292,7 +305,6 @@ final class StreamingTransformTranslator {
combineGlobally() {
return new TransformEvaluator<Combine.Globally<InputT, OutputT>>() {
- @SuppressWarnings("unchecked")
@Override
public void evaluate(
final Combine.Globally<InputT, OutputT> transform,
@@ -301,6 +313,7 @@ final class StreamingTransformTranslator {
// serializable arguments to pass.
final Coder<InputT> iCoder = context.getInput(transform).getCoder();
final Coder<OutputT> oCoder = context.getOutput(transform).getCoder();
+ @SuppressWarnings("unchecked")
final CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT> combineFn =
(CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT>)
CombineFnUtil.toFnWithContext(transform.getFn());
@@ -309,8 +322,10 @@ final class StreamingTransformTranslator {
final boolean hasDefault = transform.isInsertDefault();
final SparkPCollectionView pviews = context.getPViews();
- JavaDStream<WindowedValue<InputT>> dStream =
- ((UnboundedDataset<InputT>) context.borrowDataset(transform)).getDStream();
+ @SuppressWarnings("unchecked")
+ UnboundedDataset<InputT> unboundedDataset =
+ ((UnboundedDataset<InputT>) context.borrowDataset(transform));
+ JavaDStream<WindowedValue<InputT>> dStream = unboundedDataset.getDStream();
JavaDStream<WindowedValue<OutputT>> outStream = dStream.transform(
new Function<JavaRDD<WindowedValue<InputT>>, JavaRDD<WindowedValue<OutputT>>>() {
@@ -326,7 +341,8 @@ final class StreamingTransformTranslator {
}
});
- context.putDataset(transform, new UnboundedDataset<>(outStream));
+ context.putDataset(transform,
+ new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
}
};
}
@@ -334,7 +350,6 @@ final class StreamingTransformTranslator {
private static <K, InputT, AccumT, OutputT>
TransformEvaluator<Combine.PerKey<K, InputT, OutputT>> combinePerKey() {
return new TransformEvaluator<Combine.PerKey<K, InputT, OutputT>>() {
- @SuppressWarnings("unchecked")
@Override
public void evaluate(final Combine.PerKey<K, InputT, OutputT> transform,
final EvaluationContext context) {
@@ -342,6 +357,7 @@ final class StreamingTransformTranslator {
// serializable arguments to pass.
final KvCoder<K, InputT> inputCoder =
(KvCoder<K, InputT>) context.getInput(transform).getCoder();
+ @SuppressWarnings("unchecked")
final CombineWithContext.KeyedCombineFnWithContext<K, InputT, AccumT, OutputT> combineFn =
(CombineWithContext.KeyedCombineFnWithContext<K, InputT, AccumT, OutputT>)
CombineFnUtil.toFnWithContext(transform.getFn());
@@ -349,8 +365,10 @@ final class StreamingTransformTranslator {
final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
final SparkPCollectionView pviews = context.getPViews();
- JavaDStream<WindowedValue<KV<K, InputT>>> dStream =
- ((UnboundedDataset<KV<K, InputT>>) context.borrowDataset(transform)).getDStream();
+ @SuppressWarnings("unchecked")
+ UnboundedDataset<KV<K, InputT>> unboundedDataset =
+ ((UnboundedDataset<KV<K, InputT>>) context.borrowDataset(transform));
+ JavaDStream<WindowedValue<KV<K, InputT>>> dStream = unboundedDataset.getDStream();
JavaDStream<WindowedValue<KV<K, OutputT>>> outStream =
dStream.transform(new Function<JavaRDD<WindowedValue<KV<K, InputT>>>,
@@ -366,14 +384,14 @@ final class StreamingTransformTranslator {
windowingStrategy, sideInputs);
}
});
- context.putDataset(transform, new UnboundedDataset<>(outStream));
+ context.putDataset(transform,
+ new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
}
};
}
private static <InputT, OutputT> TransformEvaluator<ParDo.Bound<InputT, OutputT>> parDo() {
return new TransformEvaluator<ParDo.Bound<InputT, OutputT>>() {
- @SuppressWarnings("unchecked")
@Override
public void evaluate(final ParDo.Bound<InputT, OutputT> transform,
final EvaluationContext context) {
@@ -384,8 +402,11 @@ final class StreamingTransformTranslator {
final WindowingStrategy<?, ?> windowingStrategy =
context.getInput(transform).getWindowingStrategy();
final SparkPCollectionView pviews = context.getPViews();
- JavaDStream<WindowedValue<InputT>> dStream =
- ((UnboundedDataset<InputT>) context.borrowDataset(transform)).getDStream();
+
+ @SuppressWarnings("unchecked")
+ UnboundedDataset<InputT> unboundedDataset =
+ ((UnboundedDataset<InputT>) context.borrowDataset(transform));
+ JavaDStream<WindowedValue<InputT>> dStream = unboundedDataset.getDStream();
final String stepName = context.getCurrentTransform().getFullName();
@@ -409,7 +430,8 @@ final class StreamingTransformTranslator {
}
});
- context.putDataset(transform, new UnboundedDataset<>(outStream));
+ context.putDataset(transform,
+ new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
}
};
}
@@ -427,9 +449,12 @@ final class StreamingTransformTranslator {
final SparkPCollectionView pviews = context.getPViews();
final WindowingStrategy<?, ?> windowingStrategy =
context.getInput(transform).getWindowingStrategy();
+
@SuppressWarnings("unchecked")
- JavaDStream<WindowedValue<InputT>> dStream =
- ((UnboundedDataset<InputT>) context.borrowDataset(transform)).getDStream();
+ UnboundedDataset<InputT> unboundedDataset =
+ ((UnboundedDataset<InputT>) context.borrowDataset(transform));
+ JavaDStream<WindowedValue<InputT>> dStream = unboundedDataset.getDStream();
+
JavaPairDStream<TupleTag<?>, WindowedValue<?>> all = dStream.transformToPair(
new Function<JavaRDD<WindowedValue<InputT>>,
JavaPairRDD<TupleTag<?>, WindowedValue<?>>>() {
@@ -460,7 +485,8 @@ final class StreamingTransformTranslator {
JavaDStream<WindowedValue<Object>> values =
(JavaDStream<WindowedValue<Object>>)
(JavaDStream<?>) TranslationUtils.dStreamValues(filtered);
- context.putDataset(e.getValue(), new UnboundedDataset<>(values));
+ context.putDataset(e.getValue(),
+ new UnboundedDataset<>(values, unboundedDataset.getStreamingSources()));
}
}
};
http://git-wip-us.apache.org/repos/asf/beam/blob/705695eb/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
index d059c7e..80c0515 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
@@ -19,8 +19,12 @@
package org.apache.beam.runners.spark.translation.streaming;
import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.List;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nullable;
import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.translation.Dataset;
@@ -41,6 +45,7 @@ import org.slf4j.LoggerFactory;
public class UnboundedDataset<T> implements Dataset {
private static final Logger LOG = LoggerFactory.getLogger(UnboundedDataset.class);
+ private static final AtomicInteger queuedStreamIds = new AtomicInteger();
// only set if creating a DStream from a static collection
@Nullable private transient JavaStreamingContext jssc;
@@ -48,15 +53,23 @@ public class UnboundedDataset<T> implements Dataset {
private Iterable<Iterable<T>> values;
private Coder<T> coder;
private JavaDStream<WindowedValue<T>> dStream;
+ // points to the input streams that created this UnboundedDataset,
+ // should be greater > 1 in case of Flatten for example.
+ // when using GlobalWatermarkHolder this information helps to take only the relevant watermarks
+ // and reason about them accordingly.
+ private final List<Integer> streamingSources = new ArrayList<>();
- UnboundedDataset(JavaDStream<WindowedValue<T>> dStream) {
+ public UnboundedDataset(JavaDStream<WindowedValue<T>> dStream, List<Integer> streamingSources) {
this.dStream = dStream;
+ this.streamingSources.addAll(streamingSources);
}
public UnboundedDataset(Iterable<Iterable<T>> values, JavaStreamingContext jssc, Coder<T> coder) {
this.values = values;
this.jssc = jssc;
this.coder = coder;
+ // QueuedStream will have a negative (decreasing) unique id.
+ this.streamingSources.add(queuedStreamIds.decrementAndGet());
}
@SuppressWarnings("ConstantConditions")
@@ -66,7 +79,6 @@ public class UnboundedDataset<T> implements Dataset {
WindowedValue.getValueOnlyCoder(coder);
// create the DStream from queue
Queue<JavaRDD<WindowedValue<T>>> rddQueue = new LinkedBlockingQueue<>();
- JavaRDD<WindowedValue<T>> lastRDD = null;
for (Iterable<T> v : values) {
Iterable<WindowedValue<T>> windowedValues =
Iterables.transform(v, WindowingHelpers.<T>windowValueFunction());
@@ -74,18 +86,17 @@ public class UnboundedDataset<T> implements Dataset {
CoderHelpers.toByteArrays(windowedValues, windowCoder)).map(
CoderHelpers.fromByteFunction(windowCoder));
rddQueue.offer(rdd);
- lastRDD = rdd;
}
- // create DStream from queue, one at a time,
- // with last as default in case batches repeat (graceful stops for example).
- // if the stream is empty, avoid creating a default empty RDD.
- // mainly for unit test so no reason to have this configurable.
- dStream = lastRDD != null ? jssc.queueStream(rddQueue, true, lastRDD)
- : jssc.queueStream(rddQueue, true);
+ // create DStream from queue, one at a time.
+ dStream = jssc.queueStream(rddQueue, true);
}
return dStream;
}
+ public List<Integer> getStreamingSources() {
+ return streamingSources;
+ }
+
public void cache() {
dStream.cache();
}
[6/9] beam git commit: Exopse RegexMatcher as part of the SDK.
Posted by am...@apache.org.
Exopse RegexMatcher as part of the SDK.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c18f8a2c
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c18f8a2c
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c18f8a2c
Branch: refs/heads/master
Commit: c18f8a2c33a74ea00f8bbba6fb7cd766de9082f4
Parents: 705695e
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:33:55 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:15 2017 +0200
----------------------------------------------------------------------
.../runners/dataflow/util/PackageUtilTest.java | 30 +-----------------
.../apache/beam/sdk/testing/RegexMatcher.java | 32 ++++++++++++++++++++
2 files changed, 33 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c18f8a2c/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
index 800c5a9..a11872f 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
@@ -66,7 +66,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.beam.runners.dataflow.util.PackageUtil.PackageAttributes;
@@ -74,11 +73,10 @@ import org.apache.beam.sdk.options.GcsOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.ExpectedLogs;
import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
+import org.apache.beam.sdk.testing.RegexMatcher;
import org.apache.beam.sdk.util.GcsUtil;
import org.apache.beam.sdk.util.IOChannelUtils;
import org.apache.beam.sdk.util.gcsfs.GcsPath;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
import org.hamcrest.Matchers;
import org.junit.Before;
import org.junit.Rule;
@@ -107,32 +105,6 @@ public class PackageUtilTest {
// 128 bits, base64 encoded is 171 bits, rounds to 22 bytes
private static final String HASH_PATTERN = "[a-zA-Z0-9+-]{22}";
- // Hamcrest matcher to assert a string matches a pattern
- private static class RegexMatcher extends BaseMatcher<String> {
- private final Pattern pattern;
-
- public RegexMatcher(String regex) {
- this.pattern = Pattern.compile(regex);
- }
-
- @Override
- public boolean matches(Object o) {
- if (!(o instanceof String)) {
- return false;
- }
- return pattern.matcher((String) o).matches();
- }
-
- @Override
- public void describeTo(Description description) {
- description.appendText(String.format("matches regular expression %s", pattern));
- }
-
- public static RegexMatcher matches(String regex) {
- return new RegexMatcher(regex);
- }
- }
-
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
http://git-wip-us.apache.org/repos/asf/beam/blob/c18f8a2c/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
new file mode 100644
index 0000000..a4b14fe
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
@@ -0,0 +1,32 @@
+package org.apache.beam.sdk.testing;
+
+import java.util.regex.Pattern;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+
+
+/** Hamcrest matcher to assert a string matches a pattern. */
+public class RegexMatcher extends BaseMatcher<String> {
+ private final Pattern pattern;
+
+ public RegexMatcher(String regex) {
+ this.pattern = Pattern.compile(regex);
+ }
+
+ @Override
+ public boolean matches(Object o) {
+ if (!(o instanceof String)) {
+ return false;
+ }
+ return pattern.matcher((String) o).matches();
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText(String.format("matches regular expression %s", pattern));
+ }
+
+ public static RegexMatcher matches(String regex) {
+ return new RegexMatcher(regex);
+ }
+}