You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by am...@apache.org on 2017/02/20 09:57:57 UTC

[1/9] beam git commit: A global Watermark holder to update and broadcast to workers.

Repository: beam
Updated Branches:
  refs/heads/master 92190ba5d -> aa45ccb08


A global Watermark holder to update and broadcast to workers.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/a6206535
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/a6206535
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/a6206535

Branch: refs/heads/master
Commit: a6206535ec84d8c01695368c8973a6577ae6d953
Parents: 92190ba
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:28:19 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:13 2017 +0200

----------------------------------------------------------------------
 .../spark/util/GlobalWatermarkHolder.java       | 200 +++++++++++++++++++
 1 file changed, 200 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/a6206535/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
new file mode 100644
index 0000000..b215b5f
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/GlobalWatermarkHolder.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.util;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.spark.streaming.api.java.JavaStreamingContext;
+import org.apache.spark.streaming.api.java.JavaStreamingListener;
+import org.apache.spark.streaming.api.java.JavaStreamingListenerBatchCompleted;
+import org.joda.time.Instant;
+
+
+/**
+ * A {@link Broadcast} variable to hold the global watermarks for a micro-batch.
+ *
+ * <p>For each source, holds a queue for the watermarks of each micro-batch that was read,
+ * and advances the watermarks according to the queue (first-in-first-out).
+ */
+public class GlobalWatermarkHolder {
+  // the broadcast is broadcasted to the workers.
+  private static volatile Broadcast<Map<Integer, SparkWatermarks>> broadcast = null;
+  // this should only live in the driver so transient.
+  private static final transient Map<Integer, Queue<SparkWatermarks>> sourceTimes = new HashMap<>();
+
+  public static void add(int sourceId, SparkWatermarks sparkWatermarks) {
+    Queue<SparkWatermarks> timesQueue = sourceTimes.get(sourceId);
+    if (timesQueue == null) {
+      timesQueue = new ConcurrentLinkedQueue<>();
+    }
+    timesQueue.offer(sparkWatermarks);
+    sourceTimes.put(sourceId, timesQueue);
+  }
+
+  @VisibleForTesting
+  public static void addAll(Map<Integer, Queue<SparkWatermarks>> sourceTimes) {
+    for (Map.Entry<Integer, Queue<SparkWatermarks>> en: sourceTimes.entrySet()) {
+      int sourceId = en.getKey();
+      Queue<SparkWatermarks> timesQueue = en.getValue();
+      while (!timesQueue.isEmpty()) {
+        add(sourceId, timesQueue.poll());
+      }
+    }
+  }
+
+  /**
+   * Returns the {@link Broadcast} containing the {@link SparkWatermarks} mapped
+   * to their sources.
+   */
+  public static Broadcast<Map<Integer, SparkWatermarks>> get() {
+    return broadcast;
+  }
+
+  /**
+   * Advances the watermarks to the next-in-line watermarks.
+   * SparkWatermarks are monotonically increasing.
+   */
+  public static void advance(JavaSparkContext jsc) {
+    synchronized (GlobalWatermarkHolder.class){
+      if (sourceTimes.isEmpty()) {
+        return;
+      }
+
+      // update all sources' watermarks into the new broadcast.
+      Map<Integer, SparkWatermarks> newBroadcast = new HashMap<>();
+
+      for (Map.Entry<Integer, Queue<SparkWatermarks>> en: sourceTimes.entrySet()) {
+        if (en.getValue().isEmpty()) {
+          continue;
+        }
+        Integer sourceId = en.getKey();
+        Queue<SparkWatermarks> timesQueue = en.getValue();
+
+        // current state, if exists.
+        Instant currentLowWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+        Instant currentHighWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+        Instant currentSynchronizedProcessingTime = BoundedWindow.TIMESTAMP_MIN_VALUE;
+        if (broadcast != null && broadcast.getValue().containsKey(sourceId)) {
+          SparkWatermarks currentTimes = broadcast.getValue().get(sourceId);
+          currentLowWatermark = currentTimes.getLowWatermark();
+          currentHighWatermark = currentTimes.getHighWatermark();
+          currentSynchronizedProcessingTime = currentTimes.getSynchronizedProcessingTime();
+        }
+
+        SparkWatermarks next = timesQueue.poll();
+        // advance watermarks monotonically.
+        Instant nextLowWatermark = next.getLowWatermark().isAfter(currentLowWatermark)
+            ? next.getLowWatermark() : currentLowWatermark;
+        Instant nextHighWatermark = next.getHighWatermark().isAfter(currentHighWatermark)
+            ? next.getHighWatermark() : currentHighWatermark;
+        Instant nextSynchronizedProcessingTime = next.getSynchronizedProcessingTime();
+        checkState(!nextLowWatermark.isAfter(nextHighWatermark),
+            String.format(
+                "Low watermark %s cannot be later then high watermark %s",
+                nextLowWatermark, nextHighWatermark));
+        checkState(nextSynchronizedProcessingTime.isAfter(currentSynchronizedProcessingTime),
+            "Synchronized processing time must advance.");
+        newBroadcast.put(
+            sourceId,
+            new SparkWatermarks(
+                nextLowWatermark, nextHighWatermark, nextSynchronizedProcessingTime));
+      }
+
+      // update the watermarks broadcast only if something has changed.
+      if (!newBroadcast.isEmpty()) {
+        if (broadcast != null) {
+          // for now this is blocking, we could make this asynchronous
+          // but it could slow down WM propagation.
+          broadcast.unpersist(true);
+        }
+        broadcast = jsc.broadcast(newBroadcast);
+      }
+    }
+  }
+
+  @VisibleForTesting
+  public static synchronized void clear() {
+    sourceTimes.clear();
+    broadcast = null;
+  }
+
+  /**
+   * A {@link SparkWatermarks} holds the watermarks and batch time
+   * relevant to a micro-batch input from a specific source.
+   */
+  public static class SparkWatermarks implements Serializable {
+    private final Instant lowWatermark;
+    private final Instant highWatermark;
+    private final Instant synchronizedProcessingTime;
+
+    @VisibleForTesting
+    public SparkWatermarks(
+        Instant lowWatermark,
+        Instant highWatermark,
+        Instant synchronizedProcessingTime) {
+      this.lowWatermark = lowWatermark;
+      this.highWatermark = highWatermark;
+      this.synchronizedProcessingTime = synchronizedProcessingTime;
+    }
+
+    public Instant getLowWatermark() {
+      return lowWatermark;
+    }
+
+    public Instant getHighWatermark() {
+      return highWatermark;
+    }
+
+    public Instant getSynchronizedProcessingTime() {
+      return synchronizedProcessingTime;
+    }
+
+    @Override
+    public String toString() {
+      return "SparkWatermarks{"
+          + "lowWatermark=" + lowWatermark
+          + ", highWatermark=" + highWatermark
+          + ", synchronizedProcessingTime=" + synchronizedProcessingTime + '}';
+    }
+  }
+
+  /** Advance the WMs onBatchCompleted event. */
+  public static class WatermarksListener extends JavaStreamingListener {
+    private final JavaStreamingContext jssc;
+
+    public WatermarksListener(JavaStreamingContext jssc) {
+      this.jssc = jssc;
+    }
+
+    @Override
+    public void onBatchCompleted(JavaStreamingListenerBatchCompleted batchCompleted) {
+      GlobalWatermarkHolder.advance(jssc.sparkContext());
+    }
+  }
+}


[7/9] beam git commit: Streaming sources tracking test.

Posted by am...@apache.org.
Streaming sources tracking test.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/add87166
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/add87166
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/add87166

Branch: refs/heads/master
Commit: add87166fd445d634f6faf0f838f234c3f908c2e
Parents: 9784f20
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 19:23:28 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:16 2017 +0200

----------------------------------------------------------------------
 .../translation/streaming/UnboundedDataset.java |   6 +
 .../streaming/TrackStreamingSourcesTest.java    | 152 +++++++++++++++++++
 2 files changed, 158 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/add87166/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
index 80c0515..08d1ab6 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
@@ -18,6 +18,7 @@
 
 package org.apache.beam.runners.spark.translation.streaming;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
 
 import java.util.ArrayList;
@@ -72,6 +73,11 @@ public class UnboundedDataset<T> implements Dataset {
     this.streamingSources.add(queuedStreamIds.decrementAndGet());
   }
 
+  @VisibleForTesting
+  public static void resetQueuedStreamIds() {
+    queuedStreamIds.set(0);
+  }
+
   @SuppressWarnings("ConstantConditions")
   JavaDStream<WindowedValue<T>> getDStream() {
     if (dStream == null) {

http://git-wip-us.apache.org/repos/asf/beam/blob/add87166/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
new file mode 100644
index 0000000..f102ac8
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -0,0 +1,152 @@
+package org.apache.beam.runners.spark.translation.streaming;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.runners.spark.ReuseSparkContext;
+import org.apache.beam.runners.spark.SparkPipelineOptions;
+import org.apache.beam.runners.spark.SparkRunner;
+import org.apache.beam.runners.spark.io.CreateStream;
+import org.apache.beam.runners.spark.translation.Dataset;
+import org.apache.beam.runners.spark.translation.EvaluationContext;
+import org.apache.beam.runners.spark.translation.SparkContextFactory;
+import org.apache.beam.runners.spark.translation.TransformTranslator;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.spark.SparkStatusTracker;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.streaming.Duration;
+import org.apache.spark.streaming.api.java.JavaStreamingContext;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+
+/**
+ * A test suite that tests tracking of the streaming sources created an
+ * {@link org.apache.beam.runners.spark.translation.streaming.UnboundedDataset}.
+ */
+public class TrackStreamingSourcesTest {
+
+  @Rule
+  public ReuseSparkContext reuseContext = ReuseSparkContext.yes();
+
+  private static final transient SparkPipelineOptions options =
+      PipelineOptionsFactory.create().as(SparkPipelineOptions.class);
+
+  @Before
+  public void before() {
+    UnboundedDataset.resetQueuedStreamIds();
+    StreamingSourceTracker.numAssertions = 0;
+  }
+
+  @Test
+  public void testTrackSingle() {
+    options.setRunner(SparkRunner.class);
+    JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+    JavaStreamingContext jssc = new JavaStreamingContext(jsc,
+        new Duration(options.getBatchIntervalMillis()));
+
+    Pipeline p = Pipeline.create(options);
+
+    CreateStream.QueuedValues<Integer> queueStream =
+        CreateStream.fromQueue(Collections.<Iterable<Integer>>emptyList());
+
+    p.apply(queueStream).setCoder(VarIntCoder.of())
+        .apply(ParDo.of(new PassthroughFn<>()));
+
+    p.traverseTopologically(new StreamingSourceTracker(jssc, p, ParDo.Bound.class,  -1));
+    assertThat(StreamingSourceTracker.numAssertions, equalTo(1));
+  }
+
+  @Test
+  public void testTrackFlattened() {
+    options.setRunner(SparkRunner.class);
+    JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+    JavaStreamingContext jssc = new JavaStreamingContext(jsc,
+        new Duration(options.getBatchIntervalMillis()));
+
+    Pipeline p = Pipeline.create(options);
+
+    CreateStream.QueuedValues<Integer> queueStream1 =
+        CreateStream.fromQueue(Collections.<Iterable<Integer>>emptyList());
+    CreateStream.QueuedValues<Integer> queueStream2 =
+        CreateStream.fromQueue(Collections.<Iterable<Integer>>emptyList());
+
+    PCollection<Integer> pcol1 = p.apply(queueStream1).setCoder(VarIntCoder.of());
+    PCollection<Integer> pcol2 = p.apply(queueStream2).setCoder(VarIntCoder.of());
+    PCollection<Integer> flattened =
+        PCollectionList.of(pcol1).and(pcol2).apply(Flatten.<Integer>pCollections());
+    flattened.apply(ParDo.of(new PassthroughFn<>()));
+
+    p.traverseTopologically(new StreamingSourceTracker(jssc, p, ParDo.Bound.class, -1, -2));
+    assertThat(StreamingSourceTracker.numAssertions, equalTo(1));
+  }
+
+  private static class PassthroughFn<T> extends DoFn<T, T> {
+    @ProcessElement
+    public void processElement(ProcessContext c) {
+      c.output(c.element());
+    }
+  }
+
+  private static class StreamingSourceTracker extends Pipeline.PipelineVisitor.Defaults {
+    private final EvaluationContext ctxt;
+    private final SparkRunner.Evaluator evaluator;
+    private final Class<? extends PTransform> transformClassToAssert;
+    private final Integer[] expected;
+
+    private static int numAssertions = 0;
+
+    private StreamingSourceTracker(
+        JavaStreamingContext jssc,
+        Pipeline pipeline,
+        Class<? extends PTransform> transformClassToAssert,
+        Integer... expected) {
+      this.ctxt = new EvaluationContext(jssc.sparkContext(), pipeline, jssc);
+      this.evaluator = new SparkRunner.Evaluator(
+          new StreamingTransformTranslator.Translator(new TransformTranslator.Translator()), ctxt);
+      this.transformClassToAssert = transformClassToAssert;
+      this.expected = expected;
+    }
+
+    private void assertSourceIds(List<Integer> streamingSources) {
+      numAssertions++;
+      assertThat(streamingSources, containsInAnyOrder(expected));
+    }
+
+    @Override
+    public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
+      return evaluator.enterCompositeTransform(node);
+    }
+
+    @Override
+    public void visitPrimitiveTransform(TransformHierarchy.Node node) {
+      PTransform transform = node.getTransform();
+      if (transform.getClass() == transformClassToAssert) {
+        AppliedPTransform<?, ?, ?> appliedTransform = node.toAppliedPTransform();
+        ctxt.setCurrentTransform(appliedTransform);
+        //noinspection unchecked
+        Dataset dataset = ctxt.borrowDataset((PTransform<? extends PValue, ?>) transform);
+        assertSourceIds(((UnboundedDataset<?>) dataset).getStreamingSources());
+        ctxt.setCurrentTransform(null);
+      } else {
+        evaluator.visitPrimitiveTransform(node);
+      }
+    }
+  }
+
+}


[8/9] beam git commit: This relied on a wrong functionality as described in BEAM-1444 and should be revisited there.

Posted by am...@apache.org.
This relied on a wrong functionality as described in BEAM-1444 and should be revisited there.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3d25b9cc
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3d25b9cc
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3d25b9cc

Branch: refs/heads/master
Commit: 3d25b9cc794ce39a0ac3051eacf28127da138449
Parents: add8716
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 19:47:00 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:33 2017 +0200

----------------------------------------------------------------------
 .../beam/runners/spark/TestSparkRunner.java     | 11 +++++-----
 .../translation/streaming/UnboundedDataset.java |  1 -
 .../beam/runners/spark/WatermarkTest.java       | 19 +++++++++++++++--
 .../streaming/FlattenStreamingTest.java         | 22 --------------------
 .../streaming/TrackStreamingSourcesTest.java    | 18 +++++++++++++++-
 .../apache/beam/sdk/testing/RegexMatcher.java   | 17 +++++++++++++++
 6 files changed, 57 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index a634dd4..8b8f9ba 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -23,8 +23,8 @@ import static org.hamcrest.Matchers.is;
 
 import org.apache.beam.runners.core.UnboundedReadFromBoundedSource;
 import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
-import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
 import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult.State;
 import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
@@ -108,14 +108,15 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
 
   @Override
   public SparkPipelineResult run(Pipeline pipeline) {
+    // clear state of Aggregators, Metrics and Watermarks if exists.
+    AggregatorsAccumulator.clear();
+    SparkMetricsContainer.clear();
+    GlobalWatermarkHolder.clear();
+
     TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
     SparkPipelineResult result = delegate.run(pipeline);
     result.waitUntilFinish();
 
-    // clear state of Aggregators, Metrics and Watermarks.
-    AggregatorsAccumulator.clear();
-    SparkMetricsContainer.clear();
-    GlobalWatermarkHolder.clear();
 
     // make sure the test pipeline finished successfully.
     State resultState = result.getState();

http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
index 08d1ab6..8b65dca 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
@@ -20,7 +20,6 @@ package org.apache.beam.runners.spark.translation.streaming;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Queue;

http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
index 0b56403..e7a5481 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.beam.runners.spark;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -172,8 +189,6 @@ public class WatermarkTest {
 
     p.run().waitUntilFinish(Duration.millis(options.getBatchIntervalMillis()).multipliedBy(3));
 
-    System.out.println(WatermarksDoFn.strings);
-
     // this is a hacky way to assert but it will do until triggers are supported.
     assertThat(
         WatermarksDoFn.strings,

http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
index 530721b..fc40bbd 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
@@ -17,7 +17,6 @@
  */
 package org.apache.beam.runners.spark.translation.streaming;
 
-import com.google.common.collect.ImmutableList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -27,7 +26,6 @@ import org.apache.beam.runners.spark.translation.streaming.utils.PAssertStreamin
 import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
@@ -81,24 +79,4 @@ public class FlattenStreamingTest {
     PAssertStreaming.runAndAssertContents(p, union, EXPECTED_UNION, Duration.standardSeconds(1L));
   }
 
-  @Test
-  public void testFlattenBoundedUnbounded() throws Exception {
-    SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
-    options.setStreaming(true);
-
-    Pipeline p = Pipeline.create(options);
-    PCollection<String> w1 =
-        p.apply(CreateStream.fromQueue(WORDS_QUEUE_1)).setCoder(StringUtf8Coder.of());
-    PCollection<String> windowedW1 =
-        w1.apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(1))));
-    PCollection<String> w2 =
-        p.apply(Create.of(ImmutableList.copyOf(WORDS_ARRAY_2)).withCoder(StringUtf8Coder.of()));
-    PCollection<String> windowedW2 =
-        w2.apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(1))));
-    PCollectionList<String> list = PCollectionList.of(windowedW1).and(windowedW2);
-    PCollection<String> union = list.apply(Flatten.<String>pCollections());
-
-    PAssertStreaming.runAndAssertContents(p, union, EXPECTED_UNION, Duration.standardSeconds(1L));
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
index f102ac8..fbe5777 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.beam.runners.spark.translation.streaming;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -26,7 +43,6 @@ import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
 import org.apache.beam.sdk.values.PValue;
-import org.apache.spark.SparkStatusTracker;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.streaming.Duration;
 import org.apache.spark.streaming.api.java.JavaStreamingContext;

http://git-wip-us.apache.org/repos/asf/beam/blob/3d25b9cc/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
index a4b14fe..aab4a09 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.beam.sdk.testing;
 
 import java.util.regex.Pattern;


[9/9] beam git commit: This closes #1987

Posted by am...@apache.org.
This closes #1987


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/aa45ccb0
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/aa45ccb0
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/aa45ccb0

Branch: refs/heads/master
Commit: aa45ccb0800741ddf5ee7ecc7965c85c1914acf7
Parents: 92190ba 3d25b9c
Author: Sela <an...@paypal.com>
Authored: Mon Feb 20 11:40:26 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:40:26 2017 +0200

----------------------------------------------------------------------
 .../runners/dataflow/util/PackageUtilTest.java  |  30 +--
 .../apache/beam/runners/spark/SparkRunner.java  |   6 +-
 .../beam/runners/spark/TestSparkRunner.java     |  10 +-
 .../aggregators/AggregatorsAccumulator.java     |   2 +-
 .../runners/spark/io/SparkUnboundedSource.java  |  69 ++++--
 .../spark/stateful/StateSpecFunctions.java      |  31 ++-
 .../spark/translation/SparkContextFactory.java  |   2 +-
 .../streaming/StreamingTransformTranslator.java |  92 +++++---
 .../translation/streaming/UnboundedDataset.java |  34 ++-
 .../spark/util/GlobalWatermarkHolder.java       | 200 ++++++++++++++++
 .../beam/runners/spark/ClearWatermarksRule.java |  37 +++
 .../beam/runners/spark/ReuseSparkContext.java   |  46 ++++
 .../beam/runners/spark/WatermarkTest.java       | 227 +++++++++++++++++++
 .../streaming/FlattenStreamingTest.java         |  22 --
 .../streaming/TrackStreamingSourcesTest.java    | 168 ++++++++++++++
 .../apache/beam/sdk/testing/RegexMatcher.java   |  49 ++++
 16 files changed, 898 insertions(+), 127 deletions(-)
----------------------------------------------------------------------



[4/9] beam git commit: Ingest the input watermarks into the GlobalWatermarkHolder.

Posted by am...@apache.org.
Ingest the input watermarks into the GlobalWatermarkHolder.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/bbf3744d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/bbf3744d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/bbf3744d

Branch: refs/heads/master
Commit: bbf3744d4cc1a6a58712b4c54c421b0009c5bb5e
Parents: a620653
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:30:47 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:14 2017 +0200

----------------------------------------------------------------------
 .../runners/spark/io/SparkUnboundedSource.java  | 69 ++++++++++++++------
 .../spark/stateful/StateSpecFunctions.java      | 31 ++++++---
 2 files changed, 72 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/bbf3744d/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
index f03dc8c..354461f 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/SparkUnboundedSource.java
@@ -24,8 +24,13 @@ import org.apache.beam.runners.spark.SparkPipelineOptions;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.runners.spark.stateful.StateSpecFunctions;
 import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
+import org.apache.beam.runners.spark.translation.streaming.UnboundedDataset;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.spark.api.java.JavaRDD;
@@ -55,7 +60,7 @@ import scala.runtime.BoxedUnit;
  * <p>This read is a composite of the following steps:
  * <ul>
  * <li>Create a single-element (per-partition) stream, that contains the (partitioned)
- * {@link Source} and an optional {@link UnboundedSource.CheckpointMark} to start from.</li>
+ * {@link Source} and an optional {@link CheckpointMark} to start from.</li>
  * <li>Read from within a stateful operation {@link JavaPairInputDStream#mapWithState(StateSpec)}
  * using the {@link StateSpecFunctions#mapSourceFunction(SparkRuntimeContext)} mapping function,
  * which manages the state of the CheckpointMark per partition.</li>
@@ -65,10 +70,11 @@ import scala.runtime.BoxedUnit;
  */
 public class SparkUnboundedSource {
 
-  public static <T, CheckpointMarkT extends UnboundedSource.CheckpointMark>
-  JavaDStream<WindowedValue<T>> read(JavaStreamingContext jssc,
-                                     SparkRuntimeContext rc,
-                                     UnboundedSource<T, CheckpointMarkT> source) {
+  public static <T, CheckpointMarkT extends CheckpointMark> UnboundedDataset<T> read(
+      JavaStreamingContext jssc,
+      SparkRuntimeContext rc,
+      UnboundedSource<T, CheckpointMarkT> source) {
+
     SparkPipelineOptions options = rc.getPipelineOptions().as(SparkPipelineOptions.class);
     Long maxRecordsPerBatch = options.getMaxRecordsPerBatch();
     SourceDStream<T, CheckpointMarkT> sourceDStream = new SourceDStream<>(jssc.ssc(), source, rc);
@@ -82,7 +88,7 @@ public class SparkUnboundedSource {
                 JavaSparkContext$.MODULE$.<CheckpointMarkT>fakeClassTag());
 
     // call mapWithState to read from a checkpointable sources.
-    JavaMapWithStateDStream<Source<T>, CheckpointMarkT, byte[],
+    JavaMapWithStateDStream<Source<T>, CheckpointMarkT, Tuple2<byte[], Instant>,
         Tuple2<Iterable<byte[]>, Metadata>> mapWithStateDStream = inputDStream.mapWithState(
             StateSpec.function(StateSpecFunctions.<T, CheckpointMarkT>mapSourceFunction(rc)));
 
@@ -109,13 +115,14 @@ public class SparkUnboundedSource {
         WindowedValue.FullWindowedValueCoder.of(
             source.getDefaultOutputCoder(),
             GlobalWindow.Coder.INSTANCE);
-    return mapWithStateDStream.flatMap(
+    JavaDStream<WindowedValue<T>> readUnboundedStream = mapWithStateDStream.flatMap(
         new FlatMapFunction<Tuple2<Iterable<byte[]>, Metadata>, byte[]>() {
           @Override
           public Iterable<byte[]> call(Tuple2<Iterable<byte[]>, Metadata> t2) throws Exception {
             return t2._1();
           }
         }).map(CoderHelpers.fromByteFunction(coder));
+    return new UnboundedDataset<>(readUnboundedStream, Collections.singletonList(id));
   }
 
   private static <T> String getSourceName(Source<T> source, int id) {
@@ -173,30 +180,46 @@ public class SparkUnboundedSource {
       // compute parent.
       scala.Option<RDD<Metadata>> parentRDDOpt = parent.getOrCompute(validTime);
       long count = 0;
-      Instant globalWatermark = new Instant(Long.MIN_VALUE);
+      SparkWatermarks sparkWatermark = null;
+      Instant globalLowWatermarkForBatch = BoundedWindow.TIMESTAMP_MIN_VALUE;
+      Instant globalHighWatermarkForBatch = BoundedWindow.TIMESTAMP_MIN_VALUE;
       if (parentRDDOpt.isDefined()) {
         JavaRDD<Metadata> parentRDD = parentRDDOpt.get().toJavaRDD();
         for (Metadata metadata: parentRDD.collect()) {
           count += metadata.getNumRecords();
-          // a monotonically increasing watermark.
-          globalWatermark = globalWatermark.isBefore(metadata.getWatermark())
-              ? metadata.getWatermark() : globalWatermark;
+          // compute the global input watermark - advance to latest of all partitions.
+          Instant partitionLowWatermark = metadata.getLowWatermark();
+          globalLowWatermarkForBatch =
+              globalLowWatermarkForBatch.isBefore(partitionLowWatermark)
+                  ? partitionLowWatermark : globalLowWatermarkForBatch;
+          Instant partitionHighWatermark = metadata.getHighWatermark();
+          globalHighWatermarkForBatch =
+              globalHighWatermarkForBatch.isBefore(partitionHighWatermark)
+                  ? partitionHighWatermark : globalHighWatermarkForBatch;
         }
+
+        sparkWatermark =
+            new SparkWatermarks(
+                globalLowWatermarkForBatch,
+                globalHighWatermarkForBatch,
+                new Instant(validTime.milliseconds()));
+        // add to watermark queue.
+        GlobalWatermarkHolder.add(inputDStreamId, sparkWatermark);
       }
       // report - for RateEstimator and visibility.
-      report(validTime, count, globalWatermark);
+      report(validTime, count, sparkWatermark);
       return scala.Option.empty();
     }
 
-    private void report(Time batchTime, long count, Instant watermark) {
+    private void report(Time batchTime, long count, SparkWatermarks sparkWatermark) {
       // metadata - #records read and a description.
       scala.collection.immutable.Map<String, Object> metadata =
           new scala.collection.immutable.Map.Map1<String, Object>(
               StreamInputInfo.METADATA_KEY_DESCRIPTION(),
               String.format(
-                  "Read %d records with observed watermark %s, from %s for batch time: %s",
+                  "Read %d records with observed watermarks %s, from %s for batch time: %s",
                   count,
-                  watermark,
+                  sparkWatermark == null ? "N/A" : sparkWatermark,
                   sourceName,
                   batchTime));
       StreamInputInfo streamInputInfo = new StreamInputInfo(inputDStreamId, count, metadata);
@@ -209,19 +232,25 @@ public class SparkUnboundedSource {
    */
   public static class Metadata implements Serializable {
     private final long numRecords;
-    private final Instant watermark;
+    private final Instant lowWatermark;
+    private final Instant highWatermark;
 
-    public Metadata(long numRecords, Instant watermark) {
+    public Metadata(long numRecords, Instant lowWatermark, Instant highWatermark) {
       this.numRecords = numRecords;
-      this.watermark = watermark;
+      this.lowWatermark = lowWatermark;
+      this.highWatermark = highWatermark;
     }
 
     public long getNumRecords() {
       return numRecords;
     }
 
-    public Instant getWatermark() {
-      return watermark;
+    public Instant getLowWatermark() {
+      return lowWatermark;
+    }
+
+    public Instant getHighWatermark() {
+      return highWatermark;
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/bbf3744d/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
index ffe0ddd..ae5a746 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/StateSpecFunctions.java
@@ -35,6 +35,7 @@ import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
@@ -94,29 +95,36 @@ public class StateSpecFunctions {
    * @return The appropriate {@link org.apache.spark.streaming.StateSpec} function.
    */
   public static <T, CheckpointMarkT extends UnboundedSource.CheckpointMark>
-  scala.Function3<Source<T>, scala.Option<CheckpointMarkT>, /* CheckpointMarkT */State<byte[]>,
+  scala.Function3<Source<T>, scala.Option<CheckpointMarkT>, State<Tuple2<byte[], Instant>>,
       Tuple2<Iterable<byte[]>, Metadata>> mapSourceFunction(
            final SparkRuntimeContext runtimeContext) {
 
-    return new SerializableFunction3<Source<T>, Option<CheckpointMarkT>, State<byte[]>,
-        Tuple2<Iterable<byte[]>, Metadata>>() {
+    return new SerializableFunction3<Source<T>, Option<CheckpointMarkT>,
+        State<Tuple2<byte[], Instant>>, Tuple2<Iterable<byte[]>, Metadata>>() {
 
       @Override
       public Tuple2<Iterable<byte[]>, Metadata> apply(
           Source<T> source,
           scala.Option<CheckpointMarkT> startCheckpointMark,
-          State<byte[]> state) {
+          State<Tuple2<byte[], Instant>> state) {
+
         // source as MicrobatchSource
         MicrobatchSource<T, CheckpointMarkT> microbatchSource =
             (MicrobatchSource<T, CheckpointMarkT>) source;
 
+        // Initial high/low watermarks.
+        Instant lowWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE;
+        Instant highWatermark;
+
         // if state exists, use it, otherwise it's first time so use the startCheckpointMark.
         // startCheckpointMark may be EmptyCheckpointMark (the Spark Java API tries to apply
         // Optional(null)), which is handled by the UnboundedSource implementation.
         Coder<CheckpointMarkT> checkpointCoder = microbatchSource.getCheckpointMarkCoder();
         CheckpointMarkT checkpointMark;
         if (state.exists()) {
-          checkpointMark = CoderHelpers.fromByteArray(state.get(), checkpointCoder);
+          // previous (output) watermark is now the low watermark.
+          lowWatermark = state.get()._2();
+          checkpointMark = CoderHelpers.fromByteArray(state.get()._1(), checkpointCoder);
           LOG.info("Continue reading from an existing CheckpointMark.");
         } else if (startCheckpointMark.isDefined()
             && !startCheckpointMark.get().equals(EmptyCheckpointMark.get())) {
@@ -154,7 +162,10 @@ public class StateSpecFunctions {
             finished = !reader.advance();
           }
 
-          watermark = ((MicrobatchSource.Reader) reader).getWatermark();
+          // end-of-read watermark is the high watermark, but don't allow decrease.
+          Instant sourceWatermark = ((MicrobatchSource.Reader) reader).getWatermark();
+          highWatermark = sourceWatermark.isAfter(lowWatermark) ? sourceWatermark : lowWatermark;
+
           // close and checkpoint reader.
           reader.close();
           LOG.info("Source id {} spent {} msec on reading.", microbatchSource.getId(),
@@ -164,11 +175,15 @@ public class StateSpecFunctions {
           @SuppressWarnings("unchecked")
           CheckpointMarkT finishedReadCheckpointMark =
               (CheckpointMarkT) ((MicrobatchSource.Reader) reader).getCheckpointMark();
+          byte[] codedCheckpoint = new byte[0];
           if (finishedReadCheckpointMark != null) {
-            state.update(CoderHelpers.toByteArray(finishedReadCheckpointMark, checkpointCoder));
+            codedCheckpoint = CoderHelpers.toByteArray(finishedReadCheckpointMark, checkpointCoder);
           } else {
             LOG.info("Skipping checkpoint marking because the reader failed to supply one.");
           }
+          // persist the end-of-read (high) watermark for following read, where it will become
+          // the next low watermark.
+          state.update(new Tuple2<>(codedCheckpoint, highWatermark));
         } catch (IOException e) {
           throw new RuntimeException("Failed to read from reader.", e);
         }
@@ -179,7 +194,7 @@ public class StateSpecFunctions {
             return Iterators.unmodifiableIterator(readValues.iterator());
           }
         };
-        return new Tuple2<>(iterable, new Metadata(readValues.size(), watermark));
+        return new Tuple2<>(iterable, new Metadata(readValues.size(), lowWatermark, highWatermark));
       }
     };
   }


[5/9] beam git commit: Watermark tests.

Posted by am...@apache.org.
Watermark tests.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9784f204
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9784f204
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9784f204

Branch: refs/heads/master
Commit: 9784f204793ab8e8ec3ec84e3c7c8d2ca4ddaf6a
Parents: c18f8a2
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:34:30 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:15 2017 +0200

----------------------------------------------------------------------
 .../beam/runners/spark/ClearWatermarksRule.java |  37 ++++
 .../beam/runners/spark/ReuseSparkContext.java   |  46 ++++
 .../beam/runners/spark/WatermarkTest.java       | 212 +++++++++++++++++++
 3 files changed, 295 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/9784f204/runners/spark/src/test/java/org/apache/beam/runners/spark/ClearWatermarksRule.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ClearWatermarksRule.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ClearWatermarksRule.java
new file mode 100644
index 0000000..4c0c99a
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ClearWatermarksRule.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark;
+
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
+import org.junit.rules.ExternalResource;
+
+/**
+ * A rule that clears the {@link GlobalWatermarkHolder}.
+ */
+public class ClearWatermarksRule extends ExternalResource {
+
+  @Override
+  protected void before() throws Throwable {
+    clearWatermarks();
+  }
+
+  public void clearWatermarks() {
+    GlobalWatermarkHolder.clear();
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/9784f204/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java
new file mode 100644
index 0000000..027f9fd
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark;
+
+import org.apache.beam.runners.spark.translation.SparkContextFactory;
+import org.junit.rules.ExternalResource;
+
+/**
+ * Explicitly set {@link org.apache.spark.SparkContext} to be reused (or not) in tests.
+ */
+public class ReuseSparkContext extends ExternalResource {
+
+  private final boolean reuse;
+
+  private ReuseSparkContext(boolean reuse) {
+    this.reuse = reuse;
+  }
+
+  public static ReuseSparkContext no() {
+    return new ReuseSparkContext(false);
+  }
+
+  public static ReuseSparkContext yes() {
+    return new ReuseSparkContext(true);
+  }
+
+  @Override
+  protected void before() throws Throwable {
+    System.setProperty(SparkContextFactory.TEST_REUSE_SPARK_CONTEXT, Boolean.toString(reuse));
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/9784f204/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
new file mode 100644
index 0000000..0b56403
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
@@ -0,0 +1,212 @@
+package org.apache.beam.runners.spark;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.runners.spark.io.CreateStream;
+import org.apache.beam.runners.spark.translation.SparkContextFactory;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.RegexMatcher;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+
+/**
+ * A test suite for the propagation of watermarks in the Spark runner.
+ */
+public class WatermarkTest {
+
+  @Rule
+  public ClearWatermarksRule clearWatermarksRule = new ClearWatermarksRule();
+
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
+
+  @Rule
+  public ReuseSparkContext reuseContext = ReuseSparkContext.yes();
+
+  private static final SparkPipelineOptions options =
+      PipelineOptionsFactory.create().as(SparkPipelineOptions.class);
+
+  private static final String INSTANT_PATTERN =
+      "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z";
+
+  @Test
+  public void testLowHighWatermarksAdvance() {
+    JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+
+    Instant instant = new Instant(0);
+    // low == high.
+    GlobalWatermarkHolder.add(1,
+        new SparkWatermarks(
+            instant.plus(Duration.millis(5)),
+            instant.plus(Duration.millis(5)),
+            instant));
+    GlobalWatermarkHolder.advance(jsc);
+    // low < high.
+    GlobalWatermarkHolder.add(1,
+        new SparkWatermarks(
+            instant.plus(Duration.millis(10)),
+            instant.plus(Duration.millis(15)),
+            instant.plus(Duration.millis(100))));
+    GlobalWatermarkHolder.advance(jsc);
+
+    // assert watermarks in Broadcast.
+    SparkWatermarks currentWatermarks = GlobalWatermarkHolder.get().getValue().get(1);
+    assertThat(currentWatermarks.getLowWatermark(), equalTo(instant.plus(Duration.millis(10))));
+    assertThat(currentWatermarks.getHighWatermark(), equalTo(instant.plus(Duration.millis(15))));
+    assertThat(currentWatermarks.getSynchronizedProcessingTime(),
+        equalTo(instant.plus(Duration.millis(100))));
+
+    // assert illegal watermark advance.
+    thrown.expect(IllegalStateException.class);
+    thrown.expectMessage(
+        RegexMatcher.matches(
+            "Low watermark " + INSTANT_PATTERN + " cannot be later then high watermark "
+            + INSTANT_PATTERN));
+    // low > high -> not allowed!
+    GlobalWatermarkHolder.add(1,
+        new SparkWatermarks(
+            instant.plus(Duration.millis(25)),
+            instant.plus(Duration.millis(20)),
+            instant.plus(Duration.millis(200))));
+    GlobalWatermarkHolder.advance(jsc);
+  }
+
+  @Test
+  public void testSynchronizedTimeMonotonic() {
+    JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+
+    Instant instant = new Instant(0);
+    GlobalWatermarkHolder.add(1,
+        new SparkWatermarks(
+            instant.plus(Duration.millis(5)),
+            instant.plus(Duration.millis(10)),
+            instant));
+    GlobalWatermarkHolder.advance(jsc);
+
+    thrown.expect(IllegalStateException.class);
+    thrown.expectMessage("Synchronized processing time must advance.");
+    // no actual advancement of watermarks - fine by Watermarks
+    // but not by synchronized processing time.
+    GlobalWatermarkHolder.add(1,
+        new SparkWatermarks(
+            instant.plus(Duration.millis(5)),
+            instant.plus(Duration.millis(10)),
+            instant));
+    GlobalWatermarkHolder.advance(jsc);
+  }
+
+  @Test
+  public void testMultiSource() {
+    JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+
+    Instant instant = new Instant(0);
+    GlobalWatermarkHolder.add(1,
+        new SparkWatermarks(
+            instant.plus(Duration.millis(5)),
+            instant.plus(Duration.millis(10)),
+            instant));
+    GlobalWatermarkHolder.add(2,
+        new SparkWatermarks(
+            instant.plus(Duration.millis(3)),
+            instant.plus(Duration.millis(6)),
+            instant));
+
+    GlobalWatermarkHolder.advance(jsc);
+
+    // assert watermarks for source 1.
+    SparkWatermarks watermarksForSource1 = GlobalWatermarkHolder.get().getValue().get(1);
+    assertThat(watermarksForSource1.getLowWatermark(), equalTo(instant.plus(Duration.millis(5))));
+    assertThat(watermarksForSource1.getHighWatermark(), equalTo(instant.plus(Duration.millis(10))));
+
+    // assert watermarks for source 2.
+    SparkWatermarks watermarksForSource2 = GlobalWatermarkHolder.get().getValue().get(2);
+    assertThat(watermarksForSource2.getLowWatermark(), equalTo(instant.plus(Duration.millis(3))));
+    assertThat(watermarksForSource2.getHighWatermark(), equalTo(instant.plus(Duration.millis(6))));
+  }
+
+  @Test
+  public void testInDoFn() {
+    // because watermark advances onBatchCompleted.
+    Iterable<Integer> zeroBatch = Collections.emptyList();
+    Iterable<Integer> firstBatch = Collections.singletonList(1);
+    Iterable<Integer> secondBatch = Collections.singletonList(2);
+
+    Instant instant = new Instant(0);
+    GlobalWatermarkHolder.add(1,
+        new SparkWatermarks(
+            instant.plus(Duration.millis(5)),
+            instant.plus(Duration.millis(10)),
+            instant));
+    GlobalWatermarkHolder.add(1,
+        new SparkWatermarks(
+            instant.plus(Duration.millis(10)),
+            instant.plus(Duration.millis(15)),
+            instant.plus(options.getBatchIntervalMillis())));
+
+    options.setRunner(SparkRunner.class);
+    options.setStreaming(true);
+    options.setBatchIntervalMillis(500L);
+    Pipeline p = Pipeline.create(options);
+
+    CreateStream.QueuedValues<Integer> queueStream =
+        CreateStream.fromQueue(Arrays.asList(zeroBatch, firstBatch, secondBatch));
+
+    p.apply(queueStream).setCoder(VarIntCoder.of()).apply(ParDo.of(new WatermarksDoFn(1)));
+
+    p.run().waitUntilFinish(Duration.millis(options.getBatchIntervalMillis()).multipliedBy(3));
+
+    System.out.println(WatermarksDoFn.strings);
+
+    // this is a hacky way to assert but it will do until triggers are supported.
+    assertThat(
+        WatermarksDoFn.strings,
+        containsInAnyOrder(
+            "element: 1 lowWatermark: 5 highWatermark: 10 processingTime: 0",
+            "element: 2 lowWatermark: 10 highWatermark: 15 processingTime: 1000"));
+  }
+
+  private static class WatermarksDoFn extends DoFn<Integer, String> {
+    private final int sourceId;
+
+    static List<String> strings = new ArrayList<>();
+
+    private WatermarksDoFn(int sourceId) {
+      this.sourceId = sourceId;
+    }
+
+    @ProcessElement
+    public void processElement(ProcessContext c) {
+      if (GlobalWatermarkHolder.get() == null
+          || GlobalWatermarkHolder.get().getValue().get(sourceId) == null) {
+        // watermark not yet updated.
+        return;
+      }
+      SparkWatermarks sparkWatermarks = GlobalWatermarkHolder.get().getValue().get(sourceId);
+      Integer element = c.element();
+      String output =
+          "element: " + element
+          + " lowWatermark: " + sparkWatermarks.getLowWatermark().getMillis()
+          + " highWatermark: " + sparkWatermarks.getHighWatermark().getMillis()
+          + " processingTime: " + sparkWatermarks.getSynchronizedProcessingTime().getMillis();
+      strings.add(output);
+    }
+  }
+
+}


[2/9] beam git commit: Advance watermarks onBatchCompleted hook.

Posted by am...@apache.org.
Advance watermarks onBatchCompleted hook.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fa31f18e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fa31f18e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fa31f18e

Branch: refs/heads/master
Commit: fa31f18e489d4cbe44fe4a9be7ba3d7dbee7c354
Parents: bbf3744
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:31:14 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:14 2017 +0200

----------------------------------------------------------------------
 .../main/java/org/apache/beam/runners/spark/SparkRunner.java   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/fa31f18e/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index ebac375..52a080b 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -39,6 +39,7 @@ import org.apache.beam.runners.spark.translation.TransformEvaluator;
 import org.apache.beam.runners.spark.translation.TransformTranslator;
 import org.apache.beam.runners.spark.translation.streaming.Checkpoint.CheckpointDir;
 import org.apache.beam.runners.spark.translation.streaming.SparkRunnerStreamingContextFactory;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.WatermarksListener;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
@@ -191,12 +192,15 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
           new JavaStreamingListenerWrapper(
               new MetricsAccumulator.AccumulatorCheckpointingSparkListener()));
 
-      // register listeners.
+      // register user-defined listeners.
       for (JavaStreamingListener listener: mOptions.as(SparkContextOptions.class).getListeners()) {
         LOG.info("Registered listener {}." + listener.getClass().getSimpleName());
         jssc.addStreamingListener(new JavaStreamingListenerWrapper(listener));
       }
 
+      // register Watermarks listener to broadcast the advanced WMs.
+      jssc.addStreamingListener(new JavaStreamingListenerWrapper(new WatermarksListener(jssc)));
+
       startPipeline = executorService.submit(new Runnable() {
 
         @Override


[3/9] beam git commit: Handle QueuedStream (for testing) and track sources upstream.

Posted by am...@apache.org.
Handle QueuedStream (for testing) and track sources upstream.

Refactor according to changes.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/705695eb
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/705695eb
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/705695eb

Branch: refs/heads/master
Commit: 705695eb726acf086915e610cb2304bd968e3682
Parents: fa31f18
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:32:06 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:14 2017 +0200

----------------------------------------------------------------------
 .../beam/runners/spark/TestSparkRunner.java     | 11 ++-
 .../aggregators/AggregatorsAccumulator.java     |  2 +-
 .../spark/translation/SparkContextFactory.java  |  2 +-
 .../streaming/StreamingTransformTranslator.java | 92 +++++++++++++-------
 .../translation/streaming/UnboundedDataset.java | 29 ++++--
 5 files changed, 88 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/705695eb/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index e770164..a634dd4 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -22,6 +22,8 @@ import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
 
 import org.apache.beam.runners.core.UnboundedReadFromBoundedSource;
+import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
 import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult.State;
@@ -107,13 +109,14 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
   @Override
   public SparkPipelineResult run(Pipeline pipeline) {
     TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
-
-    // clear metrics singleton
-    SparkMetricsContainer.clear();
-
     SparkPipelineResult result = delegate.run(pipeline);
     result.waitUntilFinish();
 
+    // clear state of Aggregators, Metrics and Watermarks.
+    AggregatorsAccumulator.clear();
+    SparkMetricsContainer.clear();
+    GlobalWatermarkHolder.clear();
+
     // make sure the test pipeline finished successfully.
     State resultState = result.getState();
     assertThat(

http://git-wip-us.apache.org/repos/asf/beam/blob/705695eb/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
index 1b49e91..a4dfda6 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
@@ -89,7 +89,7 @@ public class AggregatorsAccumulator {
   }
 
   @VisibleForTesting
-  static void clear() {
+  public static void clear() {
     synchronized (AggregatorsAccumulator.class) {
       instance = null;
     }

http://git-wip-us.apache.org/repos/asf/beam/blob/705695eb/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
index 326838a..cdeddad 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
@@ -38,7 +38,7 @@ public final class SparkContextFactory {
    * {@code true} then the Spark context will be reused for beam pipelines.
    * This property should only be enabled for tests.
    */
-  static final String TEST_REUSE_SPARK_CONTEXT = "beam.spark.test.reuseSparkContext";
+  public static final String TEST_REUSE_SPARK_CONTEXT = "beam.spark.test.reuseSparkContext";
 
   // Spark allows only one context for JVM so this can be static.
   private static JavaSparkContext sparkContext;

http://git-wip-us.apache.org/repos/asf/beam/blob/705695eb/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index a49b959..9451df7 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -105,9 +105,12 @@ final class StreamingTransformTranslator {
     return new TransformEvaluator<Read.Unbounded<T>>() {
       @Override
       public void evaluate(Read.Unbounded<T> transform, EvaluationContext context) {
-        context.putDataset(transform,
-            new UnboundedDataset<>(SparkUnboundedSource.read(context.getStreamingContext(),
-                context.getRuntimeContext(), transform.getSource())));
+        context.putDataset(
+            transform,
+            SparkUnboundedSource.read(
+                context.getStreamingContext(),
+                context.getRuntimeContext(),
+                transform.getSource()));
       }
     };
   }
@@ -134,6 +137,7 @@ final class StreamingTransformTranslator {
         // So we could end up with an unbounded unified DStream.
         final List<JavaRDD<WindowedValue<T>>> rdds = new ArrayList<>();
         final List<JavaDStream<WindowedValue<T>>> dStreams = new ArrayList<>();
+        final List<Integer> streamingSources = new ArrayList<>();
         for (TaggedPValue pv : pcs) {
           checkArgument(
               pv.getValue() instanceof PCollection,
@@ -143,7 +147,9 @@ final class StreamingTransformTranslator {
           PCollection<T> pcol = (PCollection<T>) pv.getValue();
           Dataset dataset = context.borrowDataset(pcol);
           if (dataset instanceof UnboundedDataset) {
-            dStreams.add(((UnboundedDataset<T>) dataset).getDStream());
+            UnboundedDataset<T> unboundedDataset = (UnboundedDataset<T>) dataset;
+            streamingSources.addAll(unboundedDataset.getStreamingSources());
+            dStreams.add(unboundedDataset.getDStream());
           } else {
             rdds.add(((BoundedDataset<T>) dataset).getRDD());
           }
@@ -162,9 +168,9 @@ final class StreamingTransformTranslator {
                       return new JavaSparkContext(streamRdd.context()).union(streamRdd, rdds);
                     }
                   });
-          context.putDataset(transform, new UnboundedDataset<>(joined));
+          context.putDataset(transform, new UnboundedDataset<>(joined, streamingSources));
         } else {
-          context.putDataset(transform, new UnboundedDataset<>(unifiedStreams));
+          context.putDataset(transform, new UnboundedDataset<>(unifiedStreams, streamingSources));
         }
       }
     };
@@ -177,8 +183,9 @@ final class StreamingTransformTranslator {
         @SuppressWarnings("unchecked")
         WindowFn<? super T, W> windowFn = (WindowFn<? super T, W>) transform.getWindowFn();
         @SuppressWarnings("unchecked")
-        JavaDStream<WindowedValue<T>> dStream =
-            ((UnboundedDataset<T>) context.borrowDataset(transform)).getDStream();
+        UnboundedDataset<T> unboundedDataset =
+            ((UnboundedDataset<T>) context.borrowDataset(transform));
+        JavaDStream<WindowedValue<T>> dStream = unboundedDataset.getDStream();
         // get the right window durations.
         Duration windowDuration;
         Duration slideDuration;
@@ -197,7 +204,8 @@ final class StreamingTransformTranslator {
             dStream.window(windowDuration, slideDuration);
         //--- then we apply windowing to the elements
         if (TranslationUtils.skipAssignWindows(transform, context)) {
-          context.putDataset(transform, new UnboundedDataset<>(windowedDStream));
+          context.putDataset(transform,
+              new UnboundedDataset<>(windowedDStream, unboundedDataset.getStreamingSources()));
         } else {
           JavaDStream<WindowedValue<T>> outStream = windowedDStream.transform(
               new Function<JavaRDD<WindowedValue<T>>, JavaRDD<WindowedValue<T>>>() {
@@ -206,7 +214,8 @@ final class StreamingTransformTranslator {
               return rdd.map(new SparkAssignWindowFn<>(transform.getWindowFn()));
             }
           });
-          context.putDataset(transform, new UnboundedDataset<>(outStream));
+          context.putDataset(transform,
+              new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
         }
       }
     };
@@ -217,8 +226,9 @@ final class StreamingTransformTranslator {
       @Override
       public void evaluate(GroupByKey<K, V> transform, EvaluationContext context) {
         @SuppressWarnings("unchecked")
-        JavaDStream<WindowedValue<KV<K, V>>> dStream =
-            ((UnboundedDataset<KV<K, V>>) context.borrowDataset(transform)).getDStream();
+        UnboundedDataset<KV<K, V>> unboundedDataset =
+            ((UnboundedDataset<KV<K, V>>) context.borrowDataset(transform));
+        JavaDStream<WindowedValue<KV<K, V>>> dStream = unboundedDataset.getDStream();
 
         @SuppressWarnings("unchecked")
         final KvCoder<K, V> coder = (KvCoder<K, V>) context.getInput(transform).getCoder();
@@ -239,7 +249,8 @@ final class StreamingTransformTranslator {
                 windowingStrategy);
           }
         });
-        context.putDataset(transform, new UnboundedDataset<>(outStream));
+        context.putDataset(transform,
+            new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
       }
     };
   }
@@ -247,7 +258,6 @@ final class StreamingTransformTranslator {
   private static <K, InputT, OutputT> TransformEvaluator<Combine.GroupedValues<K, InputT, OutputT>>
   combineGrouped() {
     return new TransformEvaluator<Combine.GroupedValues<K, InputT, OutputT>>() {
-      @SuppressWarnings("unchecked")
       @Override
       public void evaluate(final Combine.GroupedValues<K, InputT, OutputT> transform,
                            EvaluationContext context) {
@@ -255,13 +265,15 @@ final class StreamingTransformTranslator {
         PCollection<? extends KV<K, ? extends Iterable<InputT>>> input =
             context.getInput(transform);
         final WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
+        @SuppressWarnings("unchecked")
         final CombineWithContext.KeyedCombineFnWithContext<K, InputT, ?, OutputT> fn =
             (CombineWithContext.KeyedCombineFnWithContext<K, InputT, ?, OutputT>)
                 CombineFnUtil.toFnWithContext(transform.getFn());
 
-        JavaDStream<WindowedValue<KV<K, Iterable<InputT>>>> dStream =
-            ((UnboundedDataset<KV<K, Iterable<InputT>>>) context.borrowDataset(transform))
-                .getDStream();
+        @SuppressWarnings("unchecked")
+        UnboundedDataset<KV<K, Iterable<InputT>>> unboundedDataset =
+            ((UnboundedDataset<KV<K, Iterable<InputT>>>) context.borrowDataset(transform));
+        JavaDStream<WindowedValue<KV<K, Iterable<InputT>>>> dStream = unboundedDataset.getDStream();
 
         final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
         final SparkPCollectionView pviews = context.getPViews();
@@ -283,7 +295,8 @@ final class StreamingTransformTranslator {
                   }
                 });
 
-        context.putDataset(transform, new UnboundedDataset<>(outStream));
+        context.putDataset(transform,
+            new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
       }
     };
   }
@@ -292,7 +305,6 @@ final class StreamingTransformTranslator {
   combineGlobally() {
     return new TransformEvaluator<Combine.Globally<InputT, OutputT>>() {
 
-      @SuppressWarnings("unchecked")
       @Override
       public void evaluate(
           final Combine.Globally<InputT, OutputT> transform,
@@ -301,6 +313,7 @@ final class StreamingTransformTranslator {
         // serializable arguments to pass.
         final Coder<InputT> iCoder = context.getInput(transform).getCoder();
         final Coder<OutputT> oCoder = context.getOutput(transform).getCoder();
+        @SuppressWarnings("unchecked")
         final CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT> combineFn =
             (CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT>)
                 CombineFnUtil.toFnWithContext(transform.getFn());
@@ -309,8 +322,10 @@ final class StreamingTransformTranslator {
         final boolean hasDefault = transform.isInsertDefault();
         final SparkPCollectionView pviews = context.getPViews();
 
-        JavaDStream<WindowedValue<InputT>> dStream =
-            ((UnboundedDataset<InputT>) context.borrowDataset(transform)).getDStream();
+        @SuppressWarnings("unchecked")
+        UnboundedDataset<InputT> unboundedDataset =
+            ((UnboundedDataset<InputT>) context.borrowDataset(transform));
+        JavaDStream<WindowedValue<InputT>> dStream = unboundedDataset.getDStream();
 
         JavaDStream<WindowedValue<OutputT>> outStream = dStream.transform(
             new Function<JavaRDD<WindowedValue<InputT>>, JavaRDD<WindowedValue<OutputT>>>() {
@@ -326,7 +341,8 @@ final class StreamingTransformTranslator {
           }
         });
 
-        context.putDataset(transform, new UnboundedDataset<>(outStream));
+        context.putDataset(transform,
+            new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
       }
     };
   }
@@ -334,7 +350,6 @@ final class StreamingTransformTranslator {
   private static <K, InputT, AccumT, OutputT>
   TransformEvaluator<Combine.PerKey<K, InputT, OutputT>> combinePerKey() {
     return new TransformEvaluator<Combine.PerKey<K, InputT, OutputT>>() {
-      @SuppressWarnings("unchecked")
       @Override
       public void evaluate(final Combine.PerKey<K, InputT, OutputT> transform,
                            final EvaluationContext context) {
@@ -342,6 +357,7 @@ final class StreamingTransformTranslator {
         // serializable arguments to pass.
         final KvCoder<K, InputT> inputCoder =
             (KvCoder<K, InputT>) context.getInput(transform).getCoder();
+        @SuppressWarnings("unchecked")
         final CombineWithContext.KeyedCombineFnWithContext<K, InputT, AccumT, OutputT> combineFn =
             (CombineWithContext.KeyedCombineFnWithContext<K, InputT, AccumT, OutputT>)
                 CombineFnUtil.toFnWithContext(transform.getFn());
@@ -349,8 +365,10 @@ final class StreamingTransformTranslator {
         final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
         final SparkPCollectionView pviews = context.getPViews();
 
-        JavaDStream<WindowedValue<KV<K, InputT>>> dStream =
-            ((UnboundedDataset<KV<K, InputT>>) context.borrowDataset(transform)).getDStream();
+        @SuppressWarnings("unchecked")
+        UnboundedDataset<KV<K, InputT>> unboundedDataset =
+            ((UnboundedDataset<KV<K, InputT>>) context.borrowDataset(transform));
+        JavaDStream<WindowedValue<KV<K, InputT>>> dStream = unboundedDataset.getDStream();
 
         JavaDStream<WindowedValue<KV<K, OutputT>>> outStream =
             dStream.transform(new Function<JavaRDD<WindowedValue<KV<K, InputT>>>,
@@ -366,14 +384,14 @@ final class StreamingTransformTranslator {
                 windowingStrategy, sideInputs);
           }
         });
-        context.putDataset(transform, new UnboundedDataset<>(outStream));
+        context.putDataset(transform,
+            new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
       }
     };
   }
 
   private static <InputT, OutputT> TransformEvaluator<ParDo.Bound<InputT, OutputT>> parDo() {
     return new TransformEvaluator<ParDo.Bound<InputT, OutputT>>() {
-      @SuppressWarnings("unchecked")
       @Override
       public void evaluate(final ParDo.Bound<InputT, OutputT> transform,
                            final EvaluationContext context) {
@@ -384,8 +402,11 @@ final class StreamingTransformTranslator {
         final WindowingStrategy<?, ?> windowingStrategy =
             context.getInput(transform).getWindowingStrategy();
         final SparkPCollectionView pviews = context.getPViews();
-        JavaDStream<WindowedValue<InputT>> dStream =
-            ((UnboundedDataset<InputT>) context.borrowDataset(transform)).getDStream();
+
+        @SuppressWarnings("unchecked")
+        UnboundedDataset<InputT> unboundedDataset =
+            ((UnboundedDataset<InputT>) context.borrowDataset(transform));
+        JavaDStream<WindowedValue<InputT>> dStream = unboundedDataset.getDStream();
 
         final String stepName = context.getCurrentTransform().getFullName();
 
@@ -409,7 +430,8 @@ final class StreamingTransformTranslator {
           }
         });
 
-        context.putDataset(transform, new UnboundedDataset<>(outStream));
+        context.putDataset(transform,
+            new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
       }
     };
   }
@@ -427,9 +449,12 @@ final class StreamingTransformTranslator {
         final SparkPCollectionView pviews = context.getPViews();
         final WindowingStrategy<?, ?> windowingStrategy =
             context.getInput(transform).getWindowingStrategy();
+
         @SuppressWarnings("unchecked")
-        JavaDStream<WindowedValue<InputT>> dStream =
-            ((UnboundedDataset<InputT>) context.borrowDataset(transform)).getDStream();
+        UnboundedDataset<InputT> unboundedDataset =
+            ((UnboundedDataset<InputT>) context.borrowDataset(transform));
+        JavaDStream<WindowedValue<InputT>> dStream = unboundedDataset.getDStream();
+
         JavaPairDStream<TupleTag<?>, WindowedValue<?>> all = dStream.transformToPair(
             new Function<JavaRDD<WindowedValue<InputT>>,
                 JavaPairRDD<TupleTag<?>, WindowedValue<?>>>() {
@@ -460,7 +485,8 @@ final class StreamingTransformTranslator {
           JavaDStream<WindowedValue<Object>> values =
               (JavaDStream<WindowedValue<Object>>)
                   (JavaDStream<?>) TranslationUtils.dStreamValues(filtered);
-          context.putDataset(e.getValue(), new UnboundedDataset<>(values));
+          context.putDataset(e.getValue(),
+              new UnboundedDataset<>(values, unboundedDataset.getStreamingSources()));
         }
       }
     };

http://git-wip-us.apache.org/repos/asf/beam/blob/705695eb/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
index d059c7e..80c0515 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
@@ -19,8 +19,12 @@
 package org.apache.beam.runners.spark.translation.streaming;
 
 import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.annotation.Nullable;
 import org.apache.beam.runners.spark.coders.CoderHelpers;
 import org.apache.beam.runners.spark.translation.Dataset;
@@ -41,6 +45,7 @@ import org.slf4j.LoggerFactory;
 public class UnboundedDataset<T> implements Dataset {
 
   private static final Logger LOG = LoggerFactory.getLogger(UnboundedDataset.class);
+  private static final AtomicInteger queuedStreamIds = new AtomicInteger();
 
   // only set if creating a DStream from a static collection
   @Nullable private transient JavaStreamingContext jssc;
@@ -48,15 +53,23 @@ public class UnboundedDataset<T> implements Dataset {
   private Iterable<Iterable<T>> values;
   private Coder<T> coder;
   private JavaDStream<WindowedValue<T>> dStream;
+  // points to the input streams that created this UnboundedDataset,
+  // should be greater > 1 in case of Flatten for example.
+  // when using GlobalWatermarkHolder this information helps to take only the relevant watermarks
+  // and reason about them accordingly.
+  private final List<Integer> streamingSources = new ArrayList<>();
 
-  UnboundedDataset(JavaDStream<WindowedValue<T>> dStream) {
+  public UnboundedDataset(JavaDStream<WindowedValue<T>> dStream, List<Integer> streamingSources) {
     this.dStream = dStream;
+    this.streamingSources.addAll(streamingSources);
   }
 
   public UnboundedDataset(Iterable<Iterable<T>> values, JavaStreamingContext jssc, Coder<T> coder) {
     this.values = values;
     this.jssc = jssc;
     this.coder = coder;
+    // QueuedStream will have a negative (decreasing) unique id.
+    this.streamingSources.add(queuedStreamIds.decrementAndGet());
   }
 
   @SuppressWarnings("ConstantConditions")
@@ -66,7 +79,6 @@ public class UnboundedDataset<T> implements Dataset {
           WindowedValue.getValueOnlyCoder(coder);
       // create the DStream from queue
       Queue<JavaRDD<WindowedValue<T>>> rddQueue = new LinkedBlockingQueue<>();
-      JavaRDD<WindowedValue<T>> lastRDD = null;
       for (Iterable<T> v : values) {
         Iterable<WindowedValue<T>> windowedValues =
             Iterables.transform(v, WindowingHelpers.<T>windowValueFunction());
@@ -74,18 +86,17 @@ public class UnboundedDataset<T> implements Dataset {
             CoderHelpers.toByteArrays(windowedValues, windowCoder)).map(
             CoderHelpers.fromByteFunction(windowCoder));
         rddQueue.offer(rdd);
-        lastRDD = rdd;
       }
-      // create DStream from queue, one at a time,
-      // with last as default in case batches repeat (graceful stops for example).
-      // if the stream is empty, avoid creating a default empty RDD.
-      // mainly for unit test so no reason to have this configurable.
-      dStream = lastRDD != null ? jssc.queueStream(rddQueue, true, lastRDD)
-          : jssc.queueStream(rddQueue, true);
+      // create DStream from queue, one at a time.
+      dStream = jssc.queueStream(rddQueue, true);
     }
     return dStream;
   }
 
+  public List<Integer> getStreamingSources() {
+    return streamingSources;
+  }
+
   public void cache() {
     dStream.cache();
   }


[6/9] beam git commit: Exopse RegexMatcher as part of the SDK.

Posted by am...@apache.org.
Exopse RegexMatcher as part of the SDK.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c18f8a2c
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c18f8a2c
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c18f8a2c

Branch: refs/heads/master
Commit: c18f8a2c33a74ea00f8bbba6fb7cd766de9082f4
Parents: 705695e
Author: Sela <an...@paypal.com>
Authored: Sun Feb 12 18:33:55 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Mon Feb 20 11:30:15 2017 +0200

----------------------------------------------------------------------
 .../runners/dataflow/util/PackageUtilTest.java  | 30 +-----------------
 .../apache/beam/sdk/testing/RegexMatcher.java   | 32 ++++++++++++++++++++
 2 files changed, 33 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/c18f8a2c/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
index 800c5a9..a11872f 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
@@ -66,7 +66,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.regex.Pattern;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import org.apache.beam.runners.dataflow.util.PackageUtil.PackageAttributes;
@@ -74,11 +73,10 @@ import org.apache.beam.sdk.options.GcsOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.ExpectedLogs;
 import org.apache.beam.sdk.testing.FastNanoClockAndSleeper;
+import org.apache.beam.sdk.testing.RegexMatcher;
 import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.junit.Rule;
@@ -107,32 +105,6 @@ public class PackageUtilTest {
   // 128 bits, base64 encoded is 171 bits, rounds to 22 bytes
   private static final String HASH_PATTERN = "[a-zA-Z0-9+-]{22}";
 
-  // Hamcrest matcher to assert a string matches a pattern
-  private static class RegexMatcher extends BaseMatcher<String> {
-    private final Pattern pattern;
-
-    public RegexMatcher(String regex) {
-      this.pattern = Pattern.compile(regex);
-    }
-
-    @Override
-    public boolean matches(Object o) {
-      if (!(o instanceof String)) {
-        return false;
-      }
-      return pattern.matcher((String) o).matches();
-    }
-
-    @Override
-    public void describeTo(Description description) {
-      description.appendText(String.format("matches regular expression %s", pattern));
-    }
-
-    public static RegexMatcher matches(String regex) {
-      return new RegexMatcher(regex);
-    }
-  }
-
   @Before
   public void setUp() {
     MockitoAnnotations.initMocks(this);

http://git-wip-us.apache.org/repos/asf/beam/blob/c18f8a2c/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
new file mode 100644
index 0000000..a4b14fe
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/RegexMatcher.java
@@ -0,0 +1,32 @@
+package org.apache.beam.sdk.testing;
+
+import java.util.regex.Pattern;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+
+
+/** Hamcrest matcher to assert a string matches a pattern. */
+public class RegexMatcher extends BaseMatcher<String> {
+  private final Pattern pattern;
+
+  public RegexMatcher(String regex) {
+    this.pattern = Pattern.compile(regex);
+  }
+
+  @Override
+  public boolean matches(Object o) {
+    if (!(o instanceof String)) {
+      return false;
+    }
+    return pattern.matcher((String) o).matches();
+  }
+
+  @Override
+  public void describeTo(Description description) {
+    description.appendText(String.format("matches regular expression %s", pattern));
+  }
+
+  public static RegexMatcher matches(String regex) {
+    return new RegexMatcher(regex);
+  }
+}