You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by am...@apache.org on 2017/02/28 22:35:08 UTC
[01/23] beam git commit: Implementation of
GroupAlsoByWindowViaWindowSet for the Spark runner.
Repository: beam
Updated Branches:
refs/heads/master a41afdc68 -> 61e31e622
Implementation of GroupAlsoByWindowViaWindowSet for the Spark runner.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/32a9d61e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/32a9d61e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/32a9d61e
Branch: refs/heads/master
Commit: 32a9d61eff79d8456f12a57c711e2b288f9c775b
Parents: bf0c119
Author: Sela <an...@paypal.com>
Authored: Mon Feb 13 16:32:21 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:17:58 2017 +0200
----------------------------------------------------------------------
.../SparkGroupAlsoByWindowViaWindowSet.java | 393 +++++++++++++++++++
1 file changed, 393 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/32a9d61e/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
new file mode 100644
index 0000000..7902d7c
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark.stateful;
+
+import com.google.common.collect.Table;
+import com.google.common.collect.AbstractIterator;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.beam.runners.core.GroupAlsoByWindowsDoFn;
+import org.apache.beam.runners.core.OutputWindowedValue;
+import org.apache.beam.runners.core.ReduceFnRunner;
+import org.apache.beam.runners.core.SystemReduceFn;
+import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
+import org.apache.beam.runners.core.triggers.TriggerStateMachines;
+import org.apache.beam.runners.spark.SparkPipelineOptions;
+import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
+import org.apache.beam.runners.spark.translation.TranslationUtils;
+import org.apache.beam.runners.spark.translation.WindowingHelpers;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
+import org.apache.beam.runners.spark.util.LateDataUtils;
+import org.apache.beam.runners.spark.util.UnsupportedSideInputReader;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.Aggregator;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Sum;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.spark.Partitioner;
+import org.apache.spark.api.java.JavaSparkContext$;
+import org.apache.spark.api.java.function.FlatMapFunction;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.streaming.Duration;
+import org.apache.spark.streaming.api.java.JavaDStream;
+import org.apache.spark.streaming.api.java.JavaPairDStream;
+import org.apache.spark.streaming.dstream.DStream;
+import org.apache.spark.streaming.dstream.PairDStreamFunctions;
+import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import scala.Function1;
+import scala.Option;
+import scala.Tuple2;
+import scala.Tuple3;
+import scala.collection.Seq;
+import scala.reflect.ClassTag;
+import scala.runtime.AbstractFunction1;
+
+/**
+ * An implementation of {@link org.apache.beam.runners.core.GroupAlsoByWindowViaWindowSetDoFn}
+ * logic for grouping by windows and controlling trigger firings and pane accumulation.
+ *
+ * <p>This implementation is a composite of Spark transformations revolving around state management
+ * using Spark's
+ * {@link PairDStreamFunctions#updateStateByKey(Function1, Partitioner, boolean, ClassTag)}
+ * to update state with new data and timers.
+ *
+ * <p>Using updateStateByKey allows to scan through the entire state visiting not just the
+ * updated state (new values for key) but also check if timers are ready to fire.
+ * Since updateStateByKey bounds the types of state and output to be the same,
+ * a (state, output) tuple is used, filtering the state (and output if no firing)
+ * in the following steps.
+ */
+public class SparkGroupAlsoByWindowViaWindowSet {
+ private static final Logger LOG = LoggerFactory.getLogger(SparkGroupAlsoByWindowViaWindowSet.class);
+
+ /**
+ * A helper class that is essentially a {@link Serializable} {@link AbstractFunction1}.
+ */
+ private abstract static class SerializableFunction1<T1, T2>
+ extends AbstractFunction1<T1, T2> implements Serializable {
+ }
+
+ public static <K, InputT, W extends BoundedWindow>
+ JavaDStream<WindowedValue<KV<K, Iterable<InputT>>>> groupAlsoByWindow(
+ JavaDStream<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>> inputDStream,
+ final Coder<InputT> iCoder,
+ final WindowingStrategy<?, W> windowingStrategy,
+ final SparkRuntimeContext runtimeContext,
+ final List<Integer> sourceIds) {
+
+ Long checkpointDuration =
+ runtimeContext.getPipelineOptions().as(SparkPipelineOptions.class)
+ .getCheckpointDurationMillis();
+
+ // we have to switch to Scala API to avoid Optional in the Java API, see: SPARK-4819.
+ // we also have a broader API for Scala (access to the actual key and entire iterator).
+ DStream<Tuple2<K, Iterable<WindowedValue<InputT>>>> pairDStream =
+ inputDStream
+ .map(WindowingHelpers.<KV<K, Iterable<WindowedValue<InputT>>>>unwindowFunction())
+ .mapToPair(TranslationUtils.<K, Iterable<WindowedValue<InputT>>>toPairFunction())
+ .dstream();
+ PairDStreamFunctions<K, Iterable<WindowedValue<InputT>>> pairDStreamFunctions =
+ DStream.toPairDStreamFunctions(
+ pairDStream,
+ JavaSparkContext$.MODULE$.<K>fakeClassTag(),
+ JavaSparkContext$.MODULE$.<Iterable<WindowedValue<InputT>>>fakeClassTag(),
+ null);
+ int defaultNumPartitions = pairDStreamFunctions.defaultPartitioner$default$1();
+ Partitioner partitioner = pairDStreamFunctions.defaultPartitioner(defaultNumPartitions);
+
+ // use updateStateByKey to scan through the state and update elements and timers.
+ DStream<Tuple2<K, Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>>
+ firedStream = pairDStreamFunctions.updateStateByKey(
+ new SerializableFunction1<
+ scala.collection.Iterator<Tuple3<K, Seq<Iterable<WindowedValue<InputT>>>,
+ Option<Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>>>,
+ scala.collection.Iterator<Tuple2<K, Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>>>>() {
+
+ @Override
+ public scala.collection.Iterator<Tuple2<K, Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>>> apply(
+ final scala.collection.Iterator<Tuple3<K, Seq<Iterable<WindowedValue<InputT>>>,
+ Option<Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>>> iter) {
+ //--- ACTUAL STATEFUL OPERATION:
+ //
+ // Input Iterator: the partition (~bundle) of a cogrouping of the input
+ // and the previous state (if exists).
+ //
+ // Output Iterator: the output key, and the updated state.
+ //
+ // possible input scenarios for (K, Seq, Option<S>):
+ // (1) Option<S>.isEmpty: new data with no previous state.
+ // (2) Seq.isEmpty: no new data, but evaluating previous state (timer-like behaviour).
+ // (3) Seq.nonEmpty && Option<S>.isDefined: new data with previous state.
+
+ final SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn =
+ SystemReduceFn.buffering(iCoder);
+ final OutputWindowedValueHolder<K, InputT> outputHolder =
+ new OutputWindowedValueHolder<>();
+ // use in memory Aggregators since Spark Accumulators are not resilient
+ // in stateful operators, once done with this partition.
+ final InMemoryLongSumAggregator droppedDueToClosedWindow = new InMemoryLongSumAggregator(
+ GroupAlsoByWindowsDoFn.DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER);
+ final InMemoryLongSumAggregator droppedDueToLateness = new InMemoryLongSumAggregator(
+ GroupAlsoByWindowsDoFn.DROPPED_DUE_TO_LATENESS_COUNTER);
+
+ AbstractIterator<
+ Tuple2<K, Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>>
+ outIter = new AbstractIterator<Tuple2<K,
+ Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>>() {
+ @Override
+ protected Tuple2<K, Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>> computeNext() {
+ // input iterator is a Spark partition (~bundle), containing keys and their
+ // (possibly) previous-state and (possibly) new data.
+ while (iter.hasNext()) {
+ // for each element in the partition:
+ Tuple3<K, Seq<Iterable<WindowedValue<InputT>>>, Option<Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>>> next = iter.next();
+ K key = next._1();
+
+ Seq<Iterable<WindowedValue<InputT>>> seq = next._2();
+
+ Option<Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>>
+ prevStateAndTimersOpt = next._3();
+
+ SparkStateInternals<K> stateInternals;
+ SparkTimerInternals timerInternals = SparkTimerInternals.forStreamFromSources(
+ sourceIds, GlobalWatermarkHolder.get());
+ // get state(internals) per key.
+ if (prevStateAndTimersOpt.isEmpty()) {
+ // no previous state.
+ stateInternals = SparkStateInternals.forKey(key);
+ } else {
+ // with pre-existing state.
+ StateAndTimers prevStateAndTimers = prevStateAndTimersOpt.get()._1();
+ stateInternals = SparkStateInternals.forKeyAndState(key,
+ prevStateAndTimers.getState());
+ timerInternals.addTimers(prevStateAndTimers.getTimers());
+ }
+
+ ReduceFnRunner<K, InputT, Iterable<InputT>, W> reduceFnRunner =
+ new ReduceFnRunner<>(
+ key,
+ windowingStrategy,
+ ExecutableTriggerStateMachine.create(
+ TriggerStateMachines.stateMachineForTrigger(
+ windowingStrategy.getTrigger())),
+ stateInternals,
+ timerInternals,
+ outputHolder,
+ new UnsupportedSideInputReader("GroupAlsoByWindow"),
+ droppedDueToClosedWindow,
+ reduceFn,
+ runtimeContext.getPipelineOptions());
+
+ outputHolder.clear(); // clear before potential use.
+ if (!seq.isEmpty()) {
+ // new input for key.
+ try {
+ Iterable<WindowedValue<InputT>> elementsIterable = seq.head();
+ Iterable<WindowedValue<InputT>> validElements =
+ LateDataUtils
+ .dropExpiredWindows(
+ key,
+ elementsIterable,
+ timerInternals,
+ windowingStrategy,
+ droppedDueToLateness);
+ reduceFnRunner.processElements(validElements);
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Failed to process element with ReduceFnRunner", e);
+ }
+ } else if (stateInternals.getState().isEmpty()) {
+ // no input and no state -> GC evict now.
+ continue;
+ }
+ try {
+ // advance the watermark to HWM to fire by timers.
+ timerInternals.advanceWatermark();
+ // call on timers that are ready.
+ reduceFnRunner.onTimers(timerInternals.getTimersReadyToProcess());
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Failed to process ReduceFnRunner onTimer.", e);
+ }
+ // this is mostly symbolic since actual persist is done by emitting output.
+ reduceFnRunner.persist();
+ // obtain output, if fired.
+ List<WindowedValue<KV<K, Iterable<InputT>>>> outputs = outputHolder.get();
+ if (!outputs.isEmpty() || !stateInternals.getState().isEmpty()) {
+ StateAndTimers updated = new StateAndTimers(stateInternals.getState(),
+ timerInternals.getTimers());
+ // persist Spark's state by outputting.
+ return new Tuple2<>(key, new Tuple2<>(updated, outputs));
+ }
+ // an empty state with no output, can be evicted completely - do nothing.
+ }
+ return endOfData();
+ }
+ };
+
+ // log if there's something to log.
+ long lateDropped = droppedDueToLateness.getSum();
+ if (lateDropped > 0) {
+ LOG.info(String.format("Dropped %d elements due to lateness.", lateDropped));
+ droppedDueToLateness.zero();
+ }
+ long closedWindowDropped = droppedDueToClosedWindow.getSum();
+ if (closedWindowDropped > 0) {
+ LOG.info(String.format("Dropped %d elements due to closed window.", closedWindowDropped));
+ droppedDueToClosedWindow.zero();
+ }
+
+ return scala.collection.JavaConversions.asScalaIterator(outIter);
+ }
+ }, partitioner, true, JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>>fakeClassTag())
+ .checkpoint(new Duration(checkpointDuration));
+
+ // go back to Java now.
+ JavaPairDStream<K, Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>
+ javaFiredStream = JavaPairDStream.fromPairDStream(
+ firedStream,
+ JavaSparkContext$.MODULE$.<K>fakeClassTag(),
+ JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>>fakeClassTag());
+
+ // filter state-only output (nothing to fire) and remove the state from the output.
+ return javaFiredStream.filter(
+ new Function<Tuple2<K, Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>>, Boolean>() {
+ @Override
+ public Boolean call(
+ Tuple2<K, Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>> t2) throws Exception {
+ // filter output if defined.
+ return !t2._2()._2().isEmpty();
+ }
+ })
+ .flatMap(
+ new FlatMapFunction<Tuple2<K, Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>>,
+ WindowedValue<KV<K, Iterable<InputT>>>>() {
+ @Override
+ public Iterable<WindowedValue<KV<K, Iterable<InputT>>>> call(
+ Tuple2<K, Tuple2<StateAndTimers,
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>> t2) throws Exception {
+ // drop the state since it is already persisted at this point.
+ return t2._2()._2();
+ }
+ });
+ }
+
+ private static class StateAndTimers {
+ //Serializable state for internals (namespace to state tag to coded value).
+ private final Table<String, String, byte[]> state;
+ private final Collection<TimerData> timers;
+
+ private StateAndTimers(
+ Table<String, String, byte[]> state, Collection<TimerData> timers) {
+ this.state = state;
+ this.timers = timers;
+ }
+
+ public Table<String, String, byte[]> getState() {
+ return state;
+ }
+
+ public Collection<TimerData> getTimers() {
+ return timers;
+ }
+ }
+
+ private static class OutputWindowedValueHolder<K, V>
+ implements OutputWindowedValue<KV<K, Iterable<V>>> {
+ private List<WindowedValue<KV<K, Iterable<V>>>> windowedValues = new ArrayList<>();
+
+ @Override
+ public void outputWindowedValue(
+ KV<K, Iterable<V>> output,
+ Instant timestamp,
+ Collection<? extends BoundedWindow> windows,
+ PaneInfo pane) {
+ windowedValues.add(WindowedValue.of(output, timestamp, windows, pane));
+ }
+
+ private List<WindowedValue<KV<K, Iterable<V>>>> get() {
+ return windowedValues;
+ }
+
+ private void clear() {
+ windowedValues.clear();
+ }
+
+ @Override
+ public <SideOutputT> void sideOutputWindowedValue(
+ TupleTag<SideOutputT> tag,
+ SideOutputT output, Instant timestamp,
+ Collection<? extends BoundedWindow> windows,
+ PaneInfo pane) {
+ throw new UnsupportedOperationException("Side outputs are not allowed in GroupAlsoByWindow.");
+ }
+ }
+
+ private static class InMemoryLongSumAggregator implements Aggregator<Long, Long> {
+ private final String name;
+ private long sum = 0;
+
+ public void zero() {
+ sum = 0;
+ }
+
+ public long getSum() {
+ return sum;
+ }
+
+ InMemoryLongSumAggregator(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public void addValue(Long value) {
+ sum += value;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Combine.CombineFn<Long, ?, Long> getCombineFn() {
+ return Sum.ofLongs();
+ }
+ }
+}
[20/23] beam git commit: Make CreateStream a TimestampedValue Source.
Posted by am...@apache.org.
Make CreateStream a TimestampedValue Source.
fixup! a nicer DSL to construct CreateStream.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/62ddca63
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/62ddca63
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/62ddca63
Branch: refs/heads/master
Commit: 62ddca638ae76918f3a0a5fac4ca8fbdb1c7fb9a
Parents: 123f482
Author: Sela <an...@paypal.com>
Authored: Mon Feb 27 23:35:10 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:09 2017 +0200
----------------------------------------------------------------------
.../beam/runners/spark/io/CreateStream.java | 66 ++++++++++++-----
.../streaming/StreamingTransformTranslator.java | 26 +++++--
.../runners/spark/SparkPipelineStateTest.java | 2 +-
.../translation/streaming/CreateStreamTest.java | 76 ++++++++------------
.../streaming/TrackStreamingSourcesTest.java | 22 +++---
5 files changed, 111 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/62ddca63/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
index 70784f1..f2e0bb3 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
@@ -19,11 +19,16 @@ package org.apache.beam.runners.spark.io;
import static com.google.common.base.Preconditions.checkArgument;
+import com.google.common.collect.Lists;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Deque;
import java.util.LinkedList;
+import java.util.List;
import java.util.Queue;
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowingStrategy;
@@ -83,20 +88,22 @@ import org.joda.time.Instant;
public final class CreateStream<T> extends PTransform<PBegin, PCollection<T>> {
private final Duration batchInterval;
- private final Instant initialSystemTime;
- private final Queue<Iterable<T>> batches = new LinkedList<>();
+ private final Queue<Iterable<TimestampedValue<T>>> batches = new LinkedList<>();
private final Deque<SparkWatermarks> times = new LinkedList<>();
+ private final Coder<T> coder;
+ private Instant initialSystemTime;
private Instant lowWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE; //for test purposes.
- private CreateStream(Duration batchInterval, Instant initialSystemTime) {
+ private CreateStream(Duration batchInterval, Instant initialSystemTime, Coder<T> coder) {
this.batchInterval = batchInterval;
this.initialSystemTime = initialSystemTime;
+ this.coder = coder;
}
/** Set the batch interval for the stream. */
- public static <T> CreateStream<T> withBatchInterval(Duration batchInterval) {
- return new CreateStream<>(batchInterval, new Instant(0));
+ public static <T> CreateStream<T> of(Coder<T> coder, Duration batchInterval) {
+ return new CreateStream<>(batchInterval, new Instant(0), coder);
}
/**
@@ -104,25 +111,46 @@ public final class CreateStream<T> extends PTransform<PBegin, PCollection<T>> {
* This is backed by a {@link Queue} so stream input order would keep the population order (FIFO).
*/
@SafeVarargs
- public final CreateStream<T> nextBatch(T... batchElements) {
+ public final CreateStream<T> nextBatch(TimestampedValue<T>... batchElements) {
// validate timestamps if timestamped elements.
- for (T element: batchElements) {
- if (element instanceof TimestampedValue) {
- TimestampedValue timestampedValue = (TimestampedValue) element;
- checkArgument(
- timestampedValue.getTimestamp().isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE),
- "Elements must have timestamps before %s. Got: %s",
- BoundedWindow.TIMESTAMP_MAX_VALUE,
- timestampedValue.getTimestamp());
- }
+ for (TimestampedValue<T> element: batchElements) {
+ TimestampedValue timestampedValue = (TimestampedValue) element;
+ checkArgument(
+ timestampedValue.getTimestamp().isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE),
+ "Elements must have timestamps before %s. Got: %s",
+ BoundedWindow.TIMESTAMP_MAX_VALUE,
+ timestampedValue.getTimestamp());
}
batches.offer(Arrays.asList(batchElements));
return this;
}
+ /**
+ * For non-timestamped elements.
+ */
+ @SafeVarargs
+ public final CreateStream<T> nextBatch(T... batchElements) {
+ List<TimestampedValue<T>> timestamped = Lists.newArrayListWithCapacity(batchElements.length);
+ // as TimestampedValue.
+ for (T element: batchElements) {
+ timestamped.add(TimestampedValue.atMinimumTimestamp(element));
+ }
+ batches.offer(timestamped);
+ return this;
+ }
+
+ /**
+ * Adds an empty batch.
+ */
+ public CreateStream<T> emptyBatch() {
+ batches.offer(Collections.<TimestampedValue<T>>emptyList());
+ return this;
+ }
+
/** Set the initial synchronized processing time. */
public CreateStream<T> initialSystemTimeAt(Instant initialSystemTime) {
- return new CreateStream<>(batchInterval, initialSystemTime);
+ this.initialSystemTime = initialSystemTime;
+ return this;
}
/**
@@ -160,7 +188,7 @@ public final class CreateStream<T> extends PTransform<PBegin, PCollection<T>> {
}
/** Get the underlying queue representing the mock stream of micro-batches. */
- public Queue<Iterable<T>> getBatches() {
+ public Queue<Iterable<TimestampedValue<T>>> getBatches() {
return batches;
}
@@ -178,4 +206,8 @@ public final class CreateStream<T> extends PTransform<PBegin, PCollection<T>> {
input.getPipeline(), WindowingStrategy.globalDefault(), PCollection.IsBounded.UNBOUNDED);
}
+ @Override
+ protected Coder<T> getDefaultOutputCoder() throws CannotProvideCoderException {
+ return coder;
+ }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/62ddca63/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index fc98781..4a07741 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
+import javax.annotation.Nonnull;
import org.apache.beam.runners.spark.aggregators.NamedAggregators;
import org.apache.beam.runners.spark.aggregators.SparkAggregators;
import org.apache.beam.runners.spark.coders.CoderHelpers;
@@ -67,6 +68,8 @@ import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.CombineFnUtil;
@@ -75,6 +78,7 @@ import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TaggedPValue;
+import org.apache.beam.sdk.values.TimestampedValue;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.spark.Accumulator;
import org.apache.spark.api.java.JavaPairRDD;
@@ -87,6 +91,7 @@ import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
+
/**
* Supports translation between a Beam transform, and Spark's operations on DStreams.
*/
@@ -127,14 +132,25 @@ final class StreamingTransformTranslator {
public void evaluate(CreateStream<T> transform, EvaluationContext context) {
Coder<T> coder = context.getOutput(transform).getCoder();
JavaStreamingContext jssc = context.getStreamingContext();
- Queue<Iterable<T>> values = transform.getBatches();
- WindowedValue.ValueOnlyWindowedValueCoder<T> windowCoder =
- WindowedValue.getValueOnlyCoder(coder);
+ Queue<Iterable<TimestampedValue<T>>> values = transform.getBatches();
+ WindowedValue.FullWindowedValueCoder<T> windowCoder =
+ WindowedValue.FullWindowedValueCoder.of(coder, GlobalWindow.Coder.INSTANCE);
// create the DStream from queue.
Queue<JavaRDD<WindowedValue<T>>> rddQueue = new LinkedBlockingQueue<>();
- for (Iterable<T> v : values) {
+ for (Iterable<TimestampedValue<T>> tv : values) {
Iterable<WindowedValue<T>> windowedValues =
- Iterables.transform(v, WindowingHelpers.<T>windowValueFunction());
+ Iterables.transform(
+ tv,
+ new com.google.common.base.Function<TimestampedValue<T>, WindowedValue<T>>() {
+ @Override
+ public WindowedValue<T> apply(@Nonnull TimestampedValue<T> timestampedValue) {
+ return WindowedValue.of(
+ timestampedValue.getValue(),
+ timestampedValue.getTimestamp(),
+ GlobalWindow.INSTANCE,
+ PaneInfo.NO_FIRING);
+ }
+ });
JavaRDD<WindowedValue<T>> rdd =
jssc.sparkContext()
.parallelize(CoderHelpers.toByteArrays(windowedValues, windowCoder))
http://git-wip-us.apache.org/repos/asf/beam/blob/62ddca63/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
index c856203..37a201c 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
@@ -75,7 +75,7 @@ public class SparkPipelineStateTest implements Serializable {
private PTransform<PBegin, PCollection<String>> getValues(final SparkPipelineOptions options) {
return options.isStreaming()
- ? CreateStream.<String>withBatchInterval(Duration.millis(1)).nextBatch("one", "two")
+ ? CreateStream.of(StringUtf8Coder.of(), Duration.millis(1)).nextBatch("one", "two")
: Create.of("one", "two");
}
http://git-wip-us.apache.org/repos/asf/beam/blob/62ddca63/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
index f2783a1..ff77535 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
@@ -32,10 +32,8 @@ import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.Values;
@@ -83,9 +81,9 @@ public class CreateStreamTest implements Serializable {
public void testLateDataAccumulating() throws IOException {
Pipeline p = pipelineRule.createPipeline();
Instant instant = new Instant(0);
- CreateStream<TimestampedValue<Integer>> source =
- CreateStream.<TimestampedValue<Integer>>withBatchInterval(pipelineRule.batchDuration())
- .nextBatch()
+ CreateStream<Integer> source =
+ CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+ .emptyBatch()
.advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(6)))
.nextBatch(
TimestampedValue.of(1, instant),
@@ -104,8 +102,7 @@ public class CreateStreamTest implements Serializable {
TimestampedValue.of(-3, instant));
PCollection<Integer> windowed = p
- .apply(source).setCoder(TimestampedValue.TimestampedValueCoder.of(VarIntCoder.of()))
- .apply(ParDo.of(new OnlyValue<Integer>()))
+ .apply(source)
.apply(Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5))).triggering(
AfterWatermark.pastEndOfWindow()
.withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
@@ -156,8 +153,8 @@ public class CreateStreamTest implements Serializable {
@Test
public void testDiscardingMode() throws IOException {
Pipeline p = pipelineRule.createPipeline();
- CreateStream<TimestampedValue<String>> source =
- CreateStream.<TimestampedValue<String>>withBatchInterval(pipelineRule.batchDuration())
+ CreateStream<String> source =
+ CreateStream.of(StringUtf8Coder.of(), pipelineRule.batchDuration())
.nextBatch(
TimestampedValue.of("firstPane", new Instant(100)),
TimestampedValue.of("alsoFirstPane", new Instant(200)))
@@ -172,8 +169,7 @@ public class CreateStreamTest implements Serializable {
FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
Duration allowedLateness = Duration.millis(5000L);
PCollection<String> values =
- p.apply(source).setCoder(TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of()))
- .apply(ParDo.of(new OnlyValue<String>()))
+ p.apply(source)
.apply(
Window.<String>into(windowFn)
.triggering(
@@ -207,9 +203,9 @@ public class CreateStreamTest implements Serializable {
public void testFirstElementLate() throws IOException {
Pipeline p = pipelineRule.createPipeline();
Instant lateElementTimestamp = new Instant(-1_000_000);
- CreateStream<TimestampedValue<String>> source =
- CreateStream.<TimestampedValue<String>>withBatchInterval(pipelineRule.batchDuration())
- .nextBatch()
+ CreateStream<String> source =
+ CreateStream.of(StringUtf8Coder.of(), pipelineRule.batchDuration())
+ .emptyBatch()
.advanceWatermarkForNextBatch(new Instant(0))
.nextBatch(
TimestampedValue.of("late", lateElementTimestamp),
@@ -219,8 +215,6 @@ public class CreateStreamTest implements Serializable {
FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
Duration allowedLateness = Duration.millis(5000L);
PCollection<String> values = p.apply(source)
- .setCoder(TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of()))
- .apply(ParDo.of(new OnlyValue<String>()))
.apply(Window.<String>into(windowFn).triggering(DefaultTrigger.of())
.discardingFiredPanes()
.withAllowedLateness(allowedLateness))
@@ -243,8 +237,8 @@ public class CreateStreamTest implements Serializable {
public void testElementsAtAlmostPositiveInfinity() throws IOException {
Pipeline p = pipelineRule.createPipeline();
Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp();
- CreateStream<TimestampedValue<String>> source =
- CreateStream.<TimestampedValue<String>>withBatchInterval(pipelineRule.batchDuration())
+ CreateStream<String> source =
+ CreateStream.of(StringUtf8Coder.of(), pipelineRule.batchDuration())
.nextBatch(
TimestampedValue.of("foo", endOfGlobalWindow),
TimestampedValue.of("bar", endOfGlobalWindow))
@@ -252,8 +246,6 @@ public class CreateStreamTest implements Serializable {
FixedWindows windows = FixedWindows.of(Duration.standardHours(6));
PCollection<String> windowedValues = p.apply(source)
- .setCoder(TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of()))
- .apply(ParDo.of(new OnlyValue<String>()))
.apply(Window.<String>into(windows))
.apply(WithKeys.<Integer, String>of(1))
.apply(GroupByKey.<Integer, String>create())
@@ -270,16 +262,16 @@ public class CreateStreamTest implements Serializable {
public void testMultipleStreams() throws IOException {
Pipeline p = pipelineRule.createPipeline();
CreateStream<String> source =
- CreateStream.<String>withBatchInterval(pipelineRule.batchDuration())
+ CreateStream.of(StringUtf8Coder.of(), pipelineRule.batchDuration())
.nextBatch("foo", "bar")
.advanceNextBatchWatermarkToInfinity();
CreateStream<Integer> other =
- CreateStream.<Integer>withBatchInterval(pipelineRule.batchDuration())
+ CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
.nextBatch(1, 2, 3, 4)
.advanceNextBatchWatermarkToInfinity();
PCollection<String> createStrings =
- p.apply("CreateStrings", source).setCoder(StringUtf8Coder.of())
+ p.apply("CreateStrings", source)
.apply("WindowStrings",
Window.<String>triggering(AfterPane.elementCountAtLeast(2))
.withAllowedLateness(Duration.ZERO)
@@ -287,7 +279,7 @@ public class CreateStreamTest implements Serializable {
PAssert.that(createStrings).containsInAnyOrder("foo", "bar");
PCollection<Integer> createInts =
- p.apply("CreateInts", other).setCoder(VarIntCoder.of())
+ p.apply("CreateInts", other)
.apply("WindowInts",
Window.<Integer>triggering(AfterPane.elementCountAtLeast(4))
.withAllowedLateness(Duration.ZERO)
@@ -301,18 +293,18 @@ public class CreateStreamTest implements Serializable {
public void testFlattenedWithWatermarkHold() throws IOException {
Pipeline p = pipelineRule.createPipeline();
Instant instant = new Instant(0);
- CreateStream<TimestampedValue<Integer>> source1 =
- CreateStream.<TimestampedValue<Integer>>withBatchInterval(pipelineRule.batchDuration())
- .nextBatch()
+ CreateStream<Integer> source1 =
+ CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+ .emptyBatch()
.advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(5)))
.nextBatch(
TimestampedValue.of(1, instant),
TimestampedValue.of(2, instant),
TimestampedValue.of(3, instant))
.advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(10)));
- CreateStream<TimestampedValue<Integer>> source2 =
- CreateStream.<TimestampedValue<Integer>>withBatchInterval(pipelineRule.batchDuration())
- .nextBatch()
+ CreateStream<Integer> source2 =
+ CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
+ .emptyBatch()
.advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(1)))
.nextBatch(
TimestampedValue.of(4, instant))
@@ -322,15 +314,13 @@ public class CreateStreamTest implements Serializable {
.advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(5)));
PCollection<Integer> windowed1 = p
- .apply(source1).setCoder(TimestampedValue.TimestampedValueCoder.of(VarIntCoder.of()))
- .apply(ParDo.of(new OnlyValue<Integer>()))
+ .apply(source1)
.apply(Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5)))
.triggering(AfterWatermark.pastEndOfWindow())
.accumulatingFiredPanes()
.withAllowedLateness(Duration.ZERO));
PCollection<Integer> windowed2 = p
- .apply(source2).setCoder(TimestampedValue.TimestampedValueCoder.of(VarIntCoder.of()))
- .apply(ParDo.of(new OnlyValue<Integer>()))
+ .apply(source2)
.apply(Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5)))
.triggering(AfterWatermark.pastEndOfWindow())
.accumulatingFiredPanes()
@@ -352,8 +342,8 @@ public class CreateStreamTest implements Serializable {
@Test
public void testElementAtPositiveInfinityThrows() {
- CreateStream<TimestampedValue<Integer>> source =
- CreateStream.<TimestampedValue<Integer>>withBatchInterval(pipelineRule.batchDuration())
+ CreateStream<Integer> source =
+ CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
.nextBatch(TimestampedValue.of(-1, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L)));
thrown.expect(IllegalArgumentException.class);
source.nextBatch(TimestampedValue.of(1, BoundedWindow.TIMESTAMP_MAX_VALUE));
@@ -362,7 +352,7 @@ public class CreateStreamTest implements Serializable {
@Test
public void testAdvanceWatermarkNonMonotonicThrows() {
CreateStream<Integer> source =
- CreateStream.<Integer>withBatchInterval(pipelineRule.batchDuration())
+ CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
.advanceWatermarkForNextBatch(new Instant(0L));
thrown.expect(IllegalArgumentException.class);
source.advanceWatermarkForNextBatch(new Instant(-1L));
@@ -371,19 +361,9 @@ public class CreateStreamTest implements Serializable {
@Test
public void testAdvanceWatermarkEqualToPositiveInfinityThrows() {
CreateStream<Integer> source =
- CreateStream.<Integer>withBatchInterval(pipelineRule.batchDuration())
+ CreateStream.of(VarIntCoder.of(), pipelineRule.batchDuration())
.advanceWatermarkForNextBatch(BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L));
thrown.expect(IllegalArgumentException.class);
source.advanceWatermarkForNextBatch(BoundedWindow.TIMESTAMP_MAX_VALUE);
}
-
- private static class OnlyValue<T> extends DoFn<TimestampedValue<T>, T> {
-
- OnlyValue() { }
-
- @ProcessElement
- public void emitTimestampedValue(ProcessContext c) {
- c.outputWithTimestamp(c.element().getValue(), c.element().getTimestamp());
- }
- }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/62ddca63/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
index 32cef7e..b181a04 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -77,11 +77,11 @@ public class TrackStreamingSourcesTest {
Pipeline p = Pipeline.create(options);
CreateStream<Integer> emptyStream =
- CreateStream.<Integer>withBatchInterval(
- Duration.millis(options.getBatchIntervalMillis())).nextBatch();
+ CreateStream.of(
+ VarIntCoder.of(),
+ Duration.millis(options.getBatchIntervalMillis())).emptyBatch();
- p.apply(emptyStream).setCoder(VarIntCoder.of())
- .apply(ParDo.of(new PassthroughFn<>()));
+ p.apply(emptyStream).apply(ParDo.of(new PassthroughFn<>()));
p.traverseTopologically(new StreamingSourceTracker(jssc, p, ParDo.Bound.class, 0));
assertThat(StreamingSourceTracker.numAssertions, equalTo(1));
@@ -97,14 +97,16 @@ public class TrackStreamingSourcesTest {
Pipeline p = Pipeline.create(options);
CreateStream<Integer> queueStream1 =
- CreateStream.<Integer>withBatchInterval(
- Duration.millis(options.getBatchIntervalMillis())).nextBatch();
+ CreateStream.of(
+ VarIntCoder.of(),
+ Duration.millis(options.getBatchIntervalMillis())).emptyBatch();
CreateStream<Integer> queueStream2 =
- CreateStream.<Integer>withBatchInterval(
- Duration.millis(options.getBatchIntervalMillis())).nextBatch();
+ CreateStream.of(
+ VarIntCoder.of(),
+ Duration.millis(options.getBatchIntervalMillis())).emptyBatch();
- PCollection<Integer> pcol1 = p.apply(queueStream1).setCoder(VarIntCoder.of());
- PCollection<Integer> pcol2 = p.apply(queueStream2).setCoder(VarIntCoder.of());
+ PCollection<Integer> pcol1 = p.apply(queueStream1);
+ PCollection<Integer> pcol2 = p.apply(queueStream2);
PCollection<Integer> flattened =
PCollectionList.of(pcol1).and(pcol2).apply(Flatten.<Integer>pCollections());
flattened.apply(ParDo.of(new PassthroughFn<>()));
[08/23] beam git commit: Test triggers,
panes and watermarks via CreateStream.
Posted by am...@apache.org.
Test triggers, panes and watermarks via CreateStream.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/96d373fe
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/96d373fe
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/96d373fe
Branch: refs/heads/master
Commit: 96d373fe954c263414e7f22ab08979f25cc83188
Parents: 9f14350
Author: Sela <an...@paypal.com>
Authored: Sat Feb 18 22:05:19 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:17:59 2017 +0200
----------------------------------------------------------------------
.../beam/runners/spark/io/CreateStream.java | 125 ++++--
.../spark/GlobalWatermarkHolderTest.java | 150 ++++++++
.../beam/runners/spark/ReuseSparkContext.java | 46 ---
.../runners/spark/ReuseSparkContextRule.java | 46 +++
.../beam/runners/spark/WatermarkTest.java | 231 -----------
.../translation/streaming/CreateStreamTest.java | 385 +++++++++++++++++++
.../ResumeFromCheckpointStreamingTest.java | 296 ++++++++------
7 files changed, 860 insertions(+), 419 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/96d373fe/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
index 7ebba90..2149372 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
@@ -17,57 +17,124 @@
*/
package org.apache.beam.runners.spark.io;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkArgument;
+import java.util.Arrays;
+import java.util.Deque;
+import java.util.LinkedList;
+import java.util.Queue;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
/**
- * Create an input stream from Queue.
+ * Create an input stream from Queue. For SparkRunner tests only.
*
- * @param <T> stream type
+ * @param <T> stream type.
*/
-public final class CreateStream<T> {
+public final class CreateStream<T> extends PTransform<PBegin, PCollection<T>> {
+
+ private final Duration batchInterval;
+ private final Instant initialSystemTime;
+ private final Queue<Iterable<T>> batches = new LinkedList<>();
+ private final Deque<SparkWatermarks> times = new LinkedList<>();
+
+ private Instant lowWatermark = BoundedWindow.TIMESTAMP_MIN_VALUE; //for test purposes.
- private CreateStream() {
+ private CreateStream(Duration batchInterval, Instant initialSystemTime) {
+ this.batchInterval = batchInterval;
+ this.initialSystemTime = initialSystemTime;
+ }
+
+ /** Set the batch interval for the stream. */
+ public static <T> CreateStream<T> withBatchInterval(Duration batchInterval) {
+ return new CreateStream<>(batchInterval, new Instant(0));
}
/**
- * Define the input stream to create from queue.
- *
- * @param queuedValues defines the input stream
- * @param <T> stream type
- * @return the queue that defines the input stream
+ * Enqueue next micro-batch elements.
+ * This is backed by a {@link Queue} so stream input order would keep the population order (FIFO).
*/
- public static <T> QueuedValues<T> fromQueue(Iterable<Iterable<T>> queuedValues) {
- return new QueuedValues<>(queuedValues);
+ @SafeVarargs
+ public final CreateStream<T> nextBatch(T... batchElements) {
+ // validate timestamps if timestamped elements.
+ for (T element: batchElements) {
+ if (element instanceof TimestampedValue) {
+ TimestampedValue timestampedValue = (TimestampedValue) element;
+ checkArgument(
+ timestampedValue.getTimestamp().isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE),
+ "Elements must have timestamps before %s. Got: %s",
+ BoundedWindow.TIMESTAMP_MAX_VALUE,
+ timestampedValue.getTimestamp());
+ }
+ }
+ batches.offer(Arrays.asList(batchElements));
+ return this;
+ }
+
+ /** Set the initial synchronized processing time. */
+ public CreateStream<T> initialSystemTimeAt(Instant initialSystemTime) {
+ return new CreateStream<>(batchInterval, initialSystemTime);
}
/**
- * {@link PTransform} for queueing values.
+ * Advances the watermark in the next batch.
*/
- public static final class QueuedValues<T> extends PTransform<PBegin, PCollection<T>> {
+ public CreateStream<T> advanceWatermarkForNextBatch(Instant newWatermark) {
+ checkArgument(
+ !newWatermark.isBefore(lowWatermark), "The watermark is not allowed to decrease!");
+ checkArgument(
+ newWatermark.isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE),
+ "The Watermark cannot progress beyond the maximum. Got: %s. Maximum: %s",
+ newWatermark,
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ return advance(newWatermark);
+ }
- private final Iterable<Iterable<T>> queuedValues;
+ /**
+ * Advances the watermark in the next batch to the end-of-time.
+ */
+ public CreateStream<T> advanceNextBatchWatermarkToInfinity() {
+ return advance(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ }
- QueuedValues(Iterable<Iterable<T>> queuedValues) {
- checkNotNull(
- queuedValues, "need to set the queuedValues of an Create.QueuedValues transform");
- this.queuedValues = queuedValues;
- }
+ private CreateStream<T> advance(Instant newWatermark) {
+ // advance the system time.
+ Instant currentSynchronizedProcessingTime = times.peekLast() == null ? initialSystemTime
+ : times.peekLast().getSynchronizedProcessingTime();
+ Instant nextSynchronizedProcessingTime = currentSynchronizedProcessingTime.plus(batchInterval);
+ checkArgument(
+ nextSynchronizedProcessingTime.isAfter(currentSynchronizedProcessingTime),
+ "Synchronized processing time must always advance.");
+ times.offer(new SparkWatermarks(lowWatermark, newWatermark, nextSynchronizedProcessingTime));
+ lowWatermark = newWatermark;
+ return this;
+ }
- public Iterable<Iterable<T>> getQueuedValues() {
- return queuedValues;
- }
+ /** Get the underlying queue representing the mock stream of micro-batches. */
+ public Queue<Iterable<T>> getBatches() {
+ return batches;
+ }
- @Override
- public PCollection<T> expand(PBegin input) {
- // Spark streaming micro batches are bounded by default
- return PCollection.createPrimitiveOutputInternal(input.getPipeline(),
- WindowingStrategy.globalDefault(), PCollection.IsBounded.UNBOUNDED);
- }
+ /**
+ * Get times so they can be pushed into the
+ * {@link org.apache.beam.runners.spark.util.GlobalWatermarkHolder}.
+ */
+ public Queue<SparkWatermarks> getTimes() {
+ return times;
+ }
+
+ @Override
+ public PCollection<T> expand(PBegin input) {
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), WindowingStrategy.globalDefault(), PCollection.IsBounded.UNBOUNDED);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/96d373fe/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java
new file mode 100644
index 0000000..c1d2944
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark;
+
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.runners.spark.translation.SparkContextFactory;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.RegexMatcher;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+
+/**
+ * A test suite for the propagation of watermarks in the Spark runner.
+ */
+public class WatermarkTest {
+
+ @Rule
+ public ClearWatermarksRule clearWatermarksRule = new ClearWatermarksRule();
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Rule
+ public ReuseSparkContextRule reuseContext = ReuseSparkContextRule.yes();
+
+ private static final SparkPipelineOptions options =
+ PipelineOptionsFactory.create().as(SparkPipelineOptions.class);
+
+ private static final String INSTANT_PATTERN =
+ "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z";
+
+ @Test
+ public void testLowHighWatermarksAdvance() {
+ JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+
+ Instant instant = new Instant(0);
+ // low == high.
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(5)),
+ instant.plus(Duration.millis(5)),
+ instant));
+ GlobalWatermarkHolder.advance(jsc);
+ // low < high.
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(10)),
+ instant.plus(Duration.millis(15)),
+ instant.plus(Duration.millis(100))));
+ GlobalWatermarkHolder.advance(jsc);
+
+ // assert watermarks in Broadcast.
+ SparkWatermarks currentWatermarks = GlobalWatermarkHolder.get().getValue().get(1);
+ assertThat(currentWatermarks.getLowWatermark(), equalTo(instant.plus(Duration.millis(10))));
+ assertThat(currentWatermarks.getHighWatermark(), equalTo(instant.plus(Duration.millis(15))));
+ assertThat(currentWatermarks.getSynchronizedProcessingTime(),
+ equalTo(instant.plus(Duration.millis(100))));
+
+ // assert illegal watermark advance.
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage(
+ RegexMatcher.matches(
+ "Low watermark " + INSTANT_PATTERN + " cannot be later then high watermark "
+ + INSTANT_PATTERN));
+ // low > high -> not allowed!
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(25)),
+ instant.plus(Duration.millis(20)),
+ instant.plus(Duration.millis(200))));
+ GlobalWatermarkHolder.advance(jsc);
+ }
+
+ @Test
+ public void testSynchronizedTimeMonotonic() {
+ JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+
+ Instant instant = new Instant(0);
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(5)),
+ instant.plus(Duration.millis(10)),
+ instant));
+ GlobalWatermarkHolder.advance(jsc);
+
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Synchronized processing time must advance.");
+ // no actual advancement of watermarks - fine by Watermarks
+ // but not by synchronized processing time.
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(5)),
+ instant.plus(Duration.millis(10)),
+ instant));
+ GlobalWatermarkHolder.advance(jsc);
+ }
+
+ @Test
+ public void testMultiSource() {
+ JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
+
+ Instant instant = new Instant(0);
+ GlobalWatermarkHolder.add(1,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(5)),
+ instant.plus(Duration.millis(10)),
+ instant));
+ GlobalWatermarkHolder.add(2,
+ new SparkWatermarks(
+ instant.plus(Duration.millis(3)),
+ instant.plus(Duration.millis(6)),
+ instant));
+
+ GlobalWatermarkHolder.advance(jsc);
+
+ // assert watermarks for source 1.
+ SparkWatermarks watermarksForSource1 = GlobalWatermarkHolder.get().getValue().get(1);
+ assertThat(watermarksForSource1.getLowWatermark(), equalTo(instant.plus(Duration.millis(5))));
+ assertThat(watermarksForSource1.getHighWatermark(), equalTo(instant.plus(Duration.millis(10))));
+
+ // assert watermarks for source 2.
+ SparkWatermarks watermarksForSource2 = GlobalWatermarkHolder.get().getValue().get(2);
+ assertThat(watermarksForSource2.getLowWatermark(), equalTo(instant.plus(Duration.millis(3))));
+ assertThat(watermarksForSource2.getHighWatermark(), equalTo(instant.plus(Duration.millis(6))));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/96d373fe/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java
deleted file mode 100644
index 027f9fd..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContext.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark;
-
-import org.apache.beam.runners.spark.translation.SparkContextFactory;
-import org.junit.rules.ExternalResource;
-
-/**
- * Explicitly set {@link org.apache.spark.SparkContext} to be reused (or not) in tests.
- */
-public class ReuseSparkContext extends ExternalResource {
-
- private final boolean reuse;
-
- private ReuseSparkContext(boolean reuse) {
- this.reuse = reuse;
- }
-
- public static ReuseSparkContext no() {
- return new ReuseSparkContext(false);
- }
-
- public static ReuseSparkContext yes() {
- return new ReuseSparkContext(true);
- }
-
- @Override
- protected void before() throws Throwable {
- System.setProperty(SparkContextFactory.TEST_REUSE_SPARK_CONTEXT, Boolean.toString(reuse));
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/96d373fe/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContextRule.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContextRule.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContextRule.java
new file mode 100644
index 0000000..027f9fd
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContextRule.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark;
+
+import org.apache.beam.runners.spark.translation.SparkContextFactory;
+import org.junit.rules.ExternalResource;
+
+/**
+ * Explicitly set {@link org.apache.spark.SparkContext} to be reused (or not) in tests.
+ */
+public class ReuseSparkContext extends ExternalResource {
+
+ private final boolean reuse;
+
+ private ReuseSparkContext(boolean reuse) {
+ this.reuse = reuse;
+ }
+
+ public static ReuseSparkContext no() {
+ return new ReuseSparkContext(false);
+ }
+
+ public static ReuseSparkContext yes() {
+ return new ReuseSparkContext(true);
+ }
+
+ @Override
+ protected void before() throws Throwable {
+ System.setProperty(SparkContextFactory.TEST_REUSE_SPARK_CONTEXT, Boolean.toString(reuse));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/96d373fe/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
deleted file mode 100644
index 0a0abf9..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/WatermarkTest.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark;
-
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import org.apache.beam.runners.spark.io.CreateStream;
-import org.apache.beam.runners.spark.translation.SparkContextFactory;
-import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
-import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.RegexMatcher;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.joda.time.Duration;
-import org.joda.time.Instant;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-
-/**
- * A test suite for the propagation of watermarks in the Spark runner.
- */
-public class WatermarkTest {
-
- @Rule
- public ClearWatermarksRule clearWatermarksRule = new ClearWatermarksRule();
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @Rule
- public ReuseSparkContext reuseContext = ReuseSparkContext.yes();
-
- private static final SparkPipelineOptions options =
- PipelineOptionsFactory.create().as(SparkPipelineOptions.class);
-
- private static final String INSTANT_PATTERN =
- "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z";
-
- @Test
- public void testLowHighWatermarksAdvance() {
- JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
-
- Instant instant = new Instant(0);
- // low == high.
- GlobalWatermarkHolder.add(1,
- new SparkWatermarks(
- instant.plus(Duration.millis(5)),
- instant.plus(Duration.millis(5)),
- instant));
- GlobalWatermarkHolder.advance(jsc);
- // low < high.
- GlobalWatermarkHolder.add(1,
- new SparkWatermarks(
- instant.plus(Duration.millis(10)),
- instant.plus(Duration.millis(15)),
- instant.plus(Duration.millis(100))));
- GlobalWatermarkHolder.advance(jsc);
-
- // assert watermarks in Broadcast.
- SparkWatermarks currentWatermarks = GlobalWatermarkHolder.get().getValue().get(1);
- assertThat(currentWatermarks.getLowWatermark(), equalTo(instant.plus(Duration.millis(10))));
- assertThat(currentWatermarks.getHighWatermark(), equalTo(instant.plus(Duration.millis(15))));
- assertThat(currentWatermarks.getSynchronizedProcessingTime(),
- equalTo(instant.plus(Duration.millis(100))));
-
- // assert illegal watermark advance.
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage(
- RegexMatcher.matches(
- "Low watermark " + INSTANT_PATTERN + " cannot be later then high watermark "
- + INSTANT_PATTERN));
- // low > high -> not allowed!
- GlobalWatermarkHolder.add(1,
- new SparkWatermarks(
- instant.plus(Duration.millis(25)),
- instant.plus(Duration.millis(20)),
- instant.plus(Duration.millis(200))));
- GlobalWatermarkHolder.advance(jsc);
- }
-
- @Test
- public void testSynchronizedTimeMonotonic() {
- JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
-
- Instant instant = new Instant(0);
- GlobalWatermarkHolder.add(1,
- new SparkWatermarks(
- instant.plus(Duration.millis(5)),
- instant.plus(Duration.millis(10)),
- instant));
- GlobalWatermarkHolder.advance(jsc);
-
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage("Synchronized processing time must advance.");
- // no actual advancement of watermarks - fine by Watermarks
- // but not by synchronized processing time.
- GlobalWatermarkHolder.add(1,
- new SparkWatermarks(
- instant.plus(Duration.millis(5)),
- instant.plus(Duration.millis(10)),
- instant));
- GlobalWatermarkHolder.advance(jsc);
- }
-
- @Test
- public void testMultiSource() {
- JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
-
- Instant instant = new Instant(0);
- GlobalWatermarkHolder.add(1,
- new SparkWatermarks(
- instant.plus(Duration.millis(5)),
- instant.plus(Duration.millis(10)),
- instant));
- GlobalWatermarkHolder.add(2,
- new SparkWatermarks(
- instant.plus(Duration.millis(3)),
- instant.plus(Duration.millis(6)),
- instant));
-
- GlobalWatermarkHolder.advance(jsc);
-
- // assert watermarks for source 1.
- SparkWatermarks watermarksForSource1 = GlobalWatermarkHolder.get().getValue().get(1);
- assertThat(watermarksForSource1.getLowWatermark(), equalTo(instant.plus(Duration.millis(5))));
- assertThat(watermarksForSource1.getHighWatermark(), equalTo(instant.plus(Duration.millis(10))));
-
- // assert watermarks for source 2.
- SparkWatermarks watermarksForSource2 = GlobalWatermarkHolder.get().getValue().get(2);
- assertThat(watermarksForSource2.getLowWatermark(), equalTo(instant.plus(Duration.millis(3))));
- assertThat(watermarksForSource2.getHighWatermark(), equalTo(instant.plus(Duration.millis(6))));
- }
-
- @Test
- @Ignore(
- "BEAM-1526 - This test is flaky, and is expected to be fixed in "
- + "https://github.com/apache/beam/pull/2050")
- public void testInDoFn() {
- // because watermark advances onBatchCompleted.
- Iterable<Integer> zeroBatch = Collections.emptyList();
- Iterable<Integer> firstBatch = Collections.singletonList(1);
- Iterable<Integer> secondBatch = Collections.singletonList(2);
-
- Instant instant = new Instant(0);
- GlobalWatermarkHolder.add(1,
- new SparkWatermarks(
- instant.plus(Duration.millis(5)),
- instant.plus(Duration.millis(10)),
- instant));
- GlobalWatermarkHolder.add(1,
- new SparkWatermarks(
- instant.plus(Duration.millis(10)),
- instant.plus(Duration.millis(15)),
- instant.plus(options.getBatchIntervalMillis())));
-
- options.setRunner(SparkRunner.class);
- options.setStreaming(true);
- options.setBatchIntervalMillis(500L);
- Pipeline p = Pipeline.create(options);
-
- CreateStream.QueuedValues<Integer> queueStream =
- CreateStream.fromQueue(Arrays.asList(zeroBatch, firstBatch, secondBatch));
-
- p.apply(queueStream).setCoder(VarIntCoder.of()).apply(ParDo.of(new WatermarksDoFn(1)));
-
- p.run().waitUntilFinish(Duration.millis(options.getBatchIntervalMillis()).multipliedBy(3));
-
- // this is a hacky way to assert but it will do until triggers are supported.
- assertThat(
- WatermarksDoFn.strings,
- containsInAnyOrder(
- "element: 1 lowWatermark: 5 highWatermark: 10 processingTime: 0",
- "element: 2 lowWatermark: 10 highWatermark: 15 processingTime: 1000"));
- }
-
- private static class WatermarksDoFn extends DoFn<Integer, String> {
- private final int sourceId;
-
- static List<String> strings = new ArrayList<>();
-
- private WatermarksDoFn(int sourceId) {
- this.sourceId = sourceId;
- }
-
- @ProcessElement
- public void processElement(ProcessContext c) {
- if (GlobalWatermarkHolder.get() == null
- || GlobalWatermarkHolder.get().getValue().get(sourceId) == null) {
- // watermark not yet updated.
- return;
- }
- SparkWatermarks sparkWatermarks = GlobalWatermarkHolder.get().getValue().get(sourceId);
- Integer element = c.element();
- String output =
- "element: " + element
- + " lowWatermark: " + sparkWatermarks.getLowWatermark().getMillis()
- + " highWatermark: " + sparkWatermarks.getHighWatermark().getMillis()
- + " processingTime: " + sparkWatermarks.getSynchronizedProcessingTime().getMillis();
- strings.add(output);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/96d373fe/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
new file mode 100644
index 0000000..0cb33ab
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark.translation.streaming;
+
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.io.Serializable;
+import org.apache.beam.runners.spark.ReuseSparkContextRule;
+import org.apache.beam.runners.spark.SparkPipelineOptions;
+import org.apache.beam.runners.spark.io.CreateStream;
+import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.Sum;
+import org.apache.beam.sdk.transforms.Values;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.AfterPane;
+import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.Never;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
+
+
+/**
+ * A test suite to test Spark runner implementation of triggers and panes.
+ *
+ * <p>Since Spark is a micro-batch engine, and will process any test-sized input
+ * within the same (first) batch, it is important to make sure inputs are ingested across
+ * micro-batches using {@link org.apache.spark.streaming.dstream.QueueInputDStream}.
+ */
+public class CreateStreamTest implements Serializable {
+
+ @Rule
+ public transient TemporaryFolder checkpointParentDir = new TemporaryFolder();
+ @Rule
+ public transient SparkTestPipelineOptionsForStreaming commonOptions =
+ new SparkTestPipelineOptionsForStreaming();
+ @Rule
+ public transient ReuseSparkContextRule noContextResue = ReuseSparkContextRule.no();
+ @Rule
+ public transient TestName testName = new TestName();
+ @Rule
+ public transient ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void testLateDataAccumulating() throws IOException {
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
+ Pipeline p = Pipeline.create(options);
+ options.setJobName(testName.getMethodName());
+ Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
+
+ Instant instant = new Instant(0);
+ CreateStream<TimestampedValue<Integer>> source =
+ CreateStream.<TimestampedValue<Integer>>withBatchInterval(batchDuration)
+ .nextBatch()
+ .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(6)))
+ .nextBatch(
+ TimestampedValue.of(1, instant),
+ TimestampedValue.of(2, instant),
+ TimestampedValue.of(3, instant))
+ .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(20)))
+ // These elements are late but within the allowed lateness
+ .nextBatch(
+ TimestampedValue.of(4, instant),
+ TimestampedValue.of(5, instant))
+ // These elements are droppably late
+ .advanceNextBatchWatermarkToInfinity()
+ .nextBatch(
+ TimestampedValue.of(-1, instant),
+ TimestampedValue.of(-2, instant),
+ TimestampedValue.of(-3, instant));
+
+ PCollection<Integer> windowed = p
+ .apply(source).setCoder(TimestampedValue.TimestampedValueCoder.of(VarIntCoder.of()))
+ .apply(ParDo.of(new OnlyValue<Integer>()))
+ .apply(Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5))).triggering(
+ AfterWatermark.pastEndOfWindow()
+ .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
+ .plusDelayOf(Duration.standardMinutes(2)))
+ .withLateFirings(AfterPane.elementCountAtLeast(1)))
+ .accumulatingFiredPanes()
+ .withAllowedLateness(Duration.standardMinutes(5), Window.ClosingBehavior.FIRE_ALWAYS));
+ PCollection<Integer> triggered = windowed.apply(WithKeys.<Integer, Integer>of(1))
+ .apply(GroupByKey.<Integer, Integer>create())
+ .apply(Values.<Iterable<Integer>>create())
+ .apply(Flatten.<Integer>iterables());
+ PCollection<Long> count = windowed.apply(Count.<Integer>globally().withoutDefaults());
+ PCollection<Integer> sum = windowed.apply(Sum.integersGlobally().withoutDefaults());
+
+ IntervalWindow window = new IntervalWindow(instant, instant.plus(Duration.standardMinutes(5L)));
+ PAssert.that(triggered)
+ .inFinalPane(window)
+ .containsInAnyOrder(1, 2, 3, 4, 5);
+ PAssert.that(triggered)
+ .inOnTimePane(window)
+ .containsInAnyOrder(1, 2, 3);
+ PAssert.that(count)
+ .inWindow(window)
+ .satisfies(new SerializableFunction<Iterable<Long>, Void>() {
+ @Override
+ public Void apply(Iterable<Long> input) {
+ for (Long count : input) {
+ assertThat(count, allOf(greaterThanOrEqualTo(3L), lessThanOrEqualTo(5L)));
+ }
+ return null;
+ }
+ });
+ PAssert.that(sum)
+ .inWindow(window)
+ .satisfies(new SerializableFunction<Iterable<Integer>, Void>() {
+ @Override
+ public Void apply(Iterable<Integer> input) {
+ for (Integer sum : input) {
+ assertThat(sum, allOf(greaterThanOrEqualTo(6), lessThanOrEqualTo(15)));
+ }
+ return null;
+ }
+ });
+
+ p.run();
+ }
+
+// @Test
+// @Category({NeedsRunner.class, UsesTestStream.class})
+// public void testProcessingTimeTrigger() {
+// TestStream<Long> source = TestStream.create(VarLongCoder.of())
+// .addElements(TimestampedValue.of(1L, new Instant(1000L)),
+// TimestampedValue.of(2L, new Instant(2000L)))
+// .advanceProcessingTime(Duration.standardMinutes(12))
+// .addElements(TimestampedValue.of(3L, new Instant(3000L)))
+// .advanceProcessingTime(Duration.standardMinutes(6))
+// .advanceWatermarkToInfinity();
+//
+// PCollection<Long> sum = p.apply(source)
+// .apply(Window.<Long>triggering(AfterWatermark.pastEndOfWindow()
+// .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
+// .plusDelayOf(Duration.standardMinutes(5)))).accumulatingFiredPanes()
+// .withAllowedLateness(Duration.ZERO))
+// .apply(Sum.longsGlobally());
+//
+// PAssert.that(sum).inEarlyGlobalWindowPanes().containsInAnyOrder(3L, 6L);
+//
+// p.run();
+// }
+
+ @Test
+ public void testDiscardingMode() throws IOException {
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
+ Pipeline p = Pipeline.create(options);
+ options.setJobName(testName.getMethodName());
+ Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
+
+ CreateStream<TimestampedValue<String>> source =
+ CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
+ .nextBatch(
+ TimestampedValue.of("firstPane", new Instant(100)),
+ TimestampedValue.of("alsoFirstPane", new Instant(200)))
+ .advanceWatermarkForNextBatch(new Instant(1001L))
+ .nextBatch(
+ TimestampedValue.of("onTimePane", new Instant(500)))
+ .advanceNextBatchWatermarkToInfinity()
+ .nextBatch(
+ TimestampedValue.of("finalLatePane", new Instant(750)),
+ TimestampedValue.of("alsoFinalLatePane", new Instant(250)));
+
+ FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
+ Duration allowedLateness = Duration.millis(5000L);
+ PCollection<String> values =
+ p.apply(source).setCoder(TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of()))
+ .apply(ParDo.of(new OnlyValue<String>()))
+ .apply(
+ Window.<String>into(windowFn)
+ .triggering(
+ AfterWatermark.pastEndOfWindow()
+ .withEarlyFirings(AfterPane.elementCountAtLeast(2))
+ .withLateFirings(Never.ever()))
+ .discardingFiredPanes()
+ .withAllowedLateness(allowedLateness))
+ .apply(WithKeys.<Integer, String>of(1))
+ .apply(GroupByKey.<Integer, String>create())
+ .apply(Values.<Iterable<String>>create())
+ .apply(Flatten.<String>iterables());
+
+ IntervalWindow window = windowFn.assignWindow(new Instant(100));
+ PAssert.that(values)
+ .inWindow(window)
+ .containsInAnyOrder(
+ "firstPane", "alsoFirstPane", "onTimePane", "finalLatePane", "alsoFinalLatePane");
+ PAssert.that(values)
+ .inCombinedNonLatePanes(window)
+ .containsInAnyOrder("firstPane", "alsoFirstPane", "onTimePane");
+ PAssert.that(values).inOnTimePane(window).containsInAnyOrder("onTimePane");
+ PAssert.that(values)
+ .inFinalPane(window)
+ .containsInAnyOrder("finalLatePane", "alsoFinalLatePane");
+
+ p.run();
+ }
+
+ @Test
+ public void testFirstElementLate() throws IOException {
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
+ Pipeline p = Pipeline.create(options);
+ options.setJobName(testName.getMethodName());
+ Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
+
+ Instant lateElementTimestamp = new Instant(-1_000_000);
+ CreateStream<TimestampedValue<String>> source =
+ CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
+ .nextBatch()
+ .advanceWatermarkForNextBatch(new Instant(-1_000_000))
+ .nextBatch(
+ TimestampedValue.of("late", lateElementTimestamp),
+ TimestampedValue.of("onTime", new Instant(100)))
+ .advanceNextBatchWatermarkToInfinity();
+
+ FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
+ Duration allowedLateness = Duration.millis(5000L);
+ PCollection<String> values = p.apply(source)
+ .setCoder(TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of()))
+ .apply(ParDo.of(new OnlyValue<String>()))
+ .apply(Window.<String>into(windowFn).triggering(DefaultTrigger.of())
+ .discardingFiredPanes()
+ .withAllowedLateness(allowedLateness))
+ .apply(WithKeys.<Integer, String>of(1))
+ .apply(GroupByKey.<Integer, String>create())
+ .apply(Values.<Iterable<String>>create())
+ .apply(Flatten.<String>iterables());
+
+ //TODO: empty panes do not emmit anything so Spark won't evaluate an "empty" assertion.
+// PAssert.that(values).inWindow(windowFn.assignWindow(lateElementTimestamp)).empty();
+ PAssert.that(values)
+ .inWindow(windowFn.assignWindow(new Instant(100)))
+ .containsInAnyOrder("onTime");
+
+ p.run();
+ }
+
+ @Test
+ public void testElementsAtAlmostPositiveInfinity() throws IOException {
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
+ Pipeline p = Pipeline.create(options);
+ options.setJobName(testName.getMethodName());
+ Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
+
+ Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp();
+ CreateStream<TimestampedValue<String>> source =
+ CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
+ .nextBatch(
+ TimestampedValue.of("foo", endOfGlobalWindow),
+ TimestampedValue.of("bar", endOfGlobalWindow))
+ .advanceNextBatchWatermarkToInfinity();
+
+ FixedWindows windows = FixedWindows.of(Duration.standardHours(6));
+ PCollection<String> windowedValues = p.apply(source)
+ .setCoder(TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of()))
+ .apply(ParDo.of(new OnlyValue<String>()))
+ .apply(Window.<String>into(windows))
+ .apply(WithKeys.<Integer, String>of(1))
+ .apply(GroupByKey.<Integer, String>create())
+ .apply(Values.<Iterable<String>>create())
+ .apply(Flatten.<String>iterables());
+
+ PAssert.that(windowedValues)
+ .inWindow(windows.assignWindow(GlobalWindow.INSTANCE.maxTimestamp()))
+ .containsInAnyOrder("foo", "bar");
+ p.run();
+ }
+
+// @Test
+// public void testMultipleStreams() throws IOException {
+// SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
+// Pipeline p = Pipeline.create(options);
+// options.setJobName(testName.getMethodName());
+// Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
+//
+// CreateStream<String> source =
+// CreateStream.<String>withBatchInterval(batchDuration)
+// .nextBatch("foo", "bar").advanceWatermarkForNextBatch(new Instant(100))
+// .nextBatch().advanceNextBatchWatermarkToInfinity();
+//
+//// CreateStream<Integer> other =
+//// CreateStream.<Integer>withBatchInterval(batchDuration)
+//// .nextBatch(1, 2, 3, 4)
+//// .advanceNextBatchWatermarkToInfinity();
+//
+// PCollection<String> createStrings =
+// p.apply("CreateStrings", source).setCoder(StringUtf8Coder.of())
+// .apply("WindowStrings",
+// Window.<String>triggering(AfterPane.elementCountAtLeast(2))
+// .withAllowedLateness(Duration.ZERO)
+// .accumulatingFiredPanes());
+// PAssert.that(createStrings).containsInAnyOrder("foo", "bar");
+//// PCollection<Integer> createInts =
+//// p.apply("CreateInts", other).setCoder(VarIntCoder.of())
+//// .apply("WindowInts",
+//// Window.<Integer>triggering(AfterPane.elementCountAtLeast(4))
+//// .withAllowedLateness(Duration.ZERO)
+//// .accumulatingFiredPanes());
+//// PAssert.that(createInts).containsInAnyOrder(1, 2, 3, 4);
+//
+// p.run();
+// }
+
+ @Test
+ public void testElementAtPositiveInfinityThrows() {
+ Duration batchDuration = Duration.millis(commonOptions.getOptions().getBatchIntervalMillis());
+ CreateStream<TimestampedValue<Integer>> source =
+ CreateStream.<TimestampedValue<Integer>>withBatchInterval(batchDuration)
+ .nextBatch(TimestampedValue.of(-1, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L)));
+ thrown.expect(IllegalArgumentException.class);
+ source.nextBatch(TimestampedValue.of(1, BoundedWindow.TIMESTAMP_MAX_VALUE));
+ }
+
+ @Test
+ public void testAdvanceWatermarkNonMonotonicThrows() {
+ Duration batchDuration = Duration.millis(commonOptions.getOptions().getBatchIntervalMillis());
+ CreateStream<Integer> source =
+ CreateStream.<Integer>withBatchInterval(batchDuration)
+ .advanceWatermarkForNextBatch(new Instant(0L));
+ thrown.expect(IllegalArgumentException.class);
+ source.advanceWatermarkForNextBatch(new Instant(-1L));
+ }
+
+ @Test
+ public void testAdvanceWatermarkEqualToPositiveInfinityThrows() {
+ Duration batchDuration = Duration.millis(commonOptions.getOptions().getBatchIntervalMillis());
+ CreateStream<Integer> source =
+ CreateStream.<Integer>withBatchInterval(batchDuration)
+ .advanceWatermarkForNextBatch(BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L));
+ thrown.expect(IllegalArgumentException.class);
+ source.advanceWatermarkForNextBatch(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ }
+
+ private static class OnlyValue<T> extends DoFn<TimestampedValue<T>, T> {
+
+ OnlyValue() { }
+
+ @ProcessElement
+ public void onlyValue(ProcessContext c) {
+ c.outputWithTimestamp(c.element().getValue(), c.element().getTimestamp());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/96d373fe/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
index 62ee672..e307363 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
@@ -17,60 +17,70 @@
*/
package org.apache.beam.runners.spark.translation.streaming;
-import static org.apache.beam.sdk.metrics.MetricMatchers.attemptedMetricsResult;
+import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.Uninterruptibles;
import java.io.IOException;
-import java.util.Arrays;
import java.util.Collections;
-import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
+import org.apache.beam.runners.spark.ReuseSparkContextRule;
import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.SparkPipelineResult;
-import org.apache.beam.runners.spark.aggregators.ClearAggregatorsRule;
+import org.apache.beam.runners.spark.SparkRunner;
+import org.apache.beam.runners.spark.aggregators.AccumulatorSingleton;
+import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.translation.streaming.utils.EmbeddedKafkaCluster;
-import org.apache.beam.runners.spark.translation.streaming.utils.PAssertStreaming;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.InstantCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.kafka.KafkaIO;
-import org.apache.beam.sdk.metrics.Counter;
-import org.apache.beam.sdk.metrics.MetricNameFilter;
-import org.apache.beam.sdk.metrics.Metrics;
-import org.apache.beam.sdk.metrics.MetricsFilter;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.Keys;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.Sum;
-import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.Values;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PDone;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.Serializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.joda.time.Duration;
+import org.joda.time.Instant;
import org.junit.AfterClass;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
/**
- * Test pipelines which are resumed from checkpoint.
+ * Tests DStream recovery from checkpoint.
+ *
+ * <p>Runs the pipeline reading from a Kafka backlog with a WM function that will move to infinity
+ * on a EOF signal.
+ * After resuming from checkpoint, a single output (guaranteed by the WM) is asserted, along with
+ * {@link Aggregator}s values that are expected to resume from previous count as well.
*/
public class ResumeFromCheckpointStreamingTest {
private static final EmbeddedKafkaCluster.EmbeddedZookeeper EMBEDDED_ZOOKEEPER =
@@ -78,142 +88,158 @@ public class ResumeFromCheckpointStreamingTest {
private static final EmbeddedKafkaCluster EMBEDDED_KAFKA_CLUSTER =
new EmbeddedKafkaCluster(EMBEDDED_ZOOKEEPER.getConnection(), new Properties());
private static final String TOPIC = "kafka_beam_test_topic";
- private static final Map<String, String> KAFKA_MESSAGES = ImmutableMap.of(
- "k1", "v1", "k2", "v2", "k3", "v3", "k4", "v4"
- );
- private static final String[] EXPECTED = {"k1,v1", "k2,v2", "k3,v3", "k4,v4"};
- private static final long EXPECTED_AGG_FIRST = 4L;
- private static final long EXPECTED_COUNTER_FIRST = 4L;
- private static final long EXPECTED_AGG_SECOND = 8L;
- private static final long EXPECTED_COUNTER_SECOND = 8L;
@Rule
- public TemporaryFolder checkpointParentDir = new TemporaryFolder();
-
+ public TemporaryFolder tmpFolder = new TemporaryFolder();
@Rule
- public SparkTestPipelineOptionsForStreaming commonOptions =
- new SparkTestPipelineOptionsForStreaming();
-
+ public ReuseSparkContextRule noContextResue = ReuseSparkContextRule.no();
@Rule
- public ClearAggregatorsRule clearAggregatorsRule = new ClearAggregatorsRule();
+ public transient TestName testName = new TestName();
@BeforeClass
public static void init() throws IOException {
EMBEDDED_ZOOKEEPER.startup();
EMBEDDED_KAFKA_CLUSTER.startup();
- /// this test actually requires to NOT reuse the context but rather to stop it and start again
- // from the checkpoint with a brand new context.
- System.setProperty("beam.spark.test.reuseSparkContext", "false");
}
- private static void produce() {
+ private static void produce(Map<String, Instant> messages) {
Properties producerProps = new Properties();
producerProps.putAll(EMBEDDED_KAFKA_CLUSTER.getProps());
producerProps.put("request.required.acks", 1);
producerProps.put("bootstrap.servers", EMBEDDED_KAFKA_CLUSTER.getBrokerList());
Serializer<String> stringSerializer = new StringSerializer();
- try (@SuppressWarnings("unchecked") KafkaProducer<String, String> kafkaProducer =
- new KafkaProducer(producerProps, stringSerializer, stringSerializer)) {
- for (Map.Entry<String, String> en : KAFKA_MESSAGES.entrySet()) {
+ Serializer<Instant> instantSerializer = new Serializer<Instant>() {
+ @Override
+ public void configure(Map<String, ?> configs, boolean isKey) { }
+
+ @Override
+ public byte[] serialize(String topic, Instant data) {
+ return CoderHelpers.toByteArray(data, InstantCoder.of());
+ }
+
+ @Override
+ public void close() { }
+ };
+
+ try (@SuppressWarnings("unchecked") KafkaProducer<String, Instant> kafkaProducer =
+ new KafkaProducer(producerProps, stringSerializer, instantSerializer)) {
+ for (Map.Entry<String, Instant> en : messages.entrySet()) {
kafkaProducer.send(new ProducerRecord<>(TOPIC, en.getKey(), en.getValue()));
}
kafkaProducer.close();
}
}
- /**
- * Tests DStream recovery from checkpoint - recreate the job and continue (from checkpoint).
- * <p>Also tests Aggregator values, which should be restored upon recovery from checkpoint.</p>
- */
@Test
- public void testRun() throws Exception {
- Duration batchIntervalDuration = Duration.standardSeconds(5);
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- // provide a generous enough batch-interval to have everything fit in one micro-batch.
- options.setBatchIntervalMillis(batchIntervalDuration.getMillis());
- // provide a very generous read time bound, we rely on num records bound here.
- options.setMinReadTimeMillis(batchIntervalDuration.minus(1).getMillis());
- // bound the read on the number of messages - 1 topic of 4 messages.
- options.setMaxRecordsPerBatch(4L);
-
- // checkpoint after first (and only) interval.
- options.setCheckpointDurationMillis(options.getBatchIntervalMillis());
-
- MetricsFilter metricsFilter =
- MetricsFilter.builder()
- .addNameFilter(MetricNameFilter.inNamespace(ResumeFromCheckpointStreamingTest.class))
- .build();
+ public void testWithResume() throws Exception {
+ SparkPipelineOptions options = PipelineOptionsFactory.create().as(SparkPipelineOptions.class);
+ options.setRunner(SparkRunner.class);
+ options.setCheckpointDir(tmpFolder.newFolder().toString());
+ options.setCheckpointDurationMillis(500L);
+ options.setJobName(testName.getMethodName());
+ options.setSparkMaster("local[*]");
+
+ // write to Kafka
+ produce(ImmutableMap.of(
+ "k1", new Instant(100),
+ "k2", new Instant(200),
+ "k3", new Instant(300),
+ "k4", new Instant(400)
+ ));
// first run will read from Kafka backlog - "auto.offset.reset=smallest"
SparkPipelineResult res = run(options);
+ res.waitUntilFinish(Duration.standardSeconds(2));
+ // assertions 1:
long processedMessages1 = res.getAggregatorValue("processedMessages", Long.class);
- assertThat(String.format("Expected %d processed messages count but "
- + "found %d", EXPECTED_AGG_FIRST, processedMessages1), processedMessages1,
- equalTo(EXPECTED_AGG_FIRST));
- assertThat(res.metrics().queryMetrics(metricsFilter).counters(),
- hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(),
- "aCounter", "formatKV", EXPECTED_COUNTER_FIRST)));
+ assertThat(
+ String.format(
+ "Expected %d processed messages count but found %d", 4, processedMessages1),
+ processedMessages1,
+ equalTo(4L));
+
+ //--- between executions:
+
+ //- clear state.
+ AccumulatorSingleton.clear();
+ GlobalWatermarkHolder.clear();
+
+ //- write a bit more.
+ produce(ImmutableMap.of(
+ "k5", new Instant(499),
+ "EOF", new Instant(500) // to be dropped from [0, 500).
+ ));
// recovery should resume from last read offset, and read the second batch of input.
res = runAgain(options);
+ res.waitUntilFinish(Duration.standardSeconds(2));
+ // assertions 2:
long processedMessages2 = res.getAggregatorValue("processedMessages", Long.class);
- assertThat(String.format("Expected %d processed messages count but "
- + "found %d", EXPECTED_AGG_SECOND, processedMessages2), processedMessages2,
- equalTo(EXPECTED_AGG_SECOND));
- assertThat(res.metrics().queryMetrics(metricsFilter).counters(),
- hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(),
- "aCounter", "formatKV", EXPECTED_COUNTER_SECOND)));
+ int successAssertions = res.getAggregatorValue(PAssert.SUCCESS_COUNTER, Integer.class);
+ assertThat(
+ String.format("Expected %d processed messages count but found %d", 1, processedMessages2),
+ processedMessages2,
+ equalTo(5L));
+ res.getAggregatorValue(PAssert.SUCCESS_COUNTER, Integer.class);
+ assertThat(
+ String.format(
+ "Expected %d successful assertions, but found %d.", 1, successAssertions),
+ successAssertions,
+ is(1));
+ // validate assertion didn't fail.
+ int failedAssertions = res.getAggregatorValue(PAssert.FAILURE_COUNTER, Integer.class);
+ assertThat(
+ String.format("Found %d failed assertions.", failedAssertions),
+ failedAssertions,
+ is(0));
+
}
private SparkPipelineResult runAgain(SparkPipelineOptions options) {
- clearAggregatorsRule.clearNamedAggregators();
// sleep before next run.
- Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
return run(options);
}
private static SparkPipelineResult run(SparkPipelineOptions options) {
- // write to Kafka
- produce();
- Map<String, Object> consumerProps = ImmutableMap.<String, Object>of(
- "auto.offset.reset", "earliest"
- );
-
- KafkaIO.Read<String, String> read = KafkaIO.<String, String>read()
+ KafkaIO.Read<String, Instant> read = KafkaIO.<String, Instant>read()
.withBootstrapServers(EMBEDDED_KAFKA_CLUSTER.getBrokerList())
.withTopics(Collections.singletonList(TOPIC))
.withKeyCoder(StringUtf8Coder.of())
- .withValueCoder(StringUtf8Coder.of())
- .updateConsumerProperties(consumerProps);
-
- Duration windowDuration = new Duration(options.getBatchIntervalMillis());
+ .withValueCoder(InstantCoder.of())
+ .updateConsumerProperties(ImmutableMap.<String, Object>of("auto.offset.reset", "earliest"))
+ .withTimestampFn(new SerializableFunction<KV<String, Instant>, Instant>() {
+ @Override
+ public Instant apply(KV<String, Instant> kv) {
+ return kv.getValue();
+ }
+ }).withWatermarkFn(new SerializableFunction<KV<String, Instant>, Instant>() {
+ @Override
+ public Instant apply(KV<String, Instant> kv) {
+ // at EOF move WM to infinity.
+ String key = kv.getKey();
+ Instant instant = kv.getValue();
+ return key.equals("EOF") ? BoundedWindow.TIMESTAMP_MAX_VALUE : instant;
+ }
+ });
Pipeline p = Pipeline.create(options);
- PCollection<String> expectedCol =
- p.apply(Create.of(ImmutableList.copyOf(EXPECTED)).withCoder(StringUtf8Coder.of()));
- final PCollectionView<List<String>> expectedView = expectedCol.apply(View.<String>asList());
-
- PCollection<String> formattedKV =
- p.apply(read.withoutMetadata())
- .apply("formatKV", ParDo.of(new DoFn<KV<String, String>, KV<String, String>>() {
- Counter counter =
- Metrics.counter(ResumeFromCheckpointStreamingTest.class, "aCounter");
-
- @ProcessElement
- public void process(ProcessContext c) {
- // Check side input is passed correctly also after resuming from checkpoint
- Assert.assertEquals(c.sideInput(expectedView), Arrays.asList(EXPECTED));
- counter.inc();
- c.output(c.element());
- }
- }).withSideInputs(expectedView))
- .apply(Window.<KV<String, String>>into(FixedWindows.of(windowDuration)))
- .apply(ParDo.of(new FormatAsText()));
-
- // graceful shutdown will make sure first batch (at least) will finish.
- Duration timeout = Duration.standardSeconds(1L);
- return PAssertStreaming.runAndAssertContents(p, formattedKV, EXPECTED, timeout);
+ PCollection<Iterable<String>> grouped = p
+ .apply(read.withoutMetadata())
+ .apply(Keys.<String>create())
+ .apply(ParDo.of(new EOFShallNotPassFn()))
+ .apply(Window.<String>into(FixedWindows.of(Duration.millis(500)))
+ .triggering(AfterWatermark.pastEndOfWindow())
+ .accumulatingFiredPanes()
+ .withAllowedLateness(Duration.ZERO))
+ .apply(WithKeys.<Integer, String>of(1))
+ .apply(GroupByKey.<Integer, String>create())
+ .apply(Values.<Iterable<String>>create());
+
+ grouped.apply(new PAssertWithoutFlatten<>("k1", "k2", "k3", "k4", "k5"));
+
+ return (SparkPipelineResult) p.run();
}
@AfterClass
@@ -222,16 +248,60 @@ public class ResumeFromCheckpointStreamingTest {
EMBEDDED_ZOOKEEPER.shutdown();
}
- private static class FormatAsText extends DoFn<KV<String, String>, String> {
-
+ /** A pass-through fn that prevents EOF event from passing. */
+ private static class EOFShallNotPassFn extends DoFn<String, String> {
private final Aggregator<Long, Long> aggregator =
createAggregator("processedMessages", Sum.ofLongs());
@ProcessElement
public void process(ProcessContext c) {
- aggregator.addValue(1L);
- String formatted = c.element().getKey() + "," + c.element().getValue();
- c.output(formatted);
+ String element = c.element();
+ if (!element.equals("EOF")) {
+ aggregator.addValue(1L);
+ c.output(c.element());
+ }
+ }
+ }
+
+ /**
+ * A custom PAssert that avoids using {@link org.apache.beam.sdk.transforms.Flatten}
+ * until BEAM-1444 is resolved.
+ */
+ private static class PAssertWithoutFlatten<T>
+ extends PTransform<PCollection<Iterable<T>>, PDone> {
+ private final T[] expected;
+
+ private PAssertWithoutFlatten(T... expected) {
+ this.expected = expected;
+ }
+
+ @Override
+ public PDone expand(PCollection<Iterable<T>> input) {
+ input.apply(ParDo.of(new AssertDoFn<>(expected)));
+ return PDone.in(input.getPipeline());
+ }
+
+ private static class AssertDoFn<T> extends DoFn<Iterable<T>, Void> {
+ private final Aggregator<Integer, Integer> success =
+ createAggregator(PAssert.SUCCESS_COUNTER, Sum.ofIntegers());
+ private final Aggregator<Integer, Integer> failure =
+ createAggregator(PAssert.FAILURE_COUNTER, Sum.ofIntegers());
+ private final T[] expected;
+
+ AssertDoFn(T[] expected) {
+ this.expected = expected;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ try {
+ assertThat(c.element(), containsInAnyOrder(expected));
+ success.addValue(1);
+ } catch (Throwable t) {
+ failure.addValue(1);
+ throw t;
+ }
+ }
}
}
[04/23] beam git commit: Use TestSparkRunner in tests.
Posted by am...@apache.org.
Use TestSparkRunner in tests.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2bcd40c2
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2bcd40c2
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2bcd40c2
Branch: refs/heads/master
Commit: 2bcd40c206d6b615135375276b249364fedf32af
Parents: c25a02f
Author: Sela <an...@paypal.com>
Authored: Sat Feb 18 22:10:50 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:17:59 2017 +0200
----------------------------------------------------------------------
.../beam/runners/spark/TestSparkRunner.java | 92 ++++++++++++--------
.../utils/SparkTestPipelineOptions.java | 4 +-
.../SparkTestPipelineOptionsForStreaming.java | 11 +++
3 files changed, 69 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/2bcd40c2/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index 8b8f9ba..24bc038 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -21,12 +21,12 @@ package org.apache.beam.runners.spark;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
+import java.io.File;
+import java.io.IOException;
import org.apache.beam.runners.core.UnboundedReadFromBoundedSource;
-import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
-import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
+import org.apache.beam.runners.spark.aggregators.AccumulatorSingleton;
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.PipelineResult.State;
import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
@@ -40,6 +40,10 @@ import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
+import org.apache.commons.io.FileUtils;
+import org.joda.time.Duration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -65,6 +69,8 @@ import org.apache.beam.sdk.values.POutput;
*/
public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
+ private static final Logger LOG = LoggerFactory.getLogger(TestSparkRunner.class);
+
private SparkRunner delegate;
private boolean isForceStreaming;
private int expectedNumberOfAssertions = 0;
@@ -87,7 +93,7 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
@SuppressWarnings("unchecked")
@Override
public <OutputT extends POutput, InputT extends PInput> OutputT apply(
- PTransform<InputT, OutputT> transform, InputT input) {
+ PTransform<InputT, OutputT> transform, InputT input) {
// if the pipeline forces execution as a streaming pipeline,
// and the source is an adapted unbounded source (as bounded),
// read it as unbounded source via UnboundedReadFromBoundedSource.
@@ -108,38 +114,52 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
@Override
public SparkPipelineResult run(Pipeline pipeline) {
- // clear state of Aggregators, Metrics and Watermarks if exists.
- AggregatorsAccumulator.clear();
- SparkMetricsContainer.clear();
- GlobalWatermarkHolder.clear();
-
- TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
- SparkPipelineResult result = delegate.run(pipeline);
- result.waitUntilFinish();
-
-
- // make sure the test pipeline finished successfully.
- State resultState = result.getState();
- assertThat(
- String.format("Test pipeline result state was %s instead of %s", resultState, State.DONE),
- resultState,
- is(State.DONE));
- assertThat(result, testPipelineOptions.getOnCreateMatcher());
- assertThat(result, testPipelineOptions.getOnSuccessMatcher());
-
- // if the pipeline was executed in streaming mode, validate aggregators.
- if (isForceStreaming) {
- // validate assertion succeeded (at least once).
- int success = result.getAggregatorValue(PAssert.SUCCESS_COUNTER, Integer.class);
- assertThat(
- String.format(
- "Expected %d successful assertions, but found %d.",
- expectedNumberOfAssertions, success),
- success,
- is(expectedNumberOfAssertions));
- // validate assertion didn't fail.
- int failure = result.getAggregatorValue(PAssert.FAILURE_COUNTER, Integer.class);
- assertThat("Failure aggregator should be zero.", failure, is(0));
+ SparkPipelineOptions sparkOptions = pipeline.getOptions().as(SparkPipelineOptions.class);
+ long timeout = sparkOptions.getForcedTimeout();
+ SparkPipelineResult result = null;
+ try {
+ // clear state of Accumulators and Aggregators.
+ AccumulatorSingleton.clear();
+ GlobalWatermarkHolder.clear();
+
+ TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
+ LOG.info("About to run test pipeline " + sparkOptions.getJobName());
+ result = delegate.run(pipeline);
+ result.waitUntilFinish(Duration.millis(timeout));
+
+ assertThat(result, testPipelineOptions.getOnCreateMatcher());
+ assertThat(result, testPipelineOptions.getOnSuccessMatcher());
+
+ // if the pipeline was executed in streaming mode, validate aggregators.
+ if (isForceStreaming) {
+ // validate assertion succeeded (at least once).
+ int successAssertions = result.getAggregatorValue(PAssert.SUCCESS_COUNTER, Integer.class);
+ assertThat(
+ String.format(
+ "Expected %d successful assertions, but found %d.",
+ expectedNumberOfAssertions, successAssertions),
+ successAssertions,
+ is(expectedNumberOfAssertions));
+ // validate assertion didn't fail.
+ int failedAssertions = result.getAggregatorValue(PAssert.FAILURE_COUNTER, Integer.class);
+ assertThat(
+ String.format("Found %d failed assertions.", failedAssertions),
+ failedAssertions,
+ is(0));
+
+ LOG.info(
+ String.format(
+ "Successfully asserted pipeline %s with %d successful assertions.",
+ sparkOptions.getJobName(),
+ successAssertions));
+ }
+ } finally {
+ try {
+ // cleanup checkpoint dir.
+ FileUtils.deleteDirectory(new File(sparkOptions.getCheckpointDir()));
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to clear checkpoint tmp dir.", e);
+ }
}
return result;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/2bcd40c2/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptions.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptions.java
index 2da9888..efc17d3 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptions.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptions.java
@@ -18,7 +18,7 @@
package org.apache.beam.runners.spark.translation.streaming.utils;
import org.apache.beam.runners.spark.SparkPipelineOptions;
-import org.apache.beam.runners.spark.SparkRunner;
+import org.apache.beam.runners.spark.TestSparkRunner;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.junit.rules.ExternalResource;
@@ -32,7 +32,7 @@ public class SparkTestPipelineOptions extends ExternalResource {
@Override
protected void before() throws Throwable {
- options.setRunner(SparkRunner.class);
+ options.setRunner(TestSparkRunner.class);
options.setEnableSparkMetricSinks(false);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/2bcd40c2/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptionsForStreaming.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptionsForStreaming.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptionsForStreaming.java
index 28f6d5d..dd3e4c8 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptionsForStreaming.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptionsForStreaming.java
@@ -20,6 +20,7 @@ package org.apache.beam.runners.spark.translation.streaming.utils;
import java.io.IOException;
import org.apache.beam.runners.spark.SparkPipelineOptions;
+import org.joda.time.Duration;
import org.junit.rules.TemporaryFolder;
@@ -28,10 +29,20 @@ import org.junit.rules.TemporaryFolder;
*/
public class SparkTestPipelineOptionsForStreaming extends SparkTestPipelineOptions {
+ private static final int DEFAULT_NUMBER_OF_BATCHES_TIMEOUT = 5;
+
public SparkPipelineOptions withTmpCheckpointDir(TemporaryFolder parent)
throws IOException {
// tests use JUnit's TemporaryFolder path in the form of: /.../junit/...
options.setCheckpointDir(parent.newFolder(options.getJobName()).toURI().toURL().toString());
+ options.setForceStreaming(true);
+ // set the default timeout to DEFAULT_NUMBER_OF_BATCHES_TIMEOUT x batchDuration
+ // to allow pipelines to finish.
+ Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
+ // set the checkpoint duration to match interval.
+ options.setCheckpointDurationMillis(batchDuration.getMillis());
+ long forcedTimeout = batchDuration.multipliedBy(DEFAULT_NUMBER_OF_BATCHES_TIMEOUT).getMillis();
+ options.setForcedTimeout(forcedTimeout);
return options;
}
}
[05/23] beam git commit: Refactor translators according to new
GroupAlsoByWindow implemenation for the Spark runnner.
Posted by am...@apache.org.
Refactor translators according to new GroupAlsoByWindow implemenation for the Spark runnner.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/96abe4f0
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/96abe4f0
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/96abe4f0
Branch: refs/heads/master
Commit: 96abe4f08be12ac10dac39b55b8f8319a227b1ea
Parents: 8c37970
Author: Sela <an...@paypal.com>
Authored: Fri Feb 17 01:19:23 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:17:59 2017 +0200
----------------------------------------------------------------------
.../SparkGroupAlsoByWindowViaWindowSet.java | 14 +-
.../spark/stateful/SparkStateInternals.java | 4 +-
.../spark/stateful/SparkTimerInternals.java | 6 +-
.../translation/GroupCombineFunctions.java | 237 ++++++------------
.../spark/translation/TransformTranslator.java | 238 +++++++++++++------
.../spark/translation/TranslationUtils.java | 22 +-
.../streaming/StreamingTransformTranslator.java | 163 ++++---------
.../translation/streaming/UnboundedDataset.java | 12 +-
.../beam/runners/spark/util/LateDataUtils.java | 2 +-
.../spark/util/UnsupportedSideInputReader.java | 2 +-
.../streaming/TrackStreamingSourcesTest.java | 2 +-
11 files changed, 314 insertions(+), 388 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 7902d7c..2fb4100 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -17,8 +17,8 @@
*/
package org.apache.beam.runners.spark.stateful;
-import com.google.common.collect.Table;
import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Table;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
@@ -84,7 +84,8 @@ import scala.runtime.AbstractFunction1;
* in the following steps.
*/
public class SparkGroupAlsoByWindowViaWindowSet {
- private static final Logger LOG = LoggerFactory.getLogger(SparkGroupAlsoByWindowViaWindowSet.class);
+ private static final Logger LOG = LoggerFactory.getLogger(
+ SparkGroupAlsoByWindowViaWindowSet.class);
/**
* A helper class that is essentially a {@link Serializable} {@link AbstractFunction1}.
@@ -101,7 +102,7 @@ public class SparkGroupAlsoByWindowViaWindowSet {
final SparkRuntimeContext runtimeContext,
final List<Integer> sourceIds) {
- Long checkpointDuration =
+ long checkpointDurationMillis =
runtimeContext.getPipelineOptions().as(SparkPipelineOptions.class)
.getCheckpointDurationMillis();
@@ -271,8 +272,11 @@ public class SparkGroupAlsoByWindowViaWindowSet {
return scala.collection.JavaConversions.asScalaIterator(outIter);
}
}, partitioner, true, JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>>fakeClassTag())
- .checkpoint(new Duration(checkpointDuration));
+ List<WindowedValue<KV<K, Iterable<InputT>>>>>>fakeClassTag());
+
+ if (checkpointDurationMillis > 0) {
+ firedStream.checkpoint(new Duration(checkpointDurationMillis));
+ }
// go back to Java now.
JavaPairDStream<K, Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
index e628d31..93b1f63 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
@@ -22,11 +22,11 @@ import com.google.common.collect.Table;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.core.StateInternals;
import org.apache.beam.runners.core.StateNamespace;
import org.apache.beam.runners.core.StateTag;
import org.apache.beam.runners.core.StateTag.StateBinder;
+import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.InstantCoder;
import org.apache.beam.sdk.coders.ListCoder;
@@ -399,4 +399,4 @@ class SparkStateInternals<K> implements StateInternals<K> {
};
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
index 65225c5..4072240 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
@@ -27,9 +27,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
-import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
import org.apache.beam.runners.core.StateNamespace;
import org.apache.beam.runners.core.TimerInternals;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.TimeDomain;
import org.apache.spark.broadcast.Broadcast;
@@ -145,7 +145,7 @@ class SparkTimerInternals implements TimerInternals {
return inputWatermark;
}
- /** Advances the watermark - since */
+ /** Advances the watermark. */
public void advanceWatermark() {
inputWatermark = highWatermark;
}
@@ -170,4 +170,4 @@ class SparkTimerInternals implements TimerInternals {
throw new UnsupportedOperationException("Deleting a timer by ID is not yet supported.");
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
index 8a41b4e..1e879ce 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
@@ -18,32 +18,21 @@
package org.apache.beam.runners.spark.translation;
-import com.google.common.collect.Lists;
-import java.util.Collections;
-import java.util.Map;
-import org.apache.beam.runners.core.SystemReduceFn;
-import org.apache.beam.runners.spark.aggregators.NamedAggregators;
+import static com.google.common.base.Preconditions.checkArgument;
+
import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.util.ByteArray;
-import org.apache.beam.runners.spark.util.SideInputBroadcast;
-import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.transforms.CombineWithContext;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowedValue.WindowedValueCoder;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.spark.Accumulator;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
-import org.apache.spark.api.java.function.PairFlatMapFunction;
-import scala.Tuple2;
@@ -53,113 +42,71 @@ import scala.Tuple2;
public class GroupCombineFunctions {
/**
- * Apply {@link org.apache.beam.sdk.transforms.GroupByKey} to a Spark RDD.
+ * An implementation of
+ * {@link org.apache.beam.runners.core.GroupByKeyViaGroupByKeyOnly.GroupByKeyOnly}
+ * for the Spark runner.
*/
- public static <K, V, W extends BoundedWindow> JavaRDD<WindowedValue<KV<K,
- Iterable<V>>>> groupByKey(JavaRDD<WindowedValue<KV<K, V>>> rdd,
- Accumulator<NamedAggregators> accum,
- KvCoder<K, V> coder,
- SparkRuntimeContext runtimeContext,
- WindowingStrategy<?, W> windowingStrategy) {
- //--- coders.
- final Coder<K> keyCoder = coder.getKeyCoder();
- final Coder<V> valueCoder = coder.getValueCoder();
- final WindowedValue.WindowedValueCoder<V> wvCoder = WindowedValue.FullWindowedValueCoder.of(
- valueCoder, windowingStrategy.getWindowFn().windowCoder());
+ public static <K, V> JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<V>>>>> groupByKeyOnly(
+ JavaRDD<WindowedValue<KV<K, V>>> rdd,
+ Coder<K> keyCoder,
+ WindowedValueCoder<V> wvCoder) {
- //--- groupByKey.
// Use coders to convert objects in the PCollection to byte arrays, so they
// can be transferred over the network for the shuffle.
- JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<V>>>>> groupedByKey =
- rdd.map(new ReifyTimestampsAndWindowsFunction<K, V>())
- .map(WindowingHelpers.<KV<K, WindowedValue<V>>>unwindowFunction())
- .mapToPair(TranslationUtils.<K, WindowedValue<V>>toPairFunction())
- .mapToPair(CoderHelpers.toByteFunction(keyCoder, wvCoder))
- .groupByKey()
- .mapToPair(CoderHelpers.fromByteFunctionIterable(keyCoder, wvCoder))
- // empty windows are OK here, see GroupByKey#evaluateHelper in the SDK
- .map(TranslationUtils.<K, Iterable<WindowedValue<V>>>fromPairFunction())
- .map(WindowingHelpers.<KV<K, Iterable<WindowedValue<V>>>>windowFunction());
-
- //--- now group also by window.
- // GroupAlsoByWindow currently uses a dummy in-memory StateInternals
- return groupedByKey.flatMap(
- new SparkGroupAlsoByWindowViaOutputBufferFn<>(
- windowingStrategy,
- new TranslationUtils.InMemoryStateInternalsFactory<K>(),
- SystemReduceFn.<K, V, W>buffering(valueCoder),
- runtimeContext,
- accum));
+ return rdd
+ .map(new ReifyTimestampsAndWindowsFunction<K, V>())
+ .map(WindowingHelpers.<KV<K, WindowedValue<V>>>unwindowFunction())
+ .mapToPair(TranslationUtils.<K, WindowedValue<V>>toPairFunction())
+ .mapToPair(CoderHelpers.toByteFunction(keyCoder, wvCoder))
+ .groupByKey()
+ .mapToPair(CoderHelpers.fromByteFunctionIterable(keyCoder, wvCoder))
+ .map(TranslationUtils.<K, Iterable<WindowedValue<V>>>fromPairFunction())
+ .map(WindowingHelpers.<KV<K, Iterable<WindowedValue<V>>>>windowFunction());
}
/**
* Apply a composite {@link org.apache.beam.sdk.transforms.Combine.Globally} transformation.
*/
- public static <InputT, AccumT, OutputT> JavaRDD<WindowedValue<OutputT>>
- combineGlobally(JavaRDD<WindowedValue<InputT>> rdd,
- final CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT> combineFn,
- final Coder<InputT> iCoder,
- final Coder<OutputT> oCoder,
- final SparkRuntimeContext runtimeContext,
- final WindowingStrategy<?, ?> windowingStrategy,
- final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>>
- sideInputs,
- boolean hasDefault) {
- // handle empty input RDD, which will natively skip the entire execution as Spark will not
- // run on empty RDDs.
- if (rdd.isEmpty()) {
- JavaSparkContext jsc = new JavaSparkContext(rdd.context());
- if (hasDefault) {
- OutputT defaultValue = combineFn.defaultValue();
- return jsc
- .parallelize(Lists.newArrayList(CoderHelpers.toByteArray(defaultValue, oCoder)))
- .map(CoderHelpers.fromByteFunction(oCoder))
- .map(WindowingHelpers.<OutputT>windowFunction());
- } else {
- return jsc.emptyRDD();
- }
- }
-
- //--- coders.
- final Coder<AccumT> aCoder;
- try {
- aCoder = combineFn.getAccumulatorCoder(runtimeContext.getCoderRegistry(), iCoder);
- } catch (CannotProvideCoderException e) {
- throw new IllegalStateException("Could not determine coder for accumulator", e);
- }
- // windowed coders.
+ public static <InputT, AccumT> Iterable<WindowedValue<AccumT>> combineGlobally(
+ JavaRDD<WindowedValue<InputT>> rdd,
+ final SparkGlobalCombineFn<InputT, AccumT, ?> sparkCombineFn,
+ final Coder<InputT> iCoder,
+ final Coder<AccumT> aCoder,
+ final WindowingStrategy<?, ?> windowingStrategy) {
+ checkArgument(!rdd.isEmpty(), "CombineGlobally computation should be skipped for empty RDDs.");
+
+ // coders.
final WindowedValue.FullWindowedValueCoder<InputT> wviCoder =
WindowedValue.FullWindowedValueCoder.of(iCoder,
windowingStrategy.getWindowFn().windowCoder());
final WindowedValue.FullWindowedValueCoder<AccumT> wvaCoder =
WindowedValue.FullWindowedValueCoder.of(aCoder,
windowingStrategy.getWindowFn().windowCoder());
- final WindowedValue.FullWindowedValueCoder<OutputT> wvoCoder =
- WindowedValue.FullWindowedValueCoder.of(oCoder,
- windowingStrategy.getWindowFn().windowCoder());
-
- final SparkGlobalCombineFn<InputT, AccumT, OutputT> sparkCombineFn =
- new SparkGlobalCombineFn<>(combineFn, runtimeContext, sideInputs, windowingStrategy);
final IterableCoder<WindowedValue<AccumT>> iterAccumCoder = IterableCoder.of(wvaCoder);
-
// Use coders to convert objects in the PCollection to byte arrays, so they
// can be transferred over the network for the shuffle.
- JavaRDD<byte[]> inRddBytes = rdd.map(CoderHelpers.toByteFunction(wviCoder));
- /*AccumT*/ byte[] acc = inRddBytes.aggregate(
+ // for readability, we add comments with actual type next to byte[].
+ // to shorten line length, we use:
+ //---- WV: WindowedValue
+ //---- Iterable: Itr
+ //---- AccumT: A
+ //---- InputT: I
+ JavaRDD<byte[]> inputRDDBytes = rdd.map(CoderHelpers.toByteFunction(wviCoder));
+ /*Itr<WV<A>>*/ byte[] accumulatedBytes = inputRDDBytes.aggregate(
CoderHelpers.toByteArray(sparkCombineFn.zeroValue(), iterAccumCoder),
- new Function2</*AccumT*/ byte[], /*InputT*/ byte[], /*AccumT*/ byte[]>() {
+ new Function2</*A*/ byte[], /*I*/ byte[], /*A*/ byte[]>() {
@Override
- public /*AccumT*/ byte[] call(/*AccumT*/ byte[] ab, /*InputT*/ byte[] ib)
+ public /*Itr<WV<A>>*/ byte[] call(/*Itr<WV<A>>*/ byte[] ab, /*WV<I>*/ byte[] ib)
throws Exception {
Iterable<WindowedValue<AccumT>> a = CoderHelpers.fromByteArray(ab, iterAccumCoder);
WindowedValue<InputT> i = CoderHelpers.fromByteArray(ib, wviCoder);
return CoderHelpers.toByteArray(sparkCombineFn.seqOp(a, i), iterAccumCoder);
}
},
- new Function2</*AccumT*/ byte[], /*AccumT*/ byte[], /*AccumT*/ byte[]>() {
+ new Function2</*Itr<WV<A>>>*/ byte[], /*Itr<WV<A>>>*/ byte[], /*Itr<WV<A>>>*/ byte[]>() {
@Override
- public /*AccumT*/ byte[] call(/*AccumT*/ byte[] a1b, /*AccumT*/ byte[] a2b)
+ public /*Itr<WV<A>>>*/ byte[] call(/*Itr<WV<A>>>*/ byte[] a1b, /*Itr<WV<A>>>*/ byte[] a2b)
throws Exception {
Iterable<WindowedValue<AccumT>> a1 = CoderHelpers.fromByteArray(a1b, iterAccumCoder);
Iterable<WindowedValue<AccumT>> a2 = CoderHelpers.fromByteArray(a2b, iterAccumCoder);
@@ -168,10 +115,7 @@ public class GroupCombineFunctions {
}
}
);
- Iterable<WindowedValue<OutputT>> output =
- sparkCombineFn.extractOutput(CoderHelpers.fromByteArray(acc, iterAccumCoder));
- return new JavaSparkContext(rdd.context()).parallelize(
- CoderHelpers.toByteArrays(output, wvoCoder)).map(CoderHelpers.fromByteFunction(wvoCoder));
+ return CoderHelpers.fromByteArray(accumulatedBytes, iterAccumCoder);
}
/**
@@ -183,31 +127,22 @@ public class GroupCombineFunctions {
* For streaming, this will be called from within a serialized context
* (DStream's transform callback), so passed arguments need to be Serializable.
*/
- public static <K, InputT, AccumT, OutputT> JavaRDD<WindowedValue<KV<K, OutputT>>>
- combinePerKey(JavaRDD<WindowedValue<KV<K, InputT>>> rdd,
- final CombineWithContext.KeyedCombineFnWithContext<K, InputT, AccumT, OutputT>
- combineFn,
- final KvCoder<K, InputT> inputCoder,
- final SparkRuntimeContext runtimeContext,
- final WindowingStrategy<?, ?> windowingStrategy,
- final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>>
- sideInputs) {
- //--- coders.
- final Coder<K> keyCoder = inputCoder.getKeyCoder();
- final Coder<InputT> viCoder = inputCoder.getValueCoder();
- final Coder<AccumT> vaCoder;
- try {
- vaCoder = combineFn.getAccumulatorCoder(runtimeContext.getCoderRegistry(), keyCoder, viCoder);
- } catch (CannotProvideCoderException e) {
- throw new IllegalStateException("Could not determine coder for accumulator", e);
- }
- // windowed coders.
+ public static <K, InputT, AccumT> JavaPairRDD<K, Iterable<WindowedValue<KV<K, AccumT>>>>
+ combinePerKey(
+ JavaRDD<WindowedValue<KV<K, InputT>>> rdd,
+ final SparkKeyedCombineFn<K, InputT, AccumT, ?> sparkCombineFn,
+ final Coder<K> keyCoder,
+ final Coder<InputT> iCoder,
+ final Coder<AccumT> aCoder,
+ final WindowingStrategy<?, ?> windowingStrategy) {
+ // coders.
final WindowedValue.FullWindowedValueCoder<KV<K, InputT>> wkviCoder =
- WindowedValue.FullWindowedValueCoder.of(KvCoder.of(keyCoder, viCoder),
+ WindowedValue.FullWindowedValueCoder.of(KvCoder.of(keyCoder, iCoder),
windowingStrategy.getWindowFn().windowCoder());
final WindowedValue.FullWindowedValueCoder<KV<K, AccumT>> wkvaCoder =
- WindowedValue.FullWindowedValueCoder.of(KvCoder.of(keyCoder, vaCoder),
+ WindowedValue.FullWindowedValueCoder.of(KvCoder.of(keyCoder, aCoder),
windowingStrategy.getWindowFn().windowCoder());
+ final IterableCoder<WindowedValue<KV<K, AccumT>>> iterAccumCoder = IterableCoder.of(wkvaCoder);
// We need to duplicate K as both the key of the JavaPairRDD as well as inside the value,
// since the functions passed to combineByKey don't receive the associated key of each
@@ -217,53 +152,46 @@ public class GroupCombineFunctions {
// we won't need to duplicate the keys anymore.
// Key has to bw windowed in order to group by window as well.
JavaPairRDD<K, WindowedValue<KV<K, InputT>>> inRddDuplicatedKeyPair =
- rdd.flatMapToPair(
- new PairFlatMapFunction<WindowedValue<KV<K, InputT>>, K,
- WindowedValue<KV<K, InputT>>>() {
- @Override
- public Iterable<Tuple2<K, WindowedValue<KV<K, InputT>>>>
- call(WindowedValue<KV<K, InputT>> wkv) {
- return Collections.singletonList(new Tuple2<>(wkv.getValue().getKey(), wkv));
- }
- });
-
- final SparkKeyedCombineFn<K, InputT, AccumT, OutputT> sparkCombineFn =
- new SparkKeyedCombineFn<>(combineFn, runtimeContext, sideInputs, windowingStrategy);
- final IterableCoder<WindowedValue<KV<K, AccumT>>> iterAccumCoder = IterableCoder.of(wkvaCoder);
+ rdd.mapToPair(TranslationUtils.<K, InputT>toPairByKeyInWindowedValue());
// Use coders to convert objects in the PCollection to byte arrays, so they
// can be transferred over the network for the shuffle.
+ // for readability, we add comments with actual type next to byte[].
+ // to shorten line length, we use:
+ //---- WV: WindowedValue
+ //---- Iterable: Itr
+ //---- AccumT: A
+ //---- InputT: I
JavaPairRDD<ByteArray, byte[]> inRddDuplicatedKeyPairBytes = inRddDuplicatedKeyPair
.mapToPair(CoderHelpers.toByteFunction(keyCoder, wkviCoder));
- // The output of combineByKey will be "AccumT" (accumulator)
- // types rather than "OutputT" (final output types) since Combine.CombineFn
- // only provides ways to merge VAs, and no way to merge VOs.
- JavaPairRDD</*K*/ ByteArray, /*KV<K, AccumT>*/ byte[]> accumulatedBytes =
+ JavaPairRDD</*K*/ ByteArray, /*Itr<WV<KV<K, A>>>*/ byte[]> accumulatedBytes =
inRddDuplicatedKeyPairBytes.combineByKey(
- new Function</*KV<K, InputT>*/ byte[], /*KV<K, AccumT>*/ byte[]>() {
+ new Function</*WV<KV<K, I>>*/ byte[], /*Itr<WV<KV<K, A>>>*/ byte[]>() {
@Override
- public /*KV<K, AccumT>*/ byte[] call(/*KV<K, InputT>*/ byte[] input) {
+ public /*Itr<WV<KV<K, A>>>*/ byte[] call(/*WV<KV<K, I>>*/ byte[] input) {
WindowedValue<KV<K, InputT>> wkvi = CoderHelpers.fromByteArray(input, wkviCoder);
return CoderHelpers.toByteArray(sparkCombineFn.createCombiner(wkvi), iterAccumCoder);
}
},
- new Function2</*KV<K, AccumT>*/ byte[], /*KV<K, InputT>*/ byte[],
- /*KV<K, AccumT>*/ byte[]>() {
+ new Function2</*Itr<WV<KV<K, A>>>*/ byte[], /*WV<KV<K, I>>*/ byte[],
+ /*Itr<WV<KV<K, A>>>*/ byte[]>() {
@Override
- public /*KV<K, AccumT>*/ byte[] call(/*KV<K, AccumT>*/ byte[] acc,
- /*KV<K, InputT>*/ byte[] input) {
+ public /*Itr<WV<KV<K, A>>>*/ byte[] call(
+ /*Itr<WV<KV<K, A>>>*/ byte[] acc,
+ /*WV<KV<K, I>>*/ byte[] input) {
Iterable<WindowedValue<KV<K, AccumT>>> wkvas =
CoderHelpers.fromByteArray(acc, iterAccumCoder);
WindowedValue<KV<K, InputT>> wkvi = CoderHelpers.fromByteArray(input, wkviCoder);
return CoderHelpers.toByteArray(sparkCombineFn.mergeValue(wkvi, wkvas), iterAccumCoder);
}
},
- new Function2</*KV<K, AccumT>*/ byte[], /*KV<K, AccumT>*/ byte[],
- /*KV<K, AccumT>*/ byte[]>() {
+ new Function2</*Itr<WV<KV<K, A>>>*/ byte[], /*Itr<WV<KV<K, A>>>*/ byte[],
+ /*Itr<WV<KV<K, A>>>*/ byte[]>() {
@Override
- public /*KV<K, AccumT>*/ byte[] call(/*KV<K, AccumT>*/ byte[] acc1,
- /*KV<K, AccumT>*/ byte[] acc2) {
+ public /*Itr<WV<KV<K, A>>>*/ byte[] call(
+ /*Itr<WV<KV<K, A>>>*/ byte[] acc1,
+ /*Itr<WV<KV<K, A>>>*/ byte[] acc2) {
Iterable<WindowedValue<KV<K, AccumT>>> wkvas1 =
CoderHelpers.fromByteArray(acc1, iterAccumCoder);
Iterable<WindowedValue<KV<K, AccumT>>> wkvas2 =
@@ -273,23 +201,6 @@ public class GroupCombineFunctions {
}
});
- JavaPairRDD<K, WindowedValue<OutputT>> extracted = accumulatedBytes
- .mapToPair(CoderHelpers.fromByteFunction(keyCoder, iterAccumCoder))
- .flatMapValues(new Function<Iterable<WindowedValue<KV<K, AccumT>>>,
- Iterable<WindowedValue<OutputT>>>() {
- @Override
- public Iterable<WindowedValue<OutputT>> call(
- Iterable<WindowedValue<KV<K, AccumT>>> accums) {
- return sparkCombineFn.extractOutput(accums);
- }
- });
- return extracted.map(TranslationUtils.<K, WindowedValue<OutputT>>fromPairFunction()).map(
- new Function<KV<K, WindowedValue<OutputT>>, WindowedValue<KV<K, OutputT>>>() {
- @Override
- public WindowedValue<KV<K, OutputT>> call(KV<K, WindowedValue<OutputT>> kwvo)
- throws Exception {
- return kwvo.getValue().withValue(KV.of(kwvo.getKey(), kwvo.getValue().getValue()));
- }
- });
+ return accumulatedBytes.mapToPair(CoderHelpers.fromByteFunction(keyCoder, iterAccumCoder));
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index 14c14dc..a643651 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -27,6 +27,7 @@ import static org.apache.beam.runners.spark.io.hadoop.ShardNameBuilder.replaceSh
import static org.apache.beam.runners.spark.translation.TranslationUtils.rejectSplittable;
import static org.apache.beam.runners.spark.translation.TranslationUtils.rejectStateAndTimers;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import java.io.IOException;
import java.util.Collections;
@@ -35,6 +36,7 @@ import java.util.Map;
import org.apache.avro.mapred.AvroKey;
import org.apache.avro.mapreduce.AvroJob;
import org.apache.avro.mapreduce.AvroKeyInputFormat;
+import org.apache.beam.runners.core.SystemReduceFn;
import org.apache.beam.runners.spark.aggregators.NamedAggregators;
import org.apache.beam.runners.spark.aggregators.SparkAggregators;
import org.apache.beam.runners.spark.coders.CoderHelpers;
@@ -46,6 +48,7 @@ import org.apache.beam.runners.spark.io.hadoop.TemplatedTextOutputFormat;
import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
import org.apache.beam.runners.spark.util.SideInputBroadcast;
+import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -63,6 +66,7 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.CombineFnUtil;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingStrategy;
@@ -119,101 +123,159 @@ public final class TransformTranslator {
};
}
- private static <K, V> TransformEvaluator<GroupByKey<K, V>> groupByKey() {
+ private static <K, V, W extends BoundedWindow> TransformEvaluator<GroupByKey<K, V>> groupByKey() {
return new TransformEvaluator<GroupByKey<K, V>>() {
@Override
public void evaluate(GroupByKey<K, V> transform, EvaluationContext context) {
@SuppressWarnings("unchecked")
JavaRDD<WindowedValue<KV<K, V>>> inRDD =
((BoundedDataset<KV<K, V>>) context.borrowDataset(transform)).getRDD();
-
@SuppressWarnings("unchecked")
final KvCoder<K, V> coder = (KvCoder<K, V>) context.getInput(transform).getCoder();
-
final Accumulator<NamedAggregators> accum =
SparkAggregators.getNamedAggregators(context.getSparkContext());
-
- context.putDataset(
- transform,
- new BoundedDataset<>(
- GroupCombineFunctions.groupByKey(
- inRDD,
- accum,
- coder,
- context.getRuntimeContext(),
- context.getInput(transform).getWindowingStrategy())));
+ @SuppressWarnings("unchecked")
+ final WindowingStrategy<?, W> windowingStrategy =
+ (WindowingStrategy<?, W>) context.getInput(transform).getWindowingStrategy();
+ @SuppressWarnings("unchecked")
+ final WindowFn<Object, W> windowFn = (WindowFn<Object, W>) windowingStrategy.getWindowFn();
+
+ //--- coders.
+ final Coder<K> keyCoder = coder.getKeyCoder();
+ final WindowedValue.WindowedValueCoder<V> wvCoder =
+ WindowedValue.FullWindowedValueCoder.of(coder.getValueCoder(), windowFn.windowCoder());
+
+ //--- group by key only.
+ JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<V>>>>> groupedByKey =
+ GroupCombineFunctions.groupByKeyOnly(inRDD, keyCoder, wvCoder);
+
+ //--- now group also by window.
+ // for batch, GroupAlsoByWindow uses an in-memory StateInternals.
+ JavaRDD<WindowedValue<KV<K, Iterable<V>>>> groupedAlsoByWindow = groupedByKey.flatMap(
+ new SparkGroupAlsoByWindowViaOutputBufferFn<>(
+ windowingStrategy,
+ new TranslationUtils.InMemoryStateInternalsFactory<K>(),
+ SystemReduceFn.<K, V, W>buffering(coder.getValueCoder()),
+ context.getRuntimeContext(),
+ accum));
+
+ context.putDataset(transform, new BoundedDataset<>(groupedAlsoByWindow));
}
};
}
private static <K, InputT, OutputT> TransformEvaluator<Combine.GroupedValues<K, InputT, OutputT>>
- combineGrouped() {
- return new TransformEvaluator<Combine.GroupedValues<K, InputT, OutputT>>() {
- @Override
- public void evaluate(Combine.GroupedValues<K, InputT, OutputT> transform,
- EvaluationContext context) {
- // get the applied combine function.
- PCollection<? extends KV<K, ? extends Iterable<InputT>>> input =
- context.getInput(transform);
- WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
- @SuppressWarnings("unchecked")
- CombineWithContext.KeyedCombineFnWithContext<K, InputT, ?, OutputT> fn =
- (CombineWithContext.KeyedCombineFnWithContext<K, InputT, ?, OutputT>)
- CombineFnUtil.toFnWithContext(transform.getFn());
-
- @SuppressWarnings("unchecked")
- JavaRDD<WindowedValue<KV<K, Iterable<InputT>>>> inRDD =
- ((BoundedDataset<KV<K, Iterable<InputT>>>)
- context.borrowDataset(transform)).getRDD();
-
- SparkKeyedCombineFn<K, InputT, ?, OutputT> combineFnWithContext =
- new SparkKeyedCombineFn<>(fn, context.getRuntimeContext(),
- TranslationUtils.getSideInputs(transform.getSideInputs(), context),
- windowingStrategy);
- context.putDataset(transform, new BoundedDataset<>(inRDD.map(new TranslationUtils
- .CombineGroupedValues<>(
- combineFnWithContext))));
- }
- };
+ combineGrouped() {
+ return new TransformEvaluator<Combine.GroupedValues<K, InputT, OutputT>>() {
+ @Override
+ public void evaluate(
+ Combine.GroupedValues<K, InputT, OutputT> transform,
+ EvaluationContext context) {
+ @SuppressWarnings("unchecked")
+ CombineWithContext.KeyedCombineFnWithContext<K, InputT, ?, OutputT> combineFn =
+ (CombineWithContext.KeyedCombineFnWithContext<K, InputT, ?, OutputT>)
+ CombineFnUtil.toFnWithContext(transform.getFn());
+ final SparkKeyedCombineFn<K, InputT, ?, OutputT> sparkCombineFn =
+ new SparkKeyedCombineFn<>(combineFn, context.getRuntimeContext(),
+ TranslationUtils.getSideInputs(transform.getSideInputs(), context),
+ context.getInput(transform).getWindowingStrategy());
+
+ @SuppressWarnings("unchecked")
+ JavaRDD<WindowedValue<KV<K, Iterable<InputT>>>> inRDD =
+ ((BoundedDataset<KV<K, Iterable<InputT>>>) context.borrowDataset(transform))
+ .getRDD();
+
+ JavaRDD<WindowedValue<KV<K, OutputT>>> outRDD = inRDD.map(
+ new Function<WindowedValue<KV<K, Iterable<InputT>>>,
+ WindowedValue<KV<K, OutputT>>>() {
+ @Override
+ public WindowedValue<KV<K, OutputT>> call(
+ WindowedValue<KV<K, Iterable<InputT>>> in) throws Exception {
+ return WindowedValue.of(
+ KV.of(in.getValue().getKey(), sparkCombineFn.apply(in)),
+ in.getTimestamp(),
+ in.getWindows(),
+ in.getPane());
+ }
+ });
+ context.putDataset(transform, new BoundedDataset<>(outRDD));
+ }
+ };
}
private static <InputT, AccumT, OutputT> TransformEvaluator<Combine.Globally<InputT, OutputT>>
- combineGlobally() {
- return new TransformEvaluator<Combine.Globally<InputT, OutputT>>() {
-
- @Override
- public void evaluate(Combine.Globally<InputT, OutputT> transform, EvaluationContext context) {
- final PCollection<InputT> input = context.getInput(transform);
- // serializable arguments to pass.
- final Coder<InputT> iCoder = context.getInput(transform).getCoder();
- final Coder<OutputT> oCoder = context.getOutput(transform).getCoder();
- @SuppressWarnings("unchecked")
- final CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT> combineFn =
- (CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT>)
- CombineFnUtil.toFnWithContext(transform.getFn());
- final WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
- final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
- final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs =
- TranslationUtils.getSideInputs(transform.getSideInputs(), context);
- final boolean hasDefault = transform.isInsertDefault();
+ combineGlobally() {
+ return new TransformEvaluator<Combine.Globally<InputT, OutputT>>() {
- @SuppressWarnings("unchecked")
- JavaRDD<WindowedValue<InputT>> inRdd =
- ((BoundedDataset<InputT>) context.borrowDataset(transform)).getRDD();
-
- context.putDataset(transform, new BoundedDataset<>(GroupCombineFunctions
- .combineGlobally(inRdd, combineFn,
- iCoder, oCoder, runtimeContext, windowingStrategy, sideInputs, hasDefault)));
- }
- };
+ @Override
+ public void evaluate(
+ Combine.Globally<InputT, OutputT> transform,
+ EvaluationContext context) {
+ final PCollection<InputT> input = context.getInput(transform);
+ final Coder<InputT> iCoder = context.getInput(transform).getCoder();
+ final Coder<OutputT> oCoder = context.getOutput(transform).getCoder();
+ final WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
+ @SuppressWarnings("unchecked")
+ final CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT> combineFn =
+ (CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT>)
+ CombineFnUtil.toFnWithContext(transform.getFn());
+ final WindowedValue.FullWindowedValueCoder<OutputT> wvoCoder =
+ WindowedValue.FullWindowedValueCoder.of(oCoder,
+ windowingStrategy.getWindowFn().windowCoder());
+ final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
+ final boolean hasDefault = transform.isInsertDefault();
+
+ final SparkGlobalCombineFn<InputT, AccumT, OutputT> sparkCombineFn =
+ new SparkGlobalCombineFn<>(
+ combineFn,
+ runtimeContext,
+ TranslationUtils.getSideInputs(transform.getSideInputs(), context),
+ windowingStrategy);
+ final Coder<AccumT> aCoder;
+ try {
+ aCoder = combineFn.getAccumulatorCoder(runtimeContext.getCoderRegistry(), iCoder);
+ } catch (CannotProvideCoderException e) {
+ throw new IllegalStateException("Could not determine coder for accumulator", e);
+ }
+
+ @SuppressWarnings("unchecked")
+ JavaRDD<WindowedValue<InputT>> inRdd =
+ ((BoundedDataset<InputT>) context.borrowDataset(transform)).getRDD();
+
+ JavaRDD<WindowedValue<OutputT>> outRdd;
+ // handle empty input RDD, which will naturally skip the entire execution
+ // as Spark will not run on empty RDDs.
+ if (inRdd.isEmpty()) {
+ JavaSparkContext jsc = new JavaSparkContext(inRdd.context());
+ if (hasDefault) {
+ OutputT defaultValue = combineFn.defaultValue();
+ outRdd = jsc
+ .parallelize(Lists.newArrayList(CoderHelpers.toByteArray(defaultValue, oCoder)))
+ .map(CoderHelpers.fromByteFunction(oCoder))
+ .map(WindowingHelpers.<OutputT>windowFunction());
+ } else {
+ outRdd = jsc.emptyRDD();
+ }
+ } else {
+ Iterable<WindowedValue<AccumT>> accumulated = GroupCombineFunctions.combineGlobally(
+ inRdd, sparkCombineFn, iCoder, aCoder, windowingStrategy);
+ Iterable<WindowedValue<OutputT>> output = sparkCombineFn.extractOutput(accumulated);
+ outRdd = context.getSparkContext()
+ .parallelize(CoderHelpers.toByteArrays(output, wvoCoder))
+ .map(CoderHelpers.fromByteFunction(wvoCoder));
+ }
+ context.putDataset(transform, new BoundedDataset<>(outRdd));
+ }
+ };
}
private static <K, InputT, AccumT, OutputT>
TransformEvaluator<Combine.PerKey<K, InputT, OutputT>> combinePerKey() {
return new TransformEvaluator<Combine.PerKey<K, InputT, OutputT>>() {
@Override
- public void evaluate(Combine.PerKey<K, InputT, OutputT> transform,
- EvaluationContext context) {
+ public void evaluate(
+ Combine.PerKey<K, InputT, OutputT> transform,
+ EvaluationContext context) {
final PCollection<KV<K, InputT>> input = context.getInput(transform);
// serializable arguments to pass.
@SuppressWarnings("unchecked")
@@ -227,14 +289,44 @@ public final class TransformTranslator {
final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs =
TranslationUtils.getSideInputs(transform.getSideInputs(), context);
+ final SparkKeyedCombineFn<K, InputT, AccumT, OutputT> sparkCombineFn =
+ new SparkKeyedCombineFn<>(combineFn, runtimeContext, sideInputs, windowingStrategy);
+ final Coder<AccumT> vaCoder;
+ try {
+ vaCoder = combineFn.getAccumulatorCoder(runtimeContext.getCoderRegistry(),
+ inputCoder.getKeyCoder(), inputCoder.getValueCoder());
+ } catch (CannotProvideCoderException e) {
+ throw new IllegalStateException("Could not determine coder for accumulator", e);
+ }
@SuppressWarnings("unchecked")
JavaRDD<WindowedValue<KV<K, InputT>>> inRdd =
((BoundedDataset<KV<K, InputT>>) context.borrowDataset(transform)).getRDD();
- context.putDataset(transform, new BoundedDataset<>(GroupCombineFunctions
- .combinePerKey(inRdd, combineFn,
- inputCoder, runtimeContext, windowingStrategy, sideInputs)));
+ JavaPairRDD<K, Iterable<WindowedValue<KV<K, AccumT>>>> accumulatePerKey =
+ GroupCombineFunctions.combinePerKey(inRdd, sparkCombineFn, inputCoder.getKeyCoder(),
+ inputCoder.getValueCoder(), vaCoder, windowingStrategy);
+
+ JavaRDD<WindowedValue<KV<K, OutputT>>> outRdd =
+ accumulatePerKey.flatMapValues(new Function<Iterable<WindowedValue<KV<K, AccumT>>>,
+ Iterable<WindowedValue<OutputT>>>() {
+ @Override
+ public Iterable<WindowedValue<OutputT>> call(
+ Iterable<WindowedValue<KV<K, AccumT>>> iter) throws Exception {
+ return sparkCombineFn.extractOutput(iter);
+ }
+ }).map(TranslationUtils.<K, WindowedValue<OutputT>>fromPairFunction())
+ .map(new Function<KV<K, WindowedValue<OutputT>>,
+ WindowedValue<KV<K, OutputT>>>() {
+ @Override
+ public WindowedValue<KV<K, OutputT>> call(
+ KV<K, WindowedValue<OutputT>> kv) throws Exception {
+ WindowedValue<OutputT> wv = kv.getValue();
+ return wv.withValue(KV.of(kv.getKey(), wv.getValue()));
+ }
+ });
+
+ context.putDataset(transform, new BoundedDataset<>(outRdd));
}
};
}
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
index 7d83230..6b27436 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
@@ -21,7 +21,6 @@ package org.apache.beam.runners.spark.translation;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import java.io.Serializable;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.beam.runners.core.InMemoryStateInternals;
@@ -42,11 +41,11 @@ import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
+
import scala.Tuple2;
/**
@@ -148,20 +147,17 @@ public final class TranslationUtils {
};
}
- /** A Flatmap iterator function, flattening iterators into their elements. */
- public static <T> FlatMapFunction<Iterator<T>, T> flattenIter() {
- return new FlatMapFunction<Iterator<T>, T>() {
- @Override
- public Iterable<T> call(final Iterator<T> t) throws Exception {
- return new Iterable<T>() {
+ /** Extract key from a {@link WindowedValue} {@link KV} into a pair. */
+ public static <K, V> PairFunction<WindowedValue<KV<K, V>>, K, WindowedValue<KV<K, V>>>
+ toPairByKeyInWindowedValue() {
+ return new PairFunction<WindowedValue<KV<K, V>>, K, WindowedValue<KV<K, V>>>() {
@Override
- public Iterator<T> iterator() {
- return t;
- }
+ public Tuple2<K, WindowedValue<KV<K, V>>> call(
+ WindowedValue<KV<K, V>> windowedKv) throws Exception {
+ return new Tuple2<>(windowedKv.getValue().getKey(), windowedKv);
+ }
};
}
- };
- }
/**
* A utility class to filter {@link TupleTag}s.
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index 9451df7..e90b490 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -31,6 +31,7 @@ import org.apache.beam.runners.spark.aggregators.SparkAggregators;
import org.apache.beam.runners.spark.io.ConsoleIO;
import org.apache.beam.runners.spark.io.CreateStream;
import org.apache.beam.runners.spark.io.SparkUnboundedSource;
+import org.apache.beam.runners.spark.stateful.SparkGroupAlsoByWindowViaWindowSet;
import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
import org.apache.beam.runners.spark.translation.BoundedDataset;
@@ -148,7 +149,7 @@ final class StreamingTransformTranslator {
Dataset dataset = context.borrowDataset(pcol);
if (dataset instanceof UnboundedDataset) {
UnboundedDataset<T> unboundedDataset = (UnboundedDataset<T>) dataset;
- streamingSources.addAll(unboundedDataset.getStreamingSources());
+ streamingSources.addAll(unboundedDataset.getStreamSources());
dStreams.add(unboundedDataset.getDStream());
} else {
rdds.add(((BoundedDataset<T>) dataset).getRDD());
@@ -205,7 +206,7 @@ final class StreamingTransformTranslator {
//--- then we apply windowing to the elements
if (TranslationUtils.skipAssignWindows(transform, context)) {
context.putDataset(transform,
- new UnboundedDataset<>(windowedDStream, unboundedDataset.getStreamingSources()));
+ new UnboundedDataset<>(windowedDStream, unboundedDataset.getStreamSources()));
} else {
JavaDStream<WindowedValue<T>> outStream = windowedDStream.transform(
new Function<JavaRDD<WindowedValue<T>>, JavaRDD<WindowedValue<T>>>() {
@@ -215,42 +216,55 @@ final class StreamingTransformTranslator {
}
});
context.putDataset(transform,
- new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
+ new UnboundedDataset<>(outStream, unboundedDataset.getStreamSources()));
}
}
};
}
- private static <K, V> TransformEvaluator<GroupByKey<K, V>> groupByKey() {
+ private static <K, V, W extends BoundedWindow> TransformEvaluator<GroupByKey<K, V>> groupByKey() {
return new TransformEvaluator<GroupByKey<K, V>>() {
@Override
public void evaluate(GroupByKey<K, V> transform, EvaluationContext context) {
- @SuppressWarnings("unchecked")
- UnboundedDataset<KV<K, V>> unboundedDataset =
- ((UnboundedDataset<KV<K, V>>) context.borrowDataset(transform));
- JavaDStream<WindowedValue<KV<K, V>>> dStream = unboundedDataset.getDStream();
-
+ @SuppressWarnings("unchecked") UnboundedDataset<KV<K, V>> inputDataset =
+ (UnboundedDataset<KV<K, V>>) context.borrowDataset(transform);
+ List<Integer> streamSources = inputDataset.getStreamSources();
+ JavaDStream<WindowedValue<KV<K, V>>> dStream = inputDataset.getDStream();
@SuppressWarnings("unchecked")
final KvCoder<K, V> coder = (KvCoder<K, V>) context.getInput(transform).getCoder();
-
final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
- final WindowingStrategy<?, ?> windowingStrategy =
- context.getInput(transform).getWindowingStrategy();
+ @SuppressWarnings("unchecked")
+ final WindowingStrategy<?, W> windowingStrategy =
+ (WindowingStrategy<?, W>) context.getInput(transform).getWindowingStrategy();
+ @SuppressWarnings("unchecked")
+ final WindowFn<Object, W> windowFn = (WindowFn<Object, W>) windowingStrategy.getWindowFn();
- JavaDStream<WindowedValue<KV<K, Iterable<V>>>> outStream =
+ //--- coders.
+ final WindowedValue.WindowedValueCoder<V> wvCoder =
+ WindowedValue.FullWindowedValueCoder.of(coder.getValueCoder(), windowFn.windowCoder());
+
+ //--- group by key only.
+ JavaDStream<WindowedValue<KV<K, Iterable<WindowedValue<V>>>>> groupedByKeyStream =
dStream.transform(new Function<JavaRDD<WindowedValue<KV<K, V>>>,
- JavaRDD<WindowedValue<KV<K, Iterable<V>>>>>() {
- @Override
- public JavaRDD<WindowedValue<KV<K, Iterable<V>>>> call(
- JavaRDD<WindowedValue<KV<K, V>>> rdd) throws Exception {
- final Accumulator<NamedAggregators> accum =
- SparkAggregators.getNamedAggregators(new JavaSparkContext(rdd.context()));
- return GroupCombineFunctions.groupByKey(rdd, accum, coder, runtimeContext,
- windowingStrategy);
- }
- });
- context.putDataset(transform,
- new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
+ JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<V>>>>>>() {
+ @Override
+ public JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<V>>>>> call(
+ JavaRDD<WindowedValue<KV<K, V>>> rdd) throws Exception {
+ return GroupCombineFunctions.groupByKeyOnly(
+ rdd, coder.getKeyCoder(), wvCoder);
+ }
+ });
+
+ //--- now group also by window.
+ JavaDStream<WindowedValue<KV<K, Iterable<V>>>> outStream =
+ SparkGroupAlsoByWindowViaWindowSet.groupAlsoByWindow(
+ groupedByKeyStream,
+ coder.getValueCoder(),
+ windowingStrategy,
+ runtimeContext,
+ streamSources);
+
+ context.putDataset(transform, new UnboundedDataset<>(outStream, streamSources));
}
};
}
@@ -296,96 +310,7 @@ final class StreamingTransformTranslator {
});
context.putDataset(transform,
- new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
- }
- };
- }
-
- private static <InputT, AccumT, OutputT> TransformEvaluator<Combine.Globally<InputT, OutputT>>
- combineGlobally() {
- return new TransformEvaluator<Combine.Globally<InputT, OutputT>>() {
-
- @Override
- public void evaluate(
- final Combine.Globally<InputT, OutputT> transform,
- EvaluationContext context) {
- final PCollection<InputT> input = context.getInput(transform);
- // serializable arguments to pass.
- final Coder<InputT> iCoder = context.getInput(transform).getCoder();
- final Coder<OutputT> oCoder = context.getOutput(transform).getCoder();
- @SuppressWarnings("unchecked")
- final CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT> combineFn =
- (CombineWithContext.CombineFnWithContext<InputT, AccumT, OutputT>)
- CombineFnUtil.toFnWithContext(transform.getFn());
- final WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
- final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
- final boolean hasDefault = transform.isInsertDefault();
- final SparkPCollectionView pviews = context.getPViews();
-
- @SuppressWarnings("unchecked")
- UnboundedDataset<InputT> unboundedDataset =
- ((UnboundedDataset<InputT>) context.borrowDataset(transform));
- JavaDStream<WindowedValue<InputT>> dStream = unboundedDataset.getDStream();
-
- JavaDStream<WindowedValue<OutputT>> outStream = dStream.transform(
- new Function<JavaRDD<WindowedValue<InputT>>, JavaRDD<WindowedValue<OutputT>>>() {
- @Override
- public JavaRDD<WindowedValue<OutputT>> call(JavaRDD<WindowedValue<InputT>> rdd)
- throws Exception {
- final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs =
- TranslationUtils.getSideInputs(transform.getSideInputs(),
- JavaSparkContext.fromSparkContext(rdd.context()),
- pviews);
- return GroupCombineFunctions.combineGlobally(rdd, combineFn, iCoder, oCoder,
- runtimeContext, windowingStrategy, sideInputs, hasDefault);
- }
- });
-
- context.putDataset(transform,
- new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
- }
- };
- }
-
- private static <K, InputT, AccumT, OutputT>
- TransformEvaluator<Combine.PerKey<K, InputT, OutputT>> combinePerKey() {
- return new TransformEvaluator<Combine.PerKey<K, InputT, OutputT>>() {
- @Override
- public void evaluate(final Combine.PerKey<K, InputT, OutputT> transform,
- final EvaluationContext context) {
- final PCollection<KV<K, InputT>> input = context.getInput(transform);
- // serializable arguments to pass.
- final KvCoder<K, InputT> inputCoder =
- (KvCoder<K, InputT>) context.getInput(transform).getCoder();
- @SuppressWarnings("unchecked")
- final CombineWithContext.KeyedCombineFnWithContext<K, InputT, AccumT, OutputT> combineFn =
- (CombineWithContext.KeyedCombineFnWithContext<K, InputT, AccumT, OutputT>)
- CombineFnUtil.toFnWithContext(transform.getFn());
- final WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
- final SparkRuntimeContext runtimeContext = context.getRuntimeContext();
- final SparkPCollectionView pviews = context.getPViews();
-
- @SuppressWarnings("unchecked")
- UnboundedDataset<KV<K, InputT>> unboundedDataset =
- ((UnboundedDataset<KV<K, InputT>>) context.borrowDataset(transform));
- JavaDStream<WindowedValue<KV<K, InputT>>> dStream = unboundedDataset.getDStream();
-
- JavaDStream<WindowedValue<KV<K, OutputT>>> outStream =
- dStream.transform(new Function<JavaRDD<WindowedValue<KV<K, InputT>>>,
- JavaRDD<WindowedValue<KV<K, OutputT>>>>() {
- @Override
- public JavaRDD<WindowedValue<KV<K, OutputT>>> call(
- JavaRDD<WindowedValue<KV<K, InputT>>> rdd) throws Exception {
- final Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs =
- TranslationUtils.getSideInputs(transform.getSideInputs(),
- JavaSparkContext.fromSparkContext(rdd.context()),
- pviews);
- return GroupCombineFunctions.combinePerKey(rdd, combineFn, inputCoder, runtimeContext,
- windowingStrategy, sideInputs);
- }
- });
- context.putDataset(transform,
- new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
+ new UnboundedDataset<>(outStream, unboundedDataset.getStreamSources()));
}
};
}
@@ -431,7 +356,7 @@ final class StreamingTransformTranslator {
});
context.putDataset(transform,
- new UnboundedDataset<>(outStream, unboundedDataset.getStreamingSources()));
+ new UnboundedDataset<>(outStream, unboundedDataset.getStreamSources()));
}
};
}
@@ -486,7 +411,7 @@ final class StreamingTransformTranslator {
(JavaDStream<WindowedValue<Object>>)
(JavaDStream<?>) TranslationUtils.dStreamValues(filtered);
context.putDataset(e.getValue(),
- new UnboundedDataset<>(values, unboundedDataset.getStreamingSources()));
+ new UnboundedDataset<>(values, unboundedDataset.getStreamSources()));
}
}
};
@@ -499,8 +424,6 @@ final class StreamingTransformTranslator {
EVALUATORS.put(Read.Unbounded.class, readUnbounded());
EVALUATORS.put(GroupByKey.class, groupByKey());
EVALUATORS.put(Combine.GroupedValues.class, combineGrouped());
- EVALUATORS.put(Combine.Globally.class, combineGlobally());
- EVALUATORS.put(Combine.PerKey.class, combinePerKey());
EVALUATORS.put(ParDo.Bound.class, parDo());
EVALUATORS.put(ParDo.BoundMulti.class, multiDo());
EVALUATORS.put(ConsoleIO.Write.Unbound.class, print());
@@ -523,7 +446,7 @@ final class StreamingTransformTranslator {
@Override
public boolean hasTranslation(Class<? extends PTransform<?, ?>> clazz) {
// streaming includes rdd/bounded transformations as well
- return EVALUATORS.containsKey(clazz) || batchTranslator.hasTranslation(clazz);
+ return EVALUATORS.containsKey(clazz);
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
index 6f5fa93..8624f41 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
@@ -56,11 +56,11 @@ public class UnboundedDataset<T> implements Dataset {
// should be greater > 1 in case of Flatten for example.
// when using GlobalWatermarkHolder this information helps to take only the relevant watermarks
// and reason about them accordingly.
- private final List<Integer> streamingSources = new ArrayList<>();
+ private final List<Integer> streamSources = new ArrayList<>();
- public UnboundedDataset(JavaDStream<WindowedValue<T>> dStream, List<Integer> streamingSources) {
+ public UnboundedDataset(JavaDStream<WindowedValue<T>> dStream, List<Integer> streamSources) {
this.dStream = dStream;
- this.streamingSources.addAll(streamingSources);
+ this.streamSources.addAll(streamSources);
}
public UnboundedDataset(Iterable<Iterable<T>> values, JavaStreamingContext jssc, Coder<T> coder) {
@@ -68,7 +68,7 @@ public class UnboundedDataset<T> implements Dataset {
this.jssc = jssc;
this.coder = coder;
// QueuedStream will have a negative (decreasing) unique id.
- this.streamingSources.add(queuedStreamIds.decrementAndGet());
+ this.streamSources.add(queuedStreamIds.decrementAndGet());
}
@VisibleForTesting
@@ -97,8 +97,8 @@ public class UnboundedDataset<T> implements Dataset {
return dStream;
}
- public List<Integer> getStreamingSources() {
- return streamingSources;
+ public List<Integer> getStreamSources() {
+ return streamSources;
}
public void cache() {
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java
index 96e6ee5..18689bd 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java
@@ -89,4 +89,4 @@ public class LateDataUtils {
}
});
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java
index 6de7e86..96d889d 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java
@@ -49,4 +49,4 @@ public class UnsupportedSideInputReader implements SideInputReader {
throw new UnsupportedOperationException(
String.format("%s does not support side inputs.", transformName));
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/96abe4f0/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
index fbe5777..8449724 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -157,7 +157,7 @@ public class TrackStreamingSourcesTest {
ctxt.setCurrentTransform(appliedTransform);
//noinspection unchecked
Dataset dataset = ctxt.borrowDataset((PTransform<? extends PValue, ?>) transform);
- assertSourceIds(((UnboundedDataset<?>) dataset).getStreamingSources());
+ assertSourceIds(((UnboundedDataset<?>) dataset).getStreamSources());
ctxt.setCurrentTransform(null);
} else {
evaluator.visitPrimitiveTransform(node);
[22/23] beam git commit: Streaming tests,
especially the ones using checkpoints, need a time buffer to finish.
Posted by am...@apache.org.
Streaming tests, especially the ones using checkpoints, need a time buffer to finish.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2c707431
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2c707431
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2c707431
Branch: refs/heads/master
Commit: 2c70743121e3da352346c0db4d718c757e36dbe8
Parents: 3c94ff2
Author: Sela <an...@paypal.com>
Authored: Tue Feb 28 16:16:00 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:11 2017 +0200
----------------------------------------------------------------------
.../translation/streaming/ResumeFromCheckpointStreamingTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/2c707431/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
index 4eea383..7706777 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
@@ -166,7 +166,7 @@ public class ResumeFromCheckpointStreamingTest {
// first run will read from Kafka backlog - "auto.offset.reset=smallest"
SparkPipelineResult res = run(options);
- res.waitUntilFinish(Duration.standardSeconds(2));
+ res.waitUntilFinish(Duration.standardSeconds(5));
// assertions 1:
long processedMessages1 = res.getAggregatorValue("processedMessages", Long.class);
assertThat(
@@ -193,7 +193,7 @@ public class ResumeFromCheckpointStreamingTest {
// recovery should resume from last read offset, and read the second batch of input.
res = runAgain(options);
- res.waitUntilFinish(Duration.standardSeconds(2));
+ res.waitUntilFinish(Duration.standardSeconds(5));
// assertions 2:
long processedMessages2 = res.getAggregatorValue("processedMessages", Long.class);
assertThat(
[19/23] beam git commit: Move LateDataUtils and
UnsupportedSideInputReader to runners-core.
Posted by am...@apache.org.
Move LateDataUtils and UnsupportedSideInputReader to runners-core.
Add missing licenses.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/123f4820
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/123f4820
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/123f4820
Branch: refs/heads/master
Commit: 123f4820b68dd13690be59a7b8b946e81031b6a7
Parents: 4ca5680
Author: Sela <an...@paypal.com>
Authored: Mon Feb 27 20:24:54 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:07 2017 +0200
----------------------------------------------------------------------
.../apache/beam/runners/core/LateDataUtils.java | 88 +++++++++++++++++++
.../core/UnsupportedSideInputReader.java | 52 +++++++++++
.../runners/spark/TestSparkPipelineOptions.java | 17 ++++
.../SparkGroupAlsoByWindowViaWindowSet.java | 4 +-
.../beam/runners/spark/util/LateDataUtils.java | 92 --------------------
.../spark/util/UnsupportedSideInputReader.java | 52 -----------
.../apache/beam/runners/spark/PipelineRule.java | 17 ++++
7 files changed, 176 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/123f4820/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
new file mode 100644
index 0000000..17bd360
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataUtils.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.transforms.Aggregator;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.WindowTracing;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+
+
+/**
+ * Utils to handle late data.
+ */
+public class LateDataUtils {
+
+ /**
+ * Returns an {@code Iterable<WindowedValue<InputT>>} that only contains non-late input elements.
+ */
+ public static <K, V> Iterable<WindowedValue<V>> dropExpiredWindows(
+ final K key,
+ Iterable<WindowedValue<V>> elements,
+ final TimerInternals timerInternals,
+ final WindowingStrategy<?, ?> windowingStrategy,
+ final Aggregator<Long, Long> droppedDueToLateness) {
+ return FluentIterable.from(elements)
+ .transformAndConcat(
+ // Explode windows to filter out expired ones
+ new Function<WindowedValue<V>, Iterable<WindowedValue<V>>>() {
+ @Override
+ public Iterable<WindowedValue<V>> apply(@Nullable WindowedValue<V> input) {
+ if (input == null) {
+ return null;
+ }
+ return input.explodeWindows();
+ }
+ })
+ .filter(
+ new Predicate<WindowedValue<V>>() {
+ @Override
+ public boolean apply(@Nullable WindowedValue<V> input) {
+ if (input == null) {
+ // drop null elements.
+ return false;
+ }
+ BoundedWindow window = Iterables.getOnlyElement(input.getWindows());
+ boolean expired =
+ window
+ .maxTimestamp()
+ .plus(windowingStrategy.getAllowedLateness())
+ .isBefore(timerInternals.currentInputWatermarkTime());
+ if (expired) {
+ // The element is too late for this window.
+ droppedDueToLateness.addValue(1L);
+ WindowTracing.debug(
+ "GroupAlsoByWindow: Dropping element at {} for key: {}; "
+ + "window: {} since it is too far behind inputWatermark: {}",
+ input.getTimestamp(),
+ key,
+ window,
+ timerInternals.currentInputWatermarkTime());
+ }
+ // Keep the element if the window is not expired.
+ return !expired;
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/123f4820/runners/core-java/src/main/java/org/apache/beam/runners/core/UnsupportedSideInputReader.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/UnsupportedSideInputReader.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/UnsupportedSideInputReader.java
new file mode 100644
index 0000000..4230f8c
--- /dev/null
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/UnsupportedSideInputReader.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.core;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.SideInputReader;
+import org.apache.beam.sdk.values.PCollectionView;
+
+
+/**
+ * An implementation of a {@link SideInputReader} that actually does not support side-inputs.
+ */
+public class UnsupportedSideInputReader implements SideInputReader {
+ private final String transformName;
+
+ public UnsupportedSideInputReader(String transformName) {
+ this.transformName = transformName;
+ }
+
+ @Override
+ public <T> T get(PCollectionView<T> view, BoundedWindow window) {
+ throw new UnsupportedOperationException(
+ String.format("%s does not support side inputs.", transformName));
+ }
+
+ @Override
+ public <T> boolean contains(PCollectionView<T> view) {
+ throw new UnsupportedOperationException(
+ String.format("%s does not support side inputs.", transformName));
+ }
+
+ @Override
+ public boolean isEmpty() {
+ throw new UnsupportedOperationException(
+ String.format("%s does not support side inputs.", transformName));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/123f4820/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkPipelineOptions.java
index 2cb58d8..d50b652 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkPipelineOptions.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark;
import org.apache.beam.sdk.options.Default;
http://git-wip-us.apache.org/repos/asf/beam/blob/123f4820/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 24b8508..2f1713a 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -24,10 +24,12 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.beam.runners.core.GroupAlsoByWindowsDoFn;
+import org.apache.beam.runners.core.LateDataUtils;
import org.apache.beam.runners.core.OutputWindowedValue;
import org.apache.beam.runners.core.ReduceFnRunner;
import org.apache.beam.runners.core.SystemReduceFn;
import org.apache.beam.runners.core.TimerInternals;
+import org.apache.beam.runners.core.UnsupportedSideInputReader;
import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachines;
import org.apache.beam.runners.spark.SparkPipelineOptions;
@@ -37,8 +39,6 @@ import org.apache.beam.runners.spark.translation.TranslationUtils;
import org.apache.beam.runners.spark.translation.WindowingHelpers;
import org.apache.beam.runners.spark.util.ByteArray;
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
-import org.apache.beam.runners.spark.util.LateDataUtils;
-import org.apache.beam.runners.spark.util.UnsupportedSideInputReader;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
import org.apache.beam.sdk.coders.KvCoder;
http://git-wip-us.apache.org/repos/asf/beam/blob/123f4820/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java
deleted file mode 100644
index 18689bd..0000000
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.util;
-
-import com.google.common.base.Function;
-import com.google.common.base.Predicate;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Iterables;
-import javax.annotation.Nullable;
-import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowTracing;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-
-
-/**
- * Utils to handle late data.
- */
-public class LateDataUtils {
-
- /**
- * Returns an {@code Iterable<WindowedValue<InputT>>} that only contains non-late input
- * elements.
- * Taken from Thomas Groh's implementation in the DirectRunner's
- * GroupAlsoByWindowEvaluatorFactory.
- */
- public static <K, V> Iterable<WindowedValue<V>> dropExpiredWindows(
- final K key,
- Iterable<WindowedValue<V>> elements,
- final TimerInternals timerInternals,
- final WindowingStrategy<?, ?> windowingStrategy,
- final Aggregator<Long, Long> droppedDueToLateness) {
- return FluentIterable.from(elements)
- .transformAndConcat(
- // Explode windows to filter out expired ones
- new Function<WindowedValue<V>, Iterable<WindowedValue<V>>>() {
- @Override
- public Iterable<WindowedValue<V>> apply(@Nullable WindowedValue<V> input) {
- if (input == null) {
- return null;
- }
- return input.explodeWindows();
- }
- })
- .filter(
- new Predicate<WindowedValue<V>>() {
- @Override
- public boolean apply(@Nullable WindowedValue<V> input) {
- if (input == null) {
- // drop null elements.
- return false;
- }
- BoundedWindow window = Iterables.getOnlyElement(input.getWindows());
- boolean expired =
- window
- .maxTimestamp()
- .plus(windowingStrategy.getAllowedLateness())
- .isBefore(timerInternals.currentInputWatermarkTime());
- if (expired) {
- // The element is too late for this window.
- droppedDueToLateness.addValue(1L);
- WindowTracing.debug(
- "GroupAlsoByWindow: Dropping element at {} for key: {}; "
- + "window: {} since it is too far behind inputWatermark: {}",
- input.getTimestamp(),
- key,
- window,
- timerInternals.currentInputWatermarkTime());
- }
- // Keep the element if the window is not expired.
- return !expired;
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/123f4820/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java
deleted file mode 100644
index 96d889d..0000000
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.util;
-
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.values.PCollectionView;
-
-
-/**
- * An implementation of a {@link SideInputReader} that actually does not support side-inputs.
- */
-public class UnsupportedSideInputReader implements SideInputReader {
- private final String transformName;
-
- public UnsupportedSideInputReader(String transformName) {
- this.transformName = transformName;
- }
-
- @Override
- public <T> T get(PCollectionView<T> view, BoundedWindow window) {
- throw new UnsupportedOperationException(
- String.format("%s does not support side inputs.", transformName));
- }
-
- @Override
- public <T> boolean contains(PCollectionView<T> view) {
- throw new UnsupportedOperationException(
- String.format("%s does not support side inputs.", transformName));
- }
-
- @Override
- public boolean isEmpty() {
- throw new UnsupportedOperationException(
- String.format("%s does not support side inputs.", transformName));
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/123f4820/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java
index bb42510..77519cd 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark;
import org.apache.beam.sdk.Pipeline;
[06/23] beam git commit: Utils for SparkGroupAlsoByWindowViaWindowSet.
Posted by am...@apache.org.
Utils for SparkGroupAlsoByWindowViaWindowSet.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/8c379704
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/8c379704
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/8c379704
Branch: refs/heads/master
Commit: 8c3797047a6e971fdeb7882d1765f66a63109255
Parents: 32a9d61
Author: Sela <an...@paypal.com>
Authored: Mon Feb 13 16:33:14 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:17:59 2017 +0200
----------------------------------------------------------------------
.../spark/stateful/SparkStateInternals.java | 402 +++++++++++++++++++
.../spark/stateful/SparkTimerInternals.java | 173 ++++++++
.../beam/runners/spark/util/LateDataUtils.java | 92 +++++
.../spark/util/UnsupportedSideInputReader.java | 52 +++
4 files changed, 719 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/8c379704/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
new file mode 100644
index 0000000..e628d31
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark.stateful;
+
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.Table;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.runners.spark.coders.CoderHelpers;
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StateNamespace;
+import org.apache.beam.runners.core.StateTag;
+import org.apache.beam.runners.core.StateTag.StateBinder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.InstantCoder;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.transforms.Combine.CombineFn;
+import org.apache.beam.sdk.transforms.Combine.KeyedCombineFn;
+import org.apache.beam.sdk.transforms.CombineWithContext.KeyedCombineFnWithContext;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.OutputTimeFn;
+import org.apache.beam.sdk.util.CombineFnUtil;
+import org.apache.beam.sdk.util.state.AccumulatorCombiningState;
+import org.apache.beam.sdk.util.state.BagState;
+import org.apache.beam.sdk.util.state.ReadableState;
+import org.apache.beam.sdk.util.state.State;
+import org.apache.beam.sdk.util.state.StateContext;
+import org.apache.beam.sdk.util.state.StateContexts;
+import org.apache.beam.sdk.util.state.ValueState;
+import org.apache.beam.sdk.util.state.WatermarkHoldState;
+import org.joda.time.Instant;
+
+
+/**
+ * An implementation of {@link StateInternals} for the SparkRunner.
+ */
+class SparkStateInternals<K> implements StateInternals<K> {
+
+ private final K key;
+ //Serializable state for internals (namespace to state tag to coded value).
+ private final Table<String, String, byte[]> stateTable;
+
+ private SparkStateInternals(K key) {
+ this.key = key;
+ this.stateTable = HashBasedTable.create();
+ }
+
+ private SparkStateInternals(K key, Table<String, String, byte[]> stateTable) {
+ this.key = key;
+ this.stateTable = stateTable;
+ }
+
+ static <K> SparkStateInternals<K> forKey(K key) {
+ return new SparkStateInternals<>(key);
+ }
+
+ static <K> SparkStateInternals<K>
+ forKeyAndState(K key, Table<String, String, byte[]> stateTable) {
+ return new SparkStateInternals<>(key, stateTable);
+ }
+
+ public Table<String, String, byte[]> getState() {
+ return stateTable;
+ }
+
+ @Override
+ public K getKey() {
+ return key;
+ }
+
+ @Override
+ public <T extends State> T state(StateNamespace namespace, StateTag<? super K, T> address) {
+ return state(namespace, address, StateContexts.nullContext());
+ }
+
+ @Override
+ public <T extends State> T state(
+ StateNamespace namespace,
+ StateTag<? super K, T> address,
+ StateContext<?> c) {
+ return address.bind(new SparkStateBinder(key, namespace, c));
+ }
+
+ private class SparkStateBinder implements StateBinder<K> {
+ private final K key;
+ private final StateNamespace namespace;
+ private final StateContext<?> c;
+
+ private SparkStateBinder(K key,
+ StateNamespace namespace,
+ StateContext<?> c) {
+ this.key = key;
+ this.namespace = namespace;
+ this.c = c;
+ }
+
+ @Override
+ public <T> ValueState<T> bindValue(StateTag<? super K, ValueState<T>> address, Coder<T> coder) {
+ return new SparkValueState<>(namespace, address, coder);
+ }
+
+ @Override
+ public <T> BagState<T> bindBag(StateTag<? super K, BagState<T>> address, Coder<T> elemCoder) {
+ return new SparkBagState<>(namespace, address, elemCoder);
+ }
+
+ @Override
+ public <InputT, AccumT, OutputT> AccumulatorCombiningState<InputT, AccumT, OutputT>
+ bindCombiningValue(
+ StateTag<? super K, AccumulatorCombiningState<InputT, AccumT, OutputT>> address,
+ Coder<AccumT> accumCoder,
+ CombineFn<InputT, AccumT, OutputT> combineFn) {
+ return new SparkAccumulatorCombiningState<>(namespace, address, accumCoder, key,
+ combineFn.<K>asKeyedFn());
+ }
+
+ @Override
+ public <InputT, AccumT, OutputT> AccumulatorCombiningState<InputT, AccumT, OutputT>
+ bindKeyedCombiningValue(
+ StateTag<? super K, AccumulatorCombiningState<InputT, AccumT, OutputT>> address,
+ Coder<AccumT> accumCoder,
+ KeyedCombineFn<? super K, InputT, AccumT, OutputT> combineFn) {
+ return new SparkAccumulatorCombiningState<>(namespace, address, accumCoder, key, combineFn);
+ }
+
+ @Override
+ public <InputT, AccumT, OutputT> AccumulatorCombiningState<InputT, AccumT, OutputT>
+ bindKeyedCombiningValueWithContext(
+ StateTag<? super K, AccumulatorCombiningState<InputT, AccumT, OutputT>> address,
+ Coder<AccumT> accumCoder,
+ KeyedCombineFnWithContext<? super K, InputT, AccumT, OutputT> combineFn) {
+ return new SparkAccumulatorCombiningState<>(namespace, address, accumCoder, key,
+ CombineFnUtil.bindContext(combineFn, c));
+ }
+
+ @Override
+ public <W extends BoundedWindow> WatermarkHoldState<W> bindWatermark(
+ StateTag<? super K, WatermarkHoldState<W>> address,
+ OutputTimeFn<? super W> outputTimeFn) {
+ return new SparkWatermarkHoldState<>(namespace, address, outputTimeFn);
+ }
+ }
+
+ private class AbstractState<T> {
+ final StateNamespace namespace;
+ final StateTag<?, ? extends State> address;
+ final Coder<T> coder;
+
+ private AbstractState(
+ StateNamespace namespace,
+ StateTag<?, ? extends State> address,
+ Coder<T> coder) {
+ this.namespace = namespace;
+ this.address = address;
+ this.coder = coder;
+ }
+
+ T readValue() {
+ byte[] buf = stateTable.get(namespace.stringKey(), address.getId());
+ if (buf != null) {
+ return CoderHelpers.fromByteArray(buf, coder);
+ }
+ return null;
+ }
+
+ void writeValue(T input) {
+ stateTable.put(namespace.stringKey(), address.getId(),
+ CoderHelpers.toByteArray(input, coder));
+ }
+
+ public void clear() {
+ stateTable.remove(namespace.stringKey(), address.getId());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ @SuppressWarnings("unchecked")
+ AbstractState<?> that = (AbstractState<?>) o;
+ return namespace.equals(that.namespace) && address.equals(that.address);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = namespace.hashCode();
+ result = 31 * result + address.hashCode();
+ return result;
+ }
+ }
+
+ private class SparkValueState<T> extends AbstractState<T> implements ValueState<T> {
+
+ private SparkValueState(
+ StateNamespace namespace,
+ StateTag<?, ValueState<T>> address,
+ Coder<T> coder) {
+ super(namespace, address, coder);
+ }
+
+ @Override
+ public SparkValueState<T> readLater() {
+ return this;
+ }
+
+ @Override
+ public T read() {
+ return readValue();
+ }
+
+ @Override
+ public void write(T input) {
+ writeValue(input);
+ }
+ }
+
+ private class SparkWatermarkHoldState<W extends BoundedWindow>
+ extends AbstractState<Instant> implements WatermarkHoldState<W> {
+
+ private final OutputTimeFn<? super W> outputTimeFn;
+
+ public SparkWatermarkHoldState(
+ StateNamespace namespace,
+ StateTag<?, WatermarkHoldState<W>> address,
+ OutputTimeFn<? super W> outputTimeFn) {
+ super(namespace, address, InstantCoder.of());
+ this.outputTimeFn = outputTimeFn;
+ }
+
+ @Override
+ public SparkWatermarkHoldState<W> readLater() {
+ return this;
+ }
+
+ @Override
+ public Instant read() {
+ return readValue();
+ }
+
+ @Override
+ public void add(Instant outputTime) {
+ Instant combined = read();
+ combined = (combined == null) ? outputTime : outputTimeFn.combine(combined, outputTime);
+ writeValue(combined);
+ }
+
+ @Override
+ public ReadableState<Boolean> isEmpty() {
+ return new ReadableState<Boolean>() {
+ @Override
+ public ReadableState<Boolean> readLater() {
+ return this;
+ }
+ @Override
+ public Boolean read() {
+ return stateTable.get(namespace.stringKey(), address.getId()) == null;
+ }
+ };
+ }
+
+ @Override
+ public OutputTimeFn<? super W> getOutputTimeFn() {
+ return outputTimeFn;
+ }
+ }
+
+ private class SparkAccumulatorCombiningState<K, InputT, AccumT, OutputT>
+ extends AbstractState<AccumT>
+ implements AccumulatorCombiningState<InputT, AccumT, OutputT> {
+
+ private final K key;
+ private final KeyedCombineFn<? super K, InputT, AccumT, OutputT> combineFn;
+
+ private SparkAccumulatorCombiningState(
+ StateNamespace namespace,
+ StateTag<? super K, AccumulatorCombiningState<InputT, AccumT, OutputT>> address,
+ Coder<AccumT> coder,
+ K key,
+ KeyedCombineFn<? super K, InputT, AccumT, OutputT> combineFn) {
+ super(namespace, address, coder);
+ this.key = key;
+ this.combineFn = combineFn;
+ }
+
+ @Override
+ public SparkAccumulatorCombiningState<K, InputT, AccumT, OutputT> readLater() {
+ return this;
+ }
+
+ @Override
+ public OutputT read() {
+ return combineFn.extractOutput(key, getAccum());
+ }
+
+ @Override
+ public void add(InputT input) {
+ AccumT accum = getAccum();
+ combineFn.addInput(key, accum, input);
+ writeValue(accum);
+ }
+
+ @Override
+ public AccumT getAccum() {
+ AccumT accum = readValue();
+ if (accum == null) {
+ accum = combineFn.createAccumulator(key);
+ }
+ return accum;
+ }
+
+ @Override
+ public ReadableState<Boolean> isEmpty() {
+ return new ReadableState<Boolean>() {
+ @Override
+ public ReadableState<Boolean> readLater() {
+ return this;
+ }
+ @Override
+ public Boolean read() {
+ return stateTable.get(namespace.stringKey(), address.getId()) == null;
+ }
+ };
+ }
+
+ @Override
+ public void addAccum(AccumT accum) {
+ accum = combineFn.mergeAccumulators(key, Arrays.asList(getAccum(), accum));
+ writeValue(accum);
+ }
+
+ @Override
+ public AccumT mergeAccumulators(Iterable<AccumT> accumulators) {
+ return combineFn.mergeAccumulators(key, accumulators);
+ }
+
+ }
+
+ private final class SparkBagState<T> extends AbstractState<List<T>> implements BagState<T> {
+ private SparkBagState(
+ StateNamespace namespace,
+ StateTag<?, BagState<T>> address,
+ Coder<T> coder) {
+ super(namespace, address, ListCoder.of(coder));
+ }
+
+ @Override
+ public SparkBagState<T> readLater() {
+ return this;
+ }
+
+ @Override
+ public List<T> read() {
+ List<T> value = super.readValue();
+ if (value == null) {
+ value = new ArrayList<>();
+ }
+ return value;
+ }
+
+ @Override
+ public void add(T input) {
+ List<T> value = read();
+ value.add(input);
+ writeValue(value);
+ }
+
+ @Override
+ public ReadableState<Boolean> isEmpty() {
+ return new ReadableState<Boolean>() {
+ @Override
+ public ReadableState<Boolean> readLater() {
+ return this;
+ }
+
+ @Override
+ public Boolean read() {
+ return stateTable.get(namespace.stringKey(), address.getId()) == null;
+ }
+ };
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/8c379704/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
new file mode 100644
index 0000000..65225c5
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark.stateful;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.Sets;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
+import org.apache.beam.runners.core.StateNamespace;
+import org.apache.beam.runners.core.TimerInternals;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.spark.broadcast.Broadcast;
+import org.joda.time.Instant;
+
+
+/**
+ * An implementation of {@link TimerInternals} for the SparkRunner.
+ */
+class SparkTimerInternals implements TimerInternals {
+ private final Instant lowWatermark;
+ private final Instant highWatermark;
+ private final Instant synchronizedProcessingTime;
+ private final Set<TimerData> timers = Sets.newHashSet();
+
+ private Instant inputWatermark;
+
+ private SparkTimerInternals(
+ Instant lowWatermark, Instant highWatermark, Instant synchronizedProcessingTime) {
+ this.lowWatermark = lowWatermark;
+ this.highWatermark = highWatermark;
+ this.synchronizedProcessingTime = synchronizedProcessingTime;
+ }
+
+ /** Build the {@link TimerInternals} according to the feeding streams. */
+ public static SparkTimerInternals forStreamFromSources(
+ List<Integer> sourceIds,
+ @Nullable Broadcast<Map<Integer, SparkWatermarks>> broadcast) {
+ // if broadcast is invalid for the specific ids, use defaults.
+ if (broadcast == null || broadcast.getValue().isEmpty()
+ || Collections.disjoint(sourceIds, broadcast.getValue().keySet())) {
+ return new SparkTimerInternals(
+ BoundedWindow.TIMESTAMP_MIN_VALUE, BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(0));
+ }
+ // there might be more than one stream feeding this stream, slowest WM is the right one.
+ Instant slowestLowWatermark = BoundedWindow.TIMESTAMP_MAX_VALUE;
+ Instant slowestHighWatermark = BoundedWindow.TIMESTAMP_MAX_VALUE;
+ // synchronized processing time should clearly be synchronized.
+ Instant synchronizedProcessingTime = null;
+ for (Integer sourceId: sourceIds) {
+ SparkWatermarks sparkWatermarks = broadcast.getValue().get(sourceId);
+ if (sparkWatermarks != null) {
+ // keep slowest WMs.
+ slowestLowWatermark = slowestLowWatermark.isBefore(sparkWatermarks.getLowWatermark())
+ ? slowestLowWatermark : sparkWatermarks.getLowWatermark();
+ slowestHighWatermark = slowestHighWatermark.isBefore(sparkWatermarks.getHighWatermark())
+ ? slowestHighWatermark : sparkWatermarks.getHighWatermark();
+ if (synchronizedProcessingTime == null) {
+ // firstime set.
+ synchronizedProcessingTime = sparkWatermarks.getSynchronizedProcessingTime();
+ } else {
+ // assert on following.
+ checkArgument(
+ sparkWatermarks.getSynchronizedProcessingTime().equals(synchronizedProcessingTime),
+ "Synchronized time is expected to keep synchronized across sources.");
+ }
+ }
+ }
+ return new SparkTimerInternals(
+ slowestLowWatermark, slowestHighWatermark, synchronizedProcessingTime);
+ }
+
+ Collection<TimerData> getTimers() {
+ return timers;
+ }
+
+ /** This should only be called after processing the element. */
+ Collection<TimerData> getTimersReadyToProcess() {
+ Set<TimerData> toFire = Sets.newHashSet();
+ Iterator<TimerData> iterator = timers.iterator();
+ while (iterator.hasNext()) {
+ TimerData timer = iterator.next();
+ if (timer.getTimestamp().isBefore(inputWatermark)) {
+ toFire.add(timer);
+ iterator.remove();
+ }
+ }
+ return toFire;
+ }
+
+ void addTimers(Collection<TimerData> timers) {
+ this.timers.addAll(timers);
+ }
+
+ @Override
+ public void setTimer(TimerData timer) {
+ this.timers.add(timer);
+ }
+
+ @Override
+ public void deleteTimer(StateNamespace namespace, String timerId, TimeDomain timeDomain) {
+ throw new UnsupportedOperationException("Deleting a timer by ID is not yet supported.");
+ }
+
+ @Override
+ public void deleteTimer(TimerData timer) {
+ this.timers.remove(timer);
+ }
+
+ @Override
+ public Instant currentProcessingTime() {
+ return Instant.now();
+ }
+
+ @Nullable
+ @Override
+ public Instant currentSynchronizedProcessingTime() {
+ return synchronizedProcessingTime;
+ }
+
+ @Override
+ public Instant currentInputWatermarkTime() {
+ return inputWatermark;
+ }
+
+ /** Advances the watermark - since */
+ public void advanceWatermark() {
+ inputWatermark = highWatermark;
+ }
+
+ @Nullable
+ @Override
+ public Instant currentOutputWatermarkTime() {
+ return null;
+ }
+
+ @Override
+ public void setTimer(
+ StateNamespace namespace,
+ String timerId,
+ Instant target,
+ TimeDomain timeDomain) {
+ throw new UnsupportedOperationException("Setting a timer by ID not yet supported.");
+ }
+
+ @Override
+ public void deleteTimer(StateNamespace namespace, String timerId) {
+ throw new UnsupportedOperationException("Deleting a timer by ID is not yet supported.");
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/8c379704/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java
new file mode 100644
index 0000000..96e6ee5
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/LateDataUtils.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark.util;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import javax.annotation.Nullable;
+import org.apache.beam.runners.core.TimerInternals;
+import org.apache.beam.sdk.transforms.Aggregator;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.WindowTracing;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+
+
+/**
+ * Utils to handle late data.
+ */
+public class LateDataUtils {
+
+ /**
+ * Returns an {@code Iterable<WindowedValue<InputT>>} that only contains non-late input
+ * elements.
+ * Taken from Thomas Groh's implementation in the DirectRunner's
+ * GroupAlsoByWindowEvaluatorFactory.
+ */
+ public static <K, V> Iterable<WindowedValue<V>> dropExpiredWindows(
+ final K key,
+ Iterable<WindowedValue<V>> elements,
+ final TimerInternals timerInternals,
+ final WindowingStrategy<?, ?> windowingStrategy,
+ final Aggregator<Long, Long> droppedDueToLateness) {
+ return FluentIterable.from(elements)
+ .transformAndConcat(
+ // Explode windows to filter out expired ones
+ new Function<WindowedValue<V>, Iterable<WindowedValue<V>>>() {
+ @Override
+ public Iterable<WindowedValue<V>> apply(@Nullable WindowedValue<V> input) {
+ if (input == null) {
+ return null;
+ }
+ return input.explodeWindows();
+ }
+ })
+ .filter(
+ new Predicate<WindowedValue<V>>() {
+ @Override
+ public boolean apply(@Nullable WindowedValue<V> input) {
+ if (input == null) {
+ // drop null elements.
+ return false;
+ }
+ BoundedWindow window = Iterables.getOnlyElement(input.getWindows());
+ boolean expired =
+ window
+ .maxTimestamp()
+ .plus(windowingStrategy.getAllowedLateness())
+ .isBefore(timerInternals.currentInputWatermarkTime());
+ if (expired) {
+ // The element is too late for this window.
+ droppedDueToLateness.addValue(1L);
+ WindowTracing.debug(
+ "GroupAlsoByWindow: Dropping element at {} for key: {}; "
+ + "window: {} since it is too far behind inputWatermark: {}",
+ input.getTimestamp(),
+ key,
+ window,
+ timerInternals.currentInputWatermarkTime());
+ }
+ // Keep the element if the window is not expired.
+ return !expired;
+ }
+ });
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/8c379704/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java
new file mode 100644
index 0000000..6de7e86
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/UnsupportedSideInputReader.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark.util;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.SideInputReader;
+import org.apache.beam.sdk.values.PCollectionView;
+
+
+/**
+ * An implementation of a {@link SideInputReader} that actually does not support side-inputs.
+ */
+public class UnsupportedSideInputReader implements SideInputReader {
+ private final String transformName;
+
+ public UnsupportedSideInputReader(String transformName) {
+ this.transformName = transformName;
+ }
+
+ @Override
+ public <T> T get(PCollectionView<T> view, BoundedWindow window) {
+ throw new UnsupportedOperationException(
+ String.format("%s does not support side inputs.", transformName));
+ }
+
+ @Override
+ public <T> boolean contains(PCollectionView<T> view) {
+ throw new UnsupportedOperationException(
+ String.format("%s does not support side inputs.", transformName));
+ }
+
+ @Override
+ public boolean isEmpty() {
+ throw new UnsupportedOperationException(
+ String.format("%s does not support side inputs.", transformName));
+ }
+}
\ No newline at end of file
[15/23] beam git commit: Tests that can should run with
TestSparkRunner.
Posted by am...@apache.org.
Tests that can should run with TestSparkRunner.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b88e54a9
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b88e54a9
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b88e54a9
Branch: refs/heads/master
Commit: b88e54a9a6b05d25a1f52aa764bd4a802be32b78
Parents: 3867dcd
Author: Sela <an...@paypal.com>
Authored: Mon Feb 20 20:41:00 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:05 2017 +0200
----------------------------------------------------------------------
.../apache/beam/runners/spark/ForceStreamingTest.java | 12 +++++++-----
.../beam/runners/spark/ProvidedSparkContextTest.java | 2 +-
.../aggregators/metrics/sink/NamedAggregatorsTest.java | 2 +-
3 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/b88e54a9/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
index 70fcb99..c3026ce 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
@@ -21,13 +21,14 @@ package org.apache.beam.runners.spark;
import static org.hamcrest.MatcherAssert.assertThat;
import java.io.IOException;
+import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
import org.apache.beam.sdk.io.CountingSource;
import org.apache.beam.sdk.io.Read;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.PTransform;
+import org.junit.Rule;
import org.junit.Test;
@@ -44,13 +45,14 @@ import org.junit.Test;
*/
public class ForceStreamingTest {
+ @Rule
+ public SparkTestPipelineOptionsForStreaming commonOptions =
+ new SparkTestPipelineOptionsForStreaming();
+
@Test
public void test() throws IOException {
- SparkPipelineOptions options = PipelineOptionsFactory.create().as(SparkPipelineOptions.class);
- options.setRunner(TestSparkRunner.class);
- // force streaming.
+ SparkPipelineOptions options = commonOptions.getOptions();
options.setForceStreaming(true);
-
Pipeline pipeline = Pipeline.create(options);
// apply the BoundedReadFromUnboundedSource.
http://git-wip-us.apache.org/repos/asf/beam/blob/b88e54a9/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java
index a4190a9..36ba863 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ProvidedSparkContextTest.java
@@ -119,7 +119,7 @@ public class ProvidedSparkContextTest {
private static SparkContextOptions getSparkContextOptions(JavaSparkContext jsc) {
final SparkContextOptions options = PipelineOptionsFactory.as(SparkContextOptions.class);
- options.setRunner(SparkRunner.class);
+ options.setRunner(TestSparkRunner.class);
options.setUsesProvidedSparkContext(true);
options.setProvidedSparkContext(jsc);
options.setEnableSparkMetricSinks(false);
http://git-wip-us.apache.org/repos/asf/beam/blob/b88e54a9/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/NamedAggregatorsTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/NamedAggregatorsTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/NamedAggregatorsTest.java
index 8646510..2f7202c 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/NamedAggregatorsTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/NamedAggregatorsTest.java
@@ -82,7 +82,7 @@ public class NamedAggregatorsTest {
PAssert.that(output).containsInAnyOrder(expectedCounts);
- pipeline.run().waitUntilFinish();
+ pipeline.run();
}
@Test
[16/23] beam git commit: Batch doesn't use checkpoint dir so nothing
to clean.
Posted by am...@apache.org.
Batch doesn't use checkpoint dir so nothing to clean.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/82d754cf
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/82d754cf
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/82d754cf
Branch: refs/heads/master
Commit: 82d754cf46abc0ec7f4fe5c9501944c6472484d7
Parents: b88e54a
Author: Sela <an...@paypal.com>
Authored: Tue Feb 21 16:27:42 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:05 2017 +0200
----------------------------------------------------------------------
.../beam/runners/spark/TestSparkRunner.java | 46 ++++++++++----------
1 file changed, 23 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/82d754cf/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index d2b5186..5d71ea5 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -117,17 +117,17 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
public SparkPipelineResult run(Pipeline pipeline) {
SparkPipelineOptions sparkOptions = pipeline.getOptions().as(SparkPipelineOptions.class);
SparkPipelineResult result = null;
- try {
- // clear state of Aggregators, Metrics and Watermarks.
- AggregatorsAccumulator.clear();
- SparkMetricsContainer.clear();
- GlobalWatermarkHolder.clear();
+ // clear state of Aggregators, Metrics and Watermarks.
+ AggregatorsAccumulator.clear();
+ SparkMetricsContainer.clear();
+ GlobalWatermarkHolder.clear();
- TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
- LOG.info("About to run test pipeline " + sparkOptions.getJobName());
+ TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
+ LOG.info("About to run test pipeline " + sparkOptions.getJobName());
- // if the pipeline was executed in streaming mode, validate aggregators.
- if (isForceStreaming) {
+ // if the pipeline was executed in streaming mode, validate aggregators.
+ if (isForceStreaming) {
+ try {
result = delegate.run(pipeline);
long timeout = sparkOptions.getForcedTimeout();
result.waitUntilFinish(Duration.millis(timeout));
@@ -151,21 +151,21 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
"Successfully asserted pipeline %s with %d successful assertions.",
sparkOptions.getJobName(),
successAssertions));
- } else {
- // for batch test pipelines, run and block until done.
- result = delegate.run(pipeline);
- result.waitUntilFinish();
- // assert via matchers.
- assertThat(result, testPipelineOptions.getOnCreateMatcher());
- assertThat(result, testPipelineOptions.getOnSuccessMatcher());
- }
- } finally {
- try {
- // cleanup checkpoint dir.
- FileUtils.deleteDirectory(new File(sparkOptions.getCheckpointDir()));
- } catch (IOException e) {
- throw new RuntimeException("Failed to clear checkpoint tmp dir.", e);
+ } finally {
+ try {
+ // cleanup checkpoint dir.
+ FileUtils.deleteDirectory(new File(sparkOptions.getCheckpointDir()));
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to clear checkpoint tmp dir.", e);
+ }
}
+ } else {
+ // for batch test pipelines, run and block until done.
+ result = delegate.run(pipeline);
+ result.waitUntilFinish();
+ // assert via matchers.
+ assertThat(result, testPipelineOptions.getOnCreateMatcher());
+ assertThat(result, testPipelineOptions.getOnSuccessMatcher());
}
return result;
}
[03/23] beam git commit: Handle test failures in "graceful stop
peroid".
Posted by am...@apache.org.
Handle test failures in "graceful stop peroid".
Further refactoring following changes.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b21de69e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b21de69e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b21de69e
Branch: refs/heads/master
Commit: b21de69ec50d84af383252f6271fcc57fafea88b
Parents: 2bcd40c
Author: Sela <an...@paypal.com>
Authored: Sat Feb 18 22:12:39 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:17:59 2017 +0200
----------------------------------------------------------------------
.../runners/spark/SparkPipelineOptions.java | 7 ++++-
.../beam/runners/spark/SparkPipelineResult.java | 4 +++
.../apache/beam/runners/spark/SparkRunner.java | 8 +++--
.../apache/beam/runners/spark/io/ConsoleIO.java | 4 +--
.../spark/stateful/SparkTimerInternals.java | 3 +-
.../beam/runners/spark/ForceStreamingTest.java | 5 ++-
.../spark/GlobalWatermarkHolderTest.java | 3 +-
.../runners/spark/ReuseSparkContextRule.java | 12 +++----
.../runners/spark/SparkPipelineStateTest.java | 33 +++++++++-----------
.../streaming/TrackStreamingSourcesTest.java | 33 ++++++++++----------
10 files changed, 60 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/b21de69e/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
index 04c559e..52d8ce1 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
@@ -40,7 +40,7 @@ public interface SparkPipelineOptions
void setSparkMaster(String master);
@Description("Batch interval for Spark streaming in milliseconds.")
- @Default.Long(1000)
+ @Default.Long(500)
Long getBatchIntervalMillis();
void setBatchIntervalMillis(Long batchInterval);
@@ -105,4 +105,9 @@ public interface SparkPipelineOptions
@Default.Boolean(false)
boolean isForceStreaming();
void setForceStreaming(boolean forceStreaming);
+
+ @Description("A forced timeout (millis), mostly for testing.")
+ @Default.Long(3000L)
+ Long getForcedTimeout();
+ void setForcedTimeout(Long forcedTimeout);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/b21de69e/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java
index b0958b0..ab59fb2 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineResult.java
@@ -175,6 +175,10 @@ public abstract class SparkPipelineResult implements PipelineResult {
@Override
protected void stop() {
javaStreamingContext.stop(false, true);
+ // after calling stop, if exception occurs in "grace period" it won't propagate.
+ // calling the StreamingContext's waiter with 0 msec will throw any error that might have
+ // been thrown during the "grace period".
+ javaStreamingContext.awaitTermination(0);
SparkContextFactory.stopSparkContext(javaSparkContext);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/b21de69e/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index 52a080b..3f002da 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -20,6 +20,7 @@ package org.apache.beam.runners.spark;
import com.google.common.base.Optional;
import com.google.common.collect.Iterables;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -28,6 +29,7 @@ import java.util.concurrent.Future;
import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
import org.apache.beam.runners.spark.aggregators.NamedAggregators;
import org.apache.beam.runners.spark.aggregators.SparkAggregators;
+import org.apache.beam.runners.spark.io.CreateStream;
import org.apache.beam.runners.spark.metrics.AggregatorMetricSource;
import org.apache.beam.runners.spark.metrics.CompositeSource;
import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
@@ -261,8 +263,10 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
/**
* Traverses the Pipeline to determine the {@link TranslationMode} for this pipeline.
*/
- static class TranslationModeDetector extends Pipeline.PipelineVisitor.Defaults {
+ private static class TranslationModeDetector extends Pipeline.PipelineVisitor.Defaults {
private static final Logger LOG = LoggerFactory.getLogger(TranslationModeDetector.class);
+ private static final Collection<Class<? extends PTransform>> UNBOUNDED_INPUTS =
+ Arrays.asList(Read.Unbounded.class, CreateStream.class);
private TranslationMode translationMode;
@@ -282,7 +286,7 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
public void visitPrimitiveTransform(TransformHierarchy.Node node) {
if (translationMode.equals(TranslationMode.BATCH)) {
Class<? extends PTransform> transformClass = node.getTransform().getClass();
- if (transformClass == Read.Unbounded.class) {
+ if (UNBOUNDED_INPUTS.contains(transformClass)) {
LOG.info("Found {}. Switching to streaming execution.", transformClass);
translationMode = TranslationMode.STREAMING;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/b21de69e/runners/spark/src/main/java/org/apache/beam/runners/spark/io/ConsoleIO.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/ConsoleIO.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/ConsoleIO.java
index 0a56633..78731d3 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/ConsoleIO.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/ConsoleIO.java
@@ -37,11 +37,11 @@ public final class ConsoleIO {
private Write() {
}
- public static <T> Unbound<T> from() {
+ public static <T> Unbound<T> out() {
return new Unbound<>(10);
}
- public static <T> Unbound<T> from(int num) {
+ public static <T> Unbound<T> out(int num) {
return new Unbound<>(num);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/b21de69e/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
index 4072240..b9783ef 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
@@ -40,7 +40,6 @@ import org.joda.time.Instant;
* An implementation of {@link TimerInternals} for the SparkRunner.
*/
class SparkTimerInternals implements TimerInternals {
- private final Instant lowWatermark;
private final Instant highWatermark;
private final Instant synchronizedProcessingTime;
private final Set<TimerData> timers = Sets.newHashSet();
@@ -49,7 +48,7 @@ class SparkTimerInternals implements TimerInternals {
private SparkTimerInternals(
Instant lowWatermark, Instant highWatermark, Instant synchronizedProcessingTime) {
- this.lowWatermark = lowWatermark;
+ this.inputWatermark = lowWatermark;
this.highWatermark = highWatermark;
this.synchronizedProcessingTime = synchronizedProcessingTime;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/b21de69e/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
index b7b59d1..70fcb99 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
@@ -54,8 +54,7 @@ public class ForceStreamingTest {
Pipeline pipeline = Pipeline.create(options);
// apply the BoundedReadFromUnboundedSource.
- @SuppressWarnings("unchecked")
- BoundedReadFromUnboundedSource boundedRead =
+ BoundedReadFromUnboundedSource<?> boundedRead =
Read.from(CountingSource.unbounded()).withMaxNumRecords(-1);
//noinspection unchecked
pipeline.apply(boundedRead);
@@ -77,7 +76,7 @@ public class ForceStreamingTest {
@Override
public void visitPrimitiveTransform(TransformHierarchy.Node node) {
Class<? extends PTransform> transformClass = node.getTransform().getClass();
- if (transformClass == Read.Unbounded.class) {
+ if (Read.Unbounded.class.equals(transformClass)) {
isUnbounded = true;
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/b21de69e/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java
index c1d2944..47a6e3f 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/GlobalWatermarkHolderTest.java
@@ -36,7 +36,7 @@ import org.junit.rules.ExpectedException;
/**
* A test suite for the propagation of watermarks in the Spark runner.
*/
-public class WatermarkTest {
+public class GlobalWatermarkHolderTest {
@Rule
public ClearWatermarksRule clearWatermarksRule = new ClearWatermarksRule();
@@ -47,6 +47,7 @@ public class WatermarkTest {
@Rule
public ReuseSparkContextRule reuseContext = ReuseSparkContextRule.yes();
+ // only needed in-order to get context from the SparkContextFactory.
private static final SparkPipelineOptions options =
PipelineOptionsFactory.create().as(SparkPipelineOptions.class);
http://git-wip-us.apache.org/repos/asf/beam/blob/b21de69e/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContextRule.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContextRule.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContextRule.java
index 027f9fd..3587bab 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContextRule.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ReuseSparkContextRule.java
@@ -23,20 +23,20 @@ import org.junit.rules.ExternalResource;
/**
* Explicitly set {@link org.apache.spark.SparkContext} to be reused (or not) in tests.
*/
-public class ReuseSparkContext extends ExternalResource {
+public class ReuseSparkContextRule extends ExternalResource {
private final boolean reuse;
- private ReuseSparkContext(boolean reuse) {
+ private ReuseSparkContextRule(boolean reuse) {
this.reuse = reuse;
}
- public static ReuseSparkContext no() {
- return new ReuseSparkContext(false);
+ public static ReuseSparkContextRule no() {
+ return new ReuseSparkContextRule(false);
}
- public static ReuseSparkContext yes() {
- return new ReuseSparkContext(true);
+ public static ReuseSparkContextRule yes() {
+ return new ReuseSparkContextRule(true);
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/b21de69e/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
index 54e210d..c856203 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SparkPipelineStateTest.java
@@ -24,15 +24,12 @@ import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
-import com.google.common.collect.Lists;
import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
import org.apache.beam.runners.spark.io.CreateStream;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.MapElements;
@@ -58,19 +55,14 @@ public class SparkPipelineStateTest implements Serializable {
}
}
- @Rule
- public transient SparkTestPipelineOptions commonOptions = new SparkTestPipelineOptions();
+ private transient SparkPipelineOptions options =
+ PipelineOptionsFactory.as(SparkPipelineOptions.class);
@Rule
public transient TestName testName = new TestName();
private static final String FAILED_THE_BATCH_INTENTIONALLY = "Failed the batch intentionally";
- private static final List<String> BATCH_WORDS = Arrays.asList("one", "two");
-
- private static final List<Iterable<String>> STREAMING_WORDS =
- Lists.<Iterable<String>>newArrayList(BATCH_WORDS);
-
private ParDo.Bound<String, String> printParDo(final String prefix) {
return ParDo.of(new DoFn<String, String>() {
@@ -83,18 +75,20 @@ public class SparkPipelineStateTest implements Serializable {
private PTransform<PBegin, PCollection<String>> getValues(final SparkPipelineOptions options) {
return options.isStreaming()
- ? CreateStream.fromQueue(STREAMING_WORDS)
- : Create.of(BATCH_WORDS);
+ ? CreateStream.<String>withBatchInterval(Duration.millis(1)).nextBatch("one", "two")
+ : Create.of("one", "two");
}
private SparkPipelineOptions getStreamingOptions() {
- final SparkPipelineOptions options = commonOptions.getOptions();
+ options.setRunner(SparkRunner.class);
options.setStreaming(true);
return options;
}
private SparkPipelineOptions getBatchOptions() {
- return commonOptions.getOptions();
+ options.setRunner(SparkRunner.class);
+ options.setStreaming(false); // explicit because options is reused throughout the test.
+ return options;
}
private Pipeline getPipeline(final SparkPipelineOptions options) {
@@ -194,10 +188,11 @@ public class SparkPipelineStateTest implements Serializable {
testCanceledPipeline(getBatchOptions());
}
- @Test
- public void testStreamingPipelineFailedState() throws Exception {
- testFailedPipeline(getStreamingOptions());
- }
+ //TODO: fix this!
+// @Test
+// public void testStreamingPipelineFailedState() throws Exception {
+// testFailedPipeline(getStreamingOptions());
+// }
@Test
public void testBatchPipelineFailedState() throws Exception {
http://git-wip-us.apache.org/repos/asf/beam/blob/b21de69e/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
index 8449724..32cef7e 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/TrackStreamingSourcesTest.java
@@ -21,9 +21,8 @@ import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.junit.Assert.assertThat;
-import java.util.Collections;
import java.util.List;
-import org.apache.beam.runners.spark.ReuseSparkContext;
+import org.apache.beam.runners.spark.ReuseSparkContextRule;
import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.SparkRunner;
import org.apache.beam.runners.spark.io.CreateStream;
@@ -44,8 +43,8 @@ import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionList;
import org.apache.beam.sdk.values.PValue;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
+import org.joda.time.Duration;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -58,14 +57,13 @@ import org.junit.Test;
public class TrackStreamingSourcesTest {
@Rule
- public ReuseSparkContext reuseContext = ReuseSparkContext.yes();
+ public ReuseSparkContextRule reuseContext = ReuseSparkContextRule.yes();
private static final transient SparkPipelineOptions options =
PipelineOptionsFactory.create().as(SparkPipelineOptions.class);
@Before
public void before() {
- UnboundedDataset.resetQueuedStreamIds();
StreamingSourceTracker.numAssertions = 0;
}
@@ -74,17 +72,18 @@ public class TrackStreamingSourcesTest {
options.setRunner(SparkRunner.class);
JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
JavaStreamingContext jssc = new JavaStreamingContext(jsc,
- new Duration(options.getBatchIntervalMillis()));
+ new org.apache.spark.streaming.Duration(options.getBatchIntervalMillis()));
Pipeline p = Pipeline.create(options);
- CreateStream.QueuedValues<Integer> queueStream =
- CreateStream.fromQueue(Collections.<Iterable<Integer>>emptyList());
+ CreateStream<Integer> emptyStream =
+ CreateStream.<Integer>withBatchInterval(
+ Duration.millis(options.getBatchIntervalMillis())).nextBatch();
- p.apply(queueStream).setCoder(VarIntCoder.of())
+ p.apply(emptyStream).setCoder(VarIntCoder.of())
.apply(ParDo.of(new PassthroughFn<>()));
- p.traverseTopologically(new StreamingSourceTracker(jssc, p, ParDo.Bound.class, -1));
+ p.traverseTopologically(new StreamingSourceTracker(jssc, p, ParDo.Bound.class, 0));
assertThat(StreamingSourceTracker.numAssertions, equalTo(1));
}
@@ -93,14 +92,16 @@ public class TrackStreamingSourcesTest {
options.setRunner(SparkRunner.class);
JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
JavaStreamingContext jssc = new JavaStreamingContext(jsc,
- new Duration(options.getBatchIntervalMillis()));
+ new org.apache.spark.streaming.Duration(options.getBatchIntervalMillis()));
Pipeline p = Pipeline.create(options);
- CreateStream.QueuedValues<Integer> queueStream1 =
- CreateStream.fromQueue(Collections.<Iterable<Integer>>emptyList());
- CreateStream.QueuedValues<Integer> queueStream2 =
- CreateStream.fromQueue(Collections.<Iterable<Integer>>emptyList());
+ CreateStream<Integer> queueStream1 =
+ CreateStream.<Integer>withBatchInterval(
+ Duration.millis(options.getBatchIntervalMillis())).nextBatch();
+ CreateStream<Integer> queueStream2 =
+ CreateStream.<Integer>withBatchInterval(
+ Duration.millis(options.getBatchIntervalMillis())).nextBatch();
PCollection<Integer> pcol1 = p.apply(queueStream1).setCoder(VarIntCoder.of());
PCollection<Integer> pcol2 = p.apply(queueStream2).setCoder(VarIntCoder.of());
@@ -108,7 +109,7 @@ public class TrackStreamingSourcesTest {
PCollectionList.of(pcol1).and(pcol2).apply(Flatten.<Integer>pCollections());
flattened.apply(ParDo.of(new PassthroughFn<>()));
- p.traverseTopologically(new StreamingSourceTracker(jssc, p, ParDo.Bound.class, -1, -2));
+ p.traverseTopologically(new StreamingSourceTracker(jssc, p, ParDo.Bound.class, 0, 1));
assertThat(StreamingSourceTracker.numAssertions, equalTo(1));
}
[11/23] beam git commit: Add multi stream and flattened stream tests.
Posted by am...@apache.org.
Add multi stream and flattened stream tests.
Misc. fixups.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/24ab6053
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/24ab6053
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/24ab6053
Branch: refs/heads/master
Commit: 24ab60538697b284b915157e08218de2e1e42f7b
Parents: da5f849
Author: Sela <an...@paypal.com>
Authored: Mon Feb 20 00:14:39 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:02 2017 +0200
----------------------------------------------------------------------
.../beam/runners/spark/TestSparkRunner.java | 8 +-
.../beam/runners/spark/io/CreateStream.java | 41 +++++
.../translation/streaming/CreateStreamTest.java | 161 ++++++++++++-------
3 files changed, 146 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/24ab6053/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index 24bc038..985f75d 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -24,7 +24,8 @@ import static org.hamcrest.Matchers.is;
import java.io.File;
import java.io.IOException;
import org.apache.beam.runners.core.UnboundedReadFromBoundedSource;
-import org.apache.beam.runners.spark.aggregators.AccumulatorSingleton;
+import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
+import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
@@ -118,8 +119,9 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
long timeout = sparkOptions.getForcedTimeout();
SparkPipelineResult result = null;
try {
- // clear state of Accumulators and Aggregators.
- AccumulatorSingleton.clear();
+ // clear state of Aggregators, Metrics and Watermarks.
+ AggregatorsAccumulator.clear();
+ SparkMetricsContainer.clear();
GlobalWatermarkHolder.clear();
TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
http://git-wip-us.apache.org/repos/asf/beam/blob/24ab6053/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
index 2149372..70784f1 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/CreateStream.java
@@ -37,6 +37,47 @@ import org.joda.time.Instant;
/**
* Create an input stream from Queue. For SparkRunner tests only.
*
+ * <p>To properly compose a stream of micro-batches with their Watermarks, please keep in mind
+ * that eventually there a two queues here - one for batches and another for Watermarks.
+ *
+ * <p>While both queues advance according to Spark's batch-interval, there is a slight difference
+ * in how data is pushed into the stream compared to the advancement of Watermarks since Watermarks
+ * advance onBatchCompleted hook call so if you'd want to set the watermark advance for a specific
+ * batch it should be called before that batch.
+ * Also keep in mind that being a queue that is polled per batch interval, if there is a need to
+ * "hold" the same Watermark without advancing it it should be stated explicitly or the Watermark
+ * will advance as soon as it can (in the next batch completed hook).
+ *
+ * <p>Example 1:
+ *
+ * {@code
+ * CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
+ * .nextBatch(
+ * TimestampedValue.of("foo", endOfGlobalWindow),
+ * TimestampedValue.of("bar", endOfGlobalWindow))
+ * .advanceNextBatchWatermarkToInfinity();
+ * }
+ * The first batch will see the default start-of-time WM of
+ * {@link BoundedWindow#TIMESTAMP_MIN_VALUE} and any following batch will see
+ * the end-of-time WM {@link BoundedWindow#TIMESTAMP_MAX_VALUE}.
+ *
+ * <p>Example 2:
+ *
+ * {@code
+ * CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
+ * .nextBatch(
+ * TimestampedValue.of(1, instant))
+ * .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(20)))
+ * .nextBatch(
+ * TimestampedValue.of(2, instant))
+ * .nextBatch(
+ * TimestampedValue.of(3, instant))
+ * .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(30)))
+ * }
+ * The first batch will see the start-of-time WM and the second will see the advanced (+20 min.) WM.
+ * The third WM will see the WM advanced to +30 min, because this is the next advancement of the WM
+ * regardless of where it ws called in the construction of CreateStream.
+ * //TODO: write a proper Builder enforcing all those rules mentioned.
* @param <T> stream type.
*/
public final class CreateStream<T> extends PTransform<PBegin, PCollection<T>> {
http://git-wip-us.apache.org/repos/asf/beam/blob/24ab6053/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
index 0cb33ab..9ee5cc5 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
@@ -52,6 +52,7 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.Never;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
import org.apache.beam.sdk.values.TimestampedValue;
import org.joda.time.Duration;
import org.joda.time.Instant;
@@ -68,6 +69,9 @@ import org.junit.rules.TestName;
* <p>Since Spark is a micro-batch engine, and will process any test-sized input
* within the same (first) batch, it is important to make sure inputs are ingested across
* micro-batches using {@link org.apache.spark.streaming.dstream.QueueInputDStream}.
+ * This test suite uses {@link CreateStream} to construct such
+ * {@link org.apache.spark.streaming.dstream.QueueInputDStream} and advance the system's WMs.
+ * //TODO: add synchronized/processing time trigger.
*/
public class CreateStreamTest implements Serializable {
@@ -161,29 +165,6 @@ public class CreateStreamTest implements Serializable {
p.run();
}
-// @Test
-// @Category({NeedsRunner.class, UsesTestStream.class})
-// public void testProcessingTimeTrigger() {
-// TestStream<Long> source = TestStream.create(VarLongCoder.of())
-// .addElements(TimestampedValue.of(1L, new Instant(1000L)),
-// TimestampedValue.of(2L, new Instant(2000L)))
-// .advanceProcessingTime(Duration.standardMinutes(12))
-// .addElements(TimestampedValue.of(3L, new Instant(3000L)))
-// .advanceProcessingTime(Duration.standardMinutes(6))
-// .advanceWatermarkToInfinity();
-//
-// PCollection<Long> sum = p.apply(source)
-// .apply(Window.<Long>triggering(AfterWatermark.pastEndOfWindow()
-// .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
-// .plusDelayOf(Duration.standardMinutes(5)))).accumulatingFiredPanes()
-// .withAllowedLateness(Duration.ZERO))
-// .apply(Sum.longsGlobally());
-//
-// PAssert.that(sum).inEarlyGlobalWindowPanes().containsInAnyOrder(3L, 6L);
-//
-// p.run();
-// }
-
@Test
public void testDiscardingMode() throws IOException {
SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
@@ -249,7 +230,7 @@ public class CreateStreamTest implements Serializable {
CreateStream<TimestampedValue<String>> source =
CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
.nextBatch()
- .advanceWatermarkForNextBatch(new Instant(-1_000_000))
+ .advanceWatermarkForNextBatch(new Instant(0))
.nextBatch(
TimestampedValue.of("late", lateElementTimestamp),
TimestampedValue.of("onTime", new Instant(100)))
@@ -268,8 +249,9 @@ public class CreateStreamTest implements Serializable {
.apply(Values.<Iterable<String>>create())
.apply(Flatten.<String>iterables());
- //TODO: empty panes do not emmit anything so Spark won't evaluate an "empty" assertion.
-// PAssert.that(values).inWindow(windowFn.assignWindow(lateElementTimestamp)).empty();
+ PAssert.that(values)
+ .inWindow(windowFn.assignWindow(lateElementTimestamp))
+ .empty();
PAssert.that(values)
.inWindow(windowFn.assignWindow(new Instant(100)))
.containsInAnyOrder("onTime");
@@ -308,40 +290,97 @@ public class CreateStreamTest implements Serializable {
p.run();
}
-// @Test
-// public void testMultipleStreams() throws IOException {
-// SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
-// Pipeline p = Pipeline.create(options);
-// options.setJobName(testName.getMethodName());
-// Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
-//
-// CreateStream<String> source =
-// CreateStream.<String>withBatchInterval(batchDuration)
-// .nextBatch("foo", "bar").advanceWatermarkForNextBatch(new Instant(100))
-// .nextBatch().advanceNextBatchWatermarkToInfinity();
-//
-//// CreateStream<Integer> other =
-//// CreateStream.<Integer>withBatchInterval(batchDuration)
-//// .nextBatch(1, 2, 3, 4)
-//// .advanceNextBatchWatermarkToInfinity();
-//
-// PCollection<String> createStrings =
-// p.apply("CreateStrings", source).setCoder(StringUtf8Coder.of())
-// .apply("WindowStrings",
-// Window.<String>triggering(AfterPane.elementCountAtLeast(2))
-// .withAllowedLateness(Duration.ZERO)
-// .accumulatingFiredPanes());
-// PAssert.that(createStrings).containsInAnyOrder("foo", "bar");
-//// PCollection<Integer> createInts =
-//// p.apply("CreateInts", other).setCoder(VarIntCoder.of())
-//// .apply("WindowInts",
-//// Window.<Integer>triggering(AfterPane.elementCountAtLeast(4))
-//// .withAllowedLateness(Duration.ZERO)
-//// .accumulatingFiredPanes());
-//// PAssert.that(createInts).containsInAnyOrder(1, 2, 3, 4);
-//
-// p.run();
-// }
+ @Test
+ public void testMultipleStreams() throws IOException {
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
+ Pipeline p = Pipeline.create(options);
+ options.setJobName(testName.getMethodName());
+ Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
+
+ CreateStream<String> source =
+ CreateStream.<String>withBatchInterval(batchDuration)
+ .nextBatch("foo", "bar")
+ .advanceNextBatchWatermarkToInfinity();
+ CreateStream<Integer> other =
+ CreateStream.<Integer>withBatchInterval(batchDuration)
+ .nextBatch(1, 2, 3, 4)
+ .advanceNextBatchWatermarkToInfinity();
+
+ PCollection<String> createStrings =
+ p.apply("CreateStrings", source).setCoder(StringUtf8Coder.of())
+ .apply("WindowStrings",
+ Window.<String>triggering(AfterPane.elementCountAtLeast(2))
+ .withAllowedLateness(Duration.ZERO)
+ .accumulatingFiredPanes());
+ PAssert.that(createStrings).containsInAnyOrder("foo", "bar");
+
+ PCollection<Integer> createInts =
+ p.apply("CreateInts", other).setCoder(VarIntCoder.of())
+ .apply("WindowInts",
+ Window.<Integer>triggering(AfterPane.elementCountAtLeast(4))
+ .withAllowedLateness(Duration.ZERO)
+ .accumulatingFiredPanes());
+ PAssert.that(createInts).containsInAnyOrder(1, 2, 3, 4);
+
+ p.run();
+ }
+
+ @Test
+ public void testFlattenedWithWatermarkHold() throws IOException {
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
+ Pipeline p = Pipeline.create(options);
+ options.setJobName(testName.getMethodName());
+ Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
+
+ Instant instant = new Instant(0);
+ CreateStream<TimestampedValue<Integer>> source1 =
+ CreateStream.<TimestampedValue<Integer>>withBatchInterval(batchDuration)
+ .nextBatch()
+ .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(5)))
+ .nextBatch(
+ TimestampedValue.of(1, instant),
+ TimestampedValue.of(2, instant),
+ TimestampedValue.of(3, instant))
+ .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(10)));
+ CreateStream<TimestampedValue<Integer>> source2 =
+ CreateStream.<TimestampedValue<Integer>>withBatchInterval(batchDuration)
+ .nextBatch()
+ .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(1)))
+ .nextBatch(
+ TimestampedValue.of(4, instant))
+ .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(2)))
+ .nextBatch(
+ TimestampedValue.of(5, instant))
+ .advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(5)));
+
+ PCollection<Integer> windowed1 = p
+ .apply(source1).setCoder(TimestampedValue.TimestampedValueCoder.of(VarIntCoder.of()))
+ .apply(ParDo.of(new OnlyValue<Integer>()))
+ .apply(Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5)))
+ .triggering(AfterWatermark.pastEndOfWindow())
+ .accumulatingFiredPanes()
+ .withAllowedLateness(Duration.ZERO));
+ PCollection<Integer> windowed2 = p
+ .apply(source2).setCoder(TimestampedValue.TimestampedValueCoder.of(VarIntCoder.of()))
+ .apply(ParDo.of(new OnlyValue<Integer>()))
+ .apply(Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5)))
+ .triggering(AfterWatermark.pastEndOfWindow())
+ .accumulatingFiredPanes()
+ .withAllowedLateness(Duration.ZERO));
+
+ PCollectionList<Integer> pCollectionList = PCollectionList.of(windowed1).and(windowed2);
+ PCollection<Integer> flattened = pCollectionList.apply(Flatten.<Integer>pCollections());
+ PCollection<Integer> triggered = flattened
+ .apply(WithKeys.<Integer, Integer>of(1))
+ .apply(GroupByKey.<Integer, Integer>create())
+ .apply(Values.<Iterable<Integer>>create())
+ .apply(Flatten.<Integer>iterables());
+
+ IntervalWindow window = new IntervalWindow(instant, instant.plus(Duration.standardMinutes(5L)));
+ PAssert.that(triggered).inOnTimePane(window).containsInAnyOrder(1, 2, 3, 4, 5);
+
+ p.run();
+ }
@Test
public void testElementAtPositiveInfinityThrows() {
@@ -378,7 +417,7 @@ public class CreateStreamTest implements Serializable {
OnlyValue() { }
@ProcessElement
- public void onlyValue(ProcessContext c) {
+ public void emitTimestampedValue(ProcessContext c) {
c.outputWithTimestamp(c.element().getValue(), c.element().getTimestamp());
}
}
[02/23] beam git commit: Better name for batch implementation of
GroupAlsoByWindow.
Posted by am...@apache.org.
Better name for batch implementation of GroupAlsoByWindow.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/bf0c119b
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/bf0c119b
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/bf0c119b
Branch: refs/heads/master
Commit: bf0c119b50c2dc45f21a8c740a6f98136771d7af
Parents: a41afdc
Author: Sela <an...@paypal.com>
Authored: Mon Feb 13 16:30:16 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:17:58 2017 +0200
----------------------------------------------------------------------
.../translation/GroupCombineFunctions.java | 2 +-
.../translation/SparkGroupAlsoByWindowFn.java | 198 -------------------
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 198 +++++++++++++++++++
.../spark/translation/TranslationUtils.java | 2 +-
4 files changed, 200 insertions(+), 200 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/bf0c119b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
index bb95065..8a41b4e 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/GroupCombineFunctions.java
@@ -84,7 +84,7 @@ public class GroupCombineFunctions {
//--- now group also by window.
// GroupAlsoByWindow currently uses a dummy in-memory StateInternals
return groupedByKey.flatMap(
- new SparkGroupAlsoByWindowFn<>(
+ new SparkGroupAlsoByWindowViaOutputBufferFn<>(
windowingStrategy,
new TranslationUtils.InMemoryStateInternalsFactory<K>(),
SystemReduceFn.<K, V, W>buffering(valueCoder),
http://git-wip-us.apache.org/repos/asf/beam/blob/bf0c119b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowFn.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowFn.java
deleted file mode 100644
index bd37fdb..0000000
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowFn.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.spark.translation;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import org.apache.beam.runners.core.GroupAlsoByWindowViaOutputBufferDoFn;
-import org.apache.beam.runners.core.GroupAlsoByWindowsDoFn;
-import org.apache.beam.runners.core.InMemoryTimerInternals;
-import org.apache.beam.runners.core.OutputWindowedValue;
-import org.apache.beam.runners.core.ReduceFnRunner;
-import org.apache.beam.runners.core.StateInternals;
-import org.apache.beam.runners.core.StateInternalsFactory;
-import org.apache.beam.runners.core.SystemReduceFn;
-import org.apache.beam.runners.core.TimerInternals;
-import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
-import org.apache.beam.runners.core.triggers.TriggerStateMachines;
-import org.apache.beam.runners.spark.aggregators.NamedAggregators;
-import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.beam.sdk.transforms.Sum;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.transforms.windowing.Triggers;
-import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.spark.Accumulator;
-import org.apache.spark.api.java.function.FlatMapFunction;
-import org.joda.time.Instant;
-
-/**
- * An implementation of {@link GroupAlsoByWindowViaOutputBufferDoFn}
- * for the Spark runner.
- */
-public class SparkGroupAlsoByWindowFn<K, InputT, W extends BoundedWindow>
- implements FlatMapFunction<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>,
- WindowedValue<KV<K, Iterable<InputT>>>> {
-
- private final WindowingStrategy<?, W> windowingStrategy;
- private final StateInternalsFactory<K> stateInternalsFactory;
- private final SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn;
- private final SparkRuntimeContext runtimeContext;
- private final Aggregator<Long, Long> droppedDueToClosedWindow;
-
-
- public SparkGroupAlsoByWindowFn(
- WindowingStrategy<?, W> windowingStrategy,
- StateInternalsFactory<K> stateInternalsFactory,
- SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn,
- SparkRuntimeContext runtimeContext,
- Accumulator<NamedAggregators> accumulator) {
- this.windowingStrategy = windowingStrategy;
- this.stateInternalsFactory = stateInternalsFactory;
- this.reduceFn = reduceFn;
- this.runtimeContext = runtimeContext;
-
- droppedDueToClosedWindow = runtimeContext.createAggregator(
- accumulator,
- GroupAlsoByWindowsDoFn.DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER,
- Sum.ofLongs());
- }
-
- @Override
- public Iterable<WindowedValue<KV<K, Iterable<InputT>>>> call(
- WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>> windowedValue) throws Exception {
- K key = windowedValue.getValue().getKey();
- Iterable<WindowedValue<InputT>> values = windowedValue.getValue().getValue();
-
- //------ based on GroupAlsoByWindowsViaOutputBufferDoFn ------//
-
- // Used with Batch, we know that all the data is available for this key. We can't use the
- // timer manager from the context because it doesn't exist. So we create one and emulate the
- // watermark, knowing that we have all data and it is in timestamp order.
- InMemoryTimerInternals timerInternals = new InMemoryTimerInternals();
- timerInternals.advanceProcessingTime(Instant.now());
- timerInternals.advanceSynchronizedProcessingTime(Instant.now());
- StateInternals<K> stateInternals = stateInternalsFactory.stateInternalsForKey(key);
- GABWOutputWindowedValue<K, InputT> outputter = new GABWOutputWindowedValue<>();
-
- ReduceFnRunner<K, InputT, Iterable<InputT>, W> reduceFnRunner =
- new ReduceFnRunner<>(
- key,
- windowingStrategy,
- ExecutableTriggerStateMachine.create(
- TriggerStateMachines.stateMachineForTrigger(
- Triggers.toProto(windowingStrategy.getTrigger()))),
- stateInternals,
- timerInternals,
- outputter,
- new SideInputReader() {
- @Override
- public <T> T get(PCollectionView<T> view, BoundedWindow sideInputWindow) {
- throw new UnsupportedOperationException(
- "GroupAlsoByWindow must not have side inputs");
- }
-
- @Override
- public <T> boolean contains(PCollectionView<T> view) {
- throw new UnsupportedOperationException(
- "GroupAlsoByWindow must not have side inputs");
- }
-
- @Override
- public boolean isEmpty() {
- throw new UnsupportedOperationException(
- "GroupAlsoByWindow must not have side inputs");
- }
- },
- droppedDueToClosedWindow,
- reduceFn,
- runtimeContext.getPipelineOptions());
-
- // Process the grouped values.
- reduceFnRunner.processElements(values);
-
- // Finish any pending windows by advancing the input watermark to infinity.
- timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
-
- // Finally, advance the processing time to infinity to fire any timers.
- timerInternals.advanceProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE);
- timerInternals.advanceSynchronizedProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE);
-
- fireEligibleTimers(timerInternals, reduceFnRunner);
-
- reduceFnRunner.persist();
-
- return outputter.getOutputs();
- }
-
- private void fireEligibleTimers(InMemoryTimerInternals timerInternals,
- ReduceFnRunner<K, InputT, Iterable<InputT>, W> reduceFnRunner) throws Exception {
- List<TimerInternals.TimerData> timers = new ArrayList<>();
- while (true) {
- TimerInternals.TimerData timer;
- while ((timer = timerInternals.removeNextEventTimer()) != null) {
- timers.add(timer);
- }
- while ((timer = timerInternals.removeNextProcessingTimer()) != null) {
- timers.add(timer);
- }
- while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) {
- timers.add(timer);
- }
- if (timers.isEmpty()) {
- break;
- }
- reduceFnRunner.onTimers(timers);
- timers.clear();
- }
- }
-
- private static class GABWOutputWindowedValue<K, V>
- implements OutputWindowedValue<KV<K, Iterable<V>>> {
- private final List<WindowedValue<KV<K, Iterable<V>>>> outputs = new ArrayList<>();
-
- @Override
- public void outputWindowedValue(
- KV<K, Iterable<V>> output,
- Instant timestamp,
- Collection<? extends BoundedWindow> windows,
- PaneInfo pane) {
- outputs.add(WindowedValue.of(output, timestamp, windows, pane));
- }
-
- @Override
- public <SideOutputT> void sideOutputWindowedValue(
- TupleTag<SideOutputT> tag,
- SideOutputT output,
- Instant timestamp,
- Collection<? extends BoundedWindow> windows, PaneInfo pane) {
- throw new UnsupportedOperationException("GroupAlsoByWindow should not use side outputs.");
- }
-
- Iterable<WindowedValue<KV<K, Iterable<V>>>> getOutputs() {
- return outputs;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/bf0c119b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
new file mode 100644
index 0000000..449e3b6
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.translation;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.beam.runners.core.GroupAlsoByWindowViaOutputBufferDoFn;
+import org.apache.beam.runners.core.GroupAlsoByWindowsDoFn;
+import org.apache.beam.runners.core.InMemoryTimerInternals;
+import org.apache.beam.runners.core.OutputWindowedValue;
+import org.apache.beam.runners.core.ReduceFnRunner;
+import org.apache.beam.runners.core.StateInternals;
+import org.apache.beam.runners.core.StateInternalsFactory;
+import org.apache.beam.runners.core.SystemReduceFn;
+import org.apache.beam.runners.core.TimerInternals;
+import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
+import org.apache.beam.runners.core.triggers.TriggerStateMachines;
+import org.apache.beam.runners.spark.aggregators.NamedAggregators;
+import org.apache.beam.sdk.transforms.Aggregator;
+import org.apache.beam.sdk.transforms.Sum;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.Triggers;
+import org.apache.beam.sdk.util.SideInputReader;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.spark.Accumulator;
+import org.apache.spark.api.java.function.FlatMapFunction;
+import org.joda.time.Instant;
+
+/**
+ * An implementation of {@link GroupAlsoByWindowViaOutputBufferDoFn}
+ * for the Spark runner.
+ */
+public class SparkGroupAlsoByWindowViaOutputBufferFn<K, InputT, W extends BoundedWindow>
+ implements FlatMapFunction<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>,
+ WindowedValue<KV<K, Iterable<InputT>>>> {
+
+ private final WindowingStrategy<?, W> windowingStrategy;
+ private final StateInternalsFactory<K> stateInternalsFactory;
+ private final SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn;
+ private final SparkRuntimeContext runtimeContext;
+ private final Aggregator<Long, Long> droppedDueToClosedWindow;
+
+
+ public SparkGroupAlsoByWindowViaOutputBufferFn(
+ WindowingStrategy<?, W> windowingStrategy,
+ StateInternalsFactory<K> stateInternalsFactory,
+ SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn,
+ SparkRuntimeContext runtimeContext,
+ Accumulator<NamedAggregators> accumulator) {
+ this.windowingStrategy = windowingStrategy;
+ this.stateInternalsFactory = stateInternalsFactory;
+ this.reduceFn = reduceFn;
+ this.runtimeContext = runtimeContext;
+
+ droppedDueToClosedWindow = runtimeContext.createAggregator(
+ accumulator,
+ GroupAlsoByWindowsDoFn.DROPPED_DUE_TO_CLOSED_WINDOW_COUNTER,
+ Sum.ofLongs());
+ }
+
+ @Override
+ public Iterable<WindowedValue<KV<K, Iterable<InputT>>>> call(
+ WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>> windowedValue) throws Exception {
+ K key = windowedValue.getValue().getKey();
+ Iterable<WindowedValue<InputT>> values = windowedValue.getValue().getValue();
+
+ //------ based on GroupAlsoByWindowsViaOutputBufferDoFn ------//
+
+ // Used with Batch, we know that all the data is available for this key. We can't use the
+ // timer manager from the context because it doesn't exist. So we create one and emulate the
+ // watermark, knowing that we have all data and it is in timestamp order.
+ InMemoryTimerInternals timerInternals = new InMemoryTimerInternals();
+ timerInternals.advanceProcessingTime(Instant.now());
+ timerInternals.advanceSynchronizedProcessingTime(Instant.now());
+ StateInternals<K> stateInternals = stateInternalsFactory.stateInternalsForKey(key);
+ GABWOutputWindowedValue<K, InputT> outputter = new GABWOutputWindowedValue<>();
+
+ ReduceFnRunner<K, InputT, Iterable<InputT>, W> reduceFnRunner =
+ new ReduceFnRunner<>(
+ key,
+ windowingStrategy,
+ ExecutableTriggerStateMachine.create(
+ TriggerStateMachines.stateMachineForTrigger(
+ Triggers.toProto(windowingStrategy.getTrigger()))),
+ stateInternals,
+ timerInternals,
+ outputter,
+ new SideInputReader() {
+ @Override
+ public <T> T get(PCollectionView<T> view, BoundedWindow sideInputWindow) {
+ throw new UnsupportedOperationException(
+ "GroupAlsoByWindow must not have side inputs");
+ }
+
+ @Override
+ public <T> boolean contains(PCollectionView<T> view) {
+ throw new UnsupportedOperationException(
+ "GroupAlsoByWindow must not have side inputs");
+ }
+
+ @Override
+ public boolean isEmpty() {
+ throw new UnsupportedOperationException(
+ "GroupAlsoByWindow must not have side inputs");
+ }
+ },
+ droppedDueToClosedWindow,
+ reduceFn,
+ runtimeContext.getPipelineOptions());
+
+ // Process the grouped values.
+ reduceFnRunner.processElements(values);
+
+ // Finish any pending windows by advancing the input watermark to infinity.
+ timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
+
+ // Finally, advance the processing time to infinity to fire any timers.
+ timerInternals.advanceProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ timerInternals.advanceSynchronizedProcessingTime(BoundedWindow.TIMESTAMP_MAX_VALUE);
+
+ fireEligibleTimers(timerInternals, reduceFnRunner);
+
+ reduceFnRunner.persist();
+
+ return outputter.getOutputs();
+ }
+
+ private void fireEligibleTimers(InMemoryTimerInternals timerInternals,
+ ReduceFnRunner<K, InputT, Iterable<InputT>, W> reduceFnRunner) throws Exception {
+ List<TimerInternals.TimerData> timers = new ArrayList<>();
+ while (true) {
+ TimerInternals.TimerData timer;
+ while ((timer = timerInternals.removeNextEventTimer()) != null) {
+ timers.add(timer);
+ }
+ while ((timer = timerInternals.removeNextProcessingTimer()) != null) {
+ timers.add(timer);
+ }
+ while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) {
+ timers.add(timer);
+ }
+ if (timers.isEmpty()) {
+ break;
+ }
+ reduceFnRunner.onTimers(timers);
+ timers.clear();
+ }
+ }
+
+ private static class GABWOutputWindowedValue<K, V>
+ implements OutputWindowedValue<KV<K, Iterable<V>>> {
+ private final List<WindowedValue<KV<K, Iterable<V>>>> outputs = new ArrayList<>();
+
+ @Override
+ public void outputWindowedValue(
+ KV<K, Iterable<V>> output,
+ Instant timestamp,
+ Collection<? extends BoundedWindow> windows,
+ PaneInfo pane) {
+ outputs.add(WindowedValue.of(output, timestamp, windows, pane));
+ }
+
+ @Override
+ public <SideOutputT> void sideOutputWindowedValue(
+ TupleTag<SideOutputT> tag,
+ SideOutputT output,
+ Instant timestamp,
+ Collection<? extends BoundedWindow> windows, PaneInfo pane) {
+ throw new UnsupportedOperationException("GroupAlsoByWindow should not use side outputs.");
+ }
+
+ Iterable<WindowedValue<KV<K, Iterable<V>>>> getOutputs() {
+ return outputs;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/bf0c119b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
index 890a91b..7d83230 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
@@ -129,7 +129,7 @@ public final class TranslationUtils {
}
/** {@link KV} to pair function. */
- static <K, V> PairFunction<KV<K, V>, K, V> toPairFunction() {
+ public static <K, V> PairFunction<KV<K, V>, K, V> toPairFunction() {
return new PairFunction<KV<K, V>, K, V>() {
@Override
public Tuple2<K, V> call(KV<K, V> kv) {
[10/23] beam git commit: Serialize state stream with coders for
shuffle and checkpointing.
Posted by am...@apache.org.
Serialize state stream with coders for shuffle and checkpointing.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/da5f8497
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/da5f8497
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/da5f8497
Branch: refs/heads/master
Commit: da5f84970a30aa5afbbe3cf93e3ed28b9df3260d
Parents: b21de69
Author: Sela <an...@paypal.com>
Authored: Mon Feb 20 00:13:23 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:01 2017 +0200
----------------------------------------------------------------------
.../beam/runners/spark/coders/CoderHelpers.java | 23 ++++
.../SparkGroupAlsoByWindowViaWindowSet.java | 135 ++++++++++++-------
.../spark/stateful/SparkTimerInternals.java | 17 ++-
.../streaming/StreamingTransformTranslator.java | 3 +-
4 files changed, 124 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/da5f8497/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java
index 0df66c2..9c46ecf 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/coders/CoderHelpers.java
@@ -18,16 +18,21 @@
package org.apache.beam.runners.spark.coders;
+import static com.google.common.base.Preconditions.checkNotNull;
+
import com.google.common.collect.Iterables;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
+import javax.annotation.Nonnull;
import org.apache.beam.runners.spark.util.ByteArray;
import org.apache.beam.sdk.coders.Coder;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
+
import scala.Tuple2;
/**
@@ -89,6 +94,24 @@ public final class CoderHelpers {
}
/**
+ * Utility method for deserializing a Iterable of byte arrays using the specified coder.
+ *
+ * @param serialized bytearrays to be deserialized.
+ * @param coder Coder to deserialize with.
+ * @param <T> Type of object to be returned.
+ * @return Iterable of deserialized objects.
+ */
+ public static <T> Iterable<T> fromByteArrays(
+ Collection<byte[]> serialized, final Coder<T> coder) {
+ return Iterables.transform(serialized, new com.google.common.base.Function<byte[], T>() {
+ @Override
+ public T apply(@Nonnull byte[] bytes) {
+ return fromByteArray(checkNotNull(bytes, "Cannot decode null values."), coder);
+ }
+ });
+ }
+
+ /**
* A function wrapper for converting an object to a bytearray.
*
* @param coder Coder to serialize with.
http://git-wip-us.apache.org/repos/asf/beam/blob/da5f8497/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 2fb4100..5589d82 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -27,23 +27,28 @@ import org.apache.beam.runners.core.GroupAlsoByWindowsDoFn;
import org.apache.beam.runners.core.OutputWindowedValue;
import org.apache.beam.runners.core.ReduceFnRunner;
import org.apache.beam.runners.core.SystemReduceFn;
-import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.runners.core.TimerInternals;
import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachines;
import org.apache.beam.runners.spark.SparkPipelineOptions;
+import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
import org.apache.beam.runners.spark.translation.TranslationUtils;
import org.apache.beam.runners.spark.translation.WindowingHelpers;
+import org.apache.beam.runners.spark.util.ByteArray;
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
import org.apache.beam.runners.spark.util.LateDataUtils;
import org.apache.beam.runners.spark.util.UnsupportedSideInputReader;
import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.TupleTag;
@@ -97,45 +102,67 @@ public class SparkGroupAlsoByWindowViaWindowSet {
public static <K, InputT, W extends BoundedWindow>
JavaDStream<WindowedValue<KV<K, Iterable<InputT>>>> groupAlsoByWindow(
JavaDStream<WindowedValue<KV<K, Iterable<WindowedValue<InputT>>>>> inputDStream,
- final Coder<InputT> iCoder,
+ final Coder<K> keyCoder,
+ final Coder<WindowedValue<InputT>> wvCoder,
final WindowingStrategy<?, W> windowingStrategy,
final SparkRuntimeContext runtimeContext,
final List<Integer> sourceIds) {
+ final IterableCoder<WindowedValue<InputT>> itrWvCoder = IterableCoder.of(wvCoder);
+ final Coder<InputT> iCoder = ((FullWindowedValueCoder<InputT>) wvCoder).getValueCoder();
+ final Coder<? extends BoundedWindow> wCoder =
+ ((FullWindowedValueCoder<InputT>) wvCoder).getWindowCoder();
+ final Coder<WindowedValue<KV<K, Iterable<InputT>>>> wvKvIterCoder =
+ FullWindowedValueCoder.of(KvCoder.of(keyCoder, IterableCoder.of(iCoder)), wCoder);
+ final TimerInternals.TimerDataCoder timerDataCoder =
+ TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder());
+
long checkpointDurationMillis =
runtimeContext.getPipelineOptions().as(SparkPipelineOptions.class)
.getCheckpointDurationMillis();
// we have to switch to Scala API to avoid Optional in the Java API, see: SPARK-4819.
// we also have a broader API for Scala (access to the actual key and entire iterator).
- DStream<Tuple2<K, Iterable<WindowedValue<InputT>>>> pairDStream =
+ // we use coders to convert objects in the PCollection to byte arrays, so they
+ // can be transferred over the network for the shuffle and be in serialized form
+ // for checkpointing.
+ // for readability, we add comments with actual type next to byte[].
+ // to shorten line length, we use:
+ //---- WV: WindowedValue
+ //---- Iterable: Itr
+ //---- AccumT: A
+ //---- InputT: I
+ DStream<Tuple2</*K*/ ByteArray, /*Itr<WV<I>>*/ byte[]>> pairDStream =
inputDStream
.map(WindowingHelpers.<KV<K, Iterable<WindowedValue<InputT>>>>unwindowFunction())
.mapToPair(TranslationUtils.<K, Iterable<WindowedValue<InputT>>>toPairFunction())
+ // move to bytes and use coders for deserialization because there's a shuffle
+ // and checkpointing involved.
+ .mapToPair(CoderHelpers.toByteFunction(keyCoder, itrWvCoder))
.dstream();
- PairDStreamFunctions<K, Iterable<WindowedValue<InputT>>> pairDStreamFunctions =
+ PairDStreamFunctions<ByteArray, byte[]> pairDStreamFunctions =
DStream.toPairDStreamFunctions(
pairDStream,
- JavaSparkContext$.MODULE$.<K>fakeClassTag(),
- JavaSparkContext$.MODULE$.<Iterable<WindowedValue<InputT>>>fakeClassTag(),
+ JavaSparkContext$.MODULE$.<ByteArray>fakeClassTag(),
+ JavaSparkContext$.MODULE$.<byte[]>fakeClassTag(),
null);
int defaultNumPartitions = pairDStreamFunctions.defaultPartitioner$default$1();
Partitioner partitioner = pairDStreamFunctions.defaultPartitioner(defaultNumPartitions);
// use updateStateByKey to scan through the state and update elements and timers.
- DStream<Tuple2<K, Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>>
+ DStream<Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>
firedStream = pairDStreamFunctions.updateStateByKey(
new SerializableFunction1<
- scala.collection.Iterator<Tuple3<K, Seq<Iterable<WindowedValue<InputT>>>,
- Option<Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>>>,
- scala.collection.Iterator<Tuple2<K, Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>>>>() {
+ scala.collection.Iterator<Tuple3</*K*/ ByteArray, Seq</*Itr<WV<I>>*/ byte[]>,
+ Option<Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>>,
+ scala.collection.Iterator<Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
+ /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>>() {
@Override
- public scala.collection.Iterator<Tuple2<K, Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>>> apply(
- final scala.collection.Iterator<Tuple3<K, Seq<Iterable<WindowedValue<InputT>>>,
- Option<Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>>> iter) {
+ public scala.collection.Iterator<Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
+ /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>> apply(
+ final scala.collection.Iterator<Tuple3</*K*/ ByteArray, Seq</*Itr<WV<I>>*/ byte[]>,
+ Option<Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>> iter) {
//--- ACTUAL STATEFUL OPERATION:
//
// Input Iterator: the partition (~bundle) of a cogrouping of the input
@@ -149,7 +176,8 @@ public class SparkGroupAlsoByWindowViaWindowSet {
// (3) Seq.nonEmpty && Option<S>.isDefined: new data with previous state.
final SystemReduceFn<K, InputT, Iterable<InputT>, Iterable<InputT>, W> reduceFn =
- SystemReduceFn.buffering(iCoder);
+ SystemReduceFn.buffering(
+ ((FullWindowedValueCoder<InputT>) wvCoder).getValueCoder());
final OutputWindowedValueHolder<K, InputT> outputHolder =
new OutputWindowedValueHolder<>();
// use in memory Aggregators since Spark Accumulators are not resilient
@@ -160,25 +188,25 @@ public class SparkGroupAlsoByWindowViaWindowSet {
GroupAlsoByWindowsDoFn.DROPPED_DUE_TO_LATENESS_COUNTER);
AbstractIterator<
- Tuple2<K, Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>>
- outIter = new AbstractIterator<Tuple2<K,
- Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>>() {
+ Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>
+ outIter = new AbstractIterator<Tuple2</*K*/ ByteArray,
+ Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>>() {
@Override
- protected Tuple2<K, Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>> computeNext() {
+ protected Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
+ /*WV<KV<K, Itr<I>>>*/ List<byte[]>>> computeNext() {
// input iterator is a Spark partition (~bundle), containing keys and their
// (possibly) previous-state and (possibly) new data.
while (iter.hasNext()) {
// for each element in the partition:
- Tuple3<K, Seq<Iterable<WindowedValue<InputT>>>, Option<Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>>> next = iter.next();
- K key = next._1();
+ Tuple3<ByteArray, Seq<byte[]>,
+ Option<Tuple2<StateAndTimers, List<byte[]>>>> next = iter.next();
+ ByteArray encodedKey = next._1();
+ K key = CoderHelpers.fromByteArray(encodedKey.getValue(), keyCoder);
- Seq<Iterable<WindowedValue<InputT>>> seq = next._2();
+ Seq<byte[]> seq = next._2();
Option<Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>>
- prevStateAndTimersOpt = next._3();
+ List<byte[]>>> prevStateAndTimersOpt = next._3();
SparkStateInternals<K> stateInternals;
SparkTimerInternals timerInternals = SparkTimerInternals.forStreamFromSources(
@@ -192,7 +220,9 @@ public class SparkGroupAlsoByWindowViaWindowSet {
StateAndTimers prevStateAndTimers = prevStateAndTimersOpt.get()._1();
stateInternals = SparkStateInternals.forKeyAndState(key,
prevStateAndTimers.getState());
- timerInternals.addTimers(prevStateAndTimers.getTimers());
+ Collection<byte[]> serTimers = prevStateAndTimers.getTimers();
+ timerInternals.addTimers(
+ SparkTimerInternals.deserializeTimers(serTimers, timerDataCoder));
}
ReduceFnRunner<K, InputT, Iterable<InputT>, W> reduceFnRunner =
@@ -214,7 +244,8 @@ public class SparkGroupAlsoByWindowViaWindowSet {
if (!seq.isEmpty()) {
// new input for key.
try {
- Iterable<WindowedValue<InputT>> elementsIterable = seq.head();
+ Iterable<WindowedValue<InputT>> elementsIterable =
+ CoderHelpers.fromByteArray(seq.head(), itrWvCoder);
Iterable<WindowedValue<InputT>> validElements =
LateDataUtils
.dropExpiredWindows(
@@ -247,9 +278,11 @@ public class SparkGroupAlsoByWindowViaWindowSet {
List<WindowedValue<KV<K, Iterable<InputT>>>> outputs = outputHolder.get();
if (!outputs.isEmpty() || !stateInternals.getState().isEmpty()) {
StateAndTimers updated = new StateAndTimers(stateInternals.getState(),
- timerInternals.getTimers());
+ SparkTimerInternals.serializeTimers(
+ timerInternals.getTimers(), timerDataCoder));
// persist Spark's state by outputting.
- return new Tuple2<>(key, new Tuple2<>(updated, outputs));
+ List<byte[]> serOutput = CoderHelpers.toByteArrays(outputs, wvKvIterCoder);
+ return new Tuple2<>(encodedKey, new Tuple2<>(updated, serOutput));
}
// an empty state with no output, can be evicted completely - do nothing.
}
@@ -271,43 +304,43 @@ public class SparkGroupAlsoByWindowViaWindowSet {
return scala.collection.JavaConversions.asScalaIterator(outIter);
}
- }, partitioner, true, JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>>fakeClassTag());
+ }, partitioner, true,
+ JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers, List<byte[]>>>fakeClassTag());
if (checkpointDurationMillis > 0) {
firedStream.checkpoint(new Duration(checkpointDurationMillis));
}
// go back to Java now.
- JavaPairDStream<K, Tuple2<StateAndTimers, List<WindowedValue<KV<K, Iterable<InputT>>>>>>
+ JavaPairDStream</*K*/ ByteArray, Tuple2<StateAndTimers, /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>
javaFiredStream = JavaPairDStream.fromPairDStream(
firedStream,
- JavaSparkContext$.MODULE$.<K>fakeClassTag(),
- JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>>fakeClassTag());
+ JavaSparkContext$.MODULE$.<ByteArray>fakeClassTag(),
+ JavaSparkContext$.MODULE$.<Tuple2<StateAndTimers, List<byte[]>>>fakeClassTag());
// filter state-only output (nothing to fire) and remove the state from the output.
return javaFiredStream.filter(
- new Function<Tuple2<K, Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>>, Boolean>() {
+ new Function<Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
+ /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>, Boolean>() {
@Override
public Boolean call(
- Tuple2<K, Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>> t2) throws Exception {
+ Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
+ /*WV<KV<K, Itr<I>>>*/ List<byte[]>>> t2) throws Exception {
// filter output if defined.
return !t2._2()._2().isEmpty();
}
})
.flatMap(
- new FlatMapFunction<Tuple2<K, Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>>,
+ new FlatMapFunction<Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
+ /*WV<KV<K, Itr<I>>>*/ List<byte[]>>>,
WindowedValue<KV<K, Iterable<InputT>>>>() {
@Override
public Iterable<WindowedValue<KV<K, Iterable<InputT>>>> call(
- Tuple2<K, Tuple2<StateAndTimers,
- List<WindowedValue<KV<K, Iterable<InputT>>>>>> t2) throws Exception {
+ Tuple2</*K*/ ByteArray, Tuple2<StateAndTimers,
+ /*WV<KV<K, Itr<I>>>*/ List<byte[]>>> t2) throws Exception {
// drop the state since it is already persisted at this point.
- return t2._2()._2();
+ // return in serialized form.
+ return CoderHelpers.fromByteArrays(t2._2()._2(), wvKvIterCoder);
}
});
}
@@ -315,20 +348,20 @@ public class SparkGroupAlsoByWindowViaWindowSet {
private static class StateAndTimers {
//Serializable state for internals (namespace to state tag to coded value).
private final Table<String, String, byte[]> state;
- private final Collection<TimerData> timers;
+ private final Collection<byte[]> serTimers;
private StateAndTimers(
- Table<String, String, byte[]> state, Collection<TimerData> timers) {
+ Table<String, String, byte[]> state, Collection<byte[]> timers) {
this.state = state;
- this.timers = timers;
+ this.serTimers = timers;
}
public Table<String, String, byte[]> getState() {
return state;
}
- public Collection<TimerData> getTimers() {
- return timers;
+ public Collection<byte[]> getTimers() {
+ return serTimers;
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/da5f8497/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
index b9783ef..1949e1d 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkTimerInternals.java
@@ -29,6 +29,7 @@ import java.util.Set;
import javax.annotation.Nullable;
import org.apache.beam.runners.core.StateNamespace;
import org.apache.beam.runners.core.TimerInternals;
+import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.TimeDomain;
@@ -109,8 +110,10 @@ class SparkTimerInternals implements TimerInternals {
return toFire;
}
- void addTimers(Collection<TimerData> timers) {
- this.timers.addAll(timers);
+ void addTimers(Iterable<TimerData> timers) {
+ for (TimerData timer: timers) {
+ this.timers.add(timer);
+ }
}
@Override
@@ -169,4 +172,14 @@ class SparkTimerInternals implements TimerInternals {
throw new UnsupportedOperationException("Deleting a timer by ID is not yet supported.");
}
+ public static Collection<byte[]> serializeTimers(
+ Collection<TimerData> timers, TimerDataCoder timerDataCoder) {
+ return CoderHelpers.toByteArrays(timers, timerDataCoder);
+ }
+
+ public static Iterable<TimerData> deserializeTimers(
+ Collection<byte[]> serTimers, TimerDataCoder timerDataCoder) {
+ return CoderHelpers.fromByteArrays(serTimers, timerDataCoder);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/beam/blob/da5f8497/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index 7abf5be..a98eff2 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -258,7 +258,8 @@ final class StreamingTransformTranslator {
JavaDStream<WindowedValue<KV<K, Iterable<V>>>> outStream =
SparkGroupAlsoByWindowViaWindowSet.groupAlsoByWindow(
groupedByKeyStream,
- coder.getValueCoder(),
+ coder.getKeyCoder(),
+ wvCoder,
windowingStrategy,
runtimeContext,
streamSources);
[13/23] beam git commit: Fix runnable-on-service profile in the Spark
runner.
Posted by am...@apache.org.
Fix runnable-on-service profile in the Spark runner.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c08e7d86
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c08e7d86
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c08e7d86
Branch: refs/heads/master
Commit: c08e7d86dd289640361e25a68ba7eb827938473e
Parents: 82d754c
Author: Sela <an...@paypal.com>
Authored: Tue Feb 21 16:57:18 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:05 2017 +0200
----------------------------------------------------------------------
runners/spark/pom.xml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c08e7d86/runners/spark/pom.xml
----------------------------------------------------------------------
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 3ef7ef4..1669833 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -57,7 +57,7 @@
<profile>
<!-- This profile adds execution of RunnableOnService integration tests
against a local Spark endpoint. -->
- <id>local-runnable-on-service-tests</id>
+ <id>runnable-on-service-tests</id>
<activation><activeByDefault>false</activeByDefault></activation>
<build>
<plugins>
@@ -79,6 +79,8 @@
org.apache.beam.sdk.testing.UsesSplittableParDo,
org.apache.beam.sdk.testing.UsesCommittedMetrics
</excludedGroups>
+ <parallel>none</parallel>
+ <threadCount>4</threadCount>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<failIfNoTests>true</failIfNoTests>
[18/23] beam git commit: Use a PipelineRule for test pipelines.
Posted by am...@apache.org.
Use a PipelineRule for test pipelines.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4ca56806
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4ca56806
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4ca56806
Branch: refs/heads/master
Commit: 4ca5680681e14c4be8b0ac1f3e1244c9d439e6d8
Parents: bd41d9a
Author: Sela <an...@paypal.com>
Authored: Mon Feb 27 19:24:45 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:07 2017 +0200
----------------------------------------------------------------------
runners/spark/pom.xml | 1 -
.../runners/spark/SparkPipelineOptions.java | 9 --
.../runners/spark/TestSparkPipelineOptions.java | 18 ++++
.../beam/runners/spark/TestSparkRunner.java | 28 ++---
.../beam/runners/spark/ForceStreamingTest.java | 8 +-
.../apache/beam/runners/spark/PipelineRule.java | 103 +++++++++++++++++++
.../metrics/sink/NamedAggregatorsTest.java | 9 +-
.../beam/runners/spark/io/AvroPipelineTest.java | 6 +-
.../beam/runners/spark/io/NumShardsTest.java | 6 +-
.../io/hadoop/HadoopFileFormatPipelineTest.java | 6 +-
.../spark/translation/StorageLevelTest.java | 8 +-
.../translation/streaming/CreateStreamTest.java | 77 ++++----------
.../utils/SparkTestPipelineOptions.java | 42 --------
.../SparkTestPipelineOptionsForStreaming.java | 48 ---------
14 files changed, 175 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/pom.xml
----------------------------------------------------------------------
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index f926bf5..409fc27 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -80,7 +80,6 @@
org.apache.beam.sdk.testing.UsesCommittedMetrics
</excludedGroups>
<parallel>none</parallel>
- <threadCount>4</threadCount>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<failIfNoTests>true</failIfNoTests>
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
index 52d8ce1..26b549b 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
@@ -101,13 +101,4 @@ public interface SparkPipelineOptions
boolean getUsesProvidedSparkContext();
void setUsesProvidedSparkContext(boolean value);
- @Description("A special flag that forces streaming in tests.")
- @Default.Boolean(false)
- boolean isForceStreaming();
- void setForceStreaming(boolean forceStreaming);
-
- @Description("A forced timeout (millis), mostly for testing.")
- @Default.Long(3000L)
- Long getForcedTimeout();
- void setForcedTimeout(Long forcedTimeout);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkPipelineOptions.java
new file mode 100644
index 0000000..2cb58d8
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkPipelineOptions.java
@@ -0,0 +1,18 @@
+package org.apache.beam.runners.spark;
+
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.testing.TestPipelineOptions;
+
+
+/**
+ * A {@link SparkPipelineOptions} for tests.
+ */
+public interface TestSparkPipelineOptions extends SparkPipelineOptions, TestPipelineOptions {
+
+ @Description("A special flag that forces streaming in tests.")
+ @Default.Boolean(false)
+ boolean isForceStreaming();
+ void setForceStreaming(boolean forceStreaming);
+
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index 5d71ea5..035da00 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -18,6 +18,7 @@
package org.apache.beam.runners.spark;
+import static com.google.common.base.Preconditions.checkNotNull;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
@@ -33,7 +34,6 @@ import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.testing.TestPipelineOptions;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.util.ValueWithRecordId;
@@ -76,15 +76,15 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
private boolean isForceStreaming;
private int expectedNumberOfAssertions = 0;
- private TestSparkRunner(SparkPipelineOptions options) {
+ private TestSparkRunner(TestSparkPipelineOptions options) {
this.delegate = SparkRunner.fromOptions(options);
this.isForceStreaming = options.isForceStreaming();
}
public static TestSparkRunner fromOptions(PipelineOptions options) {
// Default options suffice to set it up as a test runner
- SparkPipelineOptions sparkOptions =
- PipelineOptionsValidator.validate(SparkPipelineOptions.class, options);
+ TestSparkPipelineOptions sparkOptions =
+ PipelineOptionsValidator.validate(TestSparkPipelineOptions.class, options);
return new TestSparkRunner(sparkOptions);
}
@@ -115,22 +115,22 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
@Override
public SparkPipelineResult run(Pipeline pipeline) {
- SparkPipelineOptions sparkOptions = pipeline.getOptions().as(SparkPipelineOptions.class);
+ TestSparkPipelineOptions testSparkPipelineOptions =
+ pipeline.getOptions().as(TestSparkPipelineOptions.class);
SparkPipelineResult result = null;
// clear state of Aggregators, Metrics and Watermarks.
AggregatorsAccumulator.clear();
SparkMetricsContainer.clear();
GlobalWatermarkHolder.clear();
- TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
- LOG.info("About to run test pipeline " + sparkOptions.getJobName());
+ LOG.info("About to run test pipeline " + testSparkPipelineOptions.getJobName());
// if the pipeline was executed in streaming mode, validate aggregators.
if (isForceStreaming) {
try {
result = delegate.run(pipeline);
- long timeout = sparkOptions.getForcedTimeout();
- result.waitUntilFinish(Duration.millis(timeout));
+ Long timeout = testSparkPipelineOptions.getTestTimeoutSeconds();
+ result.waitUntilFinish(Duration.standardSeconds(checkNotNull(timeout)));
// validate assertion succeeded (at least once).
int successAssertions = result.getAggregatorValue(PAssert.SUCCESS_COUNTER, Integer.class);
assertThat(
@@ -149,12 +149,12 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
LOG.info(
String.format(
"Successfully asserted pipeline %s with %d successful assertions.",
- sparkOptions.getJobName(),
+ testSparkPipelineOptions.getJobName(),
successAssertions));
- } finally {
+ } finally {
try {
// cleanup checkpoint dir.
- FileUtils.deleteDirectory(new File(sparkOptions.getCheckpointDir()));
+ FileUtils.deleteDirectory(new File(testSparkPipelineOptions.getCheckpointDir()));
} catch (IOException e) {
throw new RuntimeException("Failed to clear checkpoint tmp dir.", e);
}
@@ -164,8 +164,8 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
result = delegate.run(pipeline);
result.waitUntilFinish();
// assert via matchers.
- assertThat(result, testPipelineOptions.getOnCreateMatcher());
- assertThat(result, testPipelineOptions.getOnSuccessMatcher());
+ assertThat(result, testSparkPipelineOptions.getOnCreateMatcher());
+ assertThat(result, testSparkPipelineOptions.getOnSuccessMatcher());
}
return result;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
index c3026ce..9b39558 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/ForceStreamingTest.java
@@ -21,7 +21,6 @@ package org.apache.beam.runners.spark;
import static org.hamcrest.MatcherAssert.assertThat;
import java.io.IOException;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.BoundedReadFromUnboundedSource;
import org.apache.beam.sdk.io.CountingSource;
@@ -46,14 +45,11 @@ import org.junit.Test;
public class ForceStreamingTest {
@Rule
- public SparkTestPipelineOptionsForStreaming commonOptions =
- new SparkTestPipelineOptionsForStreaming();
+ public final PipelineRule pipelineRule = PipelineRule.streaming();
@Test
public void test() throws IOException {
- SparkPipelineOptions options = commonOptions.getOptions();
- options.setForceStreaming(true);
- Pipeline pipeline = Pipeline.create(options);
+ Pipeline pipeline = pipelineRule.createPipeline();
// apply the BoundedReadFromUnboundedSource.
BoundedReadFromUnboundedSource<?> boundedRead =
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java
new file mode 100644
index 0000000..bb42510
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/PipelineRule.java
@@ -0,0 +1,103 @@
+package org.apache.beam.runners.spark;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.joda.time.Duration;
+import org.junit.rules.ExternalResource;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+/**
+ * A {@link org.junit.Rule} to provide a {@link Pipeline} instance for Spark runner tests.
+ */
+public class PipelineRule implements TestRule {
+
+ private final TestName testName = new TestName();
+
+ private final SparkPipelineRule delegate;
+ private final RuleChain chain;
+
+ private PipelineRule() {
+ this.delegate = new SparkPipelineRule(testName);
+ this.chain = RuleChain.outerRule(testName).around(this.delegate);
+ }
+
+ private PipelineRule(Duration forcedTimeout) {
+ this.delegate = new SparkStreamingPipelineRule(forcedTimeout, testName);
+ this.chain = RuleChain.outerRule(testName).around(this.delegate);
+ }
+
+ public static PipelineRule streaming() {
+ return new PipelineRule(Duration.standardSeconds(5));
+ }
+
+ public static PipelineRule batch() {
+ return new PipelineRule();
+ }
+
+ public Duration batchDuration() {
+ return Duration.millis(delegate.options.getBatchIntervalMillis());
+ }
+
+ public SparkPipelineOptions getOptions() {
+ return delegate.options;
+ }
+
+ public Pipeline createPipeline() {
+ return Pipeline.create(delegate.options);
+ }
+
+ @Override
+ public Statement apply(Statement statement, Description description) {
+ return chain.apply(statement, description);
+ }
+
+ private static class SparkStreamingPipelineRule extends SparkPipelineRule {
+
+ private final TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private final Duration forcedTimeout;
+
+ SparkStreamingPipelineRule(Duration forcedTimeout, TestName testName) {
+ super(testName);
+ this.forcedTimeout = forcedTimeout;
+ }
+
+ @Override
+ protected void before() throws Throwable {
+ super.before();
+ temporaryFolder.create();
+ options.setForceStreaming(true);
+ options.setTestTimeoutSeconds(forcedTimeout.getStandardSeconds());
+ options.setCheckpointDir(
+ temporaryFolder.newFolder(options.getJobName()).toURI().toURL().toString());
+ }
+
+ @Override
+ protected void after() {
+ temporaryFolder.delete();
+ }
+ }
+
+ private static class SparkPipelineRule extends ExternalResource {
+
+ protected final TestSparkPipelineOptions options =
+ PipelineOptionsFactory.as(TestSparkPipelineOptions.class);
+
+ private final TestName testName;
+
+ private SparkPipelineRule(TestName testName) {
+ this.testName = testName;
+ }
+
+ @Override
+ protected void before() throws Throwable {
+ options.setRunner(TestSparkRunner.class);
+ options.setEnableSparkMetricSinks(false);
+ options.setJobName(testName.getMethodName());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/NamedAggregatorsTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/NamedAggregatorsTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/NamedAggregatorsTest.java
index 2f7202c..a192807 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/NamedAggregatorsTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/NamedAggregatorsTest.java
@@ -26,12 +26,12 @@ import com.google.common.collect.ImmutableSet;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
+import org.apache.beam.runners.spark.PipelineRule;
import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.aggregators.ClearAggregatorsRule;
import org.apache.beam.runners.spark.aggregators.SparkAggregators;
import org.apache.beam.runners.spark.examples.WordCount;
import org.apache.beam.runners.spark.translation.SparkContextFactory;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -56,12 +56,11 @@ public class NamedAggregatorsTest {
public ClearAggregatorsRule clearAggregators = new ClearAggregatorsRule();
@Rule
- public final SparkTestPipelineOptions pipelineOptions = new SparkTestPipelineOptions();
+ public final PipelineRule pipelineRule = PipelineRule.batch();
private Pipeline createSparkPipeline() {
- SparkPipelineOptions options = pipelineOptions.getOptions();
- options.setEnableSparkMetricSinks(true);
- return Pipeline.create(options);
+ pipelineRule.getOptions().setEnableSparkMetricSinks(true);
+ return pipelineRule.createPipeline();
}
private void runPipeline() {
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java
index c5bb583..2a73c28 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/AvroPipelineTest.java
@@ -33,7 +33,7 @@ import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.generic.GenericRecord;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptions;
+import org.apache.beam.runners.spark.PipelineRule;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.AvroIO;
import org.apache.beam.sdk.values.PCollection;
@@ -54,7 +54,7 @@ public class AvroPipelineTest {
public final TemporaryFolder tmpDir = new TemporaryFolder();
@Rule
- public final SparkTestPipelineOptions pipelineOptions = new SparkTestPipelineOptions();
+ public final PipelineRule pipelineRule = PipelineRule.batch();
@Before
public void setUp() throws IOException {
@@ -72,7 +72,7 @@ public class AvroPipelineTest {
savedRecord.put("siblingnames", Lists.newArrayList("Jimmy", "Jane"));
populateGenericFile(Lists.newArrayList(savedRecord), schema);
- Pipeline p = Pipeline.create(pipelineOptions.getOptions());
+ Pipeline p = pipelineRule.createPipeline();
PCollection<GenericRecord> input = p.apply(
AvroIO.Read.from(inputFile.getAbsolutePath()).withSchema(schema));
input.apply(AvroIO.Write.to(outputDir.getAbsolutePath()).withSchema(schema));
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
index 34d6818..c936ed3 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
@@ -30,8 +30,8 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
+import org.apache.beam.runners.spark.PipelineRule;
import org.apache.beam.runners.spark.examples.WordCount;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.TextIO;
@@ -59,7 +59,7 @@ public class NumShardsTest {
public final TemporaryFolder tmpDir = new TemporaryFolder();
@Rule
- public final SparkTestPipelineOptions pipelineOptions = new SparkTestPipelineOptions();
+ public final PipelineRule pipelineRule = PipelineRule.batch();
@Before
public void setUp() throws IOException {
@@ -69,7 +69,7 @@ public class NumShardsTest {
@Test
public void testText() throws Exception {
- Pipeline p = Pipeline.create(pipelineOptions.getOptions());
+ Pipeline p = pipelineRule.createPipeline();
PCollection<String> inputWords = p.apply(Create.of(WORDS).withCoder(StringUtf8Coder.of()));
PCollection<String> output = inputWords.apply(new WordCount.CountWords())
.apply(MapElements.via(new WordCount.FormatAsTextFn()));
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/test/java/org/apache/beam/runners/spark/io/hadoop/HadoopFileFormatPipelineTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/hadoop/HadoopFileFormatPipelineTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/hadoop/HadoopFileFormatPipelineTest.java
index 9efc670..a5072d6 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/hadoop/HadoopFileFormatPipelineTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/hadoop/HadoopFileFormatPipelineTest.java
@@ -22,8 +22,8 @@ import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
+import org.apache.beam.runners.spark.PipelineRule;
import org.apache.beam.runners.spark.coders.WritableCoder;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.values.KV;
@@ -52,7 +52,7 @@ public class HadoopFileFormatPipelineTest {
private File outputFile;
@Rule
- public final SparkTestPipelineOptions pipelineOptions = new SparkTestPipelineOptions();
+ public final PipelineRule pipelineRule = PipelineRule.batch();
@Rule
public final TemporaryFolder tmpDir = new TemporaryFolder();
@@ -68,7 +68,7 @@ public class HadoopFileFormatPipelineTest {
public void testSequenceFile() throws Exception {
populateFile();
- Pipeline p = Pipeline.create(pipelineOptions.getOptions());
+ Pipeline p = pipelineRule.createPipeline();
@SuppressWarnings("unchecked")
Class<? extends FileInputFormat<IntWritable, Text>> inputFormatClass =
(Class<? extends FileInputFormat<IntWritable, Text>>)
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
index 48105e1..4dc5dee 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.spark.translation;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptions;
+import org.apache.beam.runners.spark.PipelineRule;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Count;
@@ -32,12 +32,12 @@ import org.junit.Test;
public class StorageLevelTest {
@Rule
- public final transient SparkTestPipelineOptions pipelineOptions = new SparkTestPipelineOptions();
+ public final transient PipelineRule pipelineRule = PipelineRule.batch();
@Test
public void test() throws Exception {
- pipelineOptions.getOptions().setStorageLevel("DISK_ONLY");
- Pipeline p = Pipeline.create(pipelineOptions.getOptions());
+ pipelineRule.getOptions().setStorageLevel("DISK_ONLY");
+ Pipeline p = pipelineRule.createPipeline();
PCollection<String> pCollection = p.apply(Create.of("foo"));
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
index 9ee5cc5..f2783a1 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamTest.java
@@ -24,10 +24,9 @@ import static org.junit.Assert.assertThat;
import java.io.IOException;
import java.io.Serializable;
+import org.apache.beam.runners.spark.PipelineRule;
import org.apache.beam.runners.spark.ReuseSparkContextRule;
-import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.io.CreateStream;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
@@ -59,8 +58,6 @@ import org.joda.time.Instant;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import org.junit.rules.TemporaryFolder;
-import org.junit.rules.TestName;
/**
@@ -76,27 +73,18 @@ import org.junit.rules.TestName;
public class CreateStreamTest implements Serializable {
@Rule
- public transient TemporaryFolder checkpointParentDir = new TemporaryFolder();
+ public final transient PipelineRule pipelineRule = PipelineRule.streaming();
@Rule
- public transient SparkTestPipelineOptionsForStreaming commonOptions =
- new SparkTestPipelineOptionsForStreaming();
+ public final transient ReuseSparkContextRule noContextResue = ReuseSparkContextRule.no();
@Rule
- public transient ReuseSparkContextRule noContextResue = ReuseSparkContextRule.no();
- @Rule
- public transient TestName testName = new TestName();
- @Rule
- public transient ExpectedException thrown = ExpectedException.none();
+ public final transient ExpectedException thrown = ExpectedException.none();
@Test
public void testLateDataAccumulating() throws IOException {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- Pipeline p = Pipeline.create(options);
- options.setJobName(testName.getMethodName());
- Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
-
+ Pipeline p = pipelineRule.createPipeline();
Instant instant = new Instant(0);
CreateStream<TimestampedValue<Integer>> source =
- CreateStream.<TimestampedValue<Integer>>withBatchInterval(batchDuration)
+ CreateStream.<TimestampedValue<Integer>>withBatchInterval(pipelineRule.batchDuration())
.nextBatch()
.advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(6)))
.nextBatch(
@@ -167,13 +155,9 @@ public class CreateStreamTest implements Serializable {
@Test
public void testDiscardingMode() throws IOException {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- Pipeline p = Pipeline.create(options);
- options.setJobName(testName.getMethodName());
- Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
-
+ Pipeline p = pipelineRule.createPipeline();
CreateStream<TimestampedValue<String>> source =
- CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
+ CreateStream.<TimestampedValue<String>>withBatchInterval(pipelineRule.batchDuration())
.nextBatch(
TimestampedValue.of("firstPane", new Instant(100)),
TimestampedValue.of("alsoFirstPane", new Instant(200)))
@@ -221,14 +205,10 @@ public class CreateStreamTest implements Serializable {
@Test
public void testFirstElementLate() throws IOException {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- Pipeline p = Pipeline.create(options);
- options.setJobName(testName.getMethodName());
- Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
-
+ Pipeline p = pipelineRule.createPipeline();
Instant lateElementTimestamp = new Instant(-1_000_000);
CreateStream<TimestampedValue<String>> source =
- CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
+ CreateStream.<TimestampedValue<String>>withBatchInterval(pipelineRule.batchDuration())
.nextBatch()
.advanceWatermarkForNextBatch(new Instant(0))
.nextBatch(
@@ -261,14 +241,10 @@ public class CreateStreamTest implements Serializable {
@Test
public void testElementsAtAlmostPositiveInfinity() throws IOException {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- Pipeline p = Pipeline.create(options);
- options.setJobName(testName.getMethodName());
- Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
-
+ Pipeline p = pipelineRule.createPipeline();
Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp();
CreateStream<TimestampedValue<String>> source =
- CreateStream.<TimestampedValue<String>>withBatchInterval(batchDuration)
+ CreateStream.<TimestampedValue<String>>withBatchInterval(pipelineRule.batchDuration())
.nextBatch(
TimestampedValue.of("foo", endOfGlobalWindow),
TimestampedValue.of("bar", endOfGlobalWindow))
@@ -292,17 +268,13 @@ public class CreateStreamTest implements Serializable {
@Test
public void testMultipleStreams() throws IOException {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- Pipeline p = Pipeline.create(options);
- options.setJobName(testName.getMethodName());
- Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
-
+ Pipeline p = pipelineRule.createPipeline();
CreateStream<String> source =
- CreateStream.<String>withBatchInterval(batchDuration)
+ CreateStream.<String>withBatchInterval(pipelineRule.batchDuration())
.nextBatch("foo", "bar")
.advanceNextBatchWatermarkToInfinity();
CreateStream<Integer> other =
- CreateStream.<Integer>withBatchInterval(batchDuration)
+ CreateStream.<Integer>withBatchInterval(pipelineRule.batchDuration())
.nextBatch(1, 2, 3, 4)
.advanceNextBatchWatermarkToInfinity();
@@ -327,14 +299,10 @@ public class CreateStreamTest implements Serializable {
@Test
public void testFlattenedWithWatermarkHold() throws IOException {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- Pipeline p = Pipeline.create(options);
- options.setJobName(testName.getMethodName());
- Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
-
+ Pipeline p = pipelineRule.createPipeline();
Instant instant = new Instant(0);
CreateStream<TimestampedValue<Integer>> source1 =
- CreateStream.<TimestampedValue<Integer>>withBatchInterval(batchDuration)
+ CreateStream.<TimestampedValue<Integer>>withBatchInterval(pipelineRule.batchDuration())
.nextBatch()
.advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(5)))
.nextBatch(
@@ -343,7 +311,7 @@ public class CreateStreamTest implements Serializable {
TimestampedValue.of(3, instant))
.advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(10)));
CreateStream<TimestampedValue<Integer>> source2 =
- CreateStream.<TimestampedValue<Integer>>withBatchInterval(batchDuration)
+ CreateStream.<TimestampedValue<Integer>>withBatchInterval(pipelineRule.batchDuration())
.nextBatch()
.advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(1)))
.nextBatch(
@@ -384,9 +352,8 @@ public class CreateStreamTest implements Serializable {
@Test
public void testElementAtPositiveInfinityThrows() {
- Duration batchDuration = Duration.millis(commonOptions.getOptions().getBatchIntervalMillis());
CreateStream<TimestampedValue<Integer>> source =
- CreateStream.<TimestampedValue<Integer>>withBatchInterval(batchDuration)
+ CreateStream.<TimestampedValue<Integer>>withBatchInterval(pipelineRule.batchDuration())
.nextBatch(TimestampedValue.of(-1, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L)));
thrown.expect(IllegalArgumentException.class);
source.nextBatch(TimestampedValue.of(1, BoundedWindow.TIMESTAMP_MAX_VALUE));
@@ -394,9 +361,8 @@ public class CreateStreamTest implements Serializable {
@Test
public void testAdvanceWatermarkNonMonotonicThrows() {
- Duration batchDuration = Duration.millis(commonOptions.getOptions().getBatchIntervalMillis());
CreateStream<Integer> source =
- CreateStream.<Integer>withBatchInterval(batchDuration)
+ CreateStream.<Integer>withBatchInterval(pipelineRule.batchDuration())
.advanceWatermarkForNextBatch(new Instant(0L));
thrown.expect(IllegalArgumentException.class);
source.advanceWatermarkForNextBatch(new Instant(-1L));
@@ -404,9 +370,8 @@ public class CreateStreamTest implements Serializable {
@Test
public void testAdvanceWatermarkEqualToPositiveInfinityThrows() {
- Duration batchDuration = Duration.millis(commonOptions.getOptions().getBatchIntervalMillis());
CreateStream<Integer> source =
- CreateStream.<Integer>withBatchInterval(batchDuration)
+ CreateStream.<Integer>withBatchInterval(pipelineRule.batchDuration())
.advanceWatermarkForNextBatch(BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L));
thrown.expect(IllegalArgumentException.class);
source.advanceWatermarkForNextBatch(BoundedWindow.TIMESTAMP_MAX_VALUE);
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptions.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptions.java
deleted file mode 100644
index efc17d3..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptions.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.translation.streaming.utils;
-
-import org.apache.beam.runners.spark.SparkPipelineOptions;
-import org.apache.beam.runners.spark.TestSparkRunner;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.junit.rules.ExternalResource;
-
-/**
- * A rule to create a common {@link SparkPipelineOptions} test options for spark-runner.
- */
-public class SparkTestPipelineOptions extends ExternalResource {
-
- protected final SparkPipelineOptions options =
- PipelineOptionsFactory.as(SparkPipelineOptions.class);
-
- @Override
- protected void before() throws Throwable {
- options.setRunner(TestSparkRunner.class);
- options.setEnableSparkMetricSinks(false);
- }
-
- public SparkPipelineOptions getOptions() {
- return options;
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/4ca56806/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptionsForStreaming.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptionsForStreaming.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptionsForStreaming.java
deleted file mode 100644
index dd3e4c8..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/SparkTestPipelineOptionsForStreaming.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.spark.translation.streaming.utils;
-
-import java.io.IOException;
-import org.apache.beam.runners.spark.SparkPipelineOptions;
-import org.joda.time.Duration;
-import org.junit.rules.TemporaryFolder;
-
-
-/**
- * A rule to create a common {@link SparkPipelineOptions} for testing streaming pipelines.
- */
-public class SparkTestPipelineOptionsForStreaming extends SparkTestPipelineOptions {
-
- private static final int DEFAULT_NUMBER_OF_BATCHES_TIMEOUT = 5;
-
- public SparkPipelineOptions withTmpCheckpointDir(TemporaryFolder parent)
- throws IOException {
- // tests use JUnit's TemporaryFolder path in the form of: /.../junit/...
- options.setCheckpointDir(parent.newFolder(options.getJobName()).toURI().toURL().toString());
- options.setForceStreaming(true);
- // set the default timeout to DEFAULT_NUMBER_OF_BATCHES_TIMEOUT x batchDuration
- // to allow pipelines to finish.
- Duration batchDuration = Duration.millis(options.getBatchIntervalMillis());
- // set the checkpoint duration to match interval.
- options.setCheckpointDurationMillis(batchDuration.getMillis());
- long forcedTimeout = batchDuration.multipliedBy(DEFAULT_NUMBER_OF_BATCHES_TIMEOUT).getMillis();
- options.setForcedTimeout(forcedTimeout);
- return options;
- }
-}
[09/23] beam git commit: Fix streaming translation of Flatten and
Window, make CreateStream eager.
Posted by am...@apache.org.
Fix streaming translation of Flatten and Window, make CreateStream eager.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9f143504
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9f143504
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9f143504
Branch: refs/heads/master
Commit: 9f143504f7ade5bb0a966dcef645a834277632ef
Parents: 96abe4f
Author: Sela <an...@paypal.com>
Authored: Sat Feb 18 22:01:58 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:17:59 2017 +0200
----------------------------------------------------------------------
.../spark/translation/EvaluationContext.java | 6 --
.../streaming/StreamingTransformTranslator.java | 101 +++++++++----------
.../translation/streaming/UnboundedDataset.java | 50 +--------
3 files changed, 52 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/9f143504/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index a35aff2..329e047 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.beam.runners.spark.SparkPipelineOptions;
-import org.apache.beam.runners.spark.translation.streaming.UnboundedDataset;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.AppliedPTransform;
@@ -136,11 +135,6 @@ public class EvaluationContext {
datasets.put(getOutput(transform), new BoundedDataset<>(values, jsc, coder));
}
- public <T> void putUnboundedDatasetFromQueue(
- PTransform<?, ? extends PValue> transform, Iterable<Iterable<T>> values, Coder<T> coder) {
- datasets.put(getOutput(transform), new UnboundedDataset<>(values, jssc, coder));
- }
-
public Dataset borrowDataset(PTransform<? extends PValue, ?> transform) {
return borrowDataset(getInput(transform));
}
http://git-wip-us.apache.org/repos/asf/beam/blob/9f143504/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index e90b490..7abf5be 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -22,12 +22,18 @@ import static com.google.common.base.Preconditions.checkState;
import static org.apache.beam.runners.spark.translation.TranslationUtils.rejectSplittable;
import static org.apache.beam.runners.spark.translation.TranslationUtils.rejectStateAndTimers;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.LinkedBlockingQueue;
import org.apache.beam.runners.spark.aggregators.NamedAggregators;
import org.apache.beam.runners.spark.aggregators.SparkAggregators;
+import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.io.ConsoleIO;
import org.apache.beam.runners.spark.io.CreateStream;
import org.apache.beam.runners.spark.io.SparkUnboundedSource;
@@ -48,6 +54,7 @@ import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
import org.apache.beam.runners.spark.translation.TransformEvaluator;
import org.apache.beam.runners.spark.translation.TranslationUtils;
import org.apache.beam.runners.spark.translation.WindowingHelpers;
+import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
import org.apache.beam.runners.spark.util.SideInputBroadcast;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
@@ -60,8 +67,6 @@ import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.CombineFnUtil;
@@ -76,10 +81,10 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
-import org.apache.spark.streaming.Duration;
-import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
+import org.apache.spark.streaming.api.java.JavaInputDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
+import org.apache.spark.streaming.api.java.JavaStreamingContext;
/**
@@ -116,13 +121,35 @@ final class StreamingTransformTranslator {
};
}
- private static <T> TransformEvaluator<CreateStream.QueuedValues<T>> createFromQueue() {
- return new TransformEvaluator<CreateStream.QueuedValues<T>>() {
+ private static <T> TransformEvaluator<CreateStream<T>> createFromQueue() {
+ return new TransformEvaluator<CreateStream<T>>() {
@Override
- public void evaluate(CreateStream.QueuedValues<T> transform, EvaluationContext context) {
- Iterable<Iterable<T>> values = transform.getQueuedValues();
+ public void evaluate(CreateStream<T> transform, EvaluationContext context) {
Coder<T> coder = context.getOutput(transform).getCoder();
- context.putUnboundedDatasetFromQueue(transform, values, coder);
+ JavaStreamingContext jssc = context.getStreamingContext();
+ Queue<Iterable<T>> values = transform.getBatches();
+ WindowedValue.ValueOnlyWindowedValueCoder<T> windowCoder =
+ WindowedValue.getValueOnlyCoder(coder);
+ // create the DStream from queue.
+ Queue<JavaRDD<WindowedValue<T>>> rddQueue = new LinkedBlockingQueue<>();
+ for (Iterable<T> v : values) {
+ Iterable<WindowedValue<T>> windowedValues =
+ Iterables.transform(v, WindowingHelpers.<T>windowValueFunction());
+ JavaRDD<WindowedValue<T>> rdd =
+ jssc.sparkContext()
+ .parallelize(CoderHelpers.toByteArrays(windowedValues, windowCoder))
+ .map(CoderHelpers.fromByteFunction(windowCoder));
+ rddQueue.offer(rdd);
+ }
+
+ JavaInputDStream<WindowedValue<T>> inputDStream = jssc.queueStream(rddQueue, true);
+ UnboundedDataset<T> unboundedDataset = new UnboundedDataset<T>(
+ inputDStream, Collections.singletonList(inputDStream.inputDStream().id()));
+ // add pre-baked Watermarks for the pre-baked batches.
+ Queue<GlobalWatermarkHolder.SparkWatermarks> times = transform.getTimes();
+ GlobalWatermarkHolder.addAll(
+ ImmutableMap.of(unboundedDataset.getStreamSources().get(0), times));
+ context.putDataset(transform, unboundedDataset);
}
};
}
@@ -136,7 +163,6 @@ final class StreamingTransformTranslator {
// since this is a streaming pipeline, at least one of the PCollections to "flatten" are
// unbounded, meaning it represents a DStream.
// So we could end up with an unbounded unified DStream.
- final List<JavaRDD<WindowedValue<T>>> rdds = new ArrayList<>();
final List<JavaDStream<WindowedValue<T>>> dStreams = new ArrayList<>();
final List<Integer> streamingSources = new ArrayList<>();
for (TaggedPValue pv : pcs) {
@@ -152,27 +178,18 @@ final class StreamingTransformTranslator {
streamingSources.addAll(unboundedDataset.getStreamSources());
dStreams.add(unboundedDataset.getDStream());
} else {
- rdds.add(((BoundedDataset<T>) dataset).getRDD());
+ // create a single RDD stream.
+ Queue<JavaRDD<WindowedValue<T>>> q = new LinkedBlockingQueue<>();
+ q.offer(((BoundedDataset) dataset).getRDD());
+ //TODO: this is not recoverable from checkpoint!
+ JavaDStream<WindowedValue<T>> dStream = context.getStreamingContext().queueStream(q);
+ dStreams.add(dStream);
}
}
// start by unifying streams into a single stream.
JavaDStream<WindowedValue<T>> unifiedStreams =
context.getStreamingContext().union(dStreams.remove(0), dStreams);
- // now unify in RDDs.
- if (rdds.size() > 0) {
- JavaDStream<WindowedValue<T>> joined =
- unifiedStreams.transform(
- new Function<JavaRDD<WindowedValue<T>>, JavaRDD<WindowedValue<T>>>() {
- @Override
- public JavaRDD<WindowedValue<T>> call(JavaRDD<WindowedValue<T>> streamRdd)
- throws Exception {
- return new JavaSparkContext(streamRdd.context()).union(streamRdd, rdds);
- }
- });
- context.putDataset(transform, new UnboundedDataset<>(joined, streamingSources));
- } else {
- context.putDataset(transform, new UnboundedDataset<>(unifiedStreams, streamingSources));
- }
+ context.putDataset(transform, new UnboundedDataset<>(unifiedStreams, streamingSources));
}
};
}
@@ -182,42 +199,24 @@ final class StreamingTransformTranslator {
@Override
public void evaluate(final Window.Bound<T> transform, EvaluationContext context) {
@SuppressWarnings("unchecked")
- WindowFn<? super T, W> windowFn = (WindowFn<? super T, W>) transform.getWindowFn();
- @SuppressWarnings("unchecked")
UnboundedDataset<T> unboundedDataset =
((UnboundedDataset<T>) context.borrowDataset(transform));
JavaDStream<WindowedValue<T>> dStream = unboundedDataset.getDStream();
- // get the right window durations.
- Duration windowDuration;
- Duration slideDuration;
- if (windowFn instanceof FixedWindows) {
- windowDuration = Durations.milliseconds(((FixedWindows) windowFn).getSize().getMillis());
- slideDuration = windowDuration;
- } else if (windowFn instanceof SlidingWindows) {
- SlidingWindows slidingWindows = (SlidingWindows) windowFn;
- windowDuration = Durations.milliseconds(slidingWindows.getSize().getMillis());
- slideDuration = Durations.milliseconds(slidingWindows.getPeriod().getMillis());
- } else {
- throw new UnsupportedOperationException(String.format("WindowFn %s is not supported.",
- windowFn.getClass().getCanonicalName()));
- }
- JavaDStream<WindowedValue<T>> windowedDStream =
- dStream.window(windowDuration, slideDuration);
- //--- then we apply windowing to the elements
+ JavaDStream<WindowedValue<T>> outputStream;
if (TranslationUtils.skipAssignWindows(transform, context)) {
- context.putDataset(transform,
- new UnboundedDataset<>(windowedDStream, unboundedDataset.getStreamSources()));
+ // do nothing.
+ outputStream = dStream;
} else {
- JavaDStream<WindowedValue<T>> outStream = windowedDStream.transform(
+ outputStream = dStream.transform(
new Function<JavaRDD<WindowedValue<T>>, JavaRDD<WindowedValue<T>>>() {
@Override
public JavaRDD<WindowedValue<T>> call(JavaRDD<WindowedValue<T>> rdd) throws Exception {
return rdd.map(new SparkAssignWindowFn<>(transform.getWindowFn()));
}
});
- context.putDataset(transform,
- new UnboundedDataset<>(outStream, unboundedDataset.getStreamSources()));
}
+ context.putDataset(transform,
+ new UnboundedDataset<>(outputStream, unboundedDataset.getStreamSources()));
}
};
}
@@ -427,7 +426,7 @@ final class StreamingTransformTranslator {
EVALUATORS.put(ParDo.Bound.class, parDo());
EVALUATORS.put(ParDo.BoundMulti.class, multiDo());
EVALUATORS.put(ConsoleIO.Write.Unbound.class, print());
- EVALUATORS.put(CreateStream.QueuedValues.class, createFromQueue());
+ EVALUATORS.put(CreateStream.class, createFromQueue());
EVALUATORS.put(Window.Bound.class, window());
EVALUATORS.put(Flatten.FlattenPCollectionList.class, flattenPColl());
}
http://git-wip-us.apache.org/repos/asf/beam/blob/9f143504/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
index 8624f41..d7f3f34 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
@@ -18,22 +18,12 @@
package org.apache.beam.runners.spark.translation.streaming;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Iterables;
import java.util.ArrayList;
import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.annotation.Nullable;
-import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.translation.Dataset;
-import org.apache.beam.runners.spark.translation.WindowingHelpers;
-import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.streaming.api.java.JavaDStream;
-import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,14 +34,8 @@ import org.slf4j.LoggerFactory;
public class UnboundedDataset<T> implements Dataset {
private static final Logger LOG = LoggerFactory.getLogger(UnboundedDataset.class);
- private static final AtomicInteger queuedStreamIds = new AtomicInteger();
- // only set if creating a DStream from a static collection
- @Nullable private transient JavaStreamingContext jssc;
-
- private Iterable<Iterable<T>> values;
- private Coder<T> coder;
- private JavaDStream<WindowedValue<T>> dStream;
+ private final JavaDStream<WindowedValue<T>> dStream;
// points to the input streams that created this UnboundedDataset,
// should be greater > 1 in case of Flatten for example.
// when using GlobalWatermarkHolder this information helps to take only the relevant watermarks
@@ -63,41 +47,11 @@ public class UnboundedDataset<T> implements Dataset {
this.streamSources.addAll(streamSources);
}
- public UnboundedDataset(Iterable<Iterable<T>> values, JavaStreamingContext jssc, Coder<T> coder) {
- this.values = values;
- this.jssc = jssc;
- this.coder = coder;
- // QueuedStream will have a negative (decreasing) unique id.
- this.streamSources.add(queuedStreamIds.decrementAndGet());
- }
-
- @VisibleForTesting
- public static void resetQueuedStreamIds() {
- queuedStreamIds.set(0);
- }
-
- @SuppressWarnings("ConstantConditions")
JavaDStream<WindowedValue<T>> getDStream() {
- if (dStream == null) {
- WindowedValue.ValueOnlyWindowedValueCoder<T> windowCoder =
- WindowedValue.getValueOnlyCoder(coder);
- // create the DStream from queue
- Queue<JavaRDD<WindowedValue<T>>> rddQueue = new LinkedBlockingQueue<>();
- for (Iterable<T> v : values) {
- Iterable<WindowedValue<T>> windowedValues =
- Iterables.transform(v, WindowingHelpers.<T>windowValueFunction());
- JavaRDD<WindowedValue<T>> rdd = jssc.sc().parallelize(
- CoderHelpers.toByteArrays(windowedValues, windowCoder)).map(
- CoderHelpers.fromByteFunction(windowCoder));
- rddQueue.offer(rdd);
- }
- // create DStream from queue, one at a time.
- dStream = jssc.queueStream(rddQueue, true);
- }
return dStream;
}
- public List<Integer> getStreamSources() {
+ List<Integer> getStreamSources() {
return streamSources;
}
[07/23] beam git commit: Remove streaming tests that were needed
before supporting the model.
Posted by am...@apache.org.
Remove streaming tests that were needed before supporting the model.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c25a02f4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c25a02f4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c25a02f4
Branch: refs/heads/master
Commit: c25a02f459a30de5b15bb404045258e486a94266
Parents: 96d373f
Author: Sela <an...@paypal.com>
Authored: Sat Feb 18 22:06:51 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:17:59 2017 +0200
----------------------------------------------------------------------
.../streaming/EmptyStreamAssertionTest.java | 87 ------
.../streaming/FlattenStreamingTest.java | 82 ------
.../streaming/KafkaStreamingTest.java | 270 -------------------
.../streaming/SimpleStreamingWordCountTest.java | 84 ------
.../utils/KafkaWriteOnBatchCompleted.java | 105 --------
.../streaming/utils/PAssertStreaming.java | 121 ---------
6 files changed, 749 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c25a02f4/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/EmptyStreamAssertionTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/EmptyStreamAssertionTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/EmptyStreamAssertionTest.java
deleted file mode 100644
index e482945..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/EmptyStreamAssertionTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.translation.streaming;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.Serializable;
-import java.util.Collections;
-import org.apache.beam.runners.spark.SparkPipelineOptions;
-import org.apache.beam.runners.spark.aggregators.ClearAggregatorsRule;
-import org.apache.beam.runners.spark.io.CreateStream;
-import org.apache.beam.runners.spark.translation.streaming.utils.PAssertStreaming;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.values.PCollection;
-import org.joda.time.Duration;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-
-/**
- * Test that {@link PAssertStreaming} can tell if the stream is empty.
- */
-public class EmptyStreamAssertionTest implements Serializable {
-
- private static final String EXPECTED_ERR =
- "Success aggregator should be greater than zero.\n"
- + "Expected: not <0>\n"
- + " but: was <0>";
-
- @Rule
- public TemporaryFolder checkpointParentDir = new TemporaryFolder();
-
- @Rule
- public SparkTestPipelineOptionsForStreaming commonOptions =
- new SparkTestPipelineOptionsForStreaming();
-
- @Rule
- public ClearAggregatorsRule clearAggregatorsRule = new ClearAggregatorsRule();
-
- @Test
- public void testAssertion() throws Exception {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- options.setStreaming(true);
-
- Duration windowDuration = new Duration(options.getBatchIntervalMillis());
-
- Pipeline pipeline = Pipeline.create(options);
-
- PCollection<String> output =
- pipeline
- .apply(CreateStream.fromQueue(Collections.<Iterable<String>>emptyList()))
- .setCoder(StringUtf8Coder.of())
- .apply(Window.<String>into(FixedWindows.of(windowDuration)));
-
- try {
- PAssertStreaming.runAndAssertContents(pipeline, output, new String[0],
- Duration.standardSeconds(1L));
- } catch (AssertionError e) {
- assertTrue("Expected error message: " + EXPECTED_ERR + " but got: " + e.getMessage(),
- e.getMessage().equals(EXPECTED_ERR));
- return;
- }
- fail("assertion should have failed");
- throw new RuntimeException("unreachable");
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/c25a02f4/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
deleted file mode 100644
index fc40bbd..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.translation.streaming;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import org.apache.beam.runners.spark.SparkPipelineOptions;
-import org.apache.beam.runners.spark.io.CreateStream;
-import org.apache.beam.runners.spark.translation.streaming.utils.PAssertStreaming;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionList;
-import org.joda.time.Duration;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-/**
- * Test Flatten (union) implementation for streaming.
- */
-public class FlattenStreamingTest {
-
- private static final String[] WORDS_ARRAY_1 = {
- "one", "two", "three", "four"};
- private static final List<Iterable<String>> WORDS_QUEUE_1 =
- Collections.<Iterable<String>>singletonList(Arrays.asList(WORDS_ARRAY_1));
- private static final String[] WORDS_ARRAY_2 = {
- "five", "six", "seven", "eight"};
- private static final List<Iterable<String>> WORDS_QUEUE_2 =
- Collections.<Iterable<String>>singletonList(Arrays.asList(WORDS_ARRAY_2));
- private static final String[] EXPECTED_UNION = {
- "one", "two", "three", "four", "five", "six", "seven", "eight"};
-
- @Rule
- public TemporaryFolder checkpointParentDir = new TemporaryFolder();
-
- @Rule
- public SparkTestPipelineOptionsForStreaming commonOptions =
- new SparkTestPipelineOptionsForStreaming();
-
- @Test
- public void testFlattenUnbounded() throws Exception {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- options.setStreaming(true);
-
- Pipeline p = Pipeline.create(options);
- PCollection<String> w1 =
- p.apply(CreateStream.fromQueue(WORDS_QUEUE_1)).setCoder(StringUtf8Coder.of());
- PCollection<String> windowedW1 =
- w1.apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(1))));
- PCollection<String> w2 =
- p.apply(CreateStream.fromQueue(WORDS_QUEUE_2)).setCoder(StringUtf8Coder.of());
- PCollection<String> windowedW2 =
- w2.apply(Window.<String>into(FixedWindows.of(Duration.standardSeconds(1))));
- PCollectionList<String> list = PCollectionList.of(windowedW1).and(windowedW2);
- PCollection<String> union = list.apply(Flatten.<String>pCollections());
-
- PAssertStreaming.runAndAssertContents(p, union, EXPECTED_UNION, Duration.standardSeconds(1L));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/c25a02f4/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
deleted file mode 100644
index 404cb5d..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.translation.streaming;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.google.common.collect.ImmutableMap;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectStreamException;
-import java.io.OutputStream;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Properties;
-import org.apache.beam.runners.spark.SparkContextOptions;
-import org.apache.beam.runners.spark.SparkPipelineOptions;
-import org.apache.beam.runners.spark.translation.streaming.utils.EmbeddedKafkaCluster;
-import org.apache.beam.runners.spark.translation.streaming.utils.KafkaWriteOnBatchCompleted;
-import org.apache.beam.runners.spark.translation.streaming.utils.PAssertStreaming;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.CustomCoder;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.kafka.KafkaIO;
-import org.apache.beam.sdk.transforms.Distinct;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.spark.streaming.api.java.JavaStreamingListener;
-import org.joda.time.Duration;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-
-/**
- * Test Kafka as input.
- */
-public class KafkaStreamingTest {
- private static final EmbeddedKafkaCluster.EmbeddedZookeeper EMBEDDED_ZOOKEEPER =
- new EmbeddedKafkaCluster.EmbeddedZookeeper();
- private static final EmbeddedKafkaCluster EMBEDDED_KAFKA_CLUSTER =
- new EmbeddedKafkaCluster(EMBEDDED_ZOOKEEPER.getConnection());
-
- @BeforeClass
- public static void init() throws IOException {
- EMBEDDED_ZOOKEEPER.startup();
- EMBEDDED_KAFKA_CLUSTER.startup();
- }
-
- @Rule
- public TemporaryFolder checkpointParentDir = new TemporaryFolder();
-
- @Rule
- public SparkTestPipelineOptionsForStreaming commonOptions =
- new SparkTestPipelineOptionsForStreaming();
-
- @Test
- public void testEarliest2Topics() throws Exception {
- Duration batchIntervalDuration = Duration.standardSeconds(5);
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
- // provide a generous enough batch-interval to have everything fit in one micro-batch.
- options.setBatchIntervalMillis(batchIntervalDuration.getMillis());
- // provide a very generous read time bound, we rely on num records bound here.
- options.setMinReadTimeMillis(batchIntervalDuration.minus(1).getMillis());
- // bound the read on the number of messages - 2 topics of 4 messages each.
- options.setMaxRecordsPerBatch(8L);
-
- //--- setup
- // two topics.
- final String topic1 = "topic1";
- final String topic2 = "topic2";
- // messages.
- final Map<String, String> messages = ImmutableMap.of(
- "k1", "v1", "k2", "v2", "k3", "v3", "k4", "v4"
- );
- // expected.
- final String[] expected = {"k1,v1", "k2,v2", "k3,v3", "k4,v4"};
-
- // write to both topics ahead.
- produce(topic1, messages);
- produce(topic2, messages);
-
- //------- test: read and dedup.
- Pipeline p = Pipeline.create(options);
-
- Map<String, Object> consumerProps = ImmutableMap.<String, Object>of(
- "auto.offset.reset", "earliest"
- );
-
- KafkaIO.Read<String, String> read = KafkaIO.<String, String>read()
- .withBootstrapServers(EMBEDDED_KAFKA_CLUSTER.getBrokerList())
- .withTopics(Arrays.asList(topic1, topic2))
- .withKeyCoder(StringUtf8Coder.of())
- .withValueCoder(StringUtf8Coder.of())
- .updateConsumerProperties(consumerProps);
-
- PCollection<String> deduped =
- p.apply(read.withoutMetadata()).setCoder(
- KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))
- .apply(Window.<KV<String, String>>into(FixedWindows.of(batchIntervalDuration)))
- .apply(ParDo.of(new FormatKVFn()))
- .apply(Distinct.<String>create());
-
- // graceful shutdown will make sure first batch (at least) will finish.
- Duration timeout = Duration.standardSeconds(1L);
- PAssertStreaming.runAndAssertContents(p, deduped, expected, timeout);
- }
-
- @Test
- public void testLatest() throws Exception {
- Duration batchIntervalDuration = Duration.standardSeconds(5);
- SparkContextOptions options =
- commonOptions.withTmpCheckpointDir(checkpointParentDir).as(SparkContextOptions.class);
- // provide a generous enough batch-interval to have everything fit in one micro-batch.
- options.setBatchIntervalMillis(batchIntervalDuration.getMillis());
- // provide a very generous read time bound, we rely on num records bound here.
- options.setMinReadTimeMillis(batchIntervalDuration.minus(1).getMillis());
- // bound the read on the number of messages - 1 topics of 4 messages.
- options.setMaxRecordsPerBatch(4L);
-
- //--- setup
- final String topic = "topic";
- // messages.
- final Map<String, String> messages = ImmutableMap.of(
- "k1", "v1", "k2", "v2", "k3", "v3", "k4", "v4"
- );
- // expected.
- final String[] expected = {"k1,v1", "k2,v2", "k3,v3", "k4,v4"};
-
- // write once first batch completes, this will guarantee latest-like behaviour.
- options.setListeners(Collections.<JavaStreamingListener>singletonList(
- KafkaWriteOnBatchCompleted.once(messages, Collections.singletonList(topic),
- EMBEDDED_KAFKA_CLUSTER.getProps(), EMBEDDED_KAFKA_CLUSTER.getBrokerList())));
-
- //------- test: read and format.
- Pipeline p = Pipeline.create(options);
-
- Map<String, Object> consumerProps = ImmutableMap.<String, Object>of(
- "auto.offset.reset", "latest"
- );
-
- KafkaIO.Read<String, String> read = KafkaIO.<String, String>read()
- .withBootstrapServers(EMBEDDED_KAFKA_CLUSTER.getBrokerList())
- .withTopics(Collections.singletonList(topic))
- .withKeyCoder(StringUtf8Coder.of())
- .withValueCoder(NonKryoSerializableStringCoder.of())
- .updateConsumerProperties(consumerProps);
-
- PCollection<String> formatted =
- p.apply(read.withoutMetadata()).setCoder(
- KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))
- .apply(Window.<KV<String, String>>into(FixedWindows.of(batchIntervalDuration)))
- .apply(ParDo.of(new FormatKVFn()));
-
- // run for more than 1 batch interval, so that reading of latest is attempted in the
- // first batch with no luck, while the OnBatchCompleted injected-input afterwards will be read
- // in the second interval.
- PAssertStreaming.runAndAssertContents(p, formatted, expected, Duration.standardSeconds(3));
- }
-
- private static void produce(String topic, Map<String, String> messages) {
- Serializer<String> stringSerializer = new StringSerializer();
- try (@SuppressWarnings("unchecked") KafkaProducer<String, String> kafkaProducer =
- new KafkaProducer(defaultProducerProps(), stringSerializer, stringSerializer)) {
- // feed topic.
- for (Map.Entry<String, String> en : messages.entrySet()) {
- kafkaProducer.send(new ProducerRecord<>(topic, en.getKey(), en.getValue()));
- }
- // await send completion.
- kafkaProducer.flush();
- }
- }
-
- private static Properties defaultProducerProps() {
- Properties producerProps = new Properties();
- producerProps.putAll(EMBEDDED_KAFKA_CLUSTER.getProps());
- producerProps.put("acks", "1");
- producerProps.put("bootstrap.servers", EMBEDDED_KAFKA_CLUSTER.getBrokerList());
- return producerProps;
- }
-
- @AfterClass
- public static void tearDown() {
- EMBEDDED_KAFKA_CLUSTER.shutdown();
- EMBEDDED_ZOOKEEPER.shutdown();
- }
-
- private static class FormatKVFn extends DoFn<KV<String, String>, String> {
- @ProcessElement
- public void processElement(ProcessContext c) {
- c.output(c.element().getKey() + "," + c.element().getValue());
- }
- }
-
- /**
- * This coder is not Kryo serializable, used to make sure
- * {@link org.apache.beam.runners.spark.coders.BeamSparkRunnerRegistrator} registers needed
- * classes to ensure Java serialization is used instead.
- */
- private static class NonKryoSerializableStringCoder extends CustomCoder<String>
- implements Serializable {
- private Coder<String> stringCoder;
- private Boolean isSerialized = false;
-
- private NonKryoSerializableStringCoder() {
- }
-
- @JsonCreator
- public static NonKryoSerializableStringCoder of() {
- return new NonKryoSerializableStringCoder();
- }
-
- private Object readResolve() throws ObjectStreamException {
- NonKryoSerializableStringCoder deserialized = new NonKryoSerializableStringCoder();
- deserialized.stringCoder = StringUtf8Coder.of();
- deserialized.isSerialized = true;
- return deserialized;
- }
-
- private Object writeReplace() throws ObjectStreamException {
- return new NonKryoSerializableStringCoder();
- }
-
- @Override
- public void encode(String value, OutputStream outStream, Context context)
- throws CoderException, IOException {
- if (!isSerialized) {
- this.stringCoder = StringUtf8Coder.of();
- }
- stringCoder.encode(value, outStream, context);
- }
-
- @Override
- public String decode(InputStream inStream, Context context) throws CoderException, IOException {
- if (!isSerialized) {
- this.stringCoder = StringUtf8Coder.of();
- }
- return stringCoder.decode(inStream, context);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/c25a02f4/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
deleted file mode 100644
index 3734cf6..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.translation.streaming;
-
-import com.google.common.collect.Lists;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-import org.apache.beam.runners.spark.SparkPipelineOptions;
-import org.apache.beam.runners.spark.examples.WordCount;
-import org.apache.beam.runners.spark.io.CreateStream;
-import org.apache.beam.runners.spark.translation.streaming.utils.PAssertStreaming;
-import org.apache.beam.runners.spark.translation.streaming.utils.SparkTestPipelineOptionsForStreaming;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.transforms.MapElements;
-import org.apache.beam.sdk.transforms.windowing.FixedWindows;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.values.PCollection;
-import org.joda.time.Duration;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-
-/**
- * Simple word count streaming test.
- */
-public class SimpleStreamingWordCountTest implements Serializable {
-
- @Rule
- public TemporaryFolder checkpointParentDir = new TemporaryFolder();
-
- @Rule
- public SparkTestPipelineOptionsForStreaming pipelineOptions =
- new SparkTestPipelineOptionsForStreaming();
-
- private static final String[] WORDS = {"hi there", "hi", "hi sue bob", "hi sue", "", "bob hi"};
-
- private static final List<Iterable<String>> MANY_WORDS =
- Lists.<Iterable<String>>newArrayList(Arrays.asList(WORDS), Arrays.asList(WORDS));
-
- private static final String[] EXPECTED_WORD_COUNTS = {"hi: 10", "there: 2", "sue: 4", "bob: 4"};
-
- private static final Duration BATCH_INTERVAL = Duration.standardSeconds(1);
-
- private static final Duration windowDuration = BATCH_INTERVAL.multipliedBy(2);
-
- @Test
- public void testFixedWindows() throws Exception {
- SparkPipelineOptions options = pipelineOptions.withTmpCheckpointDir(checkpointParentDir);
- options.setStreaming(true);
-
- // override defaults
- options.setBatchIntervalMillis(BATCH_INTERVAL.getMillis());
-
- Pipeline pipeline = Pipeline.create(options);
-
- PCollection<String> output =
- pipeline
- .apply(CreateStream.fromQueue(MANY_WORDS))
- .setCoder(StringUtf8Coder.of())
- .apply(Window.<String>into(FixedWindows.of(windowDuration)))
- .apply(new WordCount.CountWords())
- .apply(MapElements.via(new WordCount.FormatAsTextFn()));
-
- PAssertStreaming.runAndAssertContents(pipeline, output, EXPECTED_WORD_COUNTS, windowDuration);
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/c25a02f4/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/KafkaWriteOnBatchCompleted.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/KafkaWriteOnBatchCompleted.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/KafkaWriteOnBatchCompleted.java
deleted file mode 100644
index 38a5bff..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/KafkaWriteOnBatchCompleted.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.translation.streaming.utils;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.spark.streaming.api.java.JavaStreamingListener;
-import org.apache.spark.streaming.api.java.JavaStreamingListenerBatchCompleted;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Write to Kafka once the OnBatchCompleted hook is activated.
- */
-public class KafkaWriteOnBatchCompleted extends JavaStreamingListener{
- private static final Logger LOG = LoggerFactory.getLogger(KafkaWriteOnBatchCompleted.class);
-
- private final Map<String, String> messages;
- private final List<String> topics;
- private final Properties producerProps;
- private final String brokerList;
- private final boolean once;
-
- // A flag to state that no more writes should happen.
- private boolean done = false;
-
- private KafkaWriteOnBatchCompleted(Map<String, String> messages,
- List<String> topics,
- Properties producerProps,
- String brokerList,
- boolean once) {
- this.messages = messages;
- this.topics = topics;
- this.producerProps = producerProps;
- this.brokerList = brokerList;
- this.once = once;
- }
-
- public static KafkaWriteOnBatchCompleted once(Map<String, String> messages,
- List<String> topics,
- Properties producerProps,
- String brokerList) {
- return new KafkaWriteOnBatchCompleted(messages, topics, producerProps, brokerList, true);
- }
-
- public static KafkaWriteOnBatchCompleted always(Map<String, String> messages,
- List<String> topics,
- Properties producerProps,
- String brokerList) {
- return new KafkaWriteOnBatchCompleted(messages, topics, producerProps, brokerList, false);
- }
-
- @Override
- public void onBatchCompleted(JavaStreamingListenerBatchCompleted batchCompleted) {
- super.onBatchCompleted(batchCompleted);
- if (!done) {
- LOG.info("Writing to Kafka after batchTime {} has completed.",
- batchCompleted.batchInfo().batchTime());
- write();
- // once runs once.
- if (once) {
- done = true;
- }
- }
- }
-
- private void write() {
- Properties props = new Properties();
- props.putAll(producerProps);
- props.put("acks", "1");
- props.put("bootstrap.servers", brokerList);
- Serializer<String> stringSerializer = new StringSerializer();
- try (@SuppressWarnings("unchecked") KafkaProducer<String, String> kafkaProducer =
- new KafkaProducer(props, stringSerializer, stringSerializer)) {
- for (String topic: topics) {
- for (Map.Entry<String, String> en : messages.entrySet()) {
- kafkaProducer.send(new ProducerRecord<>(topic, en.getKey(), en.getValue()));
- }
- // await send completion.
- kafkaProducer.flush();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/c25a02f4/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/PAssertStreaming.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/PAssertStreaming.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/PAssertStreaming.java
deleted file mode 100644
index cd9de92..0000000
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/PAssertStreaming.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.translation.streaming.utils;
-
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
-import static org.junit.Assert.assertThat;
-
-import java.io.Serializable;
-import org.apache.beam.runners.spark.SparkPipelineResult;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.Sum;
-import org.apache.beam.sdk.transforms.Values;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.values.PCollection;
-import org.joda.time.Duration;
-import org.junit.Assert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Since PAssert doesn't propagate assert exceptions, use Aggregators to assert streaming
- * success/failure counters.
- */
-public final class PAssertStreaming implements Serializable {
- private static final Logger LOG = LoggerFactory.getLogger(PAssertStreaming.class);
-
- private PAssertStreaming() {
- }
-
- /**
- * Adds a pipeline run-time assertion that the contents of {@code actual} are {@code expected}.
- * Note that it is oblivious to windowing, so the assertion will apply indiscriminately to all
- * windows.
- */
- public static <T> SparkPipelineResult runAndAssertContents(
- Pipeline p,
- PCollection<T> actual,
- T[] expected,
- Duration timeout,
- boolean stopGracefully) {
- // Because PAssert does not support non-global windowing, but all our data is in one window,
- // we set up the assertion directly.
- actual
- .apply(WithKeys.<String, T>of("dummy"))
- .apply(GroupByKey.<String, T>create())
- .apply(Values.<Iterable<T>>create())
- .apply(ParDo.of(new AssertDoFn<>(expected)));
-
- // run the pipeline.
- SparkPipelineResult res = (SparkPipelineResult) p.run();
- res.waitUntilFinish(timeout);
- // validate assertion succeeded (at least once).
- int success = res.getAggregatorValue(PAssert.SUCCESS_COUNTER, Integer.class);
- Assert.assertThat("Success aggregator should be greater than zero.", success, not(0));
- // validate assertion didn't fail.
- int failure = res.getAggregatorValue(PAssert.FAILURE_COUNTER, Integer.class);
- Assert.assertThat("Failure aggregator should be zero.", failure, is(0));
-
- LOG.info("PAssertStreaming had {} successful assertion and {} failed.", success, failure);
- return res;
- }
-
- /**
- * Default to stop gracefully so that tests will finish processing even if slower for reasons
- * such as a slow runtime environment.
- */
- public static <T> SparkPipelineResult runAndAssertContents(
- Pipeline p,
- PCollection<T> actual,
- T[] expected,
- Duration timeout) {
- return runAndAssertContents(p, actual, expected, timeout, true);
- }
-
- private static class AssertDoFn<T> extends DoFn<Iterable<T>, Void> {
- private final Aggregator<Integer, Integer> success =
- createAggregator(PAssert.SUCCESS_COUNTER, Sum.ofIntegers());
- private final Aggregator<Integer, Integer> failure =
- createAggregator(PAssert.FAILURE_COUNTER, Sum.ofIntegers());
- private final T[] expected;
-
- AssertDoFn(T[] expected) {
- this.expected = expected;
- }
-
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- try {
- assertThat(c.element(), containsInAnyOrder(expected));
- success.addValue(1);
- } catch (Throwable t) {
- failure.addValue(1);
- LOG.error("PAssert failed expectations.", t);
- // don't throw t because it will fail this bundle and the failure count will be lost.
- }
- }
- }
-}
[21/23] beam git commit: Spark GABWVOB to use
UnsupportedSideInputReader.
Posted by am...@apache.org.
Spark GABWVOB to use UnsupportedSideInputReader.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3c94ff20
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3c94ff20
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3c94ff20
Branch: refs/heads/master
Commit: 3c94ff2069bada1ae7194879d96b5e8f12207101
Parents: 62ddca6
Author: Sela <an...@paypal.com>
Authored: Tue Feb 28 14:53:23 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:10 2017 +0200
----------------------------------------------------------------------
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 23 ++------------------
1 file changed, 2 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/3c94ff20/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
index 449e3b6..2b16c60 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkGroupAlsoByWindowViaOutputBufferFn.java
@@ -30,6 +30,7 @@ import org.apache.beam.runners.core.StateInternals;
import org.apache.beam.runners.core.StateInternalsFactory;
import org.apache.beam.runners.core.SystemReduceFn;
import org.apache.beam.runners.core.TimerInternals;
+import org.apache.beam.runners.core.UnsupportedSideInputReader;
import org.apache.beam.runners.core.triggers.ExecutableTriggerStateMachine;
import org.apache.beam.runners.core.triggers.TriggerStateMachines;
import org.apache.beam.runners.spark.aggregators.NamedAggregators;
@@ -38,11 +39,9 @@ import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.transforms.windowing.Triggers;
-import org.apache.beam.sdk.util.SideInputReader;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.spark.Accumulator;
import org.apache.spark.api.java.function.FlatMapFunction;
@@ -107,25 +106,7 @@ public class SparkGroupAlsoByWindowViaOutputBufferFn<K, InputT, W extends Bounde
stateInternals,
timerInternals,
outputter,
- new SideInputReader() {
- @Override
- public <T> T get(PCollectionView<T> view, BoundedWindow sideInputWindow) {
- throw new UnsupportedOperationException(
- "GroupAlsoByWindow must not have side inputs");
- }
-
- @Override
- public <T> boolean contains(PCollectionView<T> view) {
- throw new UnsupportedOperationException(
- "GroupAlsoByWindow must not have side inputs");
- }
-
- @Override
- public boolean isEmpty() {
- throw new UnsupportedOperationException(
- "GroupAlsoByWindow must not have side inputs");
- }
- },
+ new UnsupportedSideInputReader("GroupAlsoByWindow"),
droppedDueToClosedWindow,
reduceFn,
runtimeContext.getPipelineOptions());
[17/23] beam git commit: Build trigger state machine from Runner API
Trigger proto directly
Posted by am...@apache.org.
Build trigger state machine from Runner API Trigger proto directly
Remove printouts of stage progress in tests.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/bd41d9a3
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/bd41d9a3
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/bd41d9a3
Branch: refs/heads/master
Commit: bd41d9a323b23bf2956a34b9f42eaba4857caed7
Parents: c08e7d8
Author: Sela <an...@paypal.com>
Authored: Sat Feb 25 01:21:28 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:06 2017 +0200
----------------------------------------------------------------------
runners/spark/pom.xml | 2 ++
.../spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java | 3 ++-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/bd41d9a3/runners/spark/pom.xml
----------------------------------------------------------------------
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 1669833..f926bf5 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -97,6 +97,7 @@
</beamTestPipelineOptions>
<beam.spark.test.reuseSparkContext>true</beam.spark.test.reuseSparkContext>
<spark.ui.enabled>false</spark.ui.enabled>
+ <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
</systemPropertyVariables>
</configuration>
</execution>
@@ -339,6 +340,7 @@
<systemPropertyVariables>
<beam.spark.test.reuseSparkContext>true</beam.spark.test.reuseSparkContext>
<spark.ui.enabled>false</spark.ui.enabled>
+ <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
</systemPropertyVariables>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/beam/blob/bd41d9a3/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 5589d82..24b8508 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -47,6 +47,7 @@ import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.Triggers;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowedValue.FullWindowedValueCoder;
import org.apache.beam.sdk.util.WindowingStrategy;
@@ -231,7 +232,7 @@ public class SparkGroupAlsoByWindowViaWindowSet {
windowingStrategy,
ExecutableTriggerStateMachine.create(
TriggerStateMachines.stateMachineForTrigger(
- windowingStrategy.getTrigger())),
+ Triggers.toProto(windowingStrategy.getTrigger()))),
stateInternals,
timerInternals,
outputHolder,
[14/23] beam git commit: Batch executions should block without
timeout.
Posted by am...@apache.org.
Batch executions should block without timeout.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3867dcd7
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3867dcd7
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3867dcd7
Branch: refs/heads/master
Commit: 3867dcd793adcb030faa4713624542210b86b68d
Parents: f47e0eb
Author: Sela <an...@paypal.com>
Authored: Mon Feb 20 20:40:18 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:05 2017 +0200
----------------------------------------------------------------------
.../apache/beam/runners/spark/TestSparkRunner.java | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/3867dcd7/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index 985f75d..d2b5186 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -116,7 +116,6 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
@Override
public SparkPipelineResult run(Pipeline pipeline) {
SparkPipelineOptions sparkOptions = pipeline.getOptions().as(SparkPipelineOptions.class);
- long timeout = sparkOptions.getForcedTimeout();
SparkPipelineResult result = null;
try {
// clear state of Aggregators, Metrics and Watermarks.
@@ -126,14 +125,12 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
TestPipelineOptions testPipelineOptions = pipeline.getOptions().as(TestPipelineOptions.class);
LOG.info("About to run test pipeline " + sparkOptions.getJobName());
- result = delegate.run(pipeline);
- result.waitUntilFinish(Duration.millis(timeout));
-
- assertThat(result, testPipelineOptions.getOnCreateMatcher());
- assertThat(result, testPipelineOptions.getOnSuccessMatcher());
// if the pipeline was executed in streaming mode, validate aggregators.
if (isForceStreaming) {
+ result = delegate.run(pipeline);
+ long timeout = sparkOptions.getForcedTimeout();
+ result.waitUntilFinish(Duration.millis(timeout));
// validate assertion succeeded (at least once).
int successAssertions = result.getAggregatorValue(PAssert.SUCCESS_COUNTER, Integer.class);
assertThat(
@@ -154,6 +151,13 @@ public final class TestSparkRunner extends PipelineRunner<SparkPipelineResult> {
"Successfully asserted pipeline %s with %d successful assertions.",
sparkOptions.getJobName(),
successAssertions));
+ } else {
+ // for batch test pipelines, run and block until done.
+ result = delegate.run(pipeline);
+ result.waitUntilFinish();
+ // assert via matchers.
+ assertThat(result, testPipelineOptions.getOnCreateMatcher());
+ assertThat(result, testPipelineOptions.getOnSuccessMatcher());
}
} finally {
try {
[23/23] beam git commit: This closes #2050
Posted by am...@apache.org.
This closes #2050
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/61e31e62
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/61e31e62
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/61e31e62
Branch: refs/heads/master
Commit: 61e31e622ec5b1f3c4cd1417c859810689f2683c
Parents: a41afdc 2c70743
Author: Sela <an...@paypal.com>
Authored: Wed Mar 1 00:18:45 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:45 2017 +0200
----------------------------------------------------------------------
.../apache/beam/runners/core/LateDataUtils.java | 88 ++++
.../core/UnsupportedSideInputReader.java | 52 +++
runners/spark/pom.xml | 5 +-
.../runners/spark/SparkPipelineOptions.java | 6 +-
.../beam/runners/spark/SparkPipelineResult.java | 4 +
.../apache/beam/runners/spark/SparkRunner.java | 8 +-
.../runners/spark/TestSparkPipelineOptions.java | 35 ++
.../beam/runners/spark/TestSparkRunner.java | 88 ++--
.../aggregators/AggregatorsAccumulator.java | 2 +-
.../beam/runners/spark/coders/CoderHelpers.java | 23 +
.../apache/beam/runners/spark/io/ConsoleIO.java | 4 +-
.../beam/runners/spark/io/CreateStream.java | 198 +++++++--
.../spark/metrics/MetricsAccumulator.java | 4 -
.../spark/metrics/SparkMetricsContainer.java | 5 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 431 +++++++++++++++++++
.../spark/stateful/SparkStateInternals.java | 418 ++++++++++++++++++
.../spark/stateful/SparkTimerInternals.java | 185 ++++++++
.../spark/translation/EvaluationContext.java | 6 -
.../translation/GroupCombineFunctions.java | 237 ++++------
.../translation/SparkGroupAlsoByWindowFn.java | 198 ---------
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 179 ++++++++
.../spark/translation/TransformTranslator.java | 238 ++++++----
.../spark/translation/TranslationUtils.java | 24 +-
.../streaming/StreamingTransformTranslator.java | 277 +++++-------
.../translation/streaming/UnboundedDataset.java | 59 +--
.../beam/runners/spark/ForceStreamingTest.java | 17 +-
.../spark/GlobalWatermarkHolderTest.java | 151 +++++++
.../apache/beam/runners/spark/PipelineRule.java | 120 ++++++
.../runners/spark/ProvidedSparkContextTest.java | 2 +-
.../beam/runners/spark/ReuseSparkContext.java | 46 --
.../runners/spark/ReuseSparkContextRule.java | 46 ++
.../runners/spark/SparkPipelineStateTest.java | 33 +-
.../beam/runners/spark/WatermarkTest.java | 231 ----------
.../metrics/sink/NamedAggregatorsTest.java | 11 +-
.../beam/runners/spark/io/AvroPipelineTest.java | 6 +-
.../beam/runners/spark/io/NumShardsTest.java | 6 +-
.../io/hadoop/HadoopFileFormatPipelineTest.java | 6 +-
.../spark/translation/StorageLevelTest.java | 8 +-
.../translation/streaming/CreateStreamTest.java | 369 ++++++++++++++++
.../streaming/EmptyStreamAssertionTest.java | 87 ----
.../streaming/FlattenStreamingTest.java | 82 ----
.../streaming/KafkaStreamingTest.java | 270 ------------
.../ResumeFromCheckpointStreamingTest.java | 292 +++++++++----
.../streaming/SimpleStreamingWordCountTest.java | 84 ----
.../streaming/TrackStreamingSourcesTest.java | 45 +-
.../utils/KafkaWriteOnBatchCompleted.java | 105 -----
.../streaming/utils/PAssertStreaming.java | 121 ------
.../utils/SparkTestPipelineOptions.java | 42 --
.../SparkTestPipelineOptionsForStreaming.java | 37 --
.../windowing/TimestampTransform.java | 3 +-
50 files changed, 2971 insertions(+), 2023 deletions(-)
----------------------------------------------------------------------
[12/23] beam git commit: Make TimestampTransform Serializable.
Posted by am...@apache.org.
Make TimestampTransform Serializable.
Rebase leftover fixes.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f47e0eba
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f47e0eba
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f47e0eba
Branch: refs/heads/master
Commit: f47e0ebacd4c915e2a8c8b4e18e792926745d673
Parents: 24ab605
Author: Sela <an...@paypal.com>
Authored: Mon Feb 20 19:45:03 2017 +0200
Committer: Sela <an...@paypal.com>
Committed: Wed Mar 1 00:18:04 2017 +0200
----------------------------------------------------------------------
.../aggregators/AggregatorsAccumulator.java | 2 +-
.../spark/metrics/MetricsAccumulator.java | 4 --
.../spark/metrics/SparkMetricsContainer.java | 5 +-
.../spark/stateful/SparkStateInternals.java | 16 +++++++
.../streaming/StreamingTransformTranslator.java | 2 +-
.../translation/streaming/UnboundedDataset.java | 1 -
.../ResumeFromCheckpointStreamingTest.java | 48 ++++++++++++++++++--
.../windowing/TimestampTransform.java | 3 +-
8 files changed, 64 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/f47e0eba/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
index a4dfda6..261c327 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/AggregatorsAccumulator.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
/**
- * For resilience, {@link Accumulator}s are required to be wrapped in a Singleton.
+ * For resilience, {@link Accumulator Accumulators} are required to be wrapped in a Singleton.
* @see <a href="https://spark.apache.org/docs/1.6.3/streaming-programming-guide.html#accumulators-and-broadcast-variables">accumulators</a>
*/
public class AggregatorsAccumulator {
http://git-wip-us.apache.org/repos/asf/beam/blob/f47e0eba/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java
index f27a826..9d48289 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/MetricsAccumulator.java
@@ -53,9 +53,6 @@ public class MetricsAccumulator {
if (instance == null) {
synchronized (MetricsAccumulator.class) {
if (instance == null) {
- // TODO: currently when recovering from checkpoint, Spark does not recover the
- // last known Accumulator value. The SparkRunner should be able to persist and recover
- // the SparkMetricsContainer in order to recover metrics as well.
SparkMetricsContainer initialValue = new SparkMetricsContainer();
instance = jsc.sc().accumulator(initialValue, "Beam.Metrics",
new MetricsAccumulatorParam());
@@ -94,7 +91,6 @@ public class MetricsAccumulator {
}
}
- @SuppressWarnings("unused")
@VisibleForTesting
static void clear() {
synchronized (MetricsAccumulator.class) {
http://git-wip-us.apache.org/repos/asf/beam/blob/f47e0eba/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricsContainer.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricsContainer.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricsContainer.java
index 0bf0e70..7a4b222 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricsContainer.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/SparkMetricsContainer.java
@@ -145,10 +145,7 @@ public class SparkMetricsContainer implements Serializable {
@VisibleForTesting
public static void clear() {
try {
- SparkMetricsContainer instance = getInstance();
- instance.initializeMetricsContainers();
- instance.counters.clear();
- instance.distributions.clear();
+ MetricsAccumulator.clear();
} catch (IllegalStateException ignored) {
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/f47e0eba/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
index 93b1f63..43fb383 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkStateInternals.java
@@ -38,7 +38,9 @@ import org.apache.beam.sdk.transforms.windowing.OutputTimeFn;
import org.apache.beam.sdk.util.CombineFnUtil;
import org.apache.beam.sdk.util.state.AccumulatorCombiningState;
import org.apache.beam.sdk.util.state.BagState;
+import org.apache.beam.sdk.util.state.MapState;
import org.apache.beam.sdk.util.state.ReadableState;
+import org.apache.beam.sdk.util.state.SetState;
import org.apache.beam.sdk.util.state.State;
import org.apache.beam.sdk.util.state.StateContext;
import org.apache.beam.sdk.util.state.StateContexts;
@@ -121,6 +123,20 @@ class SparkStateInternals<K> implements StateInternals<K> {
}
@Override
+ public <T> SetState<T> bindSet(StateTag<? super K, SetState<T>> spec, Coder<T> elemCoder) {
+ throw new UnsupportedOperationException(
+ String.format("%s is not supported", SetState.class.getSimpleName()));
+ }
+
+ @Override
+ public <KeyT, ValueT> MapState<KeyT, ValueT> bindMap(
+ StateTag<? super K, MapState<KeyT, ValueT>> spec,
+ Coder<KeyT> mapKeyCoder, Coder<ValueT> mapValueCoder) {
+ throw new UnsupportedOperationException(
+ String.format("%s is not supported", MapState.class.getSimpleName()));
+ }
+
+ @Override
public <InputT, AccumT, OutputT> AccumulatorCombiningState<InputT, AccumT, OutputT>
bindCombiningValue(
StateTag<? super K, AccumulatorCombiningState<InputT, AccumT, OutputT>> address,
http://git-wip-us.apache.org/repos/asf/beam/blob/f47e0eba/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index a98eff2..fc98781 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -37,9 +37,9 @@ import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.runners.spark.io.ConsoleIO;
import org.apache.beam.runners.spark.io.CreateStream;
import org.apache.beam.runners.spark.io.SparkUnboundedSource;
-import org.apache.beam.runners.spark.stateful.SparkGroupAlsoByWindowViaWindowSet;
import org.apache.beam.runners.spark.metrics.MetricsAccumulator;
import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
+import org.apache.beam.runners.spark.stateful.SparkGroupAlsoByWindowViaWindowSet;
import org.apache.beam.runners.spark.translation.BoundedDataset;
import org.apache.beam.runners.spark.translation.Dataset;
import org.apache.beam.runners.spark.translation.DoFnFunction;
http://git-wip-us.apache.org/repos/asf/beam/blob/f47e0eba/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
index d7f3f34..e9abe93 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/UnboundedDataset.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.beam.runners.spark.translation.Dataset;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/beam/blob/f47e0eba/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
index e307363..4eea383 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
@@ -17,15 +17,19 @@
*/
package org.apache.beam.runners.spark.translation.streaming;
+import static org.apache.beam.sdk.metrics.MetricMatchers.attemptedMetricsResult;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.Uninterruptibles;
import java.io.IOException;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
@@ -33,17 +37,23 @@ import org.apache.beam.runners.spark.ReuseSparkContextRule;
import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.SparkPipelineResult;
import org.apache.beam.runners.spark.SparkRunner;
-import org.apache.beam.runners.spark.aggregators.AccumulatorSingleton;
+import org.apache.beam.runners.spark.aggregators.AggregatorsAccumulator;
import org.apache.beam.runners.spark.coders.CoderHelpers;
+import org.apache.beam.runners.spark.metrics.SparkMetricsContainer;
import org.apache.beam.runners.spark.translation.streaming.utils.EmbeddedKafkaCluster;
import org.apache.beam.runners.spark.util.GlobalWatermarkHolder;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.InstantCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.io.kafka.KafkaIO;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.MetricNameFilter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.metrics.MetricsFilter;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Aggregator;
+import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.Keys;
@@ -52,6 +62,7 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.Values;
+import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.WithKeys;
import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -59,6 +70,7 @@ import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PDone;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
@@ -147,6 +159,11 @@ public class ResumeFromCheckpointStreamingTest {
"k4", new Instant(400)
));
+ MetricsFilter metricsFilter =
+ MetricsFilter.builder()
+ .addNameFilter(MetricNameFilter.inNamespace(ResumeFromCheckpointStreamingTest.class))
+ .build();
+
// first run will read from Kafka backlog - "auto.offset.reset=smallest"
SparkPipelineResult res = run(options);
res.waitUntilFinish(Duration.standardSeconds(2));
@@ -157,11 +174,15 @@ public class ResumeFromCheckpointStreamingTest {
"Expected %d processed messages count but found %d", 4, processedMessages1),
processedMessages1,
equalTo(4L));
+ assertThat(res.metrics().queryMetrics(metricsFilter).counters(),
+ hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(),
+ "allMessages", "EOFShallNotPassFn", 4L)));
//--- between executions:
//- clear state.
- AccumulatorSingleton.clear();
+ AggregatorsAccumulator.clear();
+ SparkMetricsContainer.clear();
GlobalWatermarkHolder.clear();
//- write a bit more.
@@ -175,11 +196,14 @@ public class ResumeFromCheckpointStreamingTest {
res.waitUntilFinish(Duration.standardSeconds(2));
// assertions 2:
long processedMessages2 = res.getAggregatorValue("processedMessages", Long.class);
- int successAssertions = res.getAggregatorValue(PAssert.SUCCESS_COUNTER, Integer.class);
assertThat(
- String.format("Expected %d processed messages count but found %d", 1, processedMessages2),
+ String.format("Expected %d processed messages count but found %d", 5, processedMessages2),
processedMessages2,
equalTo(5L));
+ assertThat(res.metrics().queryMetrics(metricsFilter).counters(),
+ hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(),
+ "allMessages", "EOFShallNotPassFn", 6L)));
+ int successAssertions = res.getAggregatorValue(PAssert.SUCCESS_COUNTER, Integer.class);
res.getAggregatorValue(PAssert.SUCCESS_COUNTER, Integer.class);
assertThat(
String.format(
@@ -225,10 +249,14 @@ public class ResumeFromCheckpointStreamingTest {
Pipeline p = Pipeline.create(options);
+ PCollection<String> expectedCol =
+ p.apply(Create.of(ImmutableList.of("side1", "side2")).withCoder(StringUtf8Coder.of()));
+ PCollectionView<List<String>> view = expectedCol.apply(View.<String>asList());
+
PCollection<Iterable<String>> grouped = p
.apply(read.withoutMetadata())
.apply(Keys.<String>create())
- .apply(ParDo.of(new EOFShallNotPassFn()))
+ .apply("EOFShallNotPassFn", ParDo.of(new EOFShallNotPassFn(view)).withSideInputs(view))
.apply(Window.<String>into(FixedWindows.of(Duration.millis(500)))
.triggering(AfterWatermark.pastEndOfWindow())
.accumulatingFiredPanes()
@@ -250,12 +278,22 @@ public class ResumeFromCheckpointStreamingTest {
/** A pass-through fn that prevents EOF event from passing. */
private static class EOFShallNotPassFn extends DoFn<String, String> {
+ final PCollectionView<List<String>> view;
private final Aggregator<Long, Long> aggregator =
createAggregator("processedMessages", Sum.ofLongs());
+ Counter counter =
+ Metrics.counter(ResumeFromCheckpointStreamingTest.class, "allMessages");
+
+ private EOFShallNotPassFn(PCollectionView<List<String>> view) {
+ this.view = view;
+ }
@ProcessElement
public void process(ProcessContext c) {
String element = c.element();
+ // assert that side input is passed correctly before/after resuming from checkpoint.
+ assertThat(c.sideInput(view), containsInAnyOrder("side1", "side2"));
+ counter.inc();
if (!element.equals("EOF")) {
aggregator.addValue(1L);
c.output(c.element());
http://git-wip-us.apache.org/repos/asf/beam/blob/f47e0eba/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampTransform.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampTransform.java
index b16e968..5318592 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampTransform.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TimestampTransform.java
@@ -18,11 +18,12 @@
package org.apache.beam.sdk.transforms.windowing;
import com.google.auto.value.AutoValue;
+import java.io.Serializable;
import org.joda.time.Duration;
import org.joda.time.Instant;
/** An abstract description of a standardized transformation on timestamps. */
-public abstract class TimestampTransform {
+public abstract class TimestampTransform implements Serializable{
/** Returns a transform that shifts a timestamp later by {@code delay}. */
public static TimestampTransform delay(Duration delay) {