You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tg...@apache.org on 2017/05/04 22:55:29 UTC
[2/3] beam git commit: Split StartBundleContext and
FinishBundleContext
Split StartBundleContext and FinishBundleContext
These methods have different restrictions on their capabilities. Remove
Context.output(), and add a more precise
FinishBundleContext.output(Element, Timestamp, Window)
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d59d9b74
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d59d9b74
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d59d9b74
Branch: refs/heads/master
Commit: d59d9b74ecfd8bd936c768addab25e714a481ed0
Parents: 8af5c28
Author: Thomas Groh <tg...@google.com>
Authored: Mon May 1 18:11:18 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Thu May 4 15:55:16 2017 -0700
----------------------------------------------------------------------
.../beam/runners/core/BaseExecutionContext.java | 9 --
.../apache/beam/runners/core/DoFnAdapters.java | 111 ++++++++++---
.../beam/runners/core/ExecutionContext.java | 10 +-
...eBoundedSplittableProcessElementInvoker.java | 13 +-
.../beam/runners/core/SimpleDoFnRunner.java | 161 +++++++++++++++----
.../beam/runners/core/SplittableParDo.java | 35 ++--
.../beam/runners/core/SimpleDoFnRunnerTest.java | 4 +-
.../beam/runners/core/SplittableParDoTest.java | 4 +-
runners/google-cloud-dataflow-java/pom.xml | 2 +-
.../runners/dataflow/BatchViewOverrides.java | 2 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 7 +-
.../beam/sdk/options/PipelineOptions.java | 3 +-
.../org/apache/beam/sdk/transforms/Combine.java | 5 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 79 ++++++---
.../apache/beam/sdk/transforms/DoFnTester.java | 62 ++++---
.../beam/sdk/transforms/GroupIntoBatches.java | 2 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 5 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 19 ++-
.../sdk/transforms/reflect/DoFnInvoker.java | 19 ++-
.../sdk/transforms/reflect/DoFnSignature.java | 42 +++--
.../sdk/transforms/reflect/DoFnSignatures.java | 58 +++++--
.../beam/sdk/transforms/windowing/PaneInfo.java | 3 +-
.../apache/beam/sdk/metrics/MetricsTest.java | 4 +-
.../beam/sdk/transforms/DoFnTesterTest.java | 8 +-
.../beam/sdk/transforms/ParDoLifecycleTest.java | 16 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 75 ++-------
.../beam/sdk/transforms/SplittableDoFnTest.java | 4 +-
.../transforms/reflect/DoFnInvokersTest.java | 22 ++-
.../transforms/reflect/DoFnSignaturesTest.java | 15 +-
.../control/ProcessBundleHandlerTest.java | 17 +-
.../sdk/io/elasticsearch/ElasticsearchIO.java | 12 +-
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 4 +-
.../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 2 +-
.../io/gcp/bigquery/WriteBundlesToFiles.java | 17 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 4 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 10 +-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 4 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 4 +-
.../java/org/apache/beam/sdk/io/hdfs/Write.java | 7 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 7 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 11 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 6 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 4 +-
.../apache/beam/sdk/io/mongodb/MongoDbIO.java | 10 +-
45 files changed, 603 insertions(+), 317 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
index 8d68256..23d61f8 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/BaseExecutionContext.java
@@ -23,7 +23,6 @@ import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.DoFn.Context;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.TupleTag;
@@ -105,17 +104,9 @@ public abstract class BaseExecutionContext<T extends ExecutionContext.StepContex
return Collections.unmodifiableCollection(cachedStepContexts.values());
}
- /**
- * Hook for subclasses to implement that will be called whenever
- * {@link Context#output(Object)} is called.
- */
@Override
public void noteOutput(WindowedValue<?> output) {}
- /**
- * Hook for subclasses to implement that will be called whenever
- * {@link Context#output(TupleTag, Object)} is called.
- */
@Override
public void noteOutput(TupleTag<?> tag, WindowedValue<?> output) {}
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
index b2069e1..fd1772a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/DoFnAdapters.java
@@ -18,11 +18,13 @@
package org.apache.beam.runners.core;
import java.io.IOException;
+import org.apache.beam.runners.core.OldDoFn.Context;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.Context;
+import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
import org.apache.beam.sdk.transforms.DoFn.ProcessContext;
+import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
@@ -81,12 +83,12 @@ public class DoFnAdapters {
@Override
public void startBundle(Context c) throws Exception {
fn.prepareForProcessing();
- invoker.invokeStartBundle(new ContextAdapter<>(fn, c));
+ invoker.invokeStartBundle(new StartBundleContextAdapter<>(fn, c));
}
@Override
public void finishBundle(Context c) throws Exception {
- invoker.invokeFinishBundle(new ContextAdapter<>(fn, c));
+ invoker.invokeFinishBundle(new FinishBundleContextAdapter<>(fn, c));
}
@Override
@@ -128,42 +130,86 @@ public class DoFnAdapters {
/**
* Wraps an {@link OldDoFn.Context} as a {@link DoFnInvoker.ArgumentProvider} inside a {@link
- * DoFn.StartBundle} or {@link DoFn.FinishBundle} method, which means the extra context is
- * unavailable.
+ * DoFn.StartBundle} method, which means the extra context is unavailable.
*/
- private static class ContextAdapter<InputT, OutputT> extends DoFn<InputT, OutputT>.Context
+ private static class StartBundleContextAdapter<InputT, OutputT>
+ extends DoFn<InputT, OutputT>.StartBundleContext
implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
-
private OldDoFn<InputT, OutputT>.Context context;
- private ContextAdapter(DoFn<InputT, OutputT> fn, OldDoFn<InputT, OutputT>.Context context) {
+ private StartBundleContextAdapter(DoFn<InputT, OutputT> fn, Context context) {
fn.super();
this.context = context;
}
- @Override
public PipelineOptions getPipelineOptions() {
return context.getPipelineOptions();
}
@Override
- public void output(OutputT output) {
- context.output(output);
+ public BoundedWindow window() {
+ // The OldDoFn doesn't allow us to ask for these outside processElement, so this
+ // should be unreachable.
+ throw new UnsupportedOperationException(
+ "Can only get the window in processElement; elsewhere there is no defined window.");
}
@Override
- public void outputWithTimestamp(OutputT output, Instant timestamp) {
- context.outputWithTimestamp(output, timestamp);
+ public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ return this;
}
@Override
- public <T> void output(TupleTag<T> tag, T output) {
- context.output(tag, output);
+ public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Can only get a FinishBundleContext in finishBundle");
}
@Override
- public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
- context.outputWithTimestamp(tag, output, timestamp);
+ public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Can only get a ProcessContext in processElement");
+ }
+
+ @Override
+ public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Timers are not supported for OldDoFn");
+ }
+
+ @Override
+ public RestrictionTracker<?> restrictionTracker() {
+ throw new UnsupportedOperationException("This is a non-splittable DoFn");
+ }
+
+ @Override
+ public State state(String stateId) {
+ throw new UnsupportedOperationException("State is not supported by this runner");
+ }
+
+ @Override
+ public Timer timer(String timerId) {
+ throw new UnsupportedOperationException("Timers are not supported by this runner");
+ }
+ }
+
+ /**
+ * Wraps an {@link OldDoFn.Context} as a {@link DoFnInvoker.ArgumentProvider} inside a {@link
+ * DoFn.FinishBundle} method, which means the extra context is unavailable.
+ */
+ private static class FinishBundleContextAdapter<InputT, OutputT>
+ extends DoFn<InputT, OutputT>.FinishBundleContext
+ implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+
+ private OldDoFn<InputT, OutputT>.Context context;
+
+ private FinishBundleContextAdapter(DoFn<InputT, OutputT> fn, Context context) {
+ fn.super();
+ this.context = context;
+ }
+
+ public PipelineOptions getPipelineOptions() {
+ return context.getPipelineOptions();
}
@Override
@@ -175,7 +221,13 @@ public class DoFnAdapters {
}
@Override
- public Context context(DoFn<InputT, OutputT> doFn) {
+ public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Can only get a StartBundleContext in startBundle");
+ }
+
+ @Override
+ public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
return this;
}
@@ -205,6 +257,20 @@ public class DoFnAdapters {
public Timer timer(String timerId) {
throw new UnsupportedOperationException("Timers are not supported by this runner");
}
+
+ @Override
+ public void output(
+ OutputT output, Instant timestamp, BoundedWindow window) {
+ // Not full fidelity conversion. This should be removed as soon as possible.
+ context.outputWithTimestamp(output, timestamp);
+ }
+
+ @Override
+ public <T> void output(
+ TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
+ // Not full fidelity conversion. This should be removed as soon as possible.
+ context.outputWithTimestamp(tag, output, timestamp);
+ }
}
/**
@@ -278,8 +344,13 @@ public class DoFnAdapters {
}
@Override
- public Context context(DoFn<InputT, OutputT> doFn) {
- return this;
+ public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ return null;
+ }
+
+ @Override
+ public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
index c0d01ae..d2fdaac 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/ExecutionContext.java
@@ -20,7 +20,7 @@ package org.apache.beam.runners.core;
import java.io.IOException;
import java.util.Collection;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.DoFn.Context;
+import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.TupleTag;
@@ -42,13 +42,13 @@ public interface ExecutionContext {
/**
* Hook for subclasses to implement that will be called whenever
- * {@link Context#output(TupleTag, Object)} is called.
+ * {@link WindowedContext#output(TupleTag, Object)} is called.
*/
void noteOutput(WindowedValue<?> output);
/**
* Hook for subclasses to implement that will be called whenever
- * {@link Context#output(TupleTag, Object)} is called.
+ * {@link WindowedContext#output(TupleTag, Object)} is called.
*/
void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
@@ -69,14 +69,14 @@ public interface ExecutionContext {
/**
* Hook for subclasses to implement that will be called whenever
- * {@link org.apache.beam.sdk.transforms.DoFn.Context#output}
+ * {@link WindowedContext#output}
* is called.
*/
void noteOutput(WindowedValue<?> output);
/**
* Hook for subclasses to implement that will be called whenever
- * {@link org.apache.beam.sdk.transforms.DoFn.Context#output}
+ * {@link WindowedContext#output}
* is called.
*/
void noteOutput(TupleTag<?> tag, WindowedValue<?> output);
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
index 35d6eb7..a51041e 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
@@ -28,6 +28,8 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
+import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -117,9 +119,16 @@ public class OutputAndTimeBoundedSplittableProcessElementInvoker<
}
@Override
- public DoFn<InputT, OutputT>.Context context(DoFn<InputT, OutputT> doFn) {
+ public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
throw new IllegalStateException(
- "Should not access context() from @"
+ "Should not access startBundleContext() from @"
+ + DoFn.ProcessElement.class.getSimpleName());
+ }
+
+ @Override
+ public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new IllegalStateException(
+ "Should not access finishBundleContext() from @"
+ DoFn.ProcessElement.class.getSimpleName());
}
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 7f29a6f..768378d 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -33,9 +33,10 @@ import org.apache.beam.runners.core.ExecutionContext.StepContext;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.Context;
+import org.apache.beam.sdk.transforms.DoFn.FinishBundleContext;
import org.apache.beam.sdk.transforms.DoFn.OnTimerContext;
import org.apache.beam.sdk.transforms.DoFn.ProcessContext;
+import org.apache.beam.sdk.transforms.DoFn.StartBundleContext;
import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
@@ -137,9 +138,11 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
@Override
public void startBundle() {
+ DoFnStartBundleContext<InputT, OutputT> startBundleContext =
+ createStartBundleContext(fn, context);
// This can contain user code. Wrap it in case it throws an exception.
try {
- invoker.invokeStartBundle(context);
+ invoker.invokeStartBundle(startBundleContext);
} catch (Throwable t) {
// Exception in user code.
throw wrapUserCodeException(t);
@@ -200,15 +203,27 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
@Override
public void finishBundle() {
+ DoFnFinishBundleContext<InputT, OutputT> finishBundleContext =
+ createFinishBundleContext(fn, context);
// This can contain user code. Wrap it in case it throws an exception.
try {
- invoker.invokeFinishBundle(context);
+ invoker.invokeFinishBundle(finishBundleContext);
} catch (Throwable t) {
// Exception in user code.
throw wrapUserCodeException(t);
}
}
+ private DoFnStartBundleContext<InputT, OutputT> createStartBundleContext(
+ DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+ return new DoFnStartBundleContext<>(this.fn, this.context);
+ }
+
+ private DoFnFinishBundleContext<InputT, OutputT> createFinishBundleContext(
+ DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+ return new DoFnFinishBundleContext<>(fn, context);
+ }
+
/** Returns a new {@link DoFn.ProcessContext} for the given element. */
private DoFnProcessContext<InputT, OutputT> createProcessContext(WindowedValue<InputT> elem) {
return new DoFnProcessContext<InputT, OutputT>(fn, context, elem, allowedLateness);
@@ -228,8 +243,7 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
* @param <InputT> the type of the {@link DoFn} (main) input elements
* @param <OutputT> the type of the {@link DoFn} (main) output elements
*/
- private static class DoFnContext<InputT, OutputT> extends DoFn<InputT, OutputT>.Context
- implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+ private static class DoFnContext<InputT, OutputT> {
private static final int MAX_SIDE_OUTPUTS = 1000;
final PipelineOptions options;
@@ -255,7 +269,6 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
List<TupleTag<?>> additionalOutputTags,
StepContext stepContext,
WindowFn<?, ?> windowFn) {
- fn.super();
this.options = options;
this.fn = fn;
this.sideInputReader = sideInputReader;
@@ -274,7 +287,6 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
//////////////////////////////////////////////////////////////////////////////
- @Override
public PipelineOptions getPipelineOptions() {
return options;
}
@@ -374,30 +386,95 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
stepContext.noteOutput(tag, windowedElem);
}
}
+ }
+
+
+ /**
+ * A concrete implementation of {@link DoFn.StartBundleContext}.
+ */
+ private class DoFnStartBundleContext<InputT, OutputT>
+ extends DoFn<InputT, OutputT>.StartBundleContext
+ implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+ private final DoFn<InputT, OutputT> fn;
+ private final DoFnContext<InputT, OutputT> context;
+
+ private DoFnStartBundleContext(DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+ fn.super();
+ this.fn = fn;
+ this.context = context;
+ }
- // Following implementations of output, outputWithTimestamp, and output
- // are only accessible in DoFn.startBundle and DoFn.finishBundle, and will be shadowed by
- // ProcessContext's versions in DoFn.processElement.
@Override
- public void output(OutputT output) {
- outputWindowedValue(output, null, null, PaneInfo.NO_FIRING);
+ public PipelineOptions getPipelineOptions() {
+ return context.getPipelineOptions();
}
@Override
- public void outputWithTimestamp(OutputT output, Instant timestamp) {
- outputWindowedValue(output, timestamp, null, PaneInfo.NO_FIRING);
+ public BoundedWindow window() {
+ throw new UnsupportedOperationException(
+ "Cannot access window outside of @ProcessElement and @OnTimer methods.");
}
@Override
- public <T> void output(TupleTag<T> tag, T output) {
- checkNotNull(tag, "TupleTag passed to output cannot be null");
- outputWindowedValue(tag, output, null, null, PaneInfo.NO_FIRING);
+ public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ return this;
}
@Override
- public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
- checkNotNull(tag, "TupleTag passed to outputWithTimestamp cannot be null");
- outputWindowedValue(tag, output, timestamp, null, PaneInfo.NO_FIRING);
+ public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access FinishBundleContext outside of @FinishBundle method.");
+ }
+
+ @Override
+ public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access ProcessContext outside of @ProcessElement method.");
+ }
+
+ @Override
+ public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access OnTimerContext outside of @OnTimer methods.");
+ }
+
+ @Override
+ public RestrictionTracker<?> restrictionTracker() {
+ throw new UnsupportedOperationException(
+ "Cannot access RestrictionTracker outside of @ProcessElement method.");
+ }
+
+ @Override
+ public State state(String stateId) {
+ throw new UnsupportedOperationException(
+ "Cannot access state outside of @ProcessElement and @OnTimer methods.");
+ }
+
+ @Override
+ public Timer timer(String timerId) {
+ throw new UnsupportedOperationException(
+ "Cannot access timers outside of @ProcessElement and @OnTimer methods.");
+ }
+ }
+
+ /**
+ * B
+ * A concrete implementation of {@link DoFn.FinishBundleContext}.
+ */
+ private class DoFnFinishBundleContext<InputT, OutputT>
+ extends DoFn<InputT, OutputT>.FinishBundleContext
+ implements DoFnInvoker.ArgumentProvider<InputT, OutputT> {
+ private final DoFnContext<InputT, OutputT> context;
+
+ private DoFnFinishBundleContext(
+ DoFn<InputT, OutputT> fn, DoFnContext<InputT, OutputT> context) {
+ fn.super();
+ this.context = context;
+ }
+
+ @Override
+ public PipelineOptions getPipelineOptions() {
+ return context.getPipelineOptions();
}
@Override
@@ -407,14 +484,20 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
}
@Override
- public Context context(DoFn<InputT, OutputT> doFn) {
+ public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Cannot access StartBundleContext outside of @StartBundle method.");
+ }
+
+ @Override
+ public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
return this;
}
@Override
public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException(
- "Cannot access ProcessContext outside of @Processelement method.");
+ "Cannot access ProcessContext outside of @ProcessElement method.");
}
@Override
@@ -440,6 +523,17 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
throw new UnsupportedOperationException(
"Cannot access timers outside of @ProcessElement and @OnTimer methods.");
}
+
+ @Override
+ public void output(OutputT output, Instant timestamp, BoundedWindow window) {
+ context.outputWindowedValue(WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
+ }
+
+ @Override
+ public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
+ context.outputWindowedValue(
+ tag, WindowedValue.of(output, timestamp, window, PaneInfo.NO_FIRING));
+ }
}
/**
@@ -590,8 +684,13 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
}
@Override
- public DoFn<InputT, OutputT>.Context context(DoFn<InputT, OutputT> doFn) {
- return this;
+ public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException("StartBundleContext parameters are not supported.");
+ }
+
+ @Override
+ public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException("FinishBundleContext parameters are not supported.");
}
@Override
@@ -698,16 +797,22 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
}
@Override
- public TimeDomain timeDomain() {
- return timeDomain;
+ public StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException("StartBundleContext parameters are not supported.");
}
@Override
- public Context context(DoFn<InputT, OutputT> doFn) {
- throw new UnsupportedOperationException("Context parameters are not supported.");
+ public FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException("FinishBundleContext parameters are not supported.");
}
@Override
+ public TimeDomain timeDomain() {
+ return timeDomain;
+ }
+
+
+ @Override
public ProcessContext processContext(DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException("ProcessContext parameters are not supported.");
}
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
index ed065a6..7cf5e88 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
@@ -440,13 +440,13 @@ public class SplittableParDo<InputT, OutputT, RestrictionT>
}
@StartBundle
- public void startBundle(Context c) throws Exception {
- invoker.invokeStartBundle(wrapContext(c));
+ public void startBundle(StartBundleContext c) throws Exception {
+ invoker.invokeStartBundle(wrapContextAsStartBundle(c));
}
@FinishBundle
- public void finishBundle(Context c) throws Exception {
- invoker.invokeFinishBundle(wrapContext(c));
+ public void finishBundle(FinishBundleContext c) throws Exception {
+ invoker.invokeFinishBundle(wrapContextAsFinishBundle(c));
}
@ProcessElement
@@ -522,31 +522,39 @@ public class SplittableParDo<InputT, OutputT, RestrictionT>
stateNamespace, timerInternals.currentProcessingTime(), TimeDomain.PROCESSING_TIME));
}
- private DoFn<InputT, OutputT>.Context wrapContext(final Context baseContext) {
- return fn.new Context() {
+ private DoFn<InputT, OutputT>.StartBundleContext wrapContextAsStartBundle(
+ final StartBundleContext baseContext) {
+ return fn.new StartBundleContext() {
@Override
public PipelineOptions getPipelineOptions() {
return baseContext.getPipelineOptions();
}
- @Override
- public void output(OutputT output) {
- throwUnsupportedOutput();
+ private void throwUnsupportedOutput() {
+ throw new UnsupportedOperationException(
+ String.format(
+ "Splittable DoFn can only output from @%s",
+ ProcessElement.class.getSimpleName()));
}
+ };
+ }
+ private DoFn<InputT, OutputT>.FinishBundleContext wrapContextAsFinishBundle(
+ final FinishBundleContext baseContext) {
+ return fn.new FinishBundleContext() {
@Override
- public void outputWithTimestamp(OutputT output, Instant timestamp) {
+ public void output(OutputT output, Instant timestamp, BoundedWindow window) {
throwUnsupportedOutput();
}
@Override
- public <T> void output(TupleTag<T> tag, T output) {
+ public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
throwUnsupportedOutput();
}
@Override
- public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
- throwUnsupportedOutput();
+ public PipelineOptions getPipelineOptions() {
+ return baseContext.getPipelineOptions();
}
private void throwUnsupportedOutput() {
@@ -557,6 +565,7 @@ public class SplittableParDo<InputT, OutputT, RestrictionT>
}
};
}
+
}
/** Splits the restriction using the given {@link DoFn.SplitRestriction} method. */
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
index 9b63bab..b8fc64e 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SimpleDoFnRunnerTest.java
@@ -348,12 +348,12 @@ public class SimpleDoFnRunnerTest {
private static final TimerSpec timer = TimerSpecs.timer(TimeDomain.EVENT_TIME);
@StartBundle
- public void startBundle(Context c) throws Exception {
+ public void startBundle() throws Exception {
throw exceptionToThrow;
}
@FinishBundle
- public void finishBundle(Context c) throws Exception {
+ public void finishBundle() throws Exception {
throw exceptionToThrow;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
index a67db6d..0bb25fc 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/SplittableParDoTest.java
@@ -578,13 +578,13 @@ public class SplittableParDoTest {
}
@StartBundle
- public void startBundle(Context c) {
+ public void startBundle() {
assertEquals(State.OUTSIDE_BUNDLE, state);
state = State.INSIDE_BUNDLE;
}
@FinishBundle
- public void finishBundle(Context c) {
+ public void finishBundle() {
assertEquals(State.INSIDE_BUNDLE, state);
state = State.OUTSIDE_BUNDLE;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/runners/google-cloud-dataflow-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index b0ae757..0367bd8 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -33,7 +33,7 @@
<packaging>jar</packaging>
<properties>
- <dataflow.container_version>beam-master-20170503</dataflow.container_version>
+ <dataflow.container_version>beam-master-20170504</dataflow.container_version>
<dataflow.fnapi_environment_major_version>1</dataflow.fnapi_environment_major_version>
<dataflow.legacy_environment_major_version>6</dataflow.legacy_environment_major_version>
</properties>
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index 64fe495..b85b315 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -1021,7 +1021,7 @@ class BatchViewOverrides {
long indexInBundle;
@StartBundle
- public void startBundle(Context c) throws Exception {
+ public void startBundle() throws Exception {
indexInBundle = 0;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
index 2a057e4..5b21902 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
@@ -22,7 +22,6 @@ import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
@@ -246,6 +245,7 @@ public class WriteFiles<T> extends PTransform<PCollection<T>, PDone> {
// Writer that will write the records in this bundle. Lazily
// initialized in processElement.
private FileBasedWriter<T> writer = null;
+ private BoundedWindow window = null;
WriteBundles() {
}
@@ -263,6 +263,7 @@ public class WriteFiles<T> extends PTransform<PCollection<T>, PDone> {
} else {
writer.openUnwindowed(UUID.randomUUID().toString(), UNKNOWN_SHARDNUM, UNKNOWN_NUMSHARDS);
}
+ this.window = window;
LOG.debug("Done opening writer {} for operation {}", writer, writeOperation);
}
try {
@@ -285,10 +286,10 @@ public class WriteFiles<T> extends PTransform<PCollection<T>, PDone> {
}
@FinishBundle
- public void finishBundle(Context c) throws Exception {
+ public void finishBundle(FinishBundleContext c) throws Exception {
if (writer != null) {
FileResult result = writer.close();
- c.output(result);
+ c.output(result, window.maxTimestamp(), window);
// Reset state in case of reuse.
writer = null;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
index d423bfa..37081fd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
@@ -34,7 +34,6 @@ import org.apache.beam.sdk.options.ProxyInvocationHandler.Deserializer;
import org.apache.beam.sdk.options.ProxyInvocationHandler.Serializer;
import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.Context;
import org.apache.beam.sdk.transforms.display.HasDisplayData;
import org.joda.time.DateTimeUtils;
import org.joda.time.DateTimeZone;
@@ -56,7 +55,7 @@ import org.joda.time.format.DateTimeFormatter;
* from command-line arguments with {@link PipelineOptionsFactory#fromArgs(String[])}.
* They can be converted to another type by invoking {@link PipelineOptions#as(Class)} and
* can be accessed from within a {@link DoFn} by invoking
- * {@link Context#getPipelineOptions()}.
+ * {@code getPipelineOptions()} on the input {@link DoFn.ProcessContext Context} object.
*
* <p>For example:
* <pre>{@code
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 0be8517..9d0e97a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -1844,9 +1844,8 @@ public class Combine {
new DoFn<KV<K, InputT>, KV<K, InputT>>() {
transient int counter;
@StartBundle
- public void startBundle(Context c) {
- counter = ThreadLocalRandom.current().nextInt(
- Integer.MAX_VALUE);
+ public void startBundle() {
+ counter = ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE);
}
@ProcessElement
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index eab08f6..f3d178e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -82,15 +82,62 @@ import org.joda.time.Instant;
* @param <OutputT> the type of the (main) output elements
*/
public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayData {
+ /**
+ * Information accessible while within the {@link StartBundle} method.
+ */
+ public abstract class StartBundleContext {
+ /**
+ * Returns the {@code PipelineOptions} specified with the {@link
+ * org.apache.beam.sdk.runners.PipelineRunner} invoking this {@code DoFn}. The {@code
+ * PipelineOptions} will be the default running via {@link DoFnTester}.
+ */
+ public abstract PipelineOptions getPipelineOptions();
+ }
+
+ /**
+ * Information accessible while within the {@link FinishBundle} method.
+ */
+ public abstract class FinishBundleContext {
+ /**
+ * Returns the {@code PipelineOptions} specified with the {@link
+ * org.apache.beam.sdk.runners.PipelineRunner} invoking this {@code DoFn}. The {@code
+ * PipelineOptions} will be the default running via {@link DoFnTester}.
+ */
+ public abstract PipelineOptions getPipelineOptions();
+
+ /**
+ * Adds the given element to the main output {@code PCollection} at the given
+ * timestamp in the given window.
+ *
+ * <p>Once passed to {@code output} the element should not be modified in
+ * any way.
+ *
+ * <p><i>Note:</i> A splittable {@link DoFn} is not allowed to output from the
+ * {@link FinishBundle} method.
+ */
+ public abstract void output(OutputT output, Instant timestamp, BoundedWindow window);
- /** Information accessible to all methods in this {@code DoFn}. */
- public abstract class Context {
+ /**
+ * Adds the given element to the output {@code PCollection} with the given tag at the given
+ * timestamp in the given window.
+ *
+ * <p>Once passed to {@code output} the element should not be modified in any way.
+ *
+ * <p><i>Note:</i> A splittable {@link DoFn} is not allowed to output from the {@link
+ * FinishBundle} method.
+ */
+ public abstract <T> void output(
+ TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window);
+ }
+ /**
+ * Information accessible to all methods in this {@link DoFn} where the context is in some window.
+ */
+ public abstract class WindowedContext {
/**
- * Returns the {@code PipelineOptions} specified with the
- * {@link org.apache.beam.sdk.runners.PipelineRunner}
- * invoking this {@code DoFn}. The {@code PipelineOptions} will
- * be the default running via {@link DoFnTester}.
+ * Returns the {@code PipelineOptions} specified with the {@link
+ * org.apache.beam.sdk.runners.PipelineRunner} invoking this {@code DoFn}. The {@code
+ * PipelineOptions} will be the default running via {@link DoFnTester}.
*/
public abstract PipelineOptions getPipelineOptions();
@@ -205,7 +252,7 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
/**
* Information accessible when running a {@link DoFn.ProcessElement} method.
*/
- public abstract class ProcessContext extends Context {
+ public abstract class ProcessContext extends WindowedContext {
/**
* Returns the input element to be processed.
@@ -215,7 +262,6 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
*/
public abstract InputT element();
-
/**
* Returns the value of the side input.
*
@@ -257,7 +303,7 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
/**
* Information accessible when running a {@link DoFn.OnTimer} method.
*/
- public abstract class OnTimerContext extends Context {
+ public abstract class OnTimerContext extends WindowedContext {
/**
* Returns the timestamp of the current timer.
@@ -277,7 +323,7 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
/**
* Returns the allowed timestamp skew duration, which is the maximum duration that timestamps can
- * be shifted backward in {@link DoFn.Context#outputWithTimestamp}.
+ * be shifted backward in {@link WindowedContext#outputWithTimestamp}.
*
* <p>The default value is {@code Duration.ZERO}, in which case timestamps can only be shifted
* forward to future. For infinite skew, return {@code Duration.millis(Long.MAX_VALUE)}.
@@ -449,14 +495,9 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
* Annotation for the method to use to prepare an instance for processing a batch of elements.
* The method annotated with this must satisfy the following constraints:
* <ul>
- * <li>It must have exactly one argument.
- * <li>Its first (and only) argument must be a {@link DoFn.Context}.
+ * <li>It must have exactly zero or one arguments.
+ * <li>If it has any arguments, its only argument must be a {@link DoFn.StartBundleContext}.
* </ul>
- *
- * <p>A simple method declaration would look like:
- * <code>
- * public void setup(DoFn.Context c) { .. }
- * </code>
*/
@Documented
@Retention(RetentionPolicy.RUNTIME)
@@ -524,8 +565,8 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
* Annotation for the method to use to finish processing a batch of elements.
* The method annotated with this must satisfy the following constraints:
* <ul>
- * <li>It must have at least one argument.
- * <li>Its first (and only) argument must be a {@link DoFn.Context}.
+ * <li>It must have exactly zero or one arguments.
+ * <li>If it has any arguments, its only argument must be a {@link DoFn.FinishBundleContext}.
* </ul>
*/
@Documented
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index 70fb0ae..ca7427c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -218,9 +218,8 @@ public class DoFnTester<InputT, OutputT> implements AutoCloseable {
if (state == State.UNINITIALIZED) {
initializeState();
}
- TestContext context = new TestContext();
try {
- fnInvoker.invokeStartBundle(context);
+ fnInvoker.invokeStartBundle(new TestStartBundleContext());
} catch (UserCodeException e) {
unwrapUserCodeException(e);
}
@@ -291,9 +290,17 @@ public class DoFnTester<InputT, OutputT> implements AutoCloseable {
}
@Override
- public DoFn<InputT, OutputT>.Context context(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(
+ DoFn<InputT, OutputT> doFn) {
throw new UnsupportedOperationException(
- "Not expected to access DoFn.Context from @ProcessElement");
+ "Not expected to access DoFn.StartBundleContext from @ProcessElement");
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+ DoFn<InputT, OutputT> doFn) {
+ throw new UnsupportedOperationException(
+ "Not expected to access DoFn.FinishBundleContext from @ProcessElement");
}
@Override
@@ -303,8 +310,7 @@ public class DoFnTester<InputT, OutputT> implements AutoCloseable {
@Override
public OnTimerContext onTimerContext(DoFn<InputT, OutputT> doFn) {
- throw new UnsupportedOperationException(
- "DoFnTester doesn't support timers yet.");
+ throw new UnsupportedOperationException("DoFnTester doesn't support timers yet.");
}
@Override
@@ -344,7 +350,7 @@ public class DoFnTester<InputT, OutputT> implements AutoCloseable {
"Must be inside bundle to call finishBundle, but was: %s",
state);
try {
- fnInvoker.invokeFinishBundle(new TestContext());
+ fnInvoker.invokeFinishBundle(new TestFinishBundleContext());
} catch (UserCodeException e) {
unwrapUserCodeException(e);
}
@@ -522,8 +528,9 @@ public class DoFnTester<InputT, OutputT> implements AutoCloseable {
return mainOutputTag;
}
- private class TestContext extends DoFn<InputT, OutputT>.Context {
- TestContext() {
+ private class TestStartBundleContext extends DoFn<InputT, OutputT>.StartBundleContext {
+
+ private TestStartBundleContext() {
fn.super();
}
@@ -531,30 +538,33 @@ public class DoFnTester<InputT, OutputT> implements AutoCloseable {
public PipelineOptions getPipelineOptions() {
return options;
}
+ }
- @Override
- public void output(OutputT output) {
- throwUnsupportedOutputFromBundleMethods();
+ private class TestFinishBundleContext extends DoFn<InputT, OutputT>.FinishBundleContext {
+
+ private TestFinishBundleContext() {
+ fn.super();
}
- @Override
- public void outputWithTimestamp(OutputT output, Instant timestamp) {
- throwUnsupportedOutputFromBundleMethods();
+ private void throwUnsupportedOutputFromBundleMethods() {
+ throw new UnsupportedOperationException(
+ "DoFnTester doesn't support output from bundle methods");
}
@Override
- public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
- throwUnsupportedOutputFromBundleMethods();
+ public PipelineOptions getPipelineOptions() {
+ return options;
}
@Override
- public <T> void output(TupleTag<T> tag, T output) {
+ public void output(
+ OutputT output, Instant timestamp, BoundedWindow window) {
throwUnsupportedOutputFromBundleMethods();
}
- private void throwUnsupportedOutputFromBundleMethods() {
- throw new UnsupportedOperationException(
- "DoFnTester doesn't support output from bundle methods");
+ @Override
+ public <T> void output(TupleTag<T> tag, T output, Instant timestamp, BoundedWindow window) {
+ throwUnsupportedOutputFromBundleMethods();
}
}
@@ -564,12 +574,10 @@ public class DoFnTester<InputT, OutputT> implements AutoCloseable {
}
private class TestProcessContext extends DoFn<InputT, OutputT>.ProcessContext {
- private final TestContext context;
private final ValueInSingleWindow<InputT> element;
private TestProcessContext(ValueInSingleWindow<InputT> element) {
fn.super();
- this.context = new TestContext();
this.element = element;
}
@@ -611,7 +619,7 @@ public class DoFnTester<InputT, OutputT> implements AutoCloseable {
@Override
public PipelineOptions getPipelineOptions() {
- return context.getPipelineOptions();
+ return options;
}
@Override
@@ -634,6 +642,12 @@ public class DoFnTester<InputT, OutputT> implements AutoCloseable {
getMutableOutput(tag)
.add(ValueInSingleWindow.of(output, timestamp, element.getWindow(), element.getPane()));
}
+
+ private void throwUnsupportedOutputFromBundleMethods() {
+ throw new UnsupportedOperationException(
+ "DoFnTester doesn't support output from bundle methods");
+ }
+
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
index 564baf4..e132115 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupIntoBatches.java
@@ -195,7 +195,7 @@ public class GroupIntoBatches<K, InputT>
}
private void flushBatch(
- Context c,
+ WindowedContext c,
ValueState<K> key,
BagState<InputT> batch,
CombiningState<Long, long[], Long> numElementsInBatch) {
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index 73d78c9..7dd2cdd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -33,6 +33,7 @@ import org.apache.beam.sdk.coders.CannotProvideCoderException;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.DisplayData.Builder;
import org.apache.beam.sdk.transforms.display.DisplayData.ItemSpec;
@@ -191,8 +192,8 @@ import org.apache.beam.sdk.values.TypeDescriptor;
* necessarily need to be explicitly specified, even if the {@link DoFn}
* generates them. Within the {@link DoFn}, an element is added to the
* main output {@link PCollection} as normal, using
- * {@link DoFn.Context#output(Object)}, while an element is added to any additional output
- * {@link PCollection} using {@link DoFn.Context#output(TupleTag, Object)}. For example:
+ * {@link WindowedContext#output(Object)}, while an element is added to any additional output
+ * {@link PCollection} using {@link WindowedContext#output(TupleTag, Object)}. For example:
*
* <pre>{@code
* PCollection<String> words = ...;
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
index 6bef4df..2b6d5be 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/ByteBuddyDoFnInvokerFactory.java
@@ -67,10 +67,11 @@ import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.OnTimerMethod;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.Cases;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.ContextParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.FinishBundleContextParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.OnTimerContextParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.ProcessContextParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StartBundleContextParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
@@ -84,6 +85,8 @@ import org.apache.beam.sdk.values.TypeDescriptor;
public class ByteBuddyDoFnInvokerFactory implements DoFnInvokerFactory {
public static final String CONTEXT_PARAMETER_METHOD = "context";
+ public static final String START_BUNDLE_CONTEXT_PARAMETER_METHOD = "startBundleContext";
+ public static final String FINISH_BUNDLE_CONTEXT_PARAMETER_METHOD = "finishBundleContext";
public static final String PROCESS_CONTEXT_PARAMETER_METHOD = "processContext";
public static final String ON_TIMER_CONTEXT_PARAMETER_METHOD = "onTimerContext";
public static final String WINDOW_PARAMETER_METHOD = "window";
@@ -557,11 +560,21 @@ public class ByteBuddyDoFnInvokerFactory implements DoFnInvokerFactory {
new Cases<StackManipulation>() {
@Override
- public StackManipulation dispatch(ContextParameter p) {
+ public StackManipulation dispatch(StartBundleContextParameter p) {
return new StackManipulation.Compound(
pushDelegate,
MethodInvocation.invoke(
- getExtraContextFactoryMethodDescription(CONTEXT_PARAMETER_METHOD, DoFn.class)));
+ getExtraContextFactoryMethodDescription(
+ START_BUNDLE_CONTEXT_PARAMETER_METHOD, DoFn.class)));
+ }
+
+ @Override
+ public StackManipulation dispatch(FinishBundleContextParameter p) {
+ return new StackManipulation.Compound(
+ pushDelegate,
+ MethodInvocation.invoke(
+ getExtraContextFactoryMethodDescription(
+ FINISH_BUNDLE_CONTEXT_PARAMETER_METHOD, DoFn.class)));
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
index 0fbcc84..3c44afe 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvoker.java
@@ -41,10 +41,10 @@ public interface DoFnInvoker<InputT, OutputT> {
void invokeSetup();
/** Invoke the {@link DoFn.StartBundle} method on the bound {@link DoFn}. */
- void invokeStartBundle(DoFn<InputT, OutputT>.Context c);
+ void invokeStartBundle(DoFn<InputT, OutputT>.StartBundleContext c);
/** Invoke the {@link DoFn.FinishBundle} method on the bound {@link DoFn}. */
- void invokeFinishBundle(DoFn<InputT, OutputT>.Context c);
+ void invokeFinishBundle(DoFn<InputT, OutputT>.FinishBundleContext c);
/** Invoke the {@link DoFn.Teardown} method on the bound {@link DoFn}. */
void invokeTeardown();
@@ -100,8 +100,11 @@ public interface DoFnInvoker<InputT, OutputT> {
*/
BoundedWindow window();
- /** Provide a {@link DoFn.Context} to use with the given {@link DoFn}. */
- DoFn<InputT, OutputT>.Context context(DoFn<InputT, OutputT> doFn);
+ /** Provide a {@link DoFn.StartBundleContext} to use with the given {@link DoFn}. */
+ DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn);
+
+ /** Provide a {@link DoFn.FinishBundleContext} to use with the given {@link DoFn}. */
+ DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(DoFn<InputT, OutputT> doFn);
/** Provide a {@link DoFn.ProcessContext} to use with the given {@link DoFn}. */
DoFn<InputT, OutputT>.ProcessContext processContext(DoFn<InputT, OutputT> doFn);
@@ -135,7 +138,13 @@ public interface DoFnInvoker<InputT, OutputT> {
}
@Override
- public DoFn<InputT, OutputT>.Context context(DoFn<InputT, OutputT> doFn) {
+ public DoFn<InputT, OutputT>.StartBundleContext startBundleContext(DoFn<InputT, OutputT> doFn) {
+ return null;
+ }
+
+ @Override
+ public DoFn<InputT, OutputT>.FinishBundleContext finishBundleContext(
+ DoFn<InputT, OutputT> doFn) {
return null;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
index 1be741f..3219f96 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
@@ -177,8 +177,10 @@ public abstract class DoFnSignature {
public <ResultT> ResultT match(Cases<ResultT> cases) {
// This could be done with reflection, but since the number of cases is small and known,
// they are simply inlined.
- if (this instanceof ContextParameter) {
- return cases.dispatch((ContextParameter) this);
+ if (this instanceof StartBundleContextParameter) {
+ return cases.dispatch((StartBundleContextParameter) this);
+ } else if (this instanceof FinishBundleContextParameter) {
+ return cases.dispatch((FinishBundleContextParameter) this);
} else if (this instanceof ProcessContextParameter) {
return cases.dispatch((ProcessContextParameter) this);
} else if (this instanceof OnTimerContextParameter) {
@@ -202,7 +204,8 @@ public abstract class DoFnSignature {
* An interface for destructuring a {@link Parameter}.
*/
public interface Cases<ResultT> {
- ResultT dispatch(ContextParameter p);
+ ResultT dispatch(StartBundleContextParameter p);
+ ResultT dispatch(FinishBundleContextParameter p);
ResultT dispatch(ProcessContextParameter p);
ResultT dispatch(OnTimerContextParameter p);
ResultT dispatch(WindowParameter p);
@@ -218,7 +221,12 @@ public abstract class DoFnSignature {
protected abstract ResultT dispatchDefault(Parameter p);
@Override
- public ResultT dispatch(ContextParameter p) {
+ public ResultT dispatch(StartBundleContextParameter p) {
+ return dispatchDefault(p);
+ }
+
+ @Override
+ public ResultT dispatch(FinishBundleContextParameter p) {
return dispatchDefault(p);
}
@@ -255,18 +263,15 @@ public abstract class DoFnSignature {
}
// These parameter descriptors are constant
- private static final ContextParameter CONTEXT_PARAMETER =
- new AutoValue_DoFnSignature_Parameter_ContextParameter();
+ private static final StartBundleContextParameter START_BUNDLE_CONTEXT_PARAMETER =
+ new AutoValue_DoFnSignature_Parameter_StartBundleContextParameter();
+ private static final FinishBundleContextParameter FINISH_BUNDLE_CONTEXT_PARAMETER =
+ new AutoValue_DoFnSignature_Parameter_FinishBundleContextParameter();
private static final ProcessContextParameter PROCESS_CONTEXT_PARAMETER =
new AutoValue_DoFnSignature_Parameter_ProcessContextParameter();
private static final OnTimerContextParameter ON_TIMER_CONTEXT_PARAMETER =
new AutoValue_DoFnSignature_Parameter_OnTimerContextParameter();
- /** Returns a {@link ContextParameter}. */
- public static ContextParameter context() {
- return CONTEXT_PARAMETER;
- }
-
/** Returns a {@link ProcessContextParameter}. */
public static ProcessContextParameter processContext() {
return PROCESS_CONTEXT_PARAMETER;
@@ -301,13 +306,22 @@ public abstract class DoFnSignature {
}
/**
- * Descriptor for a {@link Parameter} of type {@link DoFn.Context}.
+ * Descriptor for a {@link Parameter} of type {@link DoFn.StartBundleContext}.
+ *
+ * <p>All such descriptors are equal.
+ */
+ @AutoValue
+ public abstract static class StartBundleContextParameter extends Parameter {
+ StartBundleContextParameter() {}
+ }
+ /**
+ * Descriptor for a {@link Parameter} of type {@link DoFn.FinishBundleContext}.
*
* <p>All such descriptors are equal.
*/
@AutoValue
- public abstract static class ContextParameter extends Parameter {
- ContextParameter() {}
+ public abstract static class FinishBundleContextParameter extends Parameter {
+ FinishBundleContextParameter() {}
}
/**
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
index 666c7f6..bac3bef 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
@@ -346,14 +346,14 @@ public class DoFnSignatures {
if (startBundleMethod != null) {
ErrorReporter startBundleErrors = errors.forMethod(DoFn.StartBundle.class, startBundleMethod);
signatureBuilder.setStartBundle(
- analyzeBundleMethod(startBundleErrors, fnT, startBundleMethod, inputT, outputT));
+ analyzeStartBundleMethod(startBundleErrors, fnT, startBundleMethod, inputT, outputT));
}
if (finishBundleMethod != null) {
ErrorReporter finishBundleErrors =
errors.forMethod(DoFn.FinishBundle.class, finishBundleMethod);
signatureBuilder.setFinishBundle(
- analyzeBundleMethod(finishBundleErrors, fnT, finishBundleMethod, inputT, outputT));
+ analyzeFinishBundleMethod(finishBundleErrors, fnT, finishBundleMethod, inputT, outputT));
}
if (setupMethod != null) {
@@ -607,12 +607,25 @@ public class DoFnSignatures {
}
/**
- * Generates a {@link TypeDescriptor} for {@code DoFn<InputT, OutputT>.Context} given {@code
- * InputT} and {@code OutputT}.
+ * Generates a {@link TypeDescriptor} for {@code DoFn<InputT, OutputT>.StartBundleContext} given
+ * {@code InputT} and {@code OutputT}.
+ */
+ private static <InputT, OutputT>
+ TypeDescriptor<DoFn<InputT, OutputT>.StartBundleContext> doFnStartBundleContextTypeOf(
+ TypeDescriptor<InputT> inputT, TypeDescriptor<OutputT> outputT) {
+ return new TypeDescriptor<DoFn<InputT, OutputT>.StartBundleContext>() {}.where(
+ new TypeParameter<InputT>() {}, inputT)
+ .where(new TypeParameter<OutputT>() {}, outputT);
+ }
+
+ /**
+ * Generates a {@link TypeDescriptor} for {@code DoFn<InputT, OutputT>.FinishBundleContext} given
+ * {@code InputT} and {@code OutputT}.
*/
- private static <InputT, OutputT> TypeDescriptor<DoFn<InputT, OutputT>.Context> doFnContextTypeOf(
- TypeDescriptor<InputT> inputT, TypeDescriptor<OutputT> outputT) {
- return new TypeDescriptor<DoFn<InputT, OutputT>.Context>() {}.where(
+ private static <InputT, OutputT>
+ TypeDescriptor<DoFn<InputT, OutputT>.FinishBundleContext> doFnFinishBundleContextTypeOf(
+ TypeDescriptor<InputT> inputT, TypeDescriptor<OutputT> outputT) {
+ return new TypeDescriptor<DoFn<InputT, OutputT>.FinishBundleContext>() {}.where(
new TypeParameter<InputT>() {}, inputT)
.where(new TypeParameter<OutputT>() {}, outputT);
}
@@ -752,7 +765,6 @@ public class DoFnSignatures {
TypeDescriptor<?> outputT) {
TypeDescriptor<?> expectedProcessContextT = doFnProcessContextTypeOf(inputT, outputT);
- TypeDescriptor<?> expectedContextT = doFnContextTypeOf(inputT, outputT);
TypeDescriptor<?> expectedOnTimerContextT = doFnOnTimerContextTypeOf(inputT, outputT);
TypeDescriptor<?> paramT = param.getType();
@@ -765,11 +777,6 @@ public class DoFnSignatures {
"ProcessContext argument must have type %s",
formatType(expectedProcessContextT));
return Parameter.processContext();
- } else if (rawType.equals(DoFn.Context.class)) {
- paramErrors.checkArgument(paramT.equals(expectedContextT),
- "Context argument must have type %s",
- formatType(expectedContextT));
- return Parameter.context();
} else if (rawType.equals(DoFn.OnTimerContext.class)) {
paramErrors.checkArgument(
paramT.equals(expectedOnTimerContextT),
@@ -921,17 +928,36 @@ public class DoFnSignatures {
}
@VisibleForTesting
- static DoFnSignature.BundleMethod analyzeBundleMethod(
+ static DoFnSignature.BundleMethod analyzeStartBundleMethod(
+ ErrorReporter errors,
+ TypeDescriptor<? extends DoFn<?, ?>> fnT,
+ Method m,
+ TypeDescriptor<?> inputT,
+ TypeDescriptor<?> outputT) {
+ errors.checkArgument(void.class.equals(m.getReturnType()), "Must return void");
+ TypeDescriptor<?> expectedContextT = doFnStartBundleContextTypeOf(inputT, outputT);
+ Type[] params = m.getGenericParameterTypes();
+ errors.checkArgument(
+ params.length == 0
+ || (params.length == 1 && fnT.resolveType(params[0]).equals(expectedContextT)),
+ "Must take a single argument of type %s",
+ formatType(expectedContextT));
+ return DoFnSignature.BundleMethod.create(m);
+ }
+
+ @VisibleForTesting
+ static DoFnSignature.BundleMethod analyzeFinishBundleMethod(
ErrorReporter errors,
TypeDescriptor<? extends DoFn<?, ?>> fnT,
Method m,
TypeDescriptor<?> inputT,
TypeDescriptor<?> outputT) {
errors.checkArgument(void.class.equals(m.getReturnType()), "Must return void");
- TypeDescriptor<?> expectedContextT = doFnContextTypeOf(inputT, outputT);
+ TypeDescriptor<?> expectedContextT = doFnFinishBundleContextTypeOf(inputT, outputT);
Type[] params = m.getGenericParameterTypes();
errors.checkArgument(
- params.length == 1 && fnT.resolveType(params[0]).equals(expectedContextT),
+ params.length == 0
+ || (params.length == 1 && fnT.resolveType(params[0]).equals(expectedContextT)),
"Must take a single argument of type %s",
formatType(expectedContextT));
return DoFnSignature.BundleMethod.create(m);
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
index b43c7a3..faf3ca9 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
@@ -30,6 +30,7 @@ import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.coders.CustomCoder;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFn.WindowedContext;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.util.VarInt;
@@ -70,7 +71,7 @@ public final class PaneInfo {
* definitions:
* <ol>
* <li>We'll call a pipeline 'simple' if it does not use
- * {@link DoFn.Context#outputWithTimestamp} in
+ * {@link WindowedContext#outputWithTimestamp} in
* any {@link DoFn}, and it uses the same
* {@link org.apache.beam.sdk.transforms.windowing.Window#withAllowedLateness}
* argument value on all windows (or uses the default of {@link org.joda.time.Duration#ZERO}).
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
index 8077c27..084c445 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/MetricsTest.java
@@ -210,7 +210,7 @@ public class MetricsTest implements Serializable {
Distribution bundleDist = Metrics.distribution(MetricsTest.class, "bundle");
@StartBundle
- public void startBundle(Context c) {
+ public void startBundle() {
bundleDist.update(10L);
}
@@ -226,7 +226,7 @@ public class MetricsTest implements Serializable {
}
@DoFn.FinishBundle
- public void finishBundle(Context c) {
+ public void finishBundle() {
bundleDist.update(40L);
}
}))
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
index d609d0e..a980f87 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
@@ -195,12 +195,12 @@ public class DoFnTesterTest {
}
@StartBundle
- public void startBundle(Context context) {
+ public void startBundle() {
++numStartBundleCalls;
}
@FinishBundle
- public void finishBundle(Context context) {
+ public void finishBundle() {
++numFinishBundleCalls;
}
}
@@ -397,7 +397,7 @@ public class DoFnTesterTest {
}
@StartBundle
- public void startBundle(Context c) {
+ public void startBundle() {
checkState(state == LifecycleState.SET_UP, "Wrong state: %s", state);
state = LifecycleState.INSIDE_BUNDLE;
startBundleCalls.inc();
@@ -412,7 +412,7 @@ public class DoFnTesterTest {
}
@FinishBundle
- public void finishBundle(Context c) {
+ public void finishBundle() {
checkState(state == LifecycleState.INSIDE_BUNDLE, "Wrong state: %s", state);
state = LifecycleState.SET_UP;
finishBundleCalls.inc();
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
index fda8947..849b874 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoLifecycleTest.java
@@ -87,7 +87,7 @@ public class ParDoLifecycleTest implements Serializable {
}
@StartBundle
- public void startBundle(Context c) {
+ public void startBundle() {
assertThat("setup should have been called", setupCalled, is(true));
assertThat(
"Even number of startBundle and finishBundle calls in startBundle",
@@ -108,7 +108,7 @@ public class ParDoLifecycleTest implements Serializable {
}
@FinishBundle
- public void finishBundle(Context c) {
+ public void finishBundle() {
assertThat("startBundle should have been called", startBundleCalls, greaterThan(0));
assertThat(
"there should be one bundle that has been started but not finished",
@@ -167,7 +167,7 @@ public class ParDoLifecycleTest implements Serializable {
}
@StartBundle
- public void begin(Context c) {
+ public void begin() {
assertThat("setup should have been called", setupCalled, is(true));
assertThat("Even number of startBundle and finishBundle calls in startBundle",
startBundleCalls,
@@ -186,7 +186,7 @@ public class ParDoLifecycleTest implements Serializable {
}
@FinishBundle
- public void end(Context c) {
+ public void end() {
assertThat("startBundle should have been called", startBundleCalls, greaterThan(0));
assertThat("there should be one bundle that has been started but not finished",
startBundleCalls,
@@ -354,7 +354,7 @@ public class ParDoLifecycleTest implements Serializable {
}
@StartBundle
- public void startBundle(Context c) throws Exception {
+ public void startBundle() throws Exception {
throwIfNecessary(MethodForException.START_BUNDLE);
}
@@ -364,7 +364,7 @@ public class ParDoLifecycleTest implements Serializable {
}
@FinishBundle
- public void finishBundle(Context c) throws Exception {
+ public void finishBundle() throws Exception {
throwIfNecessary(MethodForException.FINISH_BUNDLE);
}
@@ -401,7 +401,7 @@ public class ParDoLifecycleTest implements Serializable {
}
@StartBundle
- public void preBundle(Context c) throws Exception {
+ public void preBundle() throws Exception {
throwIfNecessary(MethodForException.START_BUNDLE);
}
@@ -411,7 +411,7 @@ public class ParDoLifecycleTest implements Serializable {
}
@FinishBundle
- public void postBundle(Context c) throws Exception {
+ public void postBundle() throws Exception {
throwIfNecessary(MethodForException.FINISH_BUNDLE);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index 1c919d4..56051a6 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -170,12 +170,11 @@ public class ParDoTest implements Serializable {
}
@StartBundle
- public void startBundle(Context c) {
+ public void startBundle() {
assertThat(state,
anyOf(equalTo(State.UNSTARTED), equalTo(State.FINISHED)));
state = State.STARTED;
- outputToAll(c, "started");
}
@ProcessElement
@@ -187,18 +186,17 @@ public class ParDoTest implements Serializable {
}
@FinishBundle
- public void finishBundle(Context c) {
+ public void finishBundle(FinishBundleContext c) {
assertThat(state,
anyOf(equalTo(State.STARTED), equalTo(State.PROCESSING)));
state = State.FINISHED;
- outputToAll(c, "finished");
- }
-
- private void outputToAll(Context c, String value) {
- c.output(value);
+ c.output("finished", BoundedWindow.TIMESTAMP_MIN_VALUE, GlobalWindow.INSTANCE);
for (TupleTag<String> additionalOutputTupleTag : additionalOutputTupleTags) {
- c.output(additionalOutputTupleTag,
- additionalOutputTupleTag.getId() + ": " + value);
+ c.output(
+ additionalOutputTupleTag,
+ additionalOutputTupleTag.getId() + ": " + "finished",
+ BoundedWindow.TIMESTAMP_MIN_VALUE,
+ GlobalWindow.INSTANCE);
}
}
@@ -220,7 +218,7 @@ public class ParDoTest implements Serializable {
static class TestStartBatchErrorDoFn extends DoFn<Integer, String> {
@StartBundle
- public void startBundle(Context c) {
+ public void startBundle() {
throw new RuntimeException("test error in initialize");
}
@@ -244,7 +242,7 @@ public class ParDoTest implements Serializable {
}
@FinishBundle
- public void finishBundle(Context c) {
+ public void finishBundle(FinishBundleContext c) {
throw new RuntimeException("test error in finalize");
}
}
@@ -1156,13 +1154,10 @@ public class ParDoTest implements Serializable {
@Override
public Void apply(Iterable<String> outputs) {
- List<String> starteds = new ArrayList<>();
List<String> processeds = new ArrayList<>();
List<String> finisheds = new ArrayList<>();
for (String output : outputs) {
- if (output.contains("started")) {
- starteds.add(output);
- } else if (output.contains("finished")) {
+ if (output.contains("finished")) {
finisheds.add(output);
} else {
processeds.add(output);
@@ -1196,10 +1191,6 @@ public class ParDoTest implements Serializable {
assertThat(processeds, contains(expectedProcessedsArray));
}
- assertEquals(starteds.size(), finisheds.size());
- for (String started : starteds) {
- assertEquals(additionalOutputPrefix + "started", started);
- }
for (String finished : finisheds) {
assertEquals(additionalOutputPrefix + "finished", finished);
}
@@ -1483,7 +1474,6 @@ public class ParDoTest implements Serializable {
private static class Checker implements SerializableFunction<Iterable<String>, Void> {
@Override
public Void apply(Iterable<String> input) {
- boolean foundStart = false;
boolean foundElement = false;
boolean foundFinish = false;
for (String str : input) {
@@ -1492,17 +1482,12 @@ public class ParDoTest implements Serializable {
throw new AssertionError("Received duplicate element");
}
foundElement = true;
- } else if (str.equals("start:2:2")) {
- foundStart = true;
} else if (str.equals("finish:3:3")) {
foundFinish = true;
} else {
throw new AssertionError("Got unexpected value: " + str);
}
}
- if (!foundStart) {
- throw new AssertionError("Missing \"start:2:2\"");
- }
if (!foundElement) {
throw new AssertionError("Missing \"elem:1:1\"");
}
@@ -1518,19 +1503,14 @@ public class ParDoTest implements Serializable {
@Category(ValidatesRunner.class)
public void testWindowingInStartAndFinishBundle() {
+ final FixedWindows windowFn = FixedWindows.of(Duration.millis(1));
PCollection<String> output =
pipeline
.apply(Create.timestamped(TimestampedValue.of("elem", new Instant(1))))
- .apply(Window.<String>into(FixedWindows.of(Duration.millis(1))))
+ .apply(Window.<String>into(windowFn))
.apply(
ParDo.of(
new DoFn<String, String>() {
- @StartBundle
- public void startBundle(Context c) {
- c.outputWithTimestamp("start", new Instant(2));
- System.out.println("Start: 2");
- }
-
@ProcessElement
public void processElement(ProcessContext c) {
c.output(c.element());
@@ -1539,8 +1519,9 @@ public class ParDoTest implements Serializable {
}
@FinishBundle
- public void finishBundle(Context c) {
- c.outputWithTimestamp("finish", new Instant(3));
+ public void finishBundle(FinishBundleContext c) {
+ Instant ts = new Instant(3);
+ c.output("finish", ts, windowFn.assignWindow(ts));
System.out.println("Finish: 3");
}
}))
@@ -1552,30 +1533,6 @@ public class ParDoTest implements Serializable {
}
@Test
- @Category(NeedsRunner.class)
- public void testWindowingInStartBundleException() {
-
- pipeline
- .apply(Create.timestamped(TimestampedValue.of("elem", new Instant(1))))
- .apply(Window.<String>into(FixedWindows.of(Duration.millis(1))))
- .apply(
- ParDo.of(
- new DoFn<String, String>() {
- @StartBundle
- public void startBundle(Context c) {
- c.output("start");
- }
-
- @ProcessElement
- public void processElement(ProcessContext c) {
- c.output(c.element());
- }
- }));
-
- thrown.expectMessage("WindowFn attempted to access input timestamp when none was available");
- pipeline.run();
- }
- @Test
public void testDoFnDisplayData() {
DoFn<String, String> fn = new DoFn<String, String>() {
@ProcessElement
http://git-wip-us.apache.org/repos/asf/beam/blob/d59d9b74/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java
index a0f1fd3..02a44d2 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SplittableDoFnTest.java
@@ -405,13 +405,13 @@ public class SplittableDoFnTest implements Serializable {
}
@StartBundle
- public void startBundle(Context c) {
+ public void startBundle() {
assertEquals(State.OUTSIDE_BUNDLE, state);
state = State.INSIDE_BUNDLE;
}
@FinishBundle
- public void finishBundle(Context c) {
+ public void finishBundle() {
assertEquals(State.INSIDE_BUNDLE, state);
state = State.OUTSIDE_BUNDLE;
}