You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2016/06/16 02:30:36 UTC
[01/12] incubator-beam git commit: Rename InProcessPipelineRunner to
DirectRunner
Repository: incubator-beam
Updated Branches:
refs/heads/master 414bb4bc3 -> 518b23c2d
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluator.java
index ba9815b..1fec9d8 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluator.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.util.WindowedValue;
/**
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
index f2d577e..b12a34c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.DoFn;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
index 81d2520..dfc1753 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
@@ -19,7 +19,7 @@ package org.apache.beam.runners.direct;
import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupAlsoByWindow;
import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.Flatten.FlattenPCollectionList;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
index fb637b4..4dd1475 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
@@ -19,7 +19,7 @@ package org.apache.beam.runners.direct;
import static com.google.common.base.Preconditions.checkState;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
index fceb20c..5030730 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
@@ -17,8 +17,8 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.io.Read.Unbounded;
import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
index ffaf3fa..f4260f5 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.PCollectionViewWriter;
+import org.apache.beam.runners.direct.DirectRunner.PCollectionViewWriter;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.transforms.AppliedPTransform;
@@ -36,7 +36,7 @@ import java.util.ArrayList;
import java.util.List;
/**
- * The {@link InProcessPipelineRunner} {@link TransformEvaluatorFactory} for the
+ * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
* {@link CreatePCollectionView} primitive {@link PTransform}.
*
* <p>The {@link ViewEvaluatorFactory} produces {@link TransformEvaluator TransformEvaluators} for
@@ -49,7 +49,7 @@ class ViewEvaluatorFactory implements TransformEvaluatorFactory {
@Override
public <T> TransformEvaluator<T> forApplication(
AppliedPTransform<?, ?, ?> application,
- InProcessPipelineRunner.CommittedBundle<?> inputBundle,
+ DirectRunner.CommittedBundle<?> inputBundle,
InProcessEvaluationContext evaluationContext) {
@SuppressWarnings({"cast", "unchecked", "rawtypes"})
TransformEvaluator<T> evaluator = createEvaluator(
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
index 628f94d..89866cc 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
@@ -17,8 +17,8 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -36,7 +36,7 @@ import java.util.Collection;
import javax.annotation.Nullable;
/**
- * The {@link InProcessPipelineRunner} {@link TransformEvaluatorFactory} for the
+ * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
* {@link Bound Window.Bound} primitive {@link PTransform}.
*/
class WindowEvaluatorFactory implements TransformEvaluatorFactory {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/AvroIOShardedWriteFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/AvroIOShardedWriteFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/AvroIOShardedWriteFactoryTest.java
index c0c1361..d94113a 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/AvroIOShardedWriteFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/AvroIOShardedWriteFactoryTest.java
@@ -114,7 +114,7 @@ public class AvroIOShardedWriteFactoryTest {
private Pipeline getPipeline() {
PipelineOptions options = TestPipeline.testingPipelineOptions();
- options.setRunner(InProcessPipelineRunner.class);
+ options.setRunner(DirectRunner.class);
return TestPipeline.fromOptions(options);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
index bcdc089..e26f860 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
@@ -25,8 +25,8 @@ import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.BigEndianLongCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.BoundedSource;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
index 0d1b464..4969a30 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
@@ -60,24 +60,24 @@ public class CommittedResultTest implements Serializable {
CommittedResult result =
CommittedResult.create(StepTransformResult.withoutHold(transform).build(),
bundleFactory.createRootBundle(created).commit(Instant.now()),
- Collections.<InProcessPipelineRunner.CommittedBundle<?>>emptyList());
+ Collections.<DirectRunner.CommittedBundle<?>>emptyList());
assertThat(result.getTransform(), Matchers.<AppliedPTransform<?, ?, ?>>equalTo(transform));
}
@Test
public void getUncommittedElementsEqualInput() {
- InProcessPipelineRunner.CommittedBundle<Integer> bundle =
+ DirectRunner.CommittedBundle<Integer> bundle =
bundleFactory.createRootBundle(created)
.add(WindowedValue.valueInGlobalWindow(2))
.commit(Instant.now());
CommittedResult result =
CommittedResult.create(StepTransformResult.withoutHold(transform).build(),
bundle,
- Collections.<InProcessPipelineRunner.CommittedBundle<?>>emptyList());
+ Collections.<DirectRunner.CommittedBundle<?>>emptyList());
assertThat(result.getUnprocessedInputs(),
- Matchers.<InProcessPipelineRunner.CommittedBundle<?>>equalTo(bundle));
+ Matchers.<DirectRunner.CommittedBundle<?>>equalTo(bundle));
}
@Test
@@ -85,14 +85,14 @@ public class CommittedResultTest implements Serializable {
CommittedResult result =
CommittedResult.create(StepTransformResult.withoutHold(transform).build(),
null,
- Collections.<InProcessPipelineRunner.CommittedBundle<?>>emptyList());
+ Collections.<DirectRunner.CommittedBundle<?>>emptyList());
assertThat(result.getUnprocessedInputs(), nullValue());
}
@Test
public void getOutputsEqualInput() {
- List<? extends InProcessPipelineRunner.CommittedBundle<?>> outputs =
+ List<? extends DirectRunner.CommittedBundle<?>> outputs =
ImmutableList.of(bundleFactory.createRootBundle(PCollection.createPrimitiveOutputInternal(p,
WindowingStrategy.globalDefault(),
PCollection.IsBounded.BOUNDED)).commit(Instant.now()),
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java
new file mode 100644
index 0000000..cd44b7e
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.apache.beam.runners.direct.DirectRegistrar.InProcessRunner;
+import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
+import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.ServiceLoader;
+
+/** Tests for {@link InProcessRunner}. */
+@RunWith(JUnit4.class)
+public class DirectRegistrarTest {
+ @Test
+ public void testCorrectOptionsAreReturned() {
+ assertEquals(
+ ImmutableList.of(DirectOptions.class),
+ new DirectRegistrar.InProcessOptions().getPipelineOptions());
+ }
+
+ @Test
+ public void testCorrectRunnersAreReturned() {
+ assertEquals(
+ ImmutableList.of(DirectRunner.class),
+ new DirectRegistrar.InProcessRunner().getPipelineRunners());
+ }
+
+ @Test
+ public void testServiceLoaderForOptions() {
+ for (PipelineOptionsRegistrar registrar :
+ Lists.newArrayList(ServiceLoader.load(PipelineOptionsRegistrar.class).iterator())) {
+ if (registrar instanceof DirectRegistrar.InProcessOptions) {
+ return;
+ }
+ }
+ fail("Expected to find " + DirectRegistrar.InProcessOptions.class);
+ }
+
+ @Test
+ public void testServiceLoaderForRunner() {
+ for (PipelineRunnerRegistrar registrar :
+ Lists.newArrayList(ServiceLoader.load(PipelineRunnerRegistrar.class).iterator())) {
+ if (registrar instanceof DirectRegistrar.InProcessRunner) {
+ return;
+ }
+ }
+ fail("Expected to find " + DirectRegistrar.InProcessRunner.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
new file mode 100644
index 0000000..1de38df
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+import org.apache.beam.runners.direct.DirectRunner.DirectPipelineResult;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.util.IllegalMutationException;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.TypeDescriptor;
+
+import com.google.common.collect.ImmutableMap;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+
+import org.hamcrest.Matchers;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.internal.matchers.ThrowableMessageMatcher;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Tests for basic {@link DirectRunner} functionality.
+ */
+@RunWith(JUnit4.class)
+public class DirectRunnerTest implements Serializable {
+ @Rule public transient ExpectedException thrown = ExpectedException.none();
+
+ private Pipeline getPipeline() {
+ PipelineOptions opts = PipelineOptionsFactory.create();
+ opts.setRunner(DirectRunner.class);
+
+ Pipeline p = Pipeline.create(opts);
+ return p;
+ }
+
+ @Test
+ public void defaultRunnerLoaded() {
+ assertThat(DirectRunner.class,
+ Matchers.<Class<? extends PipelineRunner>>equalTo(PipelineOptionsFactory.create()
+ .getRunner()));
+ }
+
+ @Test
+ public void wordCountShouldSucceed() throws Throwable {
+ Pipeline p = getPipeline();
+
+ PCollection<KV<String, Long>> counts =
+ p.apply(Create.of("foo", "bar", "foo", "baz", "bar", "foo"))
+ .apply(MapElements.via(new SimpleFunction<String, String>() {
+ @Override
+ public String apply(String input) {
+ return input;
+ }
+ }))
+ .apply(Count.<String>perElement());
+ PCollection<String> countStrs =
+ counts.apply(MapElements.via(new SimpleFunction<KV<String, Long>, String>() {
+ @Override
+ public String apply(KV<String, Long> input) {
+ String str = String.format("%s: %s", input.getKey(), input.getValue());
+ return str;
+ }
+ }));
+
+ PAssert.that(countStrs).containsInAnyOrder("baz: 1", "bar: 2", "foo: 3");
+
+ DirectPipelineResult result = ((DirectPipelineResult) p.run());
+ result.awaitCompletion();
+ }
+
+ @Test(timeout = 5000L)
+ public void byteArrayCountShouldSucceed() {
+ Pipeline p = getPipeline();
+
+ SerializableFunction<Integer, byte[]> getBytes = new SerializableFunction<Integer, byte[]>() {
+ @Override
+ public byte[] apply(Integer input) {
+ try {
+ return CoderUtils.encodeToByteArray(VarIntCoder.of(), input);
+ } catch (CoderException e) {
+ fail("Unexpected Coder Exception " + e);
+ throw new AssertionError("Unreachable");
+ }
+ }
+ };
+ TypeDescriptor<byte[]> td = new TypeDescriptor<byte[]>() {
+ };
+ PCollection<byte[]> foos =
+ p.apply(Create.of(1, 1, 1, 2, 2, 3)).apply(MapElements.via(getBytes).withOutputType(td));
+ PCollection<byte[]> msync =
+ p.apply(Create.of(1, -2, -8, -16)).apply(MapElements.via(getBytes).withOutputType(td));
+ PCollection<byte[]> bytes =
+ PCollectionList.of(foos).and(msync).apply(Flatten.<byte[]>pCollections());
+ PCollection<KV<byte[], Long>> counts = bytes.apply(Count.<byte[]>perElement());
+ PCollection<KV<Integer, Long>> countsBackToString =
+ counts.apply(MapElements.via(new SimpleFunction<KV<byte[], Long>, KV<Integer, Long>>() {
+ @Override
+ public KV<Integer, Long> apply(KV<byte[], Long> input) {
+ try {
+ return KV.of(CoderUtils.decodeFromByteArray(VarIntCoder.of(), input.getKey()),
+ input.getValue());
+ } catch (CoderException e) {
+ fail("Unexpected Coder Exception " + e);
+ throw new AssertionError("Unreachable");
+ }
+ }
+ }));
+
+ Map<Integer, Long> expected = ImmutableMap.<Integer, Long>builder().put(1, 4L)
+ .put(2, 2L)
+ .put(3, 1L)
+ .put(-2, 1L)
+ .put(-8, 1L)
+ .put(-16, 1L)
+ .build();
+ PAssert.thatMap(countsBackToString).isEqualTo(expected);
+ }
+
+ @Test
+ public void transformDisplayDataExceptionShouldFail() {
+ DoFn<Integer, Integer> brokenDoFn = new DoFn<Integer, Integer>() {
+ @Override
+ public void processElement(ProcessContext c) throws Exception {}
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ throw new RuntimeException("oh noes!");
+ }
+ };
+
+ Pipeline p = getPipeline();
+ p
+ .apply(Create.of(1, 2, 3))
+ .apply(ParDo.of(brokenDoFn));
+
+ thrown.expectMessage(brokenDoFn.getClass().getName());
+ thrown.expectCause(ThrowableMessageMatcher.hasMessage(is("oh noes!")));
+ p.run();
+ }
+
+ @Test
+ public void pipelineOptionsDisplayDataExceptionShouldFail() {
+ Object brokenValueType = new Object() {
+ @JsonValue
+ public int getValue () {
+ return 42;
+ }
+
+ @Override
+ public String toString() {
+ throw new RuntimeException("oh noes!!");
+ }
+ };
+
+ Pipeline p = getPipeline();
+ p.getOptions().as(ObjectPipelineOptions.class).setValue(brokenValueType);
+
+ p.apply(Create.of(1, 2, 3));
+
+ thrown.expectMessage(PipelineOptions.class.getName());
+ thrown.expectCause(ThrowableMessageMatcher.hasMessage(is("oh noes!!")));
+ p.run();
+ }
+
+ /** {@link PipelineOptions} to inject bad object implementations. */
+ public interface ObjectPipelineOptions extends PipelineOptions {
+ Object getValue();
+ void setValue(Object value);
+ }
+
+
+ /**
+ * Tests that a {@link DoFn} that mutates an output with a good equals() fails in the
+ * {@link DirectRunner}.
+ */
+ @Test
+ public void testMutatingOutputThenOutputDoFnError() throws Exception {
+ Pipeline pipeline = getPipeline();
+
+ pipeline
+ .apply(Create.of(42))
+ .apply(ParDo.of(new DoFn<Integer, List<Integer>>() {
+ @Override public void processElement(ProcessContext c) {
+ List<Integer> outputList = Arrays.asList(1, 2, 3, 4);
+ c.output(outputList);
+ outputList.set(0, 37);
+ c.output(outputList);
+ }
+ }));
+
+ thrown.expect(IllegalMutationException.class);
+ thrown.expectMessage("output");
+ thrown.expectMessage("must not be mutated");
+ pipeline.run();
+ }
+
+ /**
+ * Tests that a {@link DoFn} that mutates an output with a good equals() fails in the
+ * {@link DirectRunner}.
+ */
+ @Test
+ public void testMutatingOutputThenTerminateDoFnError() throws Exception {
+ Pipeline pipeline = getPipeline();
+
+ pipeline
+ .apply(Create.of(42))
+ .apply(ParDo.of(new DoFn<Integer, List<Integer>>() {
+ @Override public void processElement(ProcessContext c) {
+ List<Integer> outputList = Arrays.asList(1, 2, 3, 4);
+ c.output(outputList);
+ outputList.set(0, 37);
+ }
+ }));
+
+ thrown.expect(IllegalMutationException.class);
+ thrown.expectMessage("output");
+ thrown.expectMessage("must not be mutated");
+ pipeline.run();
+ }
+
+ /**
+ * Tests that a {@link DoFn} that mutates an output with a bad equals() still fails
+ * in the {@link DirectRunner}.
+ */
+ @Test
+ public void testMutatingOutputCoderDoFnError() throws Exception {
+ Pipeline pipeline = getPipeline();
+
+ pipeline
+ .apply(Create.of(42))
+ .apply(ParDo.of(new DoFn<Integer, byte[]>() {
+ @Override public void processElement(ProcessContext c) {
+ byte[] outputArray = new byte[]{0x1, 0x2, 0x3};
+ c.output(outputArray);
+ outputArray[0] = 0xa;
+ c.output(outputArray);
+ }
+ }));
+
+ thrown.expect(IllegalMutationException.class);
+ thrown.expectMessage("output");
+ thrown.expectMessage("must not be mutated");
+ pipeline.run();
+ }
+
+ /**
+ * Tests that a {@link DoFn} that mutates its input with a good equals() fails in the
+ * {@link DirectRunner}.
+ */
+ @Test
+ public void testMutatingInputDoFnError() throws Exception {
+ Pipeline pipeline = getPipeline();
+
+ pipeline
+ .apply(Create.of(Arrays.asList(1, 2, 3), Arrays.asList(4, 5, 6))
+ .withCoder(ListCoder.of(VarIntCoder.of())))
+ .apply(ParDo.of(new DoFn<List<Integer>, Integer>() {
+ @Override public void processElement(ProcessContext c) {
+ List<Integer> inputList = c.element();
+ inputList.set(0, 37);
+ c.output(12);
+ }
+ }));
+
+ thrown.expect(IllegalMutationException.class);
+ thrown.expectMessage("Input");
+ thrown.expectMessage("must not be mutated");
+ pipeline.run();
+ }
+
+ /**
+ * Tests that a {@link DoFn} that mutates an input with a bad equals() still fails
+ * in the {@link DirectRunner}.
+ */
+ @Test
+ public void testMutatingInputCoderDoFnError() throws Exception {
+ Pipeline pipeline = getPipeline();
+
+ pipeline
+ .apply(Create.of(new byte[]{0x1, 0x2, 0x3}, new byte[]{0x4, 0x5, 0x6}))
+ .apply(ParDo.of(new DoFn<byte[], Integer>() {
+ @Override public void processElement(ProcessContext c) {
+ byte[] inputArray = c.element();
+ inputArray[0] = 0xa;
+ c.output(13);
+ }
+ }));
+
+ thrown.expect(IllegalMutationException.class);
+ thrown.expectMessage("Input");
+ thrown.expectMessage("must not be mutated");
+ pipeline.run();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactoryTest.java
index 9a358dd..e129489 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactoryTest.java
@@ -19,7 +19,7 @@ package org.apache.beam.runners.direct;
import static org.hamcrest.Matchers.isA;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
index 66a5106..5efb090 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
@@ -23,8 +23,8 @@ import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.Create;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java
index a4f900c..b589db0 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java
@@ -22,8 +22,8 @@ import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactoryTest.java
index 20670ca..c4da86c 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactoryTest.java
@@ -20,8 +20,8 @@ package org.apache.beam.runners.direct;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertThat;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.testing.TestPipeline;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
index 6cef60d..ead9c9e 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.Count;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InMemoryWatermarkManagerTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InMemoryWatermarkManagerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InMemoryWatermarkManagerTest.java
index af08d02..21c941a 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InMemoryWatermarkManagerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InMemoryWatermarkManagerTest.java
@@ -29,8 +29,8 @@ import org.apache.beam.runners.direct.InMemoryWatermarkManager.FiredTimers;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate.TimerUpdateBuilder;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.TransformWatermarks;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarLongCoder;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessBundleFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessBundleFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessBundleFactoryTest.java
index abe2a19..3a3ac8c 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessBundleFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessBundleFactoryTest.java
@@ -21,8 +21,8 @@ import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessEvaluationContextTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessEvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessEvaluationContextTest.java
index 18db400..b1cbeb1 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessEvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessEvaluationContextTest.java
@@ -28,9 +28,9 @@ import static org.junit.Assert.assertThat;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.FiredTimers;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
import org.apache.beam.runners.direct.InProcessExecutionContext.InProcessStepContext;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.PCollectionViewWriter;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.PCollectionViewWriter;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
@@ -99,8 +99,8 @@ public class InProcessEvaluationContextTest {
@Before
public void setup() {
- InProcessPipelineRunner runner =
- InProcessPipelineRunner.fromOptions(PipelineOptionsFactory.create());
+ DirectRunner runner =
+ DirectRunner.fromOptions(PipelineOptionsFactory.create());
p = TestPipeline.create();
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactoryTest.java
index 28a3cf6..e8d4711 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactoryTest.java
@@ -22,8 +22,8 @@ import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRegistrarTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRegistrarTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRegistrarTest.java
deleted file mode 100644
index 54094c4..0000000
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRegistrarTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import org.apache.beam.runners.direct.InProcessRegistrar.InProcessRunner;
-import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-import java.util.ServiceLoader;
-
-/** Tests for {@link InProcessRunner}. */
-@RunWith(JUnit4.class)
-public class InProcessPipelineRegistrarTest {
- @Test
- public void testCorrectOptionsAreReturned() {
- assertEquals(
- ImmutableList.of(InProcessPipelineOptions.class),
- new InProcessRegistrar.InProcessOptions().getPipelineOptions());
- }
-
- @Test
- public void testCorrectRunnersAreReturned() {
- assertEquals(
- ImmutableList.of(InProcessPipelineRunner.class),
- new InProcessRegistrar.InProcessRunner().getPipelineRunners());
- }
-
- @Test
- public void testServiceLoaderForOptions() {
- for (PipelineOptionsRegistrar registrar :
- Lists.newArrayList(ServiceLoader.load(PipelineOptionsRegistrar.class).iterator())) {
- if (registrar instanceof InProcessRegistrar.InProcessOptions) {
- return;
- }
- }
- fail("Expected to find " + InProcessRegistrar.InProcessOptions.class);
- }
-
- @Test
- public void testServiceLoaderForRunner() {
- for (PipelineRunnerRegistrar registrar :
- Lists.newArrayList(ServiceLoader.load(PipelineRunnerRegistrar.class).iterator())) {
- if (registrar instanceof InProcessRegistrar.InProcessRunner) {
- return;
- }
- }
- fail("Expected to find " + InProcessRegistrar.InProcessRunner.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRunnerTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRunnerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRunnerTest.java
deleted file mode 100644
index ab26c15..0000000
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessPipelineRunnerTest.java
+++ /dev/null
@@ -1,339 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
-
-import org.apache.beam.runners.direct.InProcessPipelineRunner.InProcessPipelineResult;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.ListCoder;
-import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.runners.PipelineRunner;
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.transforms.Count;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.transforms.MapElements;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.SimpleFunction;
-import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.util.CoderUtils;
-import org.apache.beam.sdk.util.IllegalMutationException;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionList;
-import org.apache.beam.sdk.values.TypeDescriptor;
-
-import com.google.common.collect.ImmutableMap;
-
-import com.fasterxml.jackson.annotation.JsonValue;
-
-import org.hamcrest.Matchers;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.internal.matchers.ThrowableMessageMatcher;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Tests for basic {@link InProcessPipelineRunner} functionality.
- */
-@RunWith(JUnit4.class)
-public class InProcessPipelineRunnerTest implements Serializable {
- @Rule public transient ExpectedException thrown = ExpectedException.none();
-
- private Pipeline getPipeline() {
- PipelineOptions opts = PipelineOptionsFactory.create();
- opts.setRunner(InProcessPipelineRunner.class);
-
- Pipeline p = Pipeline.create(opts);
- return p;
- }
-
- @Test
- public void defaultRunnerLoaded() {
- assertThat(InProcessPipelineRunner.class,
- Matchers.<Class<? extends PipelineRunner>>equalTo(PipelineOptionsFactory.create()
- .getRunner()));
- }
-
- @Test
- public void wordCountShouldSucceed() throws Throwable {
- Pipeline p = getPipeline();
-
- PCollection<KV<String, Long>> counts =
- p.apply(Create.of("foo", "bar", "foo", "baz", "bar", "foo"))
- .apply(MapElements.via(new SimpleFunction<String, String>() {
- @Override
- public String apply(String input) {
- return input;
- }
- }))
- .apply(Count.<String>perElement());
- PCollection<String> countStrs =
- counts.apply(MapElements.via(new SimpleFunction<KV<String, Long>, String>() {
- @Override
- public String apply(KV<String, Long> input) {
- String str = String.format("%s: %s", input.getKey(), input.getValue());
- return str;
- }
- }));
-
- PAssert.that(countStrs).containsInAnyOrder("baz: 1", "bar: 2", "foo: 3");
-
- InProcessPipelineResult result = ((InProcessPipelineResult) p.run());
- result.awaitCompletion();
- }
-
- @Test(timeout = 5000L)
- public void byteArrayCountShouldSucceed() {
- Pipeline p = getPipeline();
-
- SerializableFunction<Integer, byte[]> getBytes = new SerializableFunction<Integer, byte[]>() {
- @Override
- public byte[] apply(Integer input) {
- try {
- return CoderUtils.encodeToByteArray(VarIntCoder.of(), input);
- } catch (CoderException e) {
- fail("Unexpected Coder Exception " + e);
- throw new AssertionError("Unreachable");
- }
- }
- };
- TypeDescriptor<byte[]> td = new TypeDescriptor<byte[]>() {
- };
- PCollection<byte[]> foos =
- p.apply(Create.of(1, 1, 1, 2, 2, 3)).apply(MapElements.via(getBytes).withOutputType(td));
- PCollection<byte[]> msync =
- p.apply(Create.of(1, -2, -8, -16)).apply(MapElements.via(getBytes).withOutputType(td));
- PCollection<byte[]> bytes =
- PCollectionList.of(foos).and(msync).apply(Flatten.<byte[]>pCollections());
- PCollection<KV<byte[], Long>> counts = bytes.apply(Count.<byte[]>perElement());
- PCollection<KV<Integer, Long>> countsBackToString =
- counts.apply(MapElements.via(new SimpleFunction<KV<byte[], Long>, KV<Integer, Long>>() {
- @Override
- public KV<Integer, Long> apply(KV<byte[], Long> input) {
- try {
- return KV.of(CoderUtils.decodeFromByteArray(VarIntCoder.of(), input.getKey()),
- input.getValue());
- } catch (CoderException e) {
- fail("Unexpected Coder Exception " + e);
- throw new AssertionError("Unreachable");
- }
- }
- }));
-
- Map<Integer, Long> expected = ImmutableMap.<Integer, Long>builder().put(1, 4L)
- .put(2, 2L)
- .put(3, 1L)
- .put(-2, 1L)
- .put(-8, 1L)
- .put(-16, 1L)
- .build();
- PAssert.thatMap(countsBackToString).isEqualTo(expected);
- }
-
- @Test
- public void transformDisplayDataExceptionShouldFail() {
- DoFn<Integer, Integer> brokenDoFn = new DoFn<Integer, Integer>() {
- @Override
- public void processElement(ProcessContext c) throws Exception {}
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- throw new RuntimeException("oh noes!");
- }
- };
-
- Pipeline p = getPipeline();
- p
- .apply(Create.of(1, 2, 3))
- .apply(ParDo.of(brokenDoFn));
-
- thrown.expectMessage(brokenDoFn.getClass().getName());
- thrown.expectCause(ThrowableMessageMatcher.hasMessage(is("oh noes!")));
- p.run();
- }
-
- @Test
- public void pipelineOptionsDisplayDataExceptionShouldFail() {
- Object brokenValueType = new Object() {
- @JsonValue
- public int getValue () {
- return 42;
- }
-
- @Override
- public String toString() {
- throw new RuntimeException("oh noes!!");
- }
- };
-
- Pipeline p = getPipeline();
- p.getOptions().as(ObjectPipelineOptions.class).setValue(brokenValueType);
-
- p.apply(Create.of(1, 2, 3));
-
- thrown.expectMessage(PipelineOptions.class.getName());
- thrown.expectCause(ThrowableMessageMatcher.hasMessage(is("oh noes!!")));
- p.run();
- }
-
- /** {@link PipelineOptions} to inject bad object implementations. */
- public interface ObjectPipelineOptions extends PipelineOptions {
- Object getValue();
- void setValue(Object value);
- }
-
-
- /**
- * Tests that a {@link DoFn} that mutates an output with a good equals() fails in the
- * {@link InProcessPipelineRunner}.
- */
- @Test
- public void testMutatingOutputThenOutputDoFnError() throws Exception {
- Pipeline pipeline = getPipeline();
-
- pipeline
- .apply(Create.of(42))
- .apply(ParDo.of(new DoFn<Integer, List<Integer>>() {
- @Override public void processElement(ProcessContext c) {
- List<Integer> outputList = Arrays.asList(1, 2, 3, 4);
- c.output(outputList);
- outputList.set(0, 37);
- c.output(outputList);
- }
- }));
-
- thrown.expect(IllegalMutationException.class);
- thrown.expectMessage("output");
- thrown.expectMessage("must not be mutated");
- pipeline.run();
- }
-
- /**
- * Tests that a {@link DoFn} that mutates an output with a good equals() fails in the
- * {@link InProcessPipelineRunner}.
- */
- @Test
- public void testMutatingOutputThenTerminateDoFnError() throws Exception {
- Pipeline pipeline = getPipeline();
-
- pipeline
- .apply(Create.of(42))
- .apply(ParDo.of(new DoFn<Integer, List<Integer>>() {
- @Override public void processElement(ProcessContext c) {
- List<Integer> outputList = Arrays.asList(1, 2, 3, 4);
- c.output(outputList);
- outputList.set(0, 37);
- }
- }));
-
- thrown.expect(IllegalMutationException.class);
- thrown.expectMessage("output");
- thrown.expectMessage("must not be mutated");
- pipeline.run();
- }
-
- /**
- * Tests that a {@link DoFn} that mutates an output with a bad equals() still fails
- * in the {@link InProcessPipelineRunner}.
- */
- @Test
- public void testMutatingOutputCoderDoFnError() throws Exception {
- Pipeline pipeline = getPipeline();
-
- pipeline
- .apply(Create.of(42))
- .apply(ParDo.of(new DoFn<Integer, byte[]>() {
- @Override public void processElement(ProcessContext c) {
- byte[] outputArray = new byte[]{0x1, 0x2, 0x3};
- c.output(outputArray);
- outputArray[0] = 0xa;
- c.output(outputArray);
- }
- }));
-
- thrown.expect(IllegalMutationException.class);
- thrown.expectMessage("output");
- thrown.expectMessage("must not be mutated");
- pipeline.run();
- }
-
- /**
- * Tests that a {@link DoFn} that mutates its input with a good equals() fails in the
- * {@link InProcessPipelineRunner}.
- */
- @Test
- public void testMutatingInputDoFnError() throws Exception {
- Pipeline pipeline = getPipeline();
-
- pipeline
- .apply(Create.of(Arrays.asList(1, 2, 3), Arrays.asList(4, 5, 6))
- .withCoder(ListCoder.of(VarIntCoder.of())))
- .apply(ParDo.of(new DoFn<List<Integer>, Integer>() {
- @Override public void processElement(ProcessContext c) {
- List<Integer> inputList = c.element();
- inputList.set(0, 37);
- c.output(12);
- }
- }));
-
- thrown.expect(IllegalMutationException.class);
- thrown.expectMessage("Input");
- thrown.expectMessage("must not be mutated");
- pipeline.run();
- }
-
- /**
- * Tests that a {@link DoFn} that mutates an input with a bad equals() still fails
- * in the {@link InProcessPipelineRunner}.
- */
- @Test
- public void testMutatingInputCoderDoFnError() throws Exception {
- Pipeline pipeline = getPipeline();
-
- pipeline
- .apply(Create.of(new byte[]{0x1, 0x2, 0x3}, new byte[]{0x4, 0x5, 0x6}))
- .apply(ParDo.of(new DoFn<byte[], Integer>() {
- @Override public void processElement(ProcessContext c) {
- byte[] inputArray = c.element();
- inputArray[0] = 0xa;
- c.output(13);
- }
- }));
-
- thrown.expect(IllegalMutationException.class);
- thrown.expectMessage("Input");
- thrown.expectMessage("must not be mutated");
- pipeline.run();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoInProcessEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoInProcessEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoInProcessEvaluatorTest.java
index 0f7afa1..b78eb40 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoInProcessEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoInProcessEvaluatorTest.java
@@ -25,8 +25,8 @@ import static org.mockito.Mockito.when;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
import org.apache.beam.runners.direct.InProcessExecutionContext.InProcessStepContext;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.Create;
@@ -139,7 +139,7 @@ public class ParDoInProcessEvaluatorTest {
private ParDoInProcessEvaluator<Integer> createEvaluator(
PCollectionView<Integer> singletonView,
RecorderFn fn,
- InProcessPipelineRunner.CommittedBundle<Integer> inputBundle,
+ DirectRunner.CommittedBundle<Integer> inputBundle,
PCollection<Integer> output) {
when(
evaluationContext.createSideInputReader(
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactoryTest.java
index a6f31c0..e61881e 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactoryTest.java
@@ -26,8 +26,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactoryTest.java
index a1480e5..8b8d44f 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactoryTest.java
@@ -26,8 +26,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TextIOShardedWriteFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TextIOShardedWriteFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TextIOShardedWriteFactoryTest.java
index fe9866c..5ede931 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TextIOShardedWriteFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TextIOShardedWriteFactoryTest.java
@@ -114,7 +114,7 @@ public class TextIOShardedWriteFactoryTest {
private Pipeline getPipeline() {
PipelineOptions options = TestPipeline.testingPipelineOptions();
- options.setRunner(InProcessPipelineRunner.class);
+ options.setRunner(DirectRunner.class);
return TestPipeline.fromOptions(options);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
index 0345662..a5e6cee 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
@@ -27,7 +27,7 @@ import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.AppliedPTransform;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
index 05656eb..be5c489 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
@@ -25,8 +25,8 @@ import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.BigEndianLongCoder;
import org.apache.beam.sdk.coders.Coder;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
index 859418b..714e9c9 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
@@ -23,8 +23,8 @@ import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.PCollectionViewWriter;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.PCollectionViewWriter;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VoidCoder;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
index a2f971a..8a3591b 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
@@ -21,8 +21,8 @@ import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.AppliedPTransform;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java
index 01f3070..5fdfb49 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java
@@ -21,7 +21,7 @@ package org.apache.beam.runners.spark.translation;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertThat;
-import org.apache.beam.runners.direct.InProcessPipelineRunner;
+import org.apache.beam.runners.direct.DirectRunner;
import org.apache.beam.runners.spark.SparkPipelineRunner;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
@@ -58,7 +58,7 @@ public class TransformTranslatorTest {
*/
@Test
public void testTextIOReadAndWriteTransforms() throws IOException {
- String directOut = runPipeline(InProcessPipelineRunner.class);
+ String directOut = runPipeline(DirectRunner.class);
String sparkOut = runPipeline(SparkPipelineRunner.class);
List<String> directOutput =
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
index b1b5280..456b6ae 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
@@ -277,7 +277,7 @@ public interface PipelineOptions extends HasDisplayData {
try {
@SuppressWarnings({"unchecked", "rawtypes"})
Class<? extends PipelineRunner> direct = (Class<? extends PipelineRunner>) Class.forName(
- "org.apache.beam.runners.direct.InProcessPipelineRunner");
+ "org.apache.beam.runners.direct.DirectRunner");
return direct;
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException(String.format(
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/testing/travis/test_wordcount.sh
----------------------------------------------------------------------
diff --git a/testing/travis/test_wordcount.sh b/testing/travis/test_wordcount.sh
index b00b0d6..e059a35 100755
--- a/testing/travis/test_wordcount.sh
+++ b/testing/travis/test_wordcount.sh
@@ -70,7 +70,7 @@ function run_via_mvn {
local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
local cmd='mvn exec:java -f pom.xml -pl examples/java \
-Dexec.mainClass=org.apache.beam.examples.WordCount \
- -Dexec.args="--runner=InProcessPipelineRunner --inputFile='"$input"' --output='"$outfile_prefix"'"'
+ -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"'
echo "$name: Running $cmd" >&2
sh -c "$cmd"
check_result_hash "$name" "$outfile_prefix" "$expected_hash"
@@ -84,7 +84,7 @@ function run_bundled {
local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
local cmd='java -cp '"$JAR_FILE"' \
org.apache.beam.examples.WordCount \
- --runner=InProcessPipelineRunner \
+ --runner=DirectRunner \
--inputFile='"'$input'"' \
--output='"$outfile_prefix"
echo "$name: Running $cmd" >&2
[09/12] incubator-beam git commit: Remove InProcess Prefixes
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutor.java
deleted file mode 100644
index 1cfa544..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutor.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
-
-import java.util.Collection;
-
-/**
- * An executor that schedules and executes {@link AppliedPTransform AppliedPTransforms} for both
- * source and intermediate {@link PTransform PTransforms}.
- */
-interface InProcessExecutor {
- /**
- * Starts this executor. The provided collection is the collection of root transforms to
- * initially schedule.
- *
- * @param rootTransforms
- */
- void start(Collection<AppliedPTransform<?, ?, ?>> rootTransforms);
-
- /**
- * Blocks until the job being executed enters a terminal state. A job is completed after all
- * root {@link AppliedPTransform AppliedPTransforms} have completed, and all
- * {@link CommittedBundle Bundles} have been consumed. Jobs may also terminate abnormally.
- *
- * @throws Throwable whenever an executor thread throws anything, transfers the throwable to the
- * waiting thread and rethrows it
- */
- void awaitCompletion() throws Throwable;
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupAlsoByWindowEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupAlsoByWindowEvaluatorFactory.java
deleted file mode 100644
index 53b93d0..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupAlsoByWindowEvaluatorFactory.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.runners.core.GroupAlsoByWindowViaWindowSetDoFn;
-import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupAlsoByWindow;
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly;
-import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly.GroupByKeyOnly;
-import org.apache.beam.sdk.util.KeyedWorkItem;
-import org.apache.beam.sdk.util.SystemReduceFn;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-
-import com.google.common.collect.ImmutableMap;
-
-import java.util.Collections;
-
-/**
- * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
- * {@link GroupByKeyOnly} {@link PTransform}.
- */
-class InProcessGroupAlsoByWindowEvaluatorFactory implements TransformEvaluatorFactory {
- @Override
- public <InputT> TransformEvaluator<InputT> forApplication(
- AppliedPTransform<?, ?, ?> application,
- CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext) {
- @SuppressWarnings({"cast", "unchecked", "rawtypes"})
- TransformEvaluator<InputT> evaluator =
- createEvaluator(
- (AppliedPTransform) application, (CommittedBundle) inputBundle, evaluationContext);
- return evaluator;
- }
-
- private <K, V> TransformEvaluator<KeyedWorkItem<K, V>> createEvaluator(
- AppliedPTransform<
- PCollection<KeyedWorkItem<K, V>>, PCollection<KV<K, Iterable<V>>>,
- InProcessGroupAlsoByWindow<K, V>>
- application,
- CommittedBundle<KeyedWorkItem<K, V>> inputBundle,
- InProcessEvaluationContext evaluationContext) {
- return new InProcessGroupAlsoByWindowEvaluator<K, V>(
- evaluationContext, inputBundle, application);
- }
-
- /**
- * A transform evaluator for the pseudo-primitive {@link GroupAlsoByWindow}. Windowing is ignored;
- * all input should be in the global window since all output will be as well.
- *
- * @see GroupByKeyViaGroupByKeyOnly
- */
- private static class InProcessGroupAlsoByWindowEvaluator<K, V>
- implements TransformEvaluator<KeyedWorkItem<K, V>> {
-
- private final TransformEvaluator<KeyedWorkItem<K, V>> gabwParDoEvaluator;
-
- public InProcessGroupAlsoByWindowEvaluator(
- final InProcessEvaluationContext evaluationContext,
- CommittedBundle<KeyedWorkItem<K, V>> inputBundle,
- final AppliedPTransform<
- PCollection<KeyedWorkItem<K, V>>, PCollection<KV<K, Iterable<V>>>,
- InProcessGroupAlsoByWindow<K, V>>
- application) {
-
- Coder<V> valueCoder =
- application.getTransform().getValueCoder(inputBundle.getPCollection().getCoder());
-
- @SuppressWarnings("unchecked")
- WindowingStrategy<?, BoundedWindow> windowingStrategy =
- (WindowingStrategy<?, BoundedWindow>) application.getTransform().getWindowingStrategy();
-
- DoFn<KeyedWorkItem<K, V>, KV<K, Iterable<V>>> gabwDoFn =
- GroupAlsoByWindowViaWindowSetDoFn.create(
- windowingStrategy,
- SystemReduceFn.<K, V, BoundedWindow>buffering(valueCoder));
-
- TupleTag<KV<K, Iterable<V>>> mainOutputTag = new TupleTag<KV<K, Iterable<V>>>() {};
-
- // Not technically legit, as the application is not a ParDo
- this.gabwParDoEvaluator =
- ParDoInProcessEvaluator.create(
- evaluationContext,
- inputBundle,
- application,
- gabwDoFn,
- Collections.<PCollectionView<?>>emptyList(),
- mainOutputTag,
- Collections.<TupleTag<?>>emptyList(),
- ImmutableMap.<TupleTag<?>, PCollection<?>>of(mainOutputTag, application.getOutput()));
- }
-
- @Override
- public void processElement(WindowedValue<KeyedWorkItem<K, V>> element) throws Exception {
- gabwParDoEvaluator.processElement(element);
- }
-
- @Override
- public InProcessTransformResult finishBundle() throws Exception {
- return gabwParDoEvaluator.finishBundle();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKey.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKey.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKey.java
deleted file mode 100644
index 026b4d5..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKey.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.beam.runners.direct;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.IterableCoder;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly.ReifyTimestampsAndWindows;
-import org.apache.beam.sdk.util.KeyedWorkItem;
-import org.apache.beam.sdk.util.KeyedWorkItemCoder;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-
-class InProcessGroupByKey<K, V>
- extends ForwardingPTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> {
- private final GroupByKey<K, V> original;
-
- InProcessGroupByKey(GroupByKey<K, V> from) {
- this.original = from;
- }
-
- @Override
- public PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> delegate() {
- return original;
- }
-
- @Override
- public PCollection<KV<K, Iterable<V>>> apply(PCollection<KV<K, V>> input) {
- @SuppressWarnings("unchecked")
- KvCoder<K, V> inputCoder = (KvCoder<K, V>) input.getCoder();
-
- // This operation groups by the combination of key and window,
- // merging windows as needed, using the windows assigned to the
- // key/value input elements and the window merge operation of the
- // window function associated with the input PCollection.
- WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
-
- // By default, implement GroupByKey via a series of lower-level operations.
- return input
- // Make each input element's timestamp and assigned windows
- // explicit, in the value part.
- .apply(new ReifyTimestampsAndWindows<K, V>())
- .apply(new InProcessGroupByKeyOnly<K, V>())
- .setCoder(
- KeyedWorkItemCoder.of(
- inputCoder.getKeyCoder(),
- inputCoder.getValueCoder(),
- input.getWindowingStrategy().getWindowFn().windowCoder()))
-
- // Group each key's values by window, merging windows as needed.
- .apply("GroupAlsoByWindow", new InProcessGroupAlsoByWindow<K, V>(windowingStrategy))
-
- // And update the windowing strategy as appropriate.
- .setWindowingStrategyInternal(original.updateWindowingStrategy(windowingStrategy))
- .setCoder(
- KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getValueCoder())));
- }
-
- static final class InProcessGroupByKeyOnly<K, V>
- extends PTransform<PCollection<KV<K, WindowedValue<V>>>, PCollection<KeyedWorkItem<K, V>>> {
- @Override
- public PCollection<KeyedWorkItem<K, V>> apply(PCollection<KV<K, WindowedValue<V>>> input) {
- return PCollection.<KeyedWorkItem<K, V>>createPrimitiveOutputInternal(
- input.getPipeline(), input.getWindowingStrategy(), input.isBounded());
- }
-
- InProcessGroupByKeyOnly() {}
- }
-
- static final class InProcessGroupAlsoByWindow<K, V>
- extends PTransform<PCollection<KeyedWorkItem<K, V>>, PCollection<KV<K, Iterable<V>>>> {
-
- private final WindowingStrategy<?, ?> windowingStrategy;
-
- public InProcessGroupAlsoByWindow(WindowingStrategy<?, ?> windowingStrategy) {
- this.windowingStrategy = windowingStrategy;
- }
-
- public WindowingStrategy<?, ?> getWindowingStrategy() {
- return windowingStrategy;
- }
-
- private KeyedWorkItemCoder<K, V> getKeyedWorkItemCoder(Coder<KeyedWorkItem<K, V>> inputCoder) {
- // Coder<KV<...>> --> KvCoder<...>
- checkArgument(
- inputCoder instanceof KeyedWorkItemCoder,
- "%s requires a %s<...> but got %s",
- getClass().getSimpleName(),
- KvCoder.class.getSimpleName(),
- inputCoder);
- @SuppressWarnings("unchecked")
- KeyedWorkItemCoder<K, V> kvCoder = (KeyedWorkItemCoder<K, V>) inputCoder;
- return kvCoder;
- }
-
- public Coder<K> getKeyCoder(Coder<KeyedWorkItem<K, V>> inputCoder) {
- return getKeyedWorkItemCoder(inputCoder).getKeyCoder();
- }
-
- public Coder<V> getValueCoder(Coder<KeyedWorkItem<K, V>> inputCoder) {
- return getKeyedWorkItemCoder(inputCoder).getElementCoder();
- }
-
- @Override
- public PCollection<KV<K, Iterable<V>>> apply(PCollection<KeyedWorkItem<K, V>> input) {
- return PCollection.<KV<K, Iterable<V>>>createPrimitiveOutputInternal(
- input.getPipeline(), input.getWindowingStrategy(), input.isBounded());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactory.java
deleted file mode 100644
index 3604bca..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactory.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static org.apache.beam.sdk.util.CoderUtils.encodeToByteArray;
-
-import static com.google.common.base.Preconditions.checkState;
-
-import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.runners.direct.StepTransformResult.Builder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly;
-import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly.GroupByKeyOnly;
-import org.apache.beam.sdk.util.KeyedWorkItem;
-import org.apache.beam.sdk.util.KeyedWorkItems;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
- * {@link GroupByKeyOnly} {@link PTransform}.
- */
-class InProcessGroupByKeyOnlyEvaluatorFactory implements TransformEvaluatorFactory {
- @Override
- public <InputT> TransformEvaluator<InputT> forApplication(
- AppliedPTransform<?, ?, ?> application,
- CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext) {
- @SuppressWarnings({"cast", "unchecked", "rawtypes"})
- TransformEvaluator<InputT> evaluator =
- createEvaluator(
- (AppliedPTransform) application, (CommittedBundle) inputBundle, evaluationContext);
- return evaluator;
- }
-
- private <K, V> TransformEvaluator<KV<K, WindowedValue<V>>> createEvaluator(
- final AppliedPTransform<
- PCollection<KV<K, WindowedValue<V>>>, PCollection<KeyedWorkItem<K, V>>,
- InProcessGroupByKeyOnly<K, V>>
- application,
- final CommittedBundle<KV<K, WindowedValue<V>>> inputBundle,
- final InProcessEvaluationContext evaluationContext) {
- return new InProcessGroupByKeyOnlyEvaluator<K, V>(evaluationContext, inputBundle, application);
- }
-
- /**
- * A transform evaluator for the pseudo-primitive {@link GroupByKeyOnly}. Windowing is ignored;
- * all input should be in the global window since all output will be as well.
- *
- * @see GroupByKeyViaGroupByKeyOnly
- */
- private static class InProcessGroupByKeyOnlyEvaluator<K, V>
- implements TransformEvaluator<KV<K, WindowedValue<V>>> {
- private final InProcessEvaluationContext evaluationContext;
-
- private final CommittedBundle<KV<K, WindowedValue<V>>> inputBundle;
- private final AppliedPTransform<
- PCollection<KV<K, WindowedValue<V>>>, PCollection<KeyedWorkItem<K, V>>,
- InProcessGroupByKeyOnly<K, V>>
- application;
- private final Coder<K> keyCoder;
- private Map<GroupingKey<K>, List<WindowedValue<V>>> groupingMap;
-
- public InProcessGroupByKeyOnlyEvaluator(
- InProcessEvaluationContext evaluationContext,
- CommittedBundle<KV<K, WindowedValue<V>>> inputBundle,
- AppliedPTransform<
- PCollection<KV<K, WindowedValue<V>>>, PCollection<KeyedWorkItem<K, V>>,
- InProcessGroupByKeyOnly<K, V>>
- application) {
- this.evaluationContext = evaluationContext;
- this.inputBundle = inputBundle;
- this.application = application;
- this.keyCoder = getKeyCoder(application.getInput().getCoder());
- this.groupingMap = new HashMap<>();
- }
-
- private Coder<K> getKeyCoder(Coder<KV<K, WindowedValue<V>>> coder) {
- checkState(
- coder instanceof KvCoder,
- "%s requires a coder of class %s."
- + " This is an internal error; this is checked during pipeline construction"
- + " but became corrupted.",
- getClass().getSimpleName(),
- KvCoder.class.getSimpleName());
- @SuppressWarnings("unchecked")
- Coder<K> keyCoder = ((KvCoder<K, WindowedValue<V>>) coder).getKeyCoder();
- return keyCoder;
- }
-
- @Override
- public void processElement(WindowedValue<KV<K, WindowedValue<V>>> element) {
- KV<K, WindowedValue<V>> kv = element.getValue();
- K key = kv.getKey();
- byte[] encodedKey;
- try {
- encodedKey = encodeToByteArray(keyCoder, key);
- } catch (CoderException exn) {
- // TODO: Put in better element printing:
- // truncate if too long.
- throw new IllegalArgumentException(
- String.format("unable to encode key %s of input to %s using %s", key, this, keyCoder),
- exn);
- }
- GroupingKey<K> groupingKey = new GroupingKey<>(key, encodedKey);
- List<WindowedValue<V>> values = groupingMap.get(groupingKey);
- if (values == null) {
- values = new ArrayList<WindowedValue<V>>();
- groupingMap.put(groupingKey, values);
- }
- values.add(kv.getValue());
- }
-
- @Override
- public InProcessTransformResult finishBundle() {
- Builder resultBuilder = StepTransformResult.withoutHold(application);
- for (Map.Entry<GroupingKey<K>, List<WindowedValue<V>>> groupedEntry :
- groupingMap.entrySet()) {
- K key = groupedEntry.getKey().key;
- KeyedWorkItem<K, V> groupedKv =
- KeyedWorkItems.elementsWorkItem(key, groupedEntry.getValue());
- UncommittedBundle<KeyedWorkItem<K, V>> bundle = evaluationContext.createKeyedBundle(
- inputBundle,
- StructuralKey.of(key, keyCoder),
- application.getOutput());
- bundle.add(WindowedValue.valueInGlobalWindow(groupedKv));
- resultBuilder.addOutput(bundle);
- }
- return resultBuilder.build();
- }
-
- private static class GroupingKey<K> {
- private K key;
- private byte[] encodedKey;
-
- public GroupingKey(K key, byte[] encodedKey) {
- this.key = key;
- this.encodedKey = encodedKey;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof GroupingKey) {
- GroupingKey<?> that = (GroupingKey<?>) o;
- return Arrays.equals(this.encodedKey, that.encodedKey);
- } else {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(encodedKey);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOverrideFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOverrideFactory.java
deleted file mode 100644
index 1d84bc9..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOverrideFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.POutput;
-
-/**
- * A {@link PTransformOverrideFactory} for {@link GroupByKey} PTransforms.
- */
-final class InProcessGroupByKeyOverrideFactory
- implements PTransformOverrideFactory {
- @Override
- public <InputT extends PInput, OutputT extends POutput> PTransform<InputT, OutputT> override(
- PTransform<InputT, OutputT> transform) {
- if (transform instanceof GroupByKey) {
- @SuppressWarnings({"rawtypes", "unchecked"})
- PTransform<InputT, OutputT> override =
- (PTransform) new InProcessGroupByKey((GroupByKey) transform);
- return override;
- }
- return transform;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessSideInputContainer.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessSideInputContainer.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessSideInputContainer.java
deleted file mode 100644
index f53f590..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessSideInputContainer.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.PCollectionViewWindow;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.values.PCollectionView;
-
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Optional;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
-
-import javax.annotation.Nullable;
-
-/**
- * An in-process container for {@link PCollectionView PCollectionViews}, which provides methods for
- * constructing {@link SideInputReader SideInputReaders} which block until a side input is
- * available and writing to a {@link PCollectionView}.
- */
-class InProcessSideInputContainer {
- private final Collection<PCollectionView<?>> containedViews;
- private final LoadingCache<
- PCollectionViewWindow<?>, AtomicReference<Iterable<? extends WindowedValue<?>>>>
- viewByWindows;
-
- /**
- * Create a new {@link InProcessSideInputContainer} with the provided views and the provided
- * context.
- */
- public static InProcessSideInputContainer create(
- final InProcessEvaluationContext context, Collection<PCollectionView<?>> containedViews) {
- LoadingCache<PCollectionViewWindow<?>, AtomicReference<Iterable<? extends WindowedValue<?>>>>
- viewByWindows = CacheBuilder.newBuilder().build(new CallbackSchedulingLoader(context));
- return new InProcessSideInputContainer(containedViews, viewByWindows);
- }
-
- private InProcessSideInputContainer(
- Collection<PCollectionView<?>> containedViews,
- LoadingCache<PCollectionViewWindow<?>, AtomicReference<Iterable<? extends WindowedValue<?>>>>
- viewByWindows) {
- this.containedViews = ImmutableSet.copyOf(containedViews);
- this.viewByWindows = viewByWindows;
- }
-
- /**
- * Return a view of this {@link InProcessSideInputContainer} that contains only the views in the
- * provided argument. The returned {@link InProcessSideInputContainer} is unmodifiable without
- * casting, but will change as this {@link InProcessSideInputContainer} is modified.
- */
- public ReadyCheckingSideInputReader createReaderForViews(
- Collection<PCollectionView<?>> newContainedViews) {
- if (!containedViews.containsAll(newContainedViews)) {
- Set<PCollectionView<?>> currentlyContained = ImmutableSet.copyOf(containedViews);
- Set<PCollectionView<?>> newRequested = ImmutableSet.copyOf(newContainedViews);
- throw new IllegalArgumentException("Can't create a SideInputReader with unknown views "
- + Sets.difference(newRequested, currentlyContained));
- }
- return new SideInputContainerSideInputReader(newContainedViews);
- }
-
- /**
- * Write the provided values to the provided view.
- *
- * <p>The windowed values are first exploded, then for each window the pane is determined. For
- * each window, if the pane is later than the current pane stored within this container, write
- * all of the values to the container as the new values of the {@link PCollectionView}.
- *
- * <p>The provided iterable is expected to contain only a single window and pane.
- */
- public void write(PCollectionView<?> view, Iterable<? extends WindowedValue<?>> values) {
- Map<BoundedWindow, Collection<WindowedValue<?>>> valuesPerWindow =
- indexValuesByWindow(values);
- for (Map.Entry<BoundedWindow, Collection<WindowedValue<?>>> windowValues :
- valuesPerWindow.entrySet()) {
- updatePCollectionViewWindowValues(view, windowValues.getKey(), windowValues.getValue());
- }
- }
-
- /**
- * Index the provided values by all {@link BoundedWindow windows} in which they appear.
- */
- private Map<BoundedWindow, Collection<WindowedValue<?>>> indexValuesByWindow(
- Iterable<? extends WindowedValue<?>> values) {
- Map<BoundedWindow, Collection<WindowedValue<?>>> valuesPerWindow = new HashMap<>();
- for (WindowedValue<?> value : values) {
- for (BoundedWindow window : value.getWindows()) {
- Collection<WindowedValue<?>> windowValues = valuesPerWindow.get(window);
- if (windowValues == null) {
- windowValues = new ArrayList<>();
- valuesPerWindow.put(window, windowValues);
- }
- windowValues.add(value);
- }
- }
- return valuesPerWindow;
- }
-
- /**
- * Set the value of the {@link PCollectionView} in the {@link BoundedWindow} to be based on the
- * specified values, if the values are part of a later pane than currently exist within the
- * {@link PCollectionViewWindow}.
- */
- private void updatePCollectionViewWindowValues(
- PCollectionView<?> view, BoundedWindow window, Collection<WindowedValue<?>> windowValues) {
- PCollectionViewWindow<?> windowedView = PCollectionViewWindow.of(view, window);
- AtomicReference<Iterable<? extends WindowedValue<?>>> contents =
- viewByWindows.getUnchecked(windowedView);
- if (contents.compareAndSet(null, windowValues)) {
- // the value had never been set, so we set it and are done.
- return;
- }
- PaneInfo newPane = windowValues.iterator().next().getPane();
-
- Iterable<? extends WindowedValue<?>> existingValues;
- long existingPane;
- do {
- existingValues = contents.get();
- existingPane =
- Iterables.isEmpty(existingValues)
- ? -1L
- : existingValues.iterator().next().getPane().getIndex();
- } while (newPane.getIndex() > existingPane
- && !contents.compareAndSet(existingValues, windowValues));
- }
-
- private static class CallbackSchedulingLoader extends
- CacheLoader<PCollectionViewWindow<?>, AtomicReference<Iterable<? extends WindowedValue<?>>>> {
- private final InProcessEvaluationContext context;
-
- public CallbackSchedulingLoader(
- InProcessEvaluationContext context) {
- this.context = context;
- }
-
- @Override
- public AtomicReference<Iterable<? extends WindowedValue<?>>>
- load(PCollectionViewWindow<?> view) {
-
- AtomicReference<Iterable<? extends WindowedValue<?>>> contents = new AtomicReference<>();
- WindowingStrategy<?, ?> windowingStrategy = view.getView().getWindowingStrategyInternal();
-
- context.scheduleAfterOutputWouldBeProduced(view.getView(),
- view.getWindow(),
- windowingStrategy,
- new WriteEmptyViewContents(view.getView(), view.getWindow(), contents));
- return contents;
- }
- }
-
- private static class WriteEmptyViewContents implements Runnable {
- private final PCollectionView<?> view;
- private final BoundedWindow window;
- private final AtomicReference<Iterable<? extends WindowedValue<?>>> contents;
-
- private WriteEmptyViewContents(PCollectionView<?> view, BoundedWindow window,
- AtomicReference<Iterable<? extends WindowedValue<?>>> contents) {
- this.contents = contents;
- this.view = view;
- this.window = window;
- }
-
- @Override
- public void run() {
- // The requested window has closed without producing elements, so reflect that in
- // the PCollectionView. If set has already been called, will do nothing.
- contents.compareAndSet(null, Collections.<WindowedValue<?>>emptyList());
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("view", view)
- .add("window", window)
- .toString();
- }
- }
-
- private final class SideInputContainerSideInputReader implements ReadyCheckingSideInputReader {
- private final Collection<PCollectionView<?>> readerViews;
- private final LoadingCache<
- PCollectionViewWindow<?>, Optional<? extends Iterable<? extends WindowedValue<?>>>>
- viewContents;
-
- private SideInputContainerSideInputReader(Collection<PCollectionView<?>> readerViews) {
- this.readerViews = ImmutableSet.copyOf(readerViews);
- this.viewContents = CacheBuilder.newBuilder().build(new CurrentViewContentsLoader());
- }
-
- @Override
- public boolean isReady(final PCollectionView<?> view, final BoundedWindow window) {
- checkArgument(
- readerViews.contains(view),
- "Tried to check if view %s was ready in a SideInputReader that does not contain it. "
- + "Contained views; %s",
- view,
- readerViews);
- return viewContents.getUnchecked(PCollectionViewWindow.of(view, window)).isPresent();
- }
-
- @Override
- @Nullable
- public <T> T get(final PCollectionView<T> view, final BoundedWindow window) {
- checkArgument(readerViews.contains(view),
- "call to get(PCollectionView) with unknown view: %s",
- view);
- checkArgument(
- isReady(view, window),
- "calling get() on PCollectionView %s that is not ready in window %s",
- view,
- window);
- // Safe covariant cast
- @SuppressWarnings("unchecked") Iterable<WindowedValue<?>> values =
- (Iterable<WindowedValue<?>>) viewContents.getUnchecked(PCollectionViewWindow.of(view,
- window)).get();
- return view.fromIterableInternal(values);
- }
-
- @Override
- public <T> boolean contains(PCollectionView<T> view) {
- return readerViews.contains(view);
- }
-
- @Override
- public boolean isEmpty() {
- return readerViews.isEmpty();
- }
- }
-
- /**
- * A {@link CacheLoader} that loads the current contents of a {@link PCollectionViewWindow} into
- * an optional.
- */
- private class CurrentViewContentsLoader extends CacheLoader<
- PCollectionViewWindow<?>, Optional<? extends Iterable<? extends WindowedValue<?>>>> {
-
- @Override
- public Optional<? extends Iterable<? extends WindowedValue<?>>>
- load(PCollectionViewWindow<?> key) {
- return Optional.fromNullable(viewByWindows.getUnchecked(key).get());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTimerInternals.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTimerInternals.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTimerInternals.java
deleted file mode 100644
index cd54f59..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTimerInternals.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate.TimerUpdateBuilder;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TransformWatermarks;
-import org.apache.beam.sdk.util.TimerInternals;
-
-import org.joda.time.Instant;
-
-import javax.annotation.Nullable;
-
-/**
- * An implementation of {@link TimerInternals} where all relevant data exists in memory.
- */
-public class InProcessTimerInternals implements TimerInternals {
- private final Clock processingTimeClock;
- private final TransformWatermarks watermarks;
- private final TimerUpdateBuilder timerUpdateBuilder;
-
- public static InProcessTimerInternals create(
- Clock clock, TransformWatermarks watermarks, TimerUpdateBuilder timerUpdateBuilder) {
- return new InProcessTimerInternals(clock, watermarks, timerUpdateBuilder);
- }
-
- private InProcessTimerInternals(
- Clock clock, TransformWatermarks watermarks, TimerUpdateBuilder timerUpdateBuilder) {
- this.processingTimeClock = clock;
- this.watermarks = watermarks;
- this.timerUpdateBuilder = timerUpdateBuilder;
- }
-
- @Override
- public void setTimer(TimerData timerKey) {
- timerUpdateBuilder.setTimer(timerKey);
- }
-
- @Override
- public void deleteTimer(TimerData timerKey) {
- timerUpdateBuilder.deletedTimer(timerKey);
- }
-
- public TimerUpdate getTimerUpdate() {
- return timerUpdateBuilder.build();
- }
-
- @Override
- public Instant currentProcessingTime() {
- return processingTimeClock.now();
- }
-
- @Override
- @Nullable
- public Instant currentSynchronizedProcessingTime() {
- return watermarks.getSynchronizedProcessingInputTime();
- }
-
- @Override
- public Instant currentInputWatermarkTime() {
- return watermarks.getInputWatermark();
- }
-
- @Override
- @Nullable
- public Instant currentOutputWatermarkTime() {
- return watermarks.getOutputWatermark();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTransformResult.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTransformResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTransformResult.java
deleted file mode 100644
index 92127b4..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTransformResult.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.common.CounterSet;
-import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
-
-import org.joda.time.Instant;
-
-import javax.annotation.Nullable;
-
-/**
- * The result of evaluating an {@link AppliedPTransform} with a {@link TransformEvaluator}.
- */
-public interface InProcessTransformResult {
- /**
- * Returns the {@link AppliedPTransform} that produced this result.
- */
- AppliedPTransform<?, ?, ?> getTransform();
-
- /**
- * Returns the {@link UncommittedBundle (uncommitted) Bundles} output by this transform. These
- * will be committed by the evaluation context as part of completing this result.
- */
- Iterable<? extends UncommittedBundle<?>> getOutputBundles();
-
- /**
- * Returns elements that were provided to the {@link TransformEvaluator} as input but were not
- * processed.
- */
- Iterable<? extends WindowedValue<?>> getUnprocessedElements();
-
- /**
- * Returns the {@link CounterSet} used by this {@link PTransform}, or null if this transform did
- * not use a {@link CounterSet}.
- */
- @Nullable CounterSet getCounters();
-
- /**
- * Returns the Watermark Hold for the transform at the time this result was produced.
- *
- * If the transform does not set any watermark hold, returns
- * {@link BoundedWindow#TIMESTAMP_MAX_VALUE}.
- */
- Instant getWatermarkHold();
-
- /**
- * Returns the State used by the transform.
- *
- * If this evaluation did not access state, this may return null.
- */
- @Nullable
- CopyOnAccessInMemoryStateInternals<?> getState();
-
- /**
- * Returns a TimerUpdateBuilder that was produced as a result of this evaluation. If the
- * evaluation was triggered due to the delivery of one or more timers, those timers must be added
- * to the builder before it is complete.
- *
- * <p>If this evaluation did not add or remove any timers, returns an empty TimerUpdate.
- */
- TimerUpdate getTimerUpdate();
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
index 758ee24..074619a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
@@ -30,7 +30,7 @@ import org.apache.beam.sdk.values.PCollection;
* {@link ModelEnforcement} is provided with the input bundle as part of
* {@link ModelEnforcementFactory#forBundle(CommittedBundle, AppliedPTransform)}, each element
* before and after that element is provided to an underlying {@link TransformEvaluator}, and the
- * output {@link InProcessTransformResult} and committed output bundles after the
+ * output {@link TransformResult} and committed output bundles after the
* {@link TransformEvaluator} has completed.
*
* <p>Typically, {@link ModelEnforcement} will obtain required metadata (such as the {@link Coder}
@@ -53,11 +53,11 @@ public interface ModelEnforcement<T> {
/**
* Called after a bundle has been completed and {@link TransformEvaluator#finishBundle()} has been
- * called, producing the provided {@link InProcessTransformResult} and
+ * called, producing the provided {@link TransformResult} and
* {@link CommittedBundle output bundles}.
*/
void afterFinish(
CommittedBundle<T> input,
- InProcessTransformResult result,
+ TransformResult result,
Iterable<? extends CommittedBundle<?>> outputs);
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
new file mode 100644
index 0000000..58cee4d
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluator.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.util.DoFnRunner;
+import org.apache.beam.sdk.util.DoFnRunners;
+import org.apache.beam.sdk.util.DoFnRunners.OutputManager;
+import org.apache.beam.sdk.util.PushbackSideInputDoFnRunner;
+import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
+import org.apache.beam.sdk.util.UserCodeException;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.common.CounterSet;
+import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+class ParDoEvaluator<T> implements TransformEvaluator<T> {
+ public static <InputT, OutputT> ParDoEvaluator<InputT> create(
+ EvaluationContext evaluationContext,
+ CommittedBundle<InputT> inputBundle,
+ AppliedPTransform<PCollection<InputT>, ?, ?> application,
+ DoFn<InputT, OutputT> fn,
+ List<PCollectionView<?>> sideInputs,
+ TupleTag<OutputT> mainOutputTag,
+ List<TupleTag<?>> sideOutputTags,
+ Map<TupleTag<?>, PCollection<?>> outputs) {
+ DirectExecutionContext executionContext =
+ evaluationContext.getExecutionContext(application, inputBundle.getKey());
+ String stepName = evaluationContext.getStepName(application);
+ DirectStepContext stepContext =
+ executionContext.getOrCreateStepContext(stepName, stepName);
+
+ CounterSet counters = evaluationContext.createCounterSet();
+
+ Map<TupleTag<?>, UncommittedBundle<?>> outputBundles = new HashMap<>();
+ for (Map.Entry<TupleTag<?>, PCollection<?>> outputEntry : outputs.entrySet()) {
+ outputBundles.put(
+ outputEntry.getKey(),
+ evaluationContext.createBundle(inputBundle, outputEntry.getValue()));
+ }
+
+ ReadyCheckingSideInputReader sideInputReader =
+ evaluationContext.createSideInputReader(sideInputs);
+ DoFnRunner<InputT, OutputT> underlying =
+ DoFnRunners.createDefault(
+ evaluationContext.getPipelineOptions(),
+ fn,
+ sideInputReader,
+ BundleOutputManager.create(outputBundles),
+ mainOutputTag,
+ sideOutputTags,
+ stepContext,
+ counters.getAddCounterMutator(),
+ application.getInput().getWindowingStrategy());
+ PushbackSideInputDoFnRunner<InputT, OutputT> runner =
+ PushbackSideInputDoFnRunner.create(underlying, sideInputs, sideInputReader);
+
+ try {
+ runner.startBundle();
+ } catch (Exception e) {
+ throw UserCodeException.wrap(e);
+ }
+
+ return new ParDoEvaluator<>(
+ runner, application, counters, outputBundles.values(), stepContext);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////////////////////
+
+ private final PushbackSideInputDoFnRunner<T, ?> fnRunner;
+ private final AppliedPTransform<PCollection<T>, ?, ?> transform;
+ private final CounterSet counters;
+ private final Collection<UncommittedBundle<?>> outputBundles;
+ private final DirectStepContext stepContext;
+
+ private final ImmutableList.Builder<WindowedValue<T>> unprocessedElements;
+
+ private ParDoEvaluator(
+ PushbackSideInputDoFnRunner<T, ?> fnRunner,
+ AppliedPTransform<PCollection<T>, ?, ?> transform,
+ CounterSet counters,
+ Collection<UncommittedBundle<?>> outputBundles,
+ DirectStepContext stepContext) {
+ this.fnRunner = fnRunner;
+ this.transform = transform;
+ this.counters = counters;
+ this.outputBundles = outputBundles;
+ this.stepContext = stepContext;
+
+ this.unprocessedElements = ImmutableList.builder();
+ }
+
+ @Override
+ public void processElement(WindowedValue<T> element) {
+ try {
+ Iterable<WindowedValue<T>> unprocessed = fnRunner.processElementInReadyWindows(element);
+ unprocessedElements.addAll(unprocessed);
+ } catch (Exception e) {
+ throw UserCodeException.wrap(e);
+ }
+ }
+
+ @Override
+ public TransformResult finishBundle() {
+ try {
+ fnRunner.finishBundle();
+ } catch (Exception e) {
+ throw UserCodeException.wrap(e);
+ }
+ StepTransformResult.Builder resultBuilder;
+ CopyOnAccessInMemoryStateInternals<?> state = stepContext.commitState();
+ if (state != null) {
+ resultBuilder =
+ StepTransformResult.withHold(transform, state.getEarliestWatermarkHold())
+ .withState(state);
+ } else {
+ resultBuilder = StepTransformResult.withoutHold(transform);
+ }
+ return resultBuilder
+ .addOutput(outputBundles)
+ .withTimerUpdate(stepContext.getTimerUpdate())
+ .withCounters(counters)
+ .addUnprocessedElements(unprocessedElements.build())
+ .build();
+ }
+
+ static class BundleOutputManager implements OutputManager {
+ private final Map<TupleTag<?>, UncommittedBundle<?>> bundles;
+ private final Map<TupleTag<?>, List<?>> undeclaredOutputs;
+
+ public static BundleOutputManager create(Map<TupleTag<?>, UncommittedBundle<?>> outputBundles) {
+ return new BundleOutputManager(outputBundles);
+ }
+
+ private BundleOutputManager(Map<TupleTag<?>, UncommittedBundle<?>> bundles) {
+ this.bundles = bundles;
+ undeclaredOutputs = new HashMap<>();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> void output(TupleTag<T> tag, WindowedValue<T> output) {
+ @SuppressWarnings("rawtypes")
+ UncommittedBundle bundle = bundles.get(tag);
+ if (bundle == null) {
+ List undeclaredContents = undeclaredOutputs.get(tag);
+ if (undeclaredContents == null) {
+ undeclaredContents = new ArrayList<T>();
+ undeclaredOutputs.put(tag, undeclaredContents);
+ }
+ undeclaredContents.add(output);
+ } else {
+ bundle.add(output);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoInProcessEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoInProcessEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoInProcessEvaluator.java
deleted file mode 100644
index b9f4808..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoInProcessEvaluator.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.runners.direct.InProcessExecutionContext.InProcessStepContext;
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.util.DoFnRunner;
-import org.apache.beam.sdk.util.DoFnRunners;
-import org.apache.beam.sdk.util.DoFnRunners.OutputManager;
-import org.apache.beam.sdk.util.PushbackSideInputDoFnRunner;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.UserCodeException;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.common.CounterSet;
-import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-
-import com.google.common.collect.ImmutableList;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-class ParDoInProcessEvaluator<T> implements TransformEvaluator<T> {
- public static <InputT, OutputT> ParDoInProcessEvaluator<InputT> create(
- InProcessEvaluationContext evaluationContext,
- CommittedBundle<InputT> inputBundle,
- AppliedPTransform<PCollection<InputT>, ?, ?> application,
- DoFn<InputT, OutputT> fn,
- List<PCollectionView<?>> sideInputs,
- TupleTag<OutputT> mainOutputTag,
- List<TupleTag<?>> sideOutputTags,
- Map<TupleTag<?>, PCollection<?>> outputs) {
- InProcessExecutionContext executionContext =
- evaluationContext.getExecutionContext(application, inputBundle.getKey());
- String stepName = evaluationContext.getStepName(application);
- InProcessStepContext stepContext =
- executionContext.getOrCreateStepContext(stepName, stepName);
-
- CounterSet counters = evaluationContext.createCounterSet();
-
- Map<TupleTag<?>, UncommittedBundle<?>> outputBundles = new HashMap<>();
- for (Map.Entry<TupleTag<?>, PCollection<?>> outputEntry : outputs.entrySet()) {
- outputBundles.put(
- outputEntry.getKey(),
- evaluationContext.createBundle(inputBundle, outputEntry.getValue()));
- }
-
- ReadyCheckingSideInputReader sideInputReader =
- evaluationContext.createSideInputReader(sideInputs);
- DoFnRunner<InputT, OutputT> underlying =
- DoFnRunners.createDefault(
- evaluationContext.getPipelineOptions(),
- fn,
- sideInputReader,
- BundleOutputManager.create(outputBundles),
- mainOutputTag,
- sideOutputTags,
- stepContext,
- counters.getAddCounterMutator(),
- application.getInput().getWindowingStrategy());
- PushbackSideInputDoFnRunner<InputT, OutputT> runner =
- PushbackSideInputDoFnRunner.create(underlying, sideInputs, sideInputReader);
-
- try {
- runner.startBundle();
- } catch (Exception e) {
- throw UserCodeException.wrap(e);
- }
-
- return new ParDoInProcessEvaluator<>(
- runner, application, counters, outputBundles.values(), stepContext);
- }
-
- ////////////////////////////////////////////////////////////////////////////////////////////////
-
- private final PushbackSideInputDoFnRunner<T, ?> fnRunner;
- private final AppliedPTransform<PCollection<T>, ?, ?> transform;
- private final CounterSet counters;
- private final Collection<UncommittedBundle<?>> outputBundles;
- private final InProcessStepContext stepContext;
-
- private final ImmutableList.Builder<WindowedValue<T>> unprocessedElements;
-
- private ParDoInProcessEvaluator(
- PushbackSideInputDoFnRunner<T, ?> fnRunner,
- AppliedPTransform<PCollection<T>, ?, ?> transform,
- CounterSet counters,
- Collection<UncommittedBundle<?>> outputBundles,
- InProcessStepContext stepContext) {
- this.fnRunner = fnRunner;
- this.transform = transform;
- this.counters = counters;
- this.outputBundles = outputBundles;
- this.stepContext = stepContext;
-
- this.unprocessedElements = ImmutableList.builder();
- }
-
- @Override
- public void processElement(WindowedValue<T> element) {
- try {
- Iterable<WindowedValue<T>> unprocessed = fnRunner.processElementInReadyWindows(element);
- unprocessedElements.addAll(unprocessed);
- } catch (Exception e) {
- throw UserCodeException.wrap(e);
- }
- }
-
- @Override
- public InProcessTransformResult finishBundle() {
- try {
- fnRunner.finishBundle();
- } catch (Exception e) {
- throw UserCodeException.wrap(e);
- }
- StepTransformResult.Builder resultBuilder;
- CopyOnAccessInMemoryStateInternals<?> state = stepContext.commitState();
- if (state != null) {
- resultBuilder =
- StepTransformResult.withHold(transform, state.getEarliestWatermarkHold())
- .withState(state);
- } else {
- resultBuilder = StepTransformResult.withoutHold(transform);
- }
- return resultBuilder
- .addOutput(outputBundles)
- .withTimerUpdate(stepContext.getTimerUpdate())
- .withCounters(counters)
- .addUnprocessedElements(unprocessedElements.build())
- .build();
- }
-
- static class BundleOutputManager implements OutputManager {
- private final Map<TupleTag<?>, UncommittedBundle<?>> bundles;
- private final Map<TupleTag<?>, List<?>> undeclaredOutputs;
-
- public static BundleOutputManager create(Map<TupleTag<?>, UncommittedBundle<?>> outputBundles) {
- return new BundleOutputManager(outputBundles);
- }
-
- private BundleOutputManager(Map<TupleTag<?>, UncommittedBundle<?>> bundles) {
- this.bundles = bundles;
- undeclaredOutputs = new HashMap<>();
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <T> void output(TupleTag<T> tag, WindowedValue<T> output) {
- @SuppressWarnings("rawtypes")
- UncommittedBundle bundle = bundles.get(tag);
- if (bundle == null) {
- List undeclaredContents = undeclaredOutputs.get(tag);
- if (undeclaredContents == null) {
- undeclaredContents = new ArrayList<T>();
- undeclaredOutputs.put(tag, undeclaredContents);
- }
- undeclaredContents.add(output);
- } else {
- bundle.add(output);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactory.java
index 58d6f00..e008bdc 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactory.java
@@ -47,7 +47,7 @@ class ParDoMultiEvaluatorFactory implements TransformEvaluatorFactory {
public <T> TransformEvaluator<T> forApplication(
AppliedPTransform<?, ?, ?> application,
CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext) {
+ EvaluationContext evaluationContext) {
@SuppressWarnings({"unchecked", "rawtypes"})
TransformEvaluator<T> evaluator =
createMultiEvaluator((AppliedPTransform) application, inputBundle, evaluationContext);
@@ -57,14 +57,14 @@ class ParDoMultiEvaluatorFactory implements TransformEvaluatorFactory {
private <InT, OuT> TransformEvaluator<InT> createMultiEvaluator(
AppliedPTransform<PCollection<InT>, PCollectionTuple, BoundMulti<InT, OuT>> application,
CommittedBundle<InT> inputBundle,
- InProcessEvaluationContext evaluationContext) {
+ EvaluationContext evaluationContext) {
Map<TupleTag<?>, PCollection<?>> outputs = application.getOutput().getAll();
DoFn<InT, OuT> fn = application.getTransform().getFn();
@SuppressWarnings({"unchecked", "rawtypes"}) ThreadLocal<DoFn<InT, OuT>> fnLocal =
(ThreadLocal) fnClones.getUnchecked(application.getTransform().getFn());
try {
- TransformEvaluator<InT> parDoEvaluator = ParDoInProcessEvaluator.create(evaluationContext,
+ TransformEvaluator<InT> parDoEvaluator = ParDoEvaluator.create(evaluationContext,
inputBundle,
application,
fnLocal.get(),
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactory.java
index afbb6ed..0f7fc83 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactory.java
@@ -47,7 +47,7 @@ class ParDoSingleEvaluatorFactory implements TransformEvaluatorFactory {
public <T> TransformEvaluator<T> forApplication(
final AppliedPTransform<?, ?, ?> application,
CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext) {
+ EvaluationContext evaluationContext) {
@SuppressWarnings({"unchecked", "rawtypes"})
TransformEvaluator<T> evaluator =
createSingleEvaluator((AppliedPTransform) application, inputBundle, evaluationContext);
@@ -57,13 +57,13 @@ class ParDoSingleEvaluatorFactory implements TransformEvaluatorFactory {
private <InputT, OutputT> TransformEvaluator<InputT> createSingleEvaluator(
@SuppressWarnings("rawtypes") AppliedPTransform<PCollection<InputT>, PCollection<OutputT>,
Bound<InputT, OutputT>> application,
- CommittedBundle<InputT> inputBundle, InProcessEvaluationContext evaluationContext) {
+ CommittedBundle<InputT> inputBundle, EvaluationContext evaluationContext) {
TupleTag<OutputT> mainOutputTag = new TupleTag<>("out");
@SuppressWarnings({"unchecked", "rawtypes"}) ThreadLocal<DoFn<InputT, OutputT>> fnLocal =
(ThreadLocal) fnClones.getUnchecked(application.getTransform().getFn());
try {
- ParDoInProcessEvaluator<InputT> parDoEvaluator = ParDoInProcessEvaluator.create(
+ ParDoEvaluator<InputT> parDoEvaluator = ParDoEvaluator.create(
evaluationContext,
inputBundle,
application,
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
index ba792d3..c6e10e5 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
@@ -42,7 +42,7 @@ class PassthroughTransformEvaluator<InputT> implements TransformEvaluator<InputT
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
return StepTransformResult.withoutHold(transform).addOutput(output).build();
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PipelineExecutor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PipelineExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PipelineExecutor.java
new file mode 100644
index 0000000..76df11c
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PipelineExecutor.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+
+import java.util.Collection;
+
+/**
+ * An executor that schedules and executes {@link AppliedPTransform AppliedPTransforms} for both
+ * source and intermediate {@link PTransform PTransforms}.
+ */
+interface PipelineExecutor {
+ /**
+ * Starts this executor. The provided collection is the collection of root transforms to
+ * initially schedule.
+ *
+ * @param rootTransforms
+ */
+ void start(Collection<AppliedPTransform<?, ?, ?>> rootTransforms);
+
+ /**
+ * Blocks until the job being executed enters a terminal state. A job is completed after all
+ * root {@link AppliedPTransform AppliedPTransforms} have completed, and all
+ * {@link CommittedBundle Bundles} have been consumed. Jobs may also terminate abnormally.
+ *
+ * @throws Throwable whenever an executor thread throws anything, transfers the throwable to the
+ * waiting thread and rethrows it
+ */
+ void awaitCompletion() throws Throwable;
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
new file mode 100644
index 0000000..7a19ed9
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SideInputContainer.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.util.PCollectionViewWindow;
+import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
+import org.apache.beam.sdk.util.SideInputReader;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.PCollectionView;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * An in-process container for {@link PCollectionView PCollectionViews}, which provides methods for
+ * constructing {@link SideInputReader SideInputReaders} which block until a side input is
+ * available and writing to a {@link PCollectionView}.
+ */
+class SideInputContainer {
+ private final Collection<PCollectionView<?>> containedViews;
+ private final LoadingCache<
+ PCollectionViewWindow<?>, AtomicReference<Iterable<? extends WindowedValue<?>>>>
+ viewByWindows;
+
+ /**
+ * Create a new {@link SideInputContainer} with the provided views and the provided
+ * context.
+ */
+ public static SideInputContainer create(
+ final EvaluationContext context, Collection<PCollectionView<?>> containedViews) {
+ LoadingCache<PCollectionViewWindow<?>, AtomicReference<Iterable<? extends WindowedValue<?>>>>
+ viewByWindows = CacheBuilder.newBuilder().build(new CallbackSchedulingLoader(context));
+ return new SideInputContainer(containedViews, viewByWindows);
+ }
+
+ private SideInputContainer(
+ Collection<PCollectionView<?>> containedViews,
+ LoadingCache<PCollectionViewWindow<?>, AtomicReference<Iterable<? extends WindowedValue<?>>>>
+ viewByWindows) {
+ this.containedViews = ImmutableSet.copyOf(containedViews);
+ this.viewByWindows = viewByWindows;
+ }
+
+ /**
+ * Return a view of this {@link SideInputContainer} that contains only the views in the
+ * provided argument. The returned {@link SideInputContainer} is unmodifiable without
+ * casting, but will change as this {@link SideInputContainer} is modified.
+ */
+ public ReadyCheckingSideInputReader createReaderForViews(
+ Collection<PCollectionView<?>> newContainedViews) {
+ if (!containedViews.containsAll(newContainedViews)) {
+ Set<PCollectionView<?>> currentlyContained = ImmutableSet.copyOf(containedViews);
+ Set<PCollectionView<?>> newRequested = ImmutableSet.copyOf(newContainedViews);
+ throw new IllegalArgumentException("Can't create a SideInputReader with unknown views "
+ + Sets.difference(newRequested, currentlyContained));
+ }
+ return new SideInputContainerSideInputReader(newContainedViews);
+ }
+
+ /**
+ * Write the provided values to the provided view.
+ *
+ * <p>The windowed values are first exploded, then for each window the pane is determined. For
+ * each window, if the pane is later than the current pane stored within this container, write
+ * all of the values to the container as the new values of the {@link PCollectionView}.
+ *
+ * <p>The provided iterable is expected to contain only a single window and pane.
+ */
+ public void write(PCollectionView<?> view, Iterable<? extends WindowedValue<?>> values) {
+ Map<BoundedWindow, Collection<WindowedValue<?>>> valuesPerWindow =
+ indexValuesByWindow(values);
+ for (Map.Entry<BoundedWindow, Collection<WindowedValue<?>>> windowValues :
+ valuesPerWindow.entrySet()) {
+ updatePCollectionViewWindowValues(view, windowValues.getKey(), windowValues.getValue());
+ }
+ }
+
+ /**
+ * Index the provided values by all {@link BoundedWindow windows} in which they appear.
+ */
+ private Map<BoundedWindow, Collection<WindowedValue<?>>> indexValuesByWindow(
+ Iterable<? extends WindowedValue<?>> values) {
+ Map<BoundedWindow, Collection<WindowedValue<?>>> valuesPerWindow = new HashMap<>();
+ for (WindowedValue<?> value : values) {
+ for (BoundedWindow window : value.getWindows()) {
+ Collection<WindowedValue<?>> windowValues = valuesPerWindow.get(window);
+ if (windowValues == null) {
+ windowValues = new ArrayList<>();
+ valuesPerWindow.put(window, windowValues);
+ }
+ windowValues.add(value);
+ }
+ }
+ return valuesPerWindow;
+ }
+
+ /**
+ * Set the value of the {@link PCollectionView} in the {@link BoundedWindow} to be based on the
+ * specified values, if the values are part of a later pane than currently exist within the
+ * {@link PCollectionViewWindow}.
+ */
+ private void updatePCollectionViewWindowValues(
+ PCollectionView<?> view, BoundedWindow window, Collection<WindowedValue<?>> windowValues) {
+ PCollectionViewWindow<?> windowedView = PCollectionViewWindow.of(view, window);
+ AtomicReference<Iterable<? extends WindowedValue<?>>> contents =
+ viewByWindows.getUnchecked(windowedView);
+ if (contents.compareAndSet(null, windowValues)) {
+ // the value had never been set, so we set it and are done.
+ return;
+ }
+ PaneInfo newPane = windowValues.iterator().next().getPane();
+
+ Iterable<? extends WindowedValue<?>> existingValues;
+ long existingPane;
+ do {
+ existingValues = contents.get();
+ existingPane =
+ Iterables.isEmpty(existingValues)
+ ? -1L
+ : existingValues.iterator().next().getPane().getIndex();
+ } while (newPane.getIndex() > existingPane
+ && !contents.compareAndSet(existingValues, windowValues));
+ }
+
+ private static class CallbackSchedulingLoader extends
+ CacheLoader<PCollectionViewWindow<?>, AtomicReference<Iterable<? extends WindowedValue<?>>>> {
+ private final EvaluationContext context;
+
+ public CallbackSchedulingLoader(
+ EvaluationContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public AtomicReference<Iterable<? extends WindowedValue<?>>>
+ load(PCollectionViewWindow<?> view) {
+
+ AtomicReference<Iterable<? extends WindowedValue<?>>> contents = new AtomicReference<>();
+ WindowingStrategy<?, ?> windowingStrategy = view.getView().getWindowingStrategyInternal();
+
+ context.scheduleAfterOutputWouldBeProduced(view.getView(),
+ view.getWindow(),
+ windowingStrategy,
+ new WriteEmptyViewContents(view.getView(), view.getWindow(), contents));
+ return contents;
+ }
+ }
+
+ private static class WriteEmptyViewContents implements Runnable {
+ private final PCollectionView<?> view;
+ private final BoundedWindow window;
+ private final AtomicReference<Iterable<? extends WindowedValue<?>>> contents;
+
+ private WriteEmptyViewContents(PCollectionView<?> view, BoundedWindow window,
+ AtomicReference<Iterable<? extends WindowedValue<?>>> contents) {
+ this.contents = contents;
+ this.view = view;
+ this.window = window;
+ }
+
+ @Override
+ public void run() {
+ // The requested window has closed without producing elements, so reflect that in
+ // the PCollectionView. If set has already been called, will do nothing.
+ contents.compareAndSet(null, Collections.<WindowedValue<?>>emptyList());
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("view", view)
+ .add("window", window)
+ .toString();
+ }
+ }
+
+ private final class SideInputContainerSideInputReader implements ReadyCheckingSideInputReader {
+ private final Collection<PCollectionView<?>> readerViews;
+ private final LoadingCache<
+ PCollectionViewWindow<?>, Optional<? extends Iterable<? extends WindowedValue<?>>>>
+ viewContents;
+
+ private SideInputContainerSideInputReader(Collection<PCollectionView<?>> readerViews) {
+ this.readerViews = ImmutableSet.copyOf(readerViews);
+ this.viewContents = CacheBuilder.newBuilder().build(new CurrentViewContentsLoader());
+ }
+
+ @Override
+ public boolean isReady(final PCollectionView<?> view, final BoundedWindow window) {
+ checkArgument(
+ readerViews.contains(view),
+ "Tried to check if view %s was ready in a SideInputReader that does not contain it. "
+ + "Contained views; %s",
+ view,
+ readerViews);
+ return viewContents.getUnchecked(PCollectionViewWindow.of(view, window)).isPresent();
+ }
+
+ @Override
+ @Nullable
+ public <T> T get(final PCollectionView<T> view, final BoundedWindow window) {
+ checkArgument(readerViews.contains(view),
+ "call to get(PCollectionView) with unknown view: %s",
+ view);
+ checkArgument(
+ isReady(view, window),
+ "calling get() on PCollectionView %s that is not ready in window %s",
+ view,
+ window);
+ // Safe covariant cast
+ @SuppressWarnings("unchecked") Iterable<WindowedValue<?>> values =
+ (Iterable<WindowedValue<?>>) viewContents.getUnchecked(PCollectionViewWindow.of(view,
+ window)).get();
+ return view.fromIterableInternal(values);
+ }
+
+ @Override
+ public <T> boolean contains(PCollectionView<T> view) {
+ return readerViews.contains(view);
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return readerViews.isEmpty();
+ }
+ }
+
+ /**
+ * A {@link CacheLoader} that loads the current contents of a {@link PCollectionViewWindow} into
+ * an optional.
+ */
+ private class CurrentViewContentsLoader extends CacheLoader<
+ PCollectionViewWindow<?>, Optional<? extends Iterable<? extends WindowedValue<?>>>> {
+
+ @Override
+ public Optional<? extends Iterable<? extends WindowedValue<?>>>
+ load(PCollectionViewWindow<?> key) {
+ return Optional.fromNullable(viewByWindows.getUnchecked(key).get());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
index eacea91..5706b2a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
@@ -19,8 +19,8 @@ package org.apache.beam.runners.direct;
import static com.google.common.base.Preconditions.checkNotNull;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
@@ -37,9 +37,9 @@ import java.util.Collection;
import javax.annotation.Nullable;
/**
- * An immutable {@link InProcessTransformResult}.
+ * An immutable {@link TransformResult}.
*/
-public class StepTransformResult implements InProcessTransformResult {
+public class StepTransformResult implements TransformResult {
private final AppliedPTransform<?, ?, ?> transform;
private final Iterable<? extends UncommittedBundle<?>> bundles;
private final Iterable<? extends WindowedValue<?>> unprocessedElements;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ThreadLocalInvalidatingTransformEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ThreadLocalInvalidatingTransformEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ThreadLocalInvalidatingTransformEvaluator.java
index 36bdff4..d8a6bf9 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ThreadLocalInvalidatingTransformEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ThreadLocalInvalidatingTransformEvaluator.java
@@ -52,7 +52,7 @@ class ThreadLocalInvalidatingTransformEvaluator<InputT>
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
try {
return underlying.finishBundle();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluator.java
index 1fec9d8..6c8e48b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluator.java
@@ -40,7 +40,7 @@ public interface TransformEvaluator<InputT> {
* After {@link #finishBundle()} is called, the {@link TransformEvaluator} will not be reused,
* and no more elements will be processed.
*
- * @return an {@link InProcessTransformResult} containing the results of this bundle evaluation.
+ * @return an {@link TransformResult} containing the results of this bundle evaluation.
*/
- InProcessTransformResult finishBundle() throws Exception;
+ TransformResult finishBundle() throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
index b12a34c..1973a2f 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
@@ -46,5 +46,5 @@ public interface TransformEvaluatorFactory {
*/
@Nullable <InputT> TransformEvaluator<InputT> forApplication(
AppliedPTransform<?, ?, ?> application, @Nullable CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext) throws Exception;
+ EvaluationContext evaluationContext) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
index dfc1753..f0afc3b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorRegistry.java
@@ -17,8 +17,8 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupAlsoByWindow;
-import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
+import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupAlsoByWindow;
+import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.io.Read;
import org.apache.beam.sdk.transforms.AppliedPTransform;
@@ -50,8 +50,8 @@ class TransformEvaluatorRegistry implements TransformEvaluatorFactory {
.put(ViewEvaluatorFactory.WriteView.class, new ViewEvaluatorFactory())
.put(Window.Bound.class, new WindowEvaluatorFactory())
// Runner-specific primitives used in expansion of GroupByKey
- .put(InProcessGroupByKeyOnly.class, new InProcessGroupByKeyOnlyEvaluatorFactory())
- .put(InProcessGroupAlsoByWindow.class, new InProcessGroupAlsoByWindowEvaluatorFactory())
+ .put(DirectGroupByKeyOnly.class, new GroupByKeyOnlyEvaluatorFactory())
+ .put(DirectGroupAlsoByWindow.class, new GroupAlsoByWindowEvaluatorFactory())
.build();
return new TransformEvaluatorRegistry(primitives);
}
@@ -71,7 +71,7 @@ class TransformEvaluatorRegistry implements TransformEvaluatorFactory {
public <InputT> TransformEvaluator<InputT> forApplication(
AppliedPTransform<?, ?, ?> application,
@Nullable CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext)
+ EvaluationContext evaluationContext)
throws Exception {
TransformEvaluatorFactory factory = factories.get(application.getTransform().getClass());
return factory.forApplication(application, inputBundle, evaluationContext);
[08/12] incubator-beam git commit: Remove InProcess Prefixes
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
index 4dd1475..ef31ba7 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformExecutor.java
@@ -42,7 +42,7 @@ class TransformExecutor<T> implements Runnable {
public static <T> TransformExecutor<T> create(
TransformEvaluatorFactory factory,
Iterable<? extends ModelEnforcementFactory> modelEnforcements,
- InProcessEvaluationContext evaluationContext,
+ EvaluationContext evaluationContext,
CommittedBundle<T> inputBundle,
AppliedPTransform<?, ?, ?> transform,
CompletionCallback completionCallback,
@@ -60,7 +60,7 @@ class TransformExecutor<T> implements Runnable {
private final TransformEvaluatorFactory evaluatorFactory;
private final Iterable<? extends ModelEnforcementFactory> modelEnforcements;
- private final InProcessEvaluationContext evaluationContext;
+ private final EvaluationContext evaluationContext;
/** The transform that will be evaluated. */
private final AppliedPTransform<?, ?, ?> transform;
@@ -75,7 +75,7 @@ class TransformExecutor<T> implements Runnable {
private TransformExecutor(
TransformEvaluatorFactory factory,
Iterable<? extends ModelEnforcementFactory> modelEnforcements,
- InProcessEvaluationContext evaluationContext,
+ EvaluationContext evaluationContext,
CommittedBundle<T> inputBundle,
AppliedPTransform<?, ?, ?> transform,
CompletionCallback completionCallback,
@@ -117,7 +117,7 @@ class TransformExecutor<T> implements Runnable {
processElements(evaluator, enforcements);
- InProcessTransformResult result = finishBundle(evaluator, enforcements);
+ TransformResult result = finishBundle(evaluator, enforcements);
} catch (Throwable t) {
onComplete.handleThrowable(inputBundle, t);
if (t instanceof RuntimeException) {
@@ -155,13 +155,13 @@ class TransformExecutor<T> implements Runnable {
* Finishes processing the input bundle and commit the result using the
* {@link CompletionCallback}, applying any {@link ModelEnforcement} if necessary.
*
- * @return the {@link InProcessTransformResult} produced by
+ * @return the {@link TransformResult} produced by
* {@link TransformEvaluator#finishBundle()}
*/
- private InProcessTransformResult finishBundle(
+ private TransformResult finishBundle(
TransformEvaluator<T> evaluator, Collection<ModelEnforcement<T>> enforcements)
throws Exception {
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
CommittedResult outputs = onComplete.handleResult(inputBundle, result);
for (ModelEnforcement<T> enforcement : enforcements) {
enforcement.afterFinish(inputBundle, result, outputs.getOutputs());
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java
new file mode 100644
index 0000000..c1e502d
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformResult.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.common.CounterSet;
+import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
+
+import org.joda.time.Instant;
+
+import javax.annotation.Nullable;
+
+/**
+ * The result of evaluating an {@link AppliedPTransform} with a {@link TransformEvaluator}.
+ */
+public interface TransformResult {
+ /**
+ * Returns the {@link AppliedPTransform} that produced this result.
+ */
+ AppliedPTransform<?, ?, ?> getTransform();
+
+ /**
+ * Returns the {@link UncommittedBundle (uncommitted) Bundles} output by this transform. These
+ * will be committed by the evaluation context as part of completing this result.
+ */
+ Iterable<? extends UncommittedBundle<?>> getOutputBundles();
+
+ /**
+ * Returns elements that were provided to the {@link TransformEvaluator} as input but were not
+ * processed.
+ */
+ Iterable<? extends WindowedValue<?>> getUnprocessedElements();
+
+ /**
+ * Returns the {@link CounterSet} used by this {@link PTransform}, or null if this transform did
+ * not use a {@link CounterSet}.
+ */
+ @Nullable CounterSet getCounters();
+
+ /**
+ * Returns the Watermark Hold for the transform at the time this result was produced.
+ *
+ * If the transform does not set any watermark hold, returns
+ * {@link BoundedWindow#TIMESTAMP_MAX_VALUE}.
+ */
+ Instant getWatermarkHold();
+
+ /**
+ * Returns the State used by the transform.
+ *
+ * If this evaluation did not access state, this may return null.
+ */
+ @Nullable
+ CopyOnAccessInMemoryStateInternals<?> getState();
+
+ /**
+ * Returns a TimerUpdateBuilder that was produced as a result of this evaluation. If the
+ * evaluation was triggered due to the delivery of one or more timers, those timers must be added
+ * to the builder before it is complete.
+ *
+ * <p>If this evaluation did not add or remove any timers, returns an empty TimerUpdate.
+ */
+ TimerUpdate getTimerUpdate();
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
index 5030730..3fb773e 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
@@ -69,13 +69,13 @@ class UnboundedReadEvaluatorFactory implements TransformEvaluatorFactory {
@Override
@Nullable
public <InputT> TransformEvaluator<InputT> forApplication(AppliedPTransform<?, ?, ?> application,
- @Nullable CommittedBundle<?> inputBundle, InProcessEvaluationContext evaluationContext) {
+ @Nullable CommittedBundle<?> inputBundle, EvaluationContext evaluationContext) {
return getTransformEvaluator((AppliedPTransform) application, evaluationContext);
}
private <OutputT> TransformEvaluator<?> getTransformEvaluator(
final AppliedPTransform<?, PCollection<OutputT>, Unbounded<OutputT>> transform,
- final InProcessEvaluationContext evaluationContext) {
+ final EvaluationContext evaluationContext) {
return getTransformEvaluatorQueue(transform, evaluationContext).poll();
}
@@ -90,7 +90,7 @@ class UnboundedReadEvaluatorFactory implements TransformEvaluatorFactory {
private <OutputT, CheckpointMarkT extends CheckpointMark>
Queue<UnboundedReadEvaluator<OutputT, CheckpointMarkT>> getTransformEvaluatorQueue(
final AppliedPTransform<?, PCollection<OutputT>, Unbounded<OutputT>> transform,
- final InProcessEvaluationContext evaluationContext) {
+ final EvaluationContext evaluationContext) {
// Key by the application and the context the evaluation is occurring in (which call to
// Pipeline#run).
EvaluatorKey key = new EvaluatorKey(transform, evaluationContext);
@@ -134,7 +134,7 @@ class UnboundedReadEvaluatorFactory implements TransformEvaluatorFactory {
private static final int ARBITRARY_MAX_ELEMENTS = 10;
private final AppliedPTransform<?, PCollection<OutputT>, Unbounded<OutputT>> transform;
- private final InProcessEvaluationContext evaluationContext;
+ private final EvaluationContext evaluationContext;
private final ConcurrentLinkedQueue<UnboundedReadEvaluator<OutputT, CheckpointMarkT>>
evaluatorQueue;
/**
@@ -153,7 +153,7 @@ class UnboundedReadEvaluatorFactory implements TransformEvaluatorFactory {
public UnboundedReadEvaluator(
AppliedPTransform<?, PCollection<OutputT>, Unbounded<OutputT>> transform,
- InProcessEvaluationContext evaluationContext,
+ EvaluationContext evaluationContext,
UnboundedSource<OutputT, CheckpointMarkT> source,
ConcurrentLinkedQueue<UnboundedReadEvaluator<OutputT, CheckpointMarkT>> evaluatorQueue) {
this.transform = transform;
@@ -168,7 +168,7 @@ class UnboundedReadEvaluatorFactory implements TransformEvaluatorFactory {
public void processElement(WindowedValue<Object> element) {}
@Override
- public InProcessTransformResult finishBundle() throws IOException {
+ public TransformResult finishBundle() throws IOException {
UncommittedBundle<OutputT> output = evaluationContext.createRootBundle(transform.getOutput());
try {
boolean elementAvailable = startReader();
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UncommittedBundleOutputManager.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UncommittedBundleOutputManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UncommittedBundleOutputManager.java
new file mode 100644
index 0000000..570dc90
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UncommittedBundleOutputManager.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.sdk.util.DoFnRunners.OutputManager;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+import java.util.Map;
+
+/**
+ * An {@link OutputManager} that outputs to {@link CommittedBundle Bundles} used by the
+ * {@link DirectRunner}.
+ */
+public class UncommittedBundleOutputManager implements OutputManager {
+ private final Map<TupleTag<?>, UncommittedBundle<?>> bundles;
+
+ public static UncommittedBundleOutputManager create(
+ Map<TupleTag<?>, UncommittedBundle<?>> outputBundles) {
+ return new UncommittedBundleOutputManager(outputBundles);
+ }
+
+ public UncommittedBundleOutputManager(Map<TupleTag<?>, UncommittedBundle<?>> bundles) {
+ this.bundles = bundles;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> void output(TupleTag<T> tag, WindowedValue<T> output) {
+ @SuppressWarnings("rawtypes")
+ UncommittedBundle bundle = bundles.get(tag);
+ bundle.add(output);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
index f4260f5..47eaae7 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
@@ -41,7 +41,7 @@ import java.util.List;
*
* <p>The {@link ViewEvaluatorFactory} produces {@link TransformEvaluator TransformEvaluators} for
* the {@link WriteView} {@link PTransform}, which is part of the
- * {@link InProcessCreatePCollectionView} composite transform. This transform is an override for the
+ * {@link DirectCreatePCollectionView} composite transform. This transform is an override for the
* {@link CreatePCollectionView} transform that applies windowing and triggers before the view is
* written.
*/
@@ -50,7 +50,7 @@ class ViewEvaluatorFactory implements TransformEvaluatorFactory {
public <T> TransformEvaluator<T> forApplication(
AppliedPTransform<?, ?, ?> application,
DirectRunner.CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext) {
+ EvaluationContext evaluationContext) {
@SuppressWarnings({"cast", "unchecked", "rawtypes"})
TransformEvaluator<T> evaluator = createEvaluator(
(AppliedPTransform) application, evaluationContext);
@@ -60,7 +60,7 @@ class ViewEvaluatorFactory implements TransformEvaluatorFactory {
private <InT, OuT> TransformEvaluator<Iterable<InT>> createEvaluator(
final AppliedPTransform<PCollection<Iterable<InT>>, PCollectionView<OuT>, WriteView<InT, OuT>>
application,
- InProcessEvaluationContext context) {
+ EvaluationContext context) {
PCollection<Iterable<InT>> input = application.getInput();
final PCollectionViewWriter<InT, OuT> writer =
context.createPCollectionViewWriter(input, application.getOutput());
@@ -75,14 +75,14 @@ class ViewEvaluatorFactory implements TransformEvaluatorFactory {
}
@Override
- public InProcessTransformResult finishBundle() {
+ public TransformResult finishBundle() {
writer.add(elements);
return StepTransformResult.withoutHold(application).build();
}
};
}
- public static class InProcessViewOverrideFactory implements PTransformOverrideFactory {
+ public static class ViewOverrideFactory implements PTransformOverrideFactory {
@Override
public <InputT extends PInput, OutputT extends POutput>
PTransform<InputT, OutputT> override(PTransform<InputT, OutputT> transform) {
@@ -92,7 +92,7 @@ class ViewEvaluatorFactory implements TransformEvaluatorFactory {
@SuppressWarnings({"rawtypes", "unchecked"})
PTransform<InputT, OutputT> createView =
(PTransform<InputT, OutputT>)
- new InProcessCreatePCollectionView<>((CreatePCollectionView) transform);
+ new DirectCreatePCollectionView<>((CreatePCollectionView) transform);
return createView;
}
}
@@ -100,11 +100,11 @@ class ViewEvaluatorFactory implements TransformEvaluatorFactory {
/**
* An in-process override for {@link CreatePCollectionView}.
*/
- private static class InProcessCreatePCollectionView<ElemT, ViewT>
+ private static class DirectCreatePCollectionView<ElemT, ViewT>
extends ForwardingPTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
private final CreatePCollectionView<ElemT, ViewT> og;
- private InProcessCreatePCollectionView(CreatePCollectionView<ElemT, ViewT> og) {
+ private DirectCreatePCollectionView(CreatePCollectionView<ElemT, ViewT> og) {
this.og = og;
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
new file mode 100644
index 0000000..b8f9987
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -0,0 +1,1420 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.util.TimerInternals;
+import org.apache.beam.sdk.util.TimerInternals.TimerData;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PValue;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.SortedMultiset;
+import com.google.common.collect.TreeMultiset;
+
+import org.joda.time.Instant;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Objects;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * Manages watermarks of {@link PCollection PCollections} and input and output watermarks of
+ * {@link AppliedPTransform AppliedPTransforms} to provide event-time and completion tracking for
+ * in-memory execution. {@link WatermarkManager} is designed to update and return a
+ * consistent view of watermarks in the presence of concurrent updates.
+ *
+ * <p>An {@link WatermarkManager} is provided with the collection of root
+ * {@link AppliedPTransform AppliedPTransforms} and a map of {@link PCollection PCollections} to
+ * all the {@link AppliedPTransform AppliedPTransforms} that consume them at construction time.
+ *
+ * <p>Whenever a root {@link AppliedPTransform transform} produces elements, the
+ * {@link WatermarkManager} is provided with the produced elements and the output watermark
+ * of the producing {@link AppliedPTransform transform}. The
+ * {@link WatermarkManager watermark manager} is responsible for computing the watermarks
+ * of all {@link AppliedPTransform transforms} that consume one or more
+ * {@link PCollection PCollections}.
+ *
+ * <p>Whenever a non-root {@link AppliedPTransform} finishes processing one or more in-flight
+ * elements (referred to as the input {@link CommittedBundle bundle}), the following occurs
+ * atomically:
+ * <ul>
+ * <li>All of the in-flight elements are removed from the collection of pending elements for the
+ * {@link AppliedPTransform}.</li>
+ * <li>All of the elements produced by the {@link AppliedPTransform} are added to the collection
+ * of pending elements for each {@link AppliedPTransform} that consumes them.</li>
+ * <li>The input watermark for the {@link AppliedPTransform} becomes the maximum value of
+ * <ul>
+ * <li>the previous input watermark</li>
+ * <li>the minimum of
+ * <ul>
+ * <li>the timestamps of all currently pending elements</li>
+ * <li>all input {@link PCollection} watermarks</li>
+ * </ul>
+ * </li>
+ * </ul>
+ * </li>
+ * <li>The output watermark for the {@link AppliedPTransform} becomes the maximum of
+ * <ul>
+ * <li>the previous output watermark</li>
+ * <li>the minimum of
+ * <ul>
+ * <li>the current input watermark</li>
+ * <li>the current watermark holds</li>
+ * </ul>
+ * </li>
+ * </ul>
+ * </li>
+ * <li>The watermark of the output {@link PCollection} can be advanced to the output watermark of
+ * the {@link AppliedPTransform}</li>
+ * <li>The watermark of all downstream {@link AppliedPTransform AppliedPTransforms} can be
+ * advanced.</li>
+ * </ul>
+ *
+ * <p>The watermark of a {@link PCollection} is equal to the output watermark of the
+ * {@link AppliedPTransform} that produces it.
+ *
+ * <p>The watermarks for a {@link PTransform} are updated as follows when output is committed:<pre>
+ * Watermark_In' = MAX(Watermark_In, MIN(U(TS_Pending), U(Watermark_InputPCollection)))
+ * Watermark_Out' = MAX(Watermark_Out, MIN(Watermark_In', U(StateHold)))
+ * Watermark_PCollection = Watermark_Out_ProducingPTransform
+ * </pre>
+ */
+public class WatermarkManager {
+ /**
+ * The watermark of some {@link Pipeline} element, usually a {@link PTransform} or a
+ * {@link PCollection}.
+ *
+ * <p>A watermark is a monotonically increasing value, which represents the point up to which the
+ * system believes it has received all of the data. Data that arrives with a timestamp that is
+ * before the watermark is considered late. {@link BoundedWindow#TIMESTAMP_MAX_VALUE} is a special
+ * timestamp which indicates we have received all of the data and there will be no more on-time or
+ * late data. This value is represented by {@link WatermarkManager#THE_END_OF_TIME}.
+ */
+ private static interface Watermark {
+ /**
+ * Returns the current value of this watermark.
+ */
+ Instant get();
+
+ /**
+ * Refreshes the value of this watermark from its input watermarks and watermark holds.
+ *
+ * @return true if the value of the watermark has changed (and thus dependent watermark must
+ * also be updated
+ */
+ WatermarkUpdate refresh();
+ }
+
+ /**
+ * The result of computing a {@link Watermark}.
+ */
+ private static enum WatermarkUpdate {
+ /** The watermark is later than the value at the previous time it was computed. */
+ ADVANCED(true),
+ /** The watermark is equal to the value at the previous time it was computed. */
+ NO_CHANGE(false);
+
+ private final boolean advanced;
+
+ private WatermarkUpdate(boolean advanced) {
+ this.advanced = advanced;
+ }
+
+ public boolean isAdvanced() {
+ return advanced;
+ }
+
+ /**
+ * Returns the {@link WatermarkUpdate} that is a result of combining the two watermark updates.
+ *
+ * If either of the input {@link WatermarkUpdate WatermarkUpdates} were advanced, the result
+ * {@link WatermarkUpdate} has been advanced.
+ */
+ public WatermarkUpdate union(WatermarkUpdate that) {
+ if (this.advanced) {
+ return this;
+ }
+ return that;
+ }
+
+ /**
+ * Returns the {@link WatermarkUpdate} based on the former and current
+ * {@link Instant timestamps}.
+ */
+ public static WatermarkUpdate fromTimestamps(Instant oldTime, Instant currentTime) {
+ if (currentTime.isAfter(oldTime)) {
+ return ADVANCED;
+ }
+ return NO_CHANGE;
+ }
+ }
+
+ /**
+ * The input {@link Watermark} of an {@link AppliedPTransform}.
+ *
+ * <p>At any point, the value of an {@link AppliedPTransformInputWatermark} is equal to the
+ * minimum watermark across all of its input {@link Watermark Watermarks}, and the minimum
+ * timestamp of all of the pending elements, restricted to be monotonically increasing.
+ *
+ * <p>See {@link #refresh()} for more information.
+ */
+ private static class AppliedPTransformInputWatermark implements Watermark {
+ private final Collection<? extends Watermark> inputWatermarks;
+ private final SortedMultiset<WindowedValue<?>> pendingElements;
+ private final Map<StructuralKey<?>, NavigableSet<TimerData>> objectTimers;
+
+ private AtomicReference<Instant> currentWatermark;
+
+ public AppliedPTransformInputWatermark(Collection<? extends Watermark> inputWatermarks) {
+ this.inputWatermarks = inputWatermarks;
+ this.pendingElements = TreeMultiset.create(new WindowedValueByTimestampComparator());
+ this.objectTimers = new HashMap<>();
+ currentWatermark = new AtomicReference<>(BoundedWindow.TIMESTAMP_MIN_VALUE);
+ }
+
+ @Override
+ public Instant get() {
+ return currentWatermark.get();
+ }
+
+ /**
+ * {@inheritDoc}.
+ *
+ * <p>When refresh is called, the value of the {@link AppliedPTransformInputWatermark} becomes
+ * equal to the maximum value of
+ * <ul>
+ * <li>the previous input watermark</li>
+ * <li>the minimum of
+ * <ul>
+ * <li>the timestamps of all currently pending elements</li>
+ * <li>all input {@link PCollection} watermarks</li>
+ * </ul>
+ * </li>
+ * </ul>
+ */
+ @Override
+ public synchronized WatermarkUpdate refresh() {
+ Instant oldWatermark = currentWatermark.get();
+ Instant minInputWatermark = BoundedWindow.TIMESTAMP_MAX_VALUE;
+ for (Watermark inputWatermark : inputWatermarks) {
+ minInputWatermark = INSTANT_ORDERING.min(minInputWatermark, inputWatermark.get());
+ }
+ if (!pendingElements.isEmpty()) {
+ minInputWatermark = INSTANT_ORDERING.min(
+ minInputWatermark, pendingElements.firstEntry().getElement().getTimestamp());
+ }
+ Instant newWatermark = INSTANT_ORDERING.max(oldWatermark, minInputWatermark);
+ currentWatermark.set(newWatermark);
+ return WatermarkUpdate.fromTimestamps(oldWatermark, newWatermark);
+ }
+
+ private synchronized void addPendingElements(Iterable<? extends WindowedValue<?>> newPending) {
+ for (WindowedValue<?> pendingElement : newPending) {
+ pendingElements.add(pendingElement);
+ }
+ }
+
+ private synchronized void removePendingElements(
+ Iterable<? extends WindowedValue<?>> finishedElements) {
+ for (WindowedValue<?> finishedElement : finishedElements) {
+ pendingElements.remove(finishedElement);
+ }
+ }
+
+ private synchronized void updateTimers(TimerUpdate update) {
+ NavigableSet<TimerData> keyTimers = objectTimers.get(update.key);
+ if (keyTimers == null) {
+ keyTimers = new TreeSet<>();
+ objectTimers.put(update.key, keyTimers);
+ }
+ for (TimerData timer : update.setTimers) {
+ if (TimeDomain.EVENT_TIME.equals(timer.getDomain())) {
+ keyTimers.add(timer);
+ }
+ }
+ for (TimerData timer : update.deletedTimers) {
+ if (TimeDomain.EVENT_TIME.equals(timer.getDomain())) {
+ keyTimers.remove(timer);
+ }
+ }
+ // We don't keep references to timers that have been fired and delivered via #getFiredTimers()
+ }
+
+ private synchronized Map<StructuralKey<?>, List<TimerData>> extractFiredEventTimeTimers() {
+ return extractFiredTimers(currentWatermark.get(), objectTimers);
+ }
+
+ @Override
+ public synchronized String toString() {
+ return MoreObjects.toStringHelper(AppliedPTransformInputWatermark.class)
+ .add("pendingElements", pendingElements)
+ .add("currentWatermark", currentWatermark)
+ .toString();
+ }
+ }
+
+ /**
+ * The output {@link Watermark} of an {@link AppliedPTransform}.
+ *
+ * <p>The value of an {@link AppliedPTransformOutputWatermark} is equal to the minimum of the
+ * current watermark hold and the {@link AppliedPTransformInputWatermark} for the same
+ * {@link AppliedPTransform}, restricted to be monotonically increasing. See
+ * {@link #refresh()} for more information.
+ */
+ private static class AppliedPTransformOutputWatermark implements Watermark {
+ private final Watermark inputWatermark;
+ private final PerKeyHolds holds;
+ private AtomicReference<Instant> currentWatermark;
+
+ public AppliedPTransformOutputWatermark(AppliedPTransformInputWatermark inputWatermark) {
+ this.inputWatermark = inputWatermark;
+ holds = new PerKeyHolds();
+ currentWatermark = new AtomicReference<>(BoundedWindow.TIMESTAMP_MIN_VALUE);
+ }
+
+ public synchronized void updateHold(Object key, Instant newHold) {
+ if (newHold == null) {
+ holds.removeHold(key);
+ } else {
+ holds.updateHold(key, newHold);
+ }
+ }
+
+ @Override
+ public Instant get() {
+ return currentWatermark.get();
+ }
+
+ /**
+ * {@inheritDoc}.
+ *
+ * <p>When refresh is called, the value of the {@link AppliedPTransformOutputWatermark} becomes
+ * equal to the maximum value of:
+ * <ul>
+ * <li>the previous output watermark</li>
+ * <li>the minimum of
+ * <ul>
+ * <li>the current input watermark</li>
+ * <li>the current watermark holds</li>
+ * </ul>
+ * </li>
+ * </ul>
+ */
+ @Override
+ public synchronized WatermarkUpdate refresh() {
+ Instant oldWatermark = currentWatermark.get();
+ Instant newWatermark = INSTANT_ORDERING.min(inputWatermark.get(), holds.getMinHold());
+ newWatermark = INSTANT_ORDERING.max(oldWatermark, newWatermark);
+ currentWatermark.set(newWatermark);
+ return WatermarkUpdate.fromTimestamps(oldWatermark, newWatermark);
+ }
+
+ @Override
+ public synchronized String toString() {
+ return MoreObjects.toStringHelper(AppliedPTransformOutputWatermark.class)
+ .add("holds", holds)
+ .add("currentWatermark", currentWatermark)
+ .toString();
+ }
+ }
+
+ /**
+ * The input {@link TimeDomain#SYNCHRONIZED_PROCESSING_TIME} hold for an
+ * {@link AppliedPTransform}.
+ *
+ * <p>At any point, the hold value of an {@link SynchronizedProcessingTimeInputWatermark} is equal
+ * to the minimum across all pending bundles at the {@link AppliedPTransform} and all upstream
+ * {@link TimeDomain#SYNCHRONIZED_PROCESSING_TIME} watermarks. The value of the input
+ * synchronized processing time at any step is equal to the maximum of:
+ * <ul>
+ * <li>The most recently returned synchronized processing input time
+ * <li>The minimum of
+ * <ul>
+ * <li>The current processing time
+ * <li>The current synchronized processing time input hold
+ * </ul>
+ * </ul>
+ */
+ private static class SynchronizedProcessingTimeInputWatermark implements Watermark {
+ private final Collection<? extends Watermark> inputWms;
+ private final Collection<CommittedBundle<?>> pendingBundles;
+ private final Map<StructuralKey<?>, NavigableSet<TimerData>> processingTimers;
+ private final Map<StructuralKey<?>, NavigableSet<TimerData>> synchronizedProcessingTimers;
+
+ private final PriorityQueue<TimerData> pendingTimers;
+
+ private AtomicReference<Instant> earliestHold;
+
+ public SynchronizedProcessingTimeInputWatermark(Collection<? extends Watermark> inputWms) {
+ this.inputWms = inputWms;
+ this.pendingBundles = new HashSet<>();
+ this.processingTimers = new HashMap<>();
+ this.synchronizedProcessingTimers = new HashMap<>();
+ this.pendingTimers = new PriorityQueue<>();
+ Instant initialHold = BoundedWindow.TIMESTAMP_MAX_VALUE;
+ for (Watermark wm : inputWms) {
+ initialHold = INSTANT_ORDERING.min(initialHold, wm.get());
+ }
+ earliestHold = new AtomicReference<>(initialHold);
+ }
+
+ @Override
+ public Instant get() {
+ return earliestHold.get();
+ }
+
+ /**
+ * {@inheritDoc}.
+ *
+ * <p>When refresh is called, the value of the {@link SynchronizedProcessingTimeInputWatermark}
+ * becomes equal to the minimum value of
+ * <ul>
+ * <li>the timestamps of all currently pending bundles</li>
+ * <li>all input {@link PCollection} synchronized processing time watermarks</li>
+ * </ul>
+ *
+ * <p>Note that this value is not monotonic, but the returned value for the synchronized
+ * processing time must be.
+ */
+ @Override
+ public synchronized WatermarkUpdate refresh() {
+ Instant oldHold = earliestHold.get();
+ Instant minTime = THE_END_OF_TIME.get();
+ for (Watermark input : inputWms) {
+ minTime = INSTANT_ORDERING.min(minTime, input.get());
+ }
+ for (CommittedBundle<?> bundle : pendingBundles) {
+ // TODO: Track elements in the bundle by the processing time they were output instead of
+ // entire bundles. Requried to support arbitrarily splitting and merging bundles between
+ // steps
+ minTime = INSTANT_ORDERING.min(minTime, bundle.getSynchronizedProcessingOutputWatermark());
+ }
+ earliestHold.set(minTime);
+ return WatermarkUpdate.fromTimestamps(oldHold, minTime);
+ }
+
+ public synchronized void addPending(CommittedBundle<?> bundle) {
+ pendingBundles.add(bundle);
+ }
+
+ public synchronized void removePending(CommittedBundle<?> bundle) {
+ pendingBundles.remove(bundle);
+ }
+
+ /**
+ * Return the earliest timestamp of the earliest timer that has not been completed. This is
+ * either the earliest timestamp across timers that have not been completed, or the earliest
+ * timestamp across timers that have been delivered but have not been completed.
+ */
+ public synchronized Instant getEarliestTimerTimestamp() {
+ Instant earliest = THE_END_OF_TIME.get();
+ for (NavigableSet<TimerData> timers : processingTimers.values()) {
+ if (!timers.isEmpty()) {
+ earliest = INSTANT_ORDERING.min(timers.first().getTimestamp(), earliest);
+ }
+ }
+ for (NavigableSet<TimerData> timers : synchronizedProcessingTimers.values()) {
+ if (!timers.isEmpty()) {
+ earliest = INSTANT_ORDERING.min(timers.first().getTimestamp(), earliest);
+ }
+ }
+ if (!pendingTimers.isEmpty()) {
+ earliest = INSTANT_ORDERING.min(pendingTimers.peek().getTimestamp(), earliest);
+ }
+ return earliest;
+ }
+
+ private synchronized void updateTimers(TimerUpdate update) {
+ Map<TimeDomain, NavigableSet<TimerData>> timerMap = timerMap(update.key);
+ for (TimerData addedTimer : update.setTimers) {
+ NavigableSet<TimerData> timerQueue = timerMap.get(addedTimer.getDomain());
+ if (timerQueue != null) {
+ timerQueue.add(addedTimer);
+ }
+ }
+
+ for (TimerData completedTimer : update.completedTimers) {
+ pendingTimers.remove(completedTimer);
+ }
+ for (TimerData deletedTimer : update.deletedTimers) {
+ NavigableSet<TimerData> timerQueue = timerMap.get(deletedTimer.getDomain());
+ if (timerQueue != null) {
+ timerQueue.remove(deletedTimer);
+ }
+ }
+ }
+
+ private synchronized Map<StructuralKey<?>, List<TimerData>> extractFiredDomainTimers(
+ TimeDomain domain, Instant firingTime) {
+ Map<StructuralKey<?>, List<TimerData>> firedTimers;
+ switch (domain) {
+ case PROCESSING_TIME:
+ firedTimers = extractFiredTimers(firingTime, processingTimers);
+ break;
+ case SYNCHRONIZED_PROCESSING_TIME:
+ firedTimers =
+ extractFiredTimers(
+ INSTANT_ORDERING.min(firingTime, earliestHold.get()),
+ synchronizedProcessingTimers);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Called getFiredTimers on a Synchronized Processing Time watermark"
+ + " and gave a non-processing time domain "
+ + domain);
+ }
+ for (Map.Entry<StructuralKey<?>, ? extends Collection<TimerData>> firedTimer :
+ firedTimers.entrySet()) {
+ pendingTimers.addAll(firedTimer.getValue());
+ }
+ return firedTimers;
+ }
+
+ private Map<TimeDomain, NavigableSet<TimerData>> timerMap(StructuralKey<?> key) {
+ NavigableSet<TimerData> processingQueue = processingTimers.get(key);
+ if (processingQueue == null) {
+ processingQueue = new TreeSet<>();
+ processingTimers.put(key, processingQueue);
+ }
+ NavigableSet<TimerData> synchronizedProcessingQueue =
+ synchronizedProcessingTimers.get(key);
+ if (synchronizedProcessingQueue == null) {
+ synchronizedProcessingQueue = new TreeSet<>();
+ synchronizedProcessingTimers.put(key, synchronizedProcessingQueue);
+ }
+ EnumMap<TimeDomain, NavigableSet<TimerData>> result = new EnumMap<>(TimeDomain.class);
+ result.put(TimeDomain.PROCESSING_TIME, processingQueue);
+ result.put(TimeDomain.SYNCHRONIZED_PROCESSING_TIME, synchronizedProcessingQueue);
+ return result;
+ }
+
+ @Override
+ public synchronized String toString() {
+ return MoreObjects.toStringHelper(SynchronizedProcessingTimeInputWatermark.class)
+ .add("earliestHold", earliestHold)
+ .toString();
+ }
+ }
+
+ /**
+ * The output {@link TimeDomain#SYNCHRONIZED_PROCESSING_TIME} hold for an
+ * {@link AppliedPTransform}.
+ *
+ * <p>At any point, the hold value of an {@link SynchronizedProcessingTimeOutputWatermark} is
+ * equal to the minimum across all incomplete timers at the {@link AppliedPTransform} and all
+ * upstream {@link TimeDomain#SYNCHRONIZED_PROCESSING_TIME} watermarks. The value of the output
+ * synchronized processing time at any step is equal to the maximum of:
+ * <ul>
+ * <li>The most recently returned synchronized processing output time
+ * <li>The minimum of
+ * <ul>
+ * <li>The current processing time
+ * <li>The current synchronized processing time output hold
+ * </ul>
+ * </ul>
+ */
+ private static class SynchronizedProcessingTimeOutputWatermark implements Watermark {
+ private final SynchronizedProcessingTimeInputWatermark inputWm;
+ private AtomicReference<Instant> latestRefresh;
+
+ public SynchronizedProcessingTimeOutputWatermark(
+ SynchronizedProcessingTimeInputWatermark inputWm) {
+ this.inputWm = inputWm;
+ this.latestRefresh = new AtomicReference<>(BoundedWindow.TIMESTAMP_MIN_VALUE);
+ }
+
+ @Override
+ public Instant get() {
+ return latestRefresh.get();
+ }
+
+ /**
+ * {@inheritDoc}.
+ *
+ * <p>When refresh is called, the value of the {@link SynchronizedProcessingTimeOutputWatermark}
+ * becomes equal to the minimum value of:
+ * <ul>
+ * <li>the current input watermark.
+ * <li>all {@link TimeDomain#SYNCHRONIZED_PROCESSING_TIME} timers that are based on the input
+ * watermark.
+ * <li>all {@link TimeDomain#PROCESSING_TIME} timers that are based on the input watermark.
+ * </ul>
+ *
+ * <p>Note that this value is not monotonic, but the returned value for the synchronized
+ * processing time must be.
+ */
+ @Override
+ public synchronized WatermarkUpdate refresh() {
+ // Hold the output synchronized processing time to the input watermark, which takes into
+ // account buffered bundles, and the earliest pending timer, which determines what to hold
+ // downstream timers to.
+ Instant oldRefresh = latestRefresh.get();
+ Instant newTimestamp =
+ INSTANT_ORDERING.min(inputWm.get(), inputWm.getEarliestTimerTimestamp());
+ latestRefresh.set(newTimestamp);
+ return WatermarkUpdate.fromTimestamps(oldRefresh, newTimestamp);
+ }
+
+ @Override
+ public synchronized String toString() {
+ return MoreObjects.toStringHelper(SynchronizedProcessingTimeOutputWatermark.class)
+ .add("latestRefresh", latestRefresh)
+ .toString();
+ }
+ }
+
+ /**
+ * The {@code Watermark} that is after the latest time it is possible to represent in the global
+ * window. This is a distinguished value representing a complete {@link PTransform}.
+ */
+ private static final Watermark THE_END_OF_TIME = new Watermark() {
+ @Override
+ public WatermarkUpdate refresh() {
+ // THE_END_OF_TIME is a distinguished value that cannot be advanced.
+ return WatermarkUpdate.NO_CHANGE;
+ }
+
+ @Override
+ public Instant get() {
+ return BoundedWindow.TIMESTAMP_MAX_VALUE;
+ }
+ };
+
+ private static final Ordering<Instant> INSTANT_ORDERING = Ordering.natural();
+
+ /**
+ * A function that takes a WindowedValue and returns the exploded representation of that
+ * {@link WindowedValue}.
+ */
+ private static final Function<WindowedValue<?>, ? extends Iterable<? extends WindowedValue<?>>>
+ EXPLODE_WINDOWS_FN =
+ new Function<WindowedValue<?>, Iterable<? extends WindowedValue<?>>>() {
+ @Override
+ public Iterable<? extends WindowedValue<?>> apply(WindowedValue<?> input) {
+ return input.explodeWindows();
+ }
+ };
+
+ /**
+ * For each (Object, PriorityQueue) pair in the provided map, remove each Timer that is before the
+ * latestTime argument and put in in the result with the same key, then remove all of the keys
+ * which have no more pending timers.
+ *
+ * The result collection retains ordering of timers (from earliest to latest).
+ */
+ private static Map<StructuralKey<?>, List<TimerData>> extractFiredTimers(
+ Instant latestTime, Map<StructuralKey<?>, NavigableSet<TimerData>> objectTimers) {
+ Map<StructuralKey<?>, List<TimerData>> result = new HashMap<>();
+ Set<StructuralKey<?>> emptyKeys = new HashSet<>();
+ for (Map.Entry<StructuralKey<?>, NavigableSet<TimerData>> pendingTimers :
+ objectTimers.entrySet()) {
+ NavigableSet<TimerData> timers = pendingTimers.getValue();
+ if (!timers.isEmpty() && timers.first().getTimestamp().isBefore(latestTime)) {
+ ArrayList<TimerData> keyFiredTimers = new ArrayList<>();
+ result.put(pendingTimers.getKey(), keyFiredTimers);
+ while (!timers.isEmpty() && timers.first().getTimestamp().isBefore(latestTime)) {
+ keyFiredTimers.add(timers.first());
+ timers.remove(timers.first());
+ }
+ }
+ if (timers.isEmpty()) {
+ emptyKeys.add(pendingTimers.getKey());
+ }
+ }
+ objectTimers.keySet().removeAll(emptyKeys);
+ return result;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * The {@link Clock} providing the current time in the {@link TimeDomain#PROCESSING_TIME} domain.
+ */
+ private final Clock clock;
+
+ /**
+ * A map from each {@link PCollection} to all {@link AppliedPTransform PTransform applications}
+ * that consume that {@link PCollection}.
+ */
+ private final Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> consumers;
+
+ /**
+ * The input and output watermark of each {@link AppliedPTransform}.
+ */
+ private final Map<AppliedPTransform<?, ?, ?>, TransformWatermarks> transformToWatermarks;
+
+ /**
+ * A queue of pending updates to the state of this {@link WatermarkManager}.
+ */
+ private final ConcurrentLinkedQueue<PendingWatermarkUpdate> pendingUpdates;
+
+ /**
+ * A queue of pending {@link AppliedPTransform AppliedPTransforms} that have potentially
+ * stale data.
+ */
+ private final ConcurrentLinkedQueue<AppliedPTransform<?, ?, ?>> pendingRefreshes;
+
+ /**
+ * Creates a new {@link WatermarkManager}. All watermarks within the newly created
+ * {@link WatermarkManager} start at {@link BoundedWindow#TIMESTAMP_MIN_VALUE}, the
+ * minimum watermark, with no watermark holds or pending elements.
+ *
+ * @param rootTransforms the root-level transforms of the {@link Pipeline}
+ * @param consumers a mapping between each {@link PCollection} in the {@link Pipeline} to the
+ * transforms that consume it as a part of their input
+ */
+ public static WatermarkManager create(
+ Clock clock,
+ Collection<AppliedPTransform<?, ?, ?>> rootTransforms,
+ Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> consumers) {
+ return new WatermarkManager(clock, rootTransforms, consumers);
+ }
+
+ private WatermarkManager(
+ Clock clock,
+ Collection<AppliedPTransform<?, ?, ?>> rootTransforms,
+ Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> consumers) {
+ this.clock = clock;
+ this.consumers = consumers;
+ this.pendingUpdates = new ConcurrentLinkedQueue<>();
+ this.pendingRefreshes = new ConcurrentLinkedQueue<>();
+
+ transformToWatermarks = new HashMap<>();
+
+ for (AppliedPTransform<?, ?, ?> rootTransform : rootTransforms) {
+ getTransformWatermark(rootTransform);
+ }
+ for (Collection<AppliedPTransform<?, ?, ?>> intermediateTransforms : consumers.values()) {
+ for (AppliedPTransform<?, ?, ?> transform : intermediateTransforms) {
+ getTransformWatermark(transform);
+ }
+ }
+ }
+
+ private TransformWatermarks getTransformWatermark(AppliedPTransform<?, ?, ?> transform) {
+ TransformWatermarks wms = transformToWatermarks.get(transform);
+ if (wms == null) {
+ List<Watermark> inputCollectionWatermarks = getInputWatermarks(transform);
+ AppliedPTransformInputWatermark inputWatermark =
+ new AppliedPTransformInputWatermark(inputCollectionWatermarks);
+ AppliedPTransformOutputWatermark outputWatermark =
+ new AppliedPTransformOutputWatermark(inputWatermark);
+
+ SynchronizedProcessingTimeInputWatermark inputProcessingWatermark =
+ new SynchronizedProcessingTimeInputWatermark(getInputProcessingWatermarks(transform));
+ SynchronizedProcessingTimeOutputWatermark outputProcessingWatermark =
+ new SynchronizedProcessingTimeOutputWatermark(inputProcessingWatermark);
+
+ wms =
+ new TransformWatermarks(
+ inputWatermark, outputWatermark, inputProcessingWatermark, outputProcessingWatermark);
+ transformToWatermarks.put(transform, wms);
+ }
+ return wms;
+ }
+
+ private Collection<Watermark> getInputProcessingWatermarks(
+ AppliedPTransform<?, ?, ?> transform) {
+ ImmutableList.Builder<Watermark> inputWmsBuilder = ImmutableList.builder();
+ Collection<? extends PValue> inputs = transform.getInput().expand();
+ if (inputs.isEmpty()) {
+ inputWmsBuilder.add(THE_END_OF_TIME);
+ }
+ for (PValue pvalue : inputs) {
+ Watermark producerOutputWatermark =
+ getTransformWatermark(pvalue.getProducingTransformInternal())
+ .synchronizedProcessingOutputWatermark;
+ inputWmsBuilder.add(producerOutputWatermark);
+ }
+ return inputWmsBuilder.build();
+ }
+
+ private List<Watermark> getInputWatermarks(AppliedPTransform<?, ?, ?> transform) {
+ ImmutableList.Builder<Watermark> inputWatermarksBuilder = ImmutableList.builder();
+ Collection<? extends PValue> inputs = transform.getInput().expand();
+ if (inputs.isEmpty()) {
+ inputWatermarksBuilder.add(THE_END_OF_TIME);
+ }
+ for (PValue pvalue : inputs) {
+ Watermark producerOutputWatermark =
+ getTransformWatermark(pvalue.getProducingTransformInternal()).outputWatermark;
+ inputWatermarksBuilder.add(producerOutputWatermark);
+ }
+ List<Watermark> inputCollectionWatermarks = inputWatermarksBuilder.build();
+ return inputCollectionWatermarks;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * Gets the input and output watermarks for an {@link AppliedPTransform}. If the
+ * {@link AppliedPTransform PTransform} has not processed any elements, return a watermark of
+ * {@link BoundedWindow#TIMESTAMP_MIN_VALUE}.
+ *
+ * @return a snapshot of the input watermark and output watermark for the provided transform
+ */
+ public TransformWatermarks getWatermarks(AppliedPTransform<?, ?, ?> transform) {
+ return transformToWatermarks.get(transform);
+ }
+
+ /**
+ * Updates the watermarks of a transform with one or more inputs.
+ *
+ * <p>Each transform has two monotonically increasing watermarks: the input watermark, which can,
+ * at any time, be updated to equal:
+ * <pre>
+ * MAX(CurrentInputWatermark, MIN(PendingElements, InputPCollectionWatermarks))
+ * </pre>
+ * and the output watermark, which can, at any time, be updated to equal:
+ * <pre>
+ * MAX(CurrentOutputWatermark, MIN(InputWatermark, WatermarkHolds))
+ * </pre>.
+ *
+ * @param completed the input that has completed
+ * @param timerUpdate the timers that were added, removed, and completed as part of producing
+ * this update
+ * @param result the result that was produced by processing the input
+ * @param earliestHold the earliest watermark hold in the transform's state. {@code null} if there
+ * is no hold
+ */
+ public void updateWatermarks(
+ @Nullable CommittedBundle<?> completed,
+ TimerUpdate timerUpdate,
+ CommittedResult result,
+ Instant earliestHold) {
+ pendingUpdates.offer(PendingWatermarkUpdate.create(completed,
+ timerUpdate,
+ result,
+ earliestHold));
+ }
+
+ /**
+ * Applies all pending updates to this {@link WatermarkManager}, causing the pending state
+ * of all {@link TransformWatermarks} to be advanced as far as possible.
+ */
+ private void applyPendingUpdates() {
+ Set<AppliedPTransform<?, ?, ?>> updatedTransforms = new HashSet<>();
+ PendingWatermarkUpdate pending = pendingUpdates.poll();
+ while (pending != null) {
+ applyPendingUpdate(pending);
+ updatedTransforms.add(pending.getTransform());
+ pending = pendingUpdates.poll();
+ }
+ pendingRefreshes.addAll(updatedTransforms);
+ }
+
+ private void applyPendingUpdate(PendingWatermarkUpdate pending) {
+ CommittedResult result = pending.getResult();
+ AppliedPTransform transform = result.getTransform();
+ CommittedBundle<?> inputBundle = pending.getInputBundle();
+
+ updatePending(inputBundle, pending.getTimerUpdate(), result);
+
+ TransformWatermarks transformWms = transformToWatermarks.get(transform);
+ transformWms.setEventTimeHold(inputBundle == null ? null : inputBundle.getKey(),
+ pending.getEarliestHold());
+ }
+
+ /**
+ * First adds all produced elements to the queue of pending elements for each consumer, then adds
+ * all pending timers to the collection of pending timers, then removes all completed and deleted
+ * timers from the collection of pending timers, then removes all completed elements from the
+ * pending queue of the transform.
+ *
+ * <p>It is required that all newly pending elements are added to the queue of pending elements
+ * for each consumer prior to the completed elements being removed, as doing otherwise could cause
+ * a Watermark to appear in a state in which the upstream (completed) element does not hold the
+ * watermark but the element it produced is not yet pending. This can cause the watermark to
+ * erroneously advance.
+ */
+ private void updatePending(
+ CommittedBundle<?> input,
+ TimerUpdate timerUpdate,
+ CommittedResult result) {
+ // Newly pending elements must be added before completed elements are removed, as the two
+ // do not share a Mutex within this call and thus can be interleaved with external calls to
+ // refresh.
+ for (CommittedBundle<?> bundle : result.getOutputs()) {
+ for (AppliedPTransform<?, ?, ?> consumer : consumers.get(bundle.getPCollection())) {
+ TransformWatermarks watermarks = transformToWatermarks.get(consumer);
+ watermarks.addPending(bundle);
+ }
+ }
+
+ TransformWatermarks completedTransform = transformToWatermarks.get(result.getTransform());
+ if (input != null) {
+ // Add the unprocessed inputs
+ completedTransform.addPending(result.getUnprocessedInputs());
+ }
+ completedTransform.updateTimers(timerUpdate);
+ if (input != null) {
+ completedTransform.removePending(input);
+ }
+ }
+
+ /**
+ * Refresh the watermarks contained within this {@link WatermarkManager}, causing all
+ * watermarks to be advanced as far as possible.
+ */
+ synchronized void refreshAll() {
+ applyPendingUpdates();
+ while (!pendingRefreshes.isEmpty()) {
+ refreshWatermarks(pendingRefreshes.poll());
+ }
+ }
+
+ private void refreshWatermarks(AppliedPTransform<?, ?, ?> toRefresh) {
+ TransformWatermarks myWatermarks = transformToWatermarks.get(toRefresh);
+ WatermarkUpdate updateResult = myWatermarks.refresh();
+ Set<AppliedPTransform<?, ?, ?>> additionalRefreshes = new HashSet<>();
+ if (updateResult.isAdvanced()) {
+ for (PValue outputPValue : toRefresh.getOutput().expand()) {
+ additionalRefreshes.addAll(consumers.get(outputPValue));
+ }
+ }
+ pendingRefreshes.addAll(additionalRefreshes);
+ }
+
+ /**
+ * Returns a map of each {@link PTransform} that has pending timers to those timers. All of the
+ * pending timers will be removed from this {@link WatermarkManager}.
+ */
+ public Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> extractFiredTimers() {
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> allTimers = new HashMap<>();
+ for (Map.Entry<AppliedPTransform<?, ?, ?>, TransformWatermarks> watermarksEntry :
+ transformToWatermarks.entrySet()) {
+ Map<StructuralKey<?>, FiredTimers> keyFiredTimers =
+ watermarksEntry.getValue().extractFiredTimers();
+ if (!keyFiredTimers.isEmpty()) {
+ allTimers.put(watermarksEntry.getKey(), keyFiredTimers);
+ }
+ }
+ return allTimers;
+ }
+
+ /**
+ * A (key, Instant) pair that holds the watermark. Holds are per-key, but the watermark is global,
+ * and as such the watermark manager must track holds and the release of holds on a per-key basis.
+ *
+ * <p>The {@link #compareTo(KeyedHold)} method of {@link KeyedHold} is not consistent with equals,
+ * as the key is arbitrarily ordered via identity, rather than object equality.
+ */
+ private static final class KeyedHold implements Comparable<KeyedHold> {
+ private static final Ordering<Object> KEY_ORDERING = Ordering.arbitrary().nullsLast();
+
+ private final Object key;
+ private final Instant timestamp;
+
+ /**
+ * Create a new KeyedHold with the specified key and timestamp.
+ */
+ public static KeyedHold of(Object key, Instant timestamp) {
+ return new KeyedHold(key, MoreObjects.firstNonNull(timestamp, THE_END_OF_TIME.get()));
+ }
+
+ private KeyedHold(Object key, Instant timestamp) {
+ this.key = key;
+ this.timestamp = timestamp;
+ }
+
+ @Override
+ public int compareTo(KeyedHold that) {
+ return ComparisonChain.start()
+ .compare(this.timestamp, that.timestamp)
+ .compare(this.key, that.key, KEY_ORDERING)
+ .result();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(timestamp, key);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null || !(other instanceof KeyedHold)) {
+ return false;
+ }
+ KeyedHold that = (KeyedHold) other;
+ return Objects.equals(this.timestamp, that.timestamp) && Objects.equals(this.key, that.key);
+ }
+
+ /**
+ * Get the value of this {@link KeyedHold}.
+ */
+ public Instant getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(KeyedHold.class)
+ .add("key", key)
+ .add("hold", timestamp)
+ .toString();
+ }
+ }
+
+ private static class PerKeyHolds {
+ private final Map<Object, KeyedHold> keyedHolds;
+ private final PriorityQueue<KeyedHold> allHolds;
+
+ private PerKeyHolds() {
+ this.keyedHolds = new HashMap<>();
+ this.allHolds = new PriorityQueue<>();
+ }
+
+ /**
+ * Gets the minimum hold across all keys in this {@link PerKeyHolds}, or THE_END_OF_TIME if
+ * there are no holds within this {@link PerKeyHolds}.
+ */
+ public Instant getMinHold() {
+ return allHolds.isEmpty() ? THE_END_OF_TIME.get() : allHolds.peek().getTimestamp();
+ }
+
+ /**
+ * Updates the hold of the provided key to the provided value, removing any other holds for
+ * the same key.
+ */
+ public void updateHold(@Nullable Object key, Instant newHold) {
+ removeHold(key);
+ KeyedHold newKeyedHold = KeyedHold.of(key, newHold);
+ keyedHolds.put(key, newKeyedHold);
+ allHolds.offer(newKeyedHold);
+ }
+
+ /**
+ * Removes the hold of the provided key.
+ */
+ public void removeHold(Object key) {
+ KeyedHold oldHold = keyedHolds.get(key);
+ if (oldHold != null) {
+ allHolds.remove(oldHold);
+ }
+ }
+ }
+
+ /**
+ * A reference to the input and output watermarks of an {@link AppliedPTransform}.
+ */
+ public class TransformWatermarks {
+ private final AppliedPTransformInputWatermark inputWatermark;
+ private final AppliedPTransformOutputWatermark outputWatermark;
+
+ private final SynchronizedProcessingTimeInputWatermark synchronizedProcessingInputWatermark;
+ private final SynchronizedProcessingTimeOutputWatermark synchronizedProcessingOutputWatermark;
+
+ private Instant latestSynchronizedInputWm;
+ private Instant latestSynchronizedOutputWm;
+
+ private TransformWatermarks(
+ AppliedPTransformInputWatermark inputWatermark,
+ AppliedPTransformOutputWatermark outputWatermark,
+ SynchronizedProcessingTimeInputWatermark inputSynchProcessingWatermark,
+ SynchronizedProcessingTimeOutputWatermark outputSynchProcessingWatermark) {
+ this.inputWatermark = inputWatermark;
+ this.outputWatermark = outputWatermark;
+
+ this.synchronizedProcessingInputWatermark = inputSynchProcessingWatermark;
+ this.synchronizedProcessingOutputWatermark = outputSynchProcessingWatermark;
+ this.latestSynchronizedInputWm = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ this.latestSynchronizedOutputWm = BoundedWindow.TIMESTAMP_MIN_VALUE;
+ }
+
+ /**
+ * Returns the input watermark of the {@link AppliedPTransform}.
+ */
+ public Instant getInputWatermark() {
+ return Preconditions.checkNotNull(inputWatermark.get());
+ }
+
+ /**
+ * Returns the output watermark of the {@link AppliedPTransform}.
+ */
+ public Instant getOutputWatermark() {
+ return outputWatermark.get();
+ }
+
+ /**
+ * Returns the synchronized processing input time of the {@link AppliedPTransform}.
+ *
+ * <p>The returned value is guaranteed to be monotonically increasing, and outside of the
+ * presence of holds, will increase as the system time progresses.
+ */
+ public synchronized Instant getSynchronizedProcessingInputTime() {
+ latestSynchronizedInputWm = INSTANT_ORDERING.max(
+ latestSynchronizedInputWm,
+ INSTANT_ORDERING.min(clock.now(), synchronizedProcessingInputWatermark.get()));
+ return latestSynchronizedInputWm;
+ }
+
+ /**
+ * Returns the synchronized processing output time of the {@link AppliedPTransform}.
+ *
+ * <p>The returned value is guaranteed to be monotonically increasing, and outside of the
+ * presence of holds, will increase as the system time progresses.
+ */
+ public synchronized Instant getSynchronizedProcessingOutputTime() {
+ latestSynchronizedOutputWm = INSTANT_ORDERING.max(
+ latestSynchronizedOutputWm,
+ INSTANT_ORDERING.min(clock.now(), synchronizedProcessingOutputWatermark.get()));
+ return latestSynchronizedOutputWm;
+ }
+
+ private WatermarkUpdate refresh() {
+ inputWatermark.refresh();
+ synchronizedProcessingInputWatermark.refresh();
+ WatermarkUpdate eventOutputUpdate = outputWatermark.refresh();
+ WatermarkUpdate syncOutputUpdate = synchronizedProcessingOutputWatermark.refresh();
+ return eventOutputUpdate.union(syncOutputUpdate);
+ }
+
+ private void setEventTimeHold(Object key, Instant newHold) {
+ outputWatermark.updateHold(key, newHold);
+ }
+
+ private void removePending(CommittedBundle<?> bundle) {
+ inputWatermark.removePendingElements(elementsFromBundle(bundle));
+ synchronizedProcessingInputWatermark.removePending(bundle);
+ }
+
+ private void addPending(CommittedBundle<?> bundle) {
+ inputWatermark.addPendingElements(elementsFromBundle(bundle));
+ synchronizedProcessingInputWatermark.addPending(bundle);
+ }
+
+ private Iterable<? extends WindowedValue<?>> elementsFromBundle(CommittedBundle<?> bundle) {
+ return FluentIterable.from(bundle.getElements()).transformAndConcat(EXPLODE_WINDOWS_FN);
+ }
+
+ private Map<StructuralKey<?>, FiredTimers> extractFiredTimers() {
+ Map<StructuralKey<?>, List<TimerData>> eventTimeTimers =
+ inputWatermark.extractFiredEventTimeTimers();
+ Map<StructuralKey<?>, List<TimerData>> processingTimers;
+ Map<StructuralKey<?>, List<TimerData>> synchronizedTimers;
+ if (inputWatermark.get().equals(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
+ processingTimers = synchronizedProcessingInputWatermark.extractFiredDomainTimers(
+ TimeDomain.PROCESSING_TIME, BoundedWindow.TIMESTAMP_MAX_VALUE);
+ synchronizedTimers = synchronizedProcessingInputWatermark.extractFiredDomainTimers(
+ TimeDomain.PROCESSING_TIME, BoundedWindow.TIMESTAMP_MAX_VALUE);
+ } else {
+ processingTimers = synchronizedProcessingInputWatermark.extractFiredDomainTimers(
+ TimeDomain.PROCESSING_TIME, clock.now());
+ synchronizedTimers = synchronizedProcessingInputWatermark.extractFiredDomainTimers(
+ TimeDomain.SYNCHRONIZED_PROCESSING_TIME, getSynchronizedProcessingInputTime());
+ }
+ Map<StructuralKey<?>, Map<TimeDomain, List<TimerData>>> groupedTimers = new HashMap<>();
+ groupFiredTimers(groupedTimers, eventTimeTimers, processingTimers, synchronizedTimers);
+
+ Map<StructuralKey<?>, FiredTimers> keyFiredTimers = new HashMap<>();
+ for (Map.Entry<StructuralKey<?>, Map<TimeDomain, List<TimerData>>> firedTimers :
+ groupedTimers.entrySet()) {
+ keyFiredTimers.put(firedTimers.getKey(), new FiredTimers(firedTimers.getValue()));
+ }
+ return keyFiredTimers;
+ }
+
+ @SafeVarargs
+ private final void groupFiredTimers(
+ Map<StructuralKey<?>, Map<TimeDomain, List<TimerData>>> groupedToMutate,
+ Map<StructuralKey<?>, List<TimerData>>... timersToGroup) {
+ for (Map<StructuralKey<?>, List<TimerData>> subGroup : timersToGroup) {
+ for (Map.Entry<StructuralKey<?>, List<TimerData>> newTimers : subGroup.entrySet()) {
+ Map<TimeDomain, List<TimerData>> grouped = groupedToMutate.get(newTimers.getKey());
+ if (grouped == null) {
+ grouped = new HashMap<>();
+ groupedToMutate.put(newTimers.getKey(), grouped);
+ }
+ grouped.put(newTimers.getValue().get(0).getDomain(), newTimers.getValue());
+ }
+ }
+ }
+
+ private void updateTimers(TimerUpdate update) {
+ inputWatermark.updateTimers(update);
+ synchronizedProcessingInputWatermark.updateTimers(update);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(TransformWatermarks.class)
+ .add("inputWatermark", inputWatermark)
+ .add("outputWatermark", outputWatermark)
+ .add("inputProcessingTime", synchronizedProcessingInputWatermark)
+ .add("outputProcessingTime", synchronizedProcessingOutputWatermark)
+ .toString();
+ }
+ }
+
+ /**
+ * A collection of newly set, deleted, and completed timers.
+ *
+ * <p>setTimers and deletedTimers are collections of {@link TimerData} that have been added to the
+ * {@link TimerInternals} of an executed step. completedTimers are timers that were delivered as
+ * the input to the executed step.
+ */
+ public static class TimerUpdate {
+ private final StructuralKey<?> key;
+ private final Iterable<? extends TimerData> completedTimers;
+
+ private final Iterable<? extends TimerData> setTimers;
+ private final Iterable<? extends TimerData> deletedTimers;
+
+ /**
+ * Returns a TimerUpdate for a null key with no timers.
+ */
+ public static TimerUpdate empty() {
+ return new TimerUpdate(
+ null,
+ Collections.<TimerData>emptyList(),
+ Collections.<TimerData>emptyList(),
+ Collections.<TimerData>emptyList());
+ }
+
+ /**
+ * Creates a new {@link TimerUpdate} builder with the provided completed timers that needs the
+ * set and deleted timers to be added to it.
+ */
+ public static TimerUpdateBuilder builder(StructuralKey<?> key) {
+ return new TimerUpdateBuilder(key);
+ }
+
+ /**
+ * A {@link TimerUpdate} builder that needs to be provided with set timers and deleted timers.
+ */
+ public static final class TimerUpdateBuilder {
+ private final StructuralKey<?> key;
+ private final Collection<TimerData> completedTimers;
+ private final Collection<TimerData> setTimers;
+ private final Collection<TimerData> deletedTimers;
+
+ private TimerUpdateBuilder(StructuralKey<?> key) {
+ this.key = key;
+ this.completedTimers = new HashSet<>();
+ this.setTimers = new HashSet<>();
+ this.deletedTimers = new HashSet<>();
+ }
+
+ /**
+ * Adds all of the provided timers to the collection of completed timers, and returns this
+ * {@link TimerUpdateBuilder}.
+ */
+ public TimerUpdateBuilder withCompletedTimers(Iterable<TimerData> completedTimers) {
+ Iterables.addAll(this.completedTimers, completedTimers);
+ return this;
+ }
+
+ /**
+ * Adds the provided timer to the collection of set timers, removing it from deleted timers if
+ * it has previously been deleted. Returns this {@link TimerUpdateBuilder}.
+ */
+ public TimerUpdateBuilder setTimer(TimerData setTimer) {
+ deletedTimers.remove(setTimer);
+ setTimers.add(setTimer);
+ return this;
+ }
+
+ /**
+ * Adds the provided timer to the collection of deleted timers, removing it from set timers if
+ * it has previously been set. Returns this {@link TimerUpdateBuilder}.
+ */
+ public TimerUpdateBuilder deletedTimer(TimerData deletedTimer) {
+ deletedTimers.add(deletedTimer);
+ setTimers.remove(deletedTimer);
+ return this;
+ }
+
+ /**
+ * Returns a new {@link TimerUpdate} with the most recently set completedTimers, setTimers,
+ * and deletedTimers.
+ */
+ public TimerUpdate build() {
+ return new TimerUpdate(
+ key,
+ ImmutableSet.copyOf(completedTimers),
+ ImmutableSet.copyOf(setTimers),
+ ImmutableSet.copyOf(deletedTimers));
+ }
+ }
+
+ private TimerUpdate(
+ StructuralKey<?> key,
+ Iterable<? extends TimerData> completedTimers,
+ Iterable<? extends TimerData> setTimers,
+ Iterable<? extends TimerData> deletedTimers) {
+ this.key = key;
+ this.completedTimers = completedTimers;
+ this.setTimers = setTimers;
+ this.deletedTimers = deletedTimers;
+ }
+
+ @VisibleForTesting
+ StructuralKey<?> getKey() {
+ return key;
+ }
+
+ @VisibleForTesting
+ Iterable<? extends TimerData> getCompletedTimers() {
+ return completedTimers;
+ }
+
+ @VisibleForTesting
+ Iterable<? extends TimerData> getSetTimers() {
+ return setTimers;
+ }
+
+ @VisibleForTesting
+ Iterable<? extends TimerData> getDeletedTimers() {
+ return deletedTimers;
+ }
+
+ /**
+ * Returns a {@link TimerUpdate} that is like this one, but with the specified completed timers.
+ */
+ public TimerUpdate withCompletedTimers(Iterable<TimerData> completedTimers) {
+ return new TimerUpdate(this.key, completedTimers, setTimers, deletedTimers);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(key, completedTimers, setTimers, deletedTimers);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null || !(other instanceof TimerUpdate)) {
+ return false;
+ }
+ TimerUpdate that = (TimerUpdate) other;
+ return Objects.equals(this.key, that.key)
+ && Objects.equals(this.completedTimers, that.completedTimers)
+ && Objects.equals(this.setTimers, that.setTimers)
+ && Objects.equals(this.deletedTimers, that.deletedTimers);
+ }
+ }
+
+ /**
+ * A pair of {@link TimerData} and key which can be delivered to the appropriate
+ * {@link AppliedPTransform}. A timer fires at the transform that set it with a specific key when
+ * the time domain in which it lives progresses past a specified time, as determined by the
+ * {@link WatermarkManager}.
+ */
+ public static class FiredTimers {
+ private final Map<TimeDomain, ? extends Collection<TimerData>> timers;
+
+ private FiredTimers(Map<TimeDomain, ? extends Collection<TimerData>> timers) {
+ this.timers = timers;
+ }
+
+ /**
+ * Gets all of the timers that have fired within the provided {@link TimeDomain}. If no timers
+ * fired within the provided domain, return an empty collection.
+ *
+ * <p>Timers within a {@link TimeDomain} are guaranteed to be in order of increasing timestamp.
+ */
+ public Collection<TimerData> getTimers(TimeDomain domain) {
+ Collection<TimerData> domainTimers = timers.get(domain);
+ if (domainTimers == null) {
+ return Collections.emptyList();
+ }
+ return domainTimers;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(FiredTimers.class).add("timers", timers).toString();
+ }
+ }
+
+ private static class WindowedValueByTimestampComparator extends Ordering<WindowedValue<?>> {
+ @Override
+ public int compare(WindowedValue<?> o1, WindowedValue<?> o2) {
+ return ComparisonChain.start()
+ .compare(o1.getTimestamp(), o2.getTimestamp())
+ .result();
+ }
+ }
+
+ public Set<AppliedPTransform<?, ?, ?>> getCompletedTransforms() {
+ Set<AppliedPTransform<?, ?, ?>> result = new HashSet<>();
+ for (Map.Entry<AppliedPTransform<?, ?, ?>, TransformWatermarks> wms :
+ transformToWatermarks.entrySet()) {
+ if (wms.getValue().getOutputWatermark().equals(THE_END_OF_TIME.get())) {
+ result.add(wms.getKey());
+ }
+ }
+ return result;
+ }
+
+ @AutoValue
+ abstract static class PendingWatermarkUpdate {
+ @Nullable
+ public abstract CommittedBundle<?> getInputBundle();
+ public abstract TimerUpdate getTimerUpdate();
+ public abstract CommittedResult getResult();
+ public abstract Instant getEarliestHold();
+
+ /**
+ * Gets the {@link AppliedPTransform} that generated this result.
+ */
+ public AppliedPTransform<?, ?, ?> getTransform() {
+ return getResult().getTransform();
+ }
+
+ public static PendingWatermarkUpdate create(
+ CommittedBundle<?> inputBundle,
+ TimerUpdate timerUpdate,
+ CommittedResult result, Instant earliestHold) {
+ return new AutoValue_WatermarkManager_PendingWatermarkUpdate(inputBundle,
+ timerUpdate,
+ result,
+ earliestHold);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
index 89866cc..b07b58a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
@@ -45,7 +45,7 @@ class WindowEvaluatorFactory implements TransformEvaluatorFactory {
public <InputT> TransformEvaluator<InputT> forApplication(
AppliedPTransform<?, ?, ?> application,
@Nullable CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext)
+ EvaluationContext evaluationContext)
throws Exception {
return createTransformEvaluator(
(AppliedPTransform) application, inputBundle, evaluationContext);
@@ -54,7 +54,7 @@ class WindowEvaluatorFactory implements TransformEvaluatorFactory {
private <InputT> TransformEvaluator<InputT> createTransformEvaluator(
AppliedPTransform<PCollection<InputT>, PCollection<InputT>, Window.Bound<InputT>> transform,
CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext) {
+ EvaluationContext evaluationContext) {
WindowFn<? super InputT, ?> fn = transform.getTransform().getWindowFn();
UncommittedBundle<InputT> outputBundle =
evaluationContext.createBundle(inputBundle, transform.getOutput());
@@ -92,22 +92,22 @@ class WindowEvaluatorFactory implements TransformEvaluatorFactory {
private <W extends BoundedWindow> Collection<? extends BoundedWindow> assignWindows(
WindowFn<InputT, W> windowFn, WindowedValue<InputT> element) throws Exception {
WindowFn<InputT, W>.AssignContext assignContext =
- new InProcessAssignContext<>(windowFn, element);
+ new DirectAssignContext<>(windowFn, element);
Collection<? extends BoundedWindow> windows = windowFn.assignWindows(assignContext);
return windows;
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
return StepTransformResult.withoutHold(transform).addOutput(outputBundle).build();
}
}
- private static class InProcessAssignContext<InputT, W extends BoundedWindow>
+ private static class DirectAssignContext<InputT, W extends BoundedWindow>
extends WindowFn<InputT, W>.AssignContext {
private final WindowedValue<InputT> value;
- public InProcessAssignContext(WindowFn<InputT, W> fn, WindowedValue<InputT> value) {
+ public DirectAssignContext(WindowFn<InputT, W> fn, WindowedValue<InputT> value) {
fn.super();
this.value = value;
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
index e26f860..9bc4f7b 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactoryTest.java
@@ -64,7 +64,7 @@ public class BoundedReadEvaluatorFactoryTest {
private BoundedSource<Long> source;
private PCollection<Long> longs;
private TransformEvaluatorFactory factory;
- @Mock private InProcessEvaluationContext context;
+ @Mock private EvaluationContext context;
private BundleFactory bundleFactory;
@Before
@@ -75,7 +75,7 @@ public class BoundedReadEvaluatorFactoryTest {
longs = p.apply(Read.from(source));
factory = new BoundedReadEvaluatorFactory();
- bundleFactory = InProcessBundleFactory.create();
+ bundleFactory = ImmutableListBundleFactory.create();
}
@Test
@@ -85,7 +85,7 @@ public class BoundedReadEvaluatorFactoryTest {
TransformEvaluator<?> evaluator =
factory.forApplication(longs.getProducingTransformInternal(), null, context);
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(result.getWatermarkHold(), equalTo(BoundedWindow.TIMESTAMP_MAX_VALUE));
assertThat(
output.commit(BoundedWindow.TIMESTAMP_MAX_VALUE).getElements(),
@@ -105,7 +105,7 @@ public class BoundedReadEvaluatorFactoryTest {
TransformEvaluator<?> evaluator =
factory.forApplication(longs.getProducingTransformInternal(), null, context);
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(result.getWatermarkHold(), equalTo(BoundedWindow.TIMESTAMP_MAX_VALUE));
Iterable<? extends WindowedValue<Long>> outputElements =
output.commit(BoundedWindow.TIMESTAMP_MAX_VALUE).getElements();
@@ -138,7 +138,7 @@ public class BoundedReadEvaluatorFactoryTest {
factory.forApplication(longs.getProducingTransformInternal(), null, context);
assertThat(secondEvaluator, nullValue());
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(result.getWatermarkHold(), equalTo(BoundedWindow.TIMESTAMP_MAX_VALUE));
Iterable<? extends WindowedValue<Long>> outputElements =
output.commit(BoundedWindow.TIMESTAMP_MAX_VALUE).getElements();
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
index 4969a30..1e51b55 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/CommittedResultTest.java
@@ -53,7 +53,7 @@ public class CommittedResultTest implements Serializable {
private transient AppliedPTransform<?, ?, ?> transform =
AppliedPTransform.of("foo", p.begin(), PDone.in(p), new PTransform<PBegin, PDone>() {
});
- private transient BundleFactory bundleFactory = InProcessBundleFactory.create();
+ private transient BundleFactory bundleFactory = ImmutableListBundleFactory.create();
@Test
public void getTransformExtractsFromResult() {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java
index cd44b7e..5c89f1b 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRegistrarTest.java
@@ -20,7 +20,7 @@ package org.apache.beam.runners.direct;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
-import org.apache.beam.runners.direct.DirectRegistrar.InProcessRunner;
+import org.apache.beam.runners.direct.DirectRegistrar.DirectRunner;
import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
@@ -33,42 +33,42 @@ import org.junit.runners.JUnit4;
import java.util.ServiceLoader;
-/** Tests for {@link InProcessRunner}. */
+/** Tests for {@link DirectRunner}. */
@RunWith(JUnit4.class)
public class DirectRegistrarTest {
@Test
public void testCorrectOptionsAreReturned() {
assertEquals(
ImmutableList.of(DirectOptions.class),
- new DirectRegistrar.InProcessOptions().getPipelineOptions());
+ new DirectRegistrar.DirectOptions().getPipelineOptions());
}
@Test
public void testCorrectRunnersAreReturned() {
assertEquals(
- ImmutableList.of(DirectRunner.class),
- new DirectRegistrar.InProcessRunner().getPipelineRunners());
+ ImmutableList.of(org.apache.beam.runners.direct.DirectRunner.class),
+ new DirectRunner().getPipelineRunners());
}
@Test
public void testServiceLoaderForOptions() {
for (PipelineOptionsRegistrar registrar :
Lists.newArrayList(ServiceLoader.load(PipelineOptionsRegistrar.class).iterator())) {
- if (registrar instanceof DirectRegistrar.InProcessOptions) {
+ if (registrar instanceof DirectRegistrar.DirectOptions) {
return;
}
}
- fail("Expected to find " + DirectRegistrar.InProcessOptions.class);
+ fail("Expected to find " + DirectRegistrar.DirectOptions.class);
}
@Test
public void testServiceLoaderForRunner() {
for (PipelineRunnerRegistrar registrar :
Lists.newArrayList(ServiceLoader.load(PipelineRunnerRegistrar.class).iterator())) {
- if (registrar instanceof DirectRegistrar.InProcessRunner) {
+ if (registrar instanceof DirectRunner) {
return;
}
}
- fail("Expected to find " + DirectRegistrar.InProcessRunner.class);
+ fail("Expected to find " + DirectRunner.class);
}
}
[05/12] incubator-beam git commit: Remove InProcess Prefixes
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessEvaluationContextTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessEvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessEvaluationContextTest.java
deleted file mode 100644
index b1cbeb1..0000000
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessEvaluationContextTest.java
+++ /dev/null
@@ -1,545 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.emptyIterable;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
-import static org.junit.Assert.assertThat;
-
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.FiredTimers;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.InProcessExecutionContext.InProcessStepContext;
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.runners.direct.DirectRunner.PCollectionViewWriter;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.sdk.coders.ByteArrayCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.io.CountingInput;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
-import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.TimerInternals.TimerData;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.common.Counter;
-import org.apache.beam.sdk.util.common.Counter.AggregationKind;
-import org.apache.beam.sdk.util.common.CounterSet;
-import org.apache.beam.sdk.util.state.BagState;
-import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
-import org.apache.beam.sdk.util.state.StateNamespaces;
-import org.apache.beam.sdk.util.state.StateTag;
-import org.apache.beam.sdk.util.state.StateTags;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollection.IsBounded;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PValue;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-
-import org.hamcrest.Matchers;
-import org.joda.time.Instant;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Tests for {@link InProcessEvaluationContext}.
- */
-@RunWith(JUnit4.class)
-public class InProcessEvaluationContextTest {
- private TestPipeline p;
- private InProcessEvaluationContext context;
-
- private PCollection<Integer> created;
- private PCollection<KV<String, Integer>> downstream;
- private PCollectionView<Iterable<Integer>> view;
- private PCollection<Long> unbounded;
- private Collection<AppliedPTransform<?, ?, ?>> rootTransforms;
- private Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> valueToConsumers;
-
- private BundleFactory bundleFactory;
-
- @Before
- public void setup() {
- DirectRunner runner =
- DirectRunner.fromOptions(PipelineOptionsFactory.create());
-
- p = TestPipeline.create();
-
- created = p.apply(Create.of(1, 2, 3));
- downstream = created.apply(WithKeys.<String, Integer>of("foo"));
- view = created.apply(View.<Integer>asIterable());
- unbounded = p.apply(CountingInput.unbounded());
-
- ConsumerTrackingPipelineVisitor cVis = new ConsumerTrackingPipelineVisitor();
- p.traverseTopologically(cVis);
- rootTransforms = cVis.getRootTransforms();
- valueToConsumers = cVis.getValueToConsumers();
-
- bundleFactory = InProcessBundleFactory.create();
-
- context =
- InProcessEvaluationContext.create(
- runner.getPipelineOptions(),
- InProcessBundleFactory.create(),
- rootTransforms,
- valueToConsumers,
- cVis.getStepNames(),
- cVis.getViews());
- }
-
- @Test
- public void writeToViewWriterThenReadReads() {
- PCollectionViewWriter<Integer, Iterable<Integer>> viewWriter =
- context.createPCollectionViewWriter(
- PCollection.<Iterable<Integer>>createPrimitiveOutputInternal(
- p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED),
- view);
- BoundedWindow window = new TestBoundedWindow(new Instant(1024L));
- BoundedWindow second = new TestBoundedWindow(new Instant(899999L));
- WindowedValue<Integer> firstValue =
- WindowedValue.of(1, new Instant(1222), window, PaneInfo.ON_TIME_AND_ONLY_FIRING);
- WindowedValue<Integer> secondValue =
- WindowedValue.of(
- 2, new Instant(8766L), second, PaneInfo.createPane(true, false, Timing.ON_TIME, 0, 0));
- Iterable<WindowedValue<Integer>> values = ImmutableList.of(firstValue, secondValue);
- viewWriter.add(values);
-
- SideInputReader reader =
- context.createSideInputReader(ImmutableList.<PCollectionView<?>>of(view));
- assertThat(reader.get(view, window), containsInAnyOrder(1));
- assertThat(reader.get(view, second), containsInAnyOrder(2));
-
- WindowedValue<Integer> overrittenSecondValue =
- WindowedValue.of(
- 4444, new Instant(8677L), second, PaneInfo.createPane(false, true, Timing.LATE, 1, 1));
- viewWriter.add(Collections.singleton(overrittenSecondValue));
- assertThat(reader.get(view, second), containsInAnyOrder(2));
- // The cached value is served in the earlier reader
- reader = context.createSideInputReader(ImmutableList.<PCollectionView<?>>of(view));
- assertThat(reader.get(view, second), containsInAnyOrder(4444));
- }
-
- @Test
- public void getExecutionContextSameStepSameKeyState() {
- InProcessExecutionContext fooContext =
- context.getExecutionContext(created.getProducingTransformInternal(),
- StructuralKey.of("foo", StringUtf8Coder.of()));
-
- StateTag<Object, BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
-
- InProcessStepContext stepContext = fooContext.getOrCreateStepContext("s1", "s1");
- stepContext.stateInternals().state(StateNamespaces.global(), intBag).add(1);
-
- context.handleResult(InProcessBundleFactory.create()
- .createKeyedBundle(null, StructuralKey.of("foo", StringUtf8Coder.of()), created)
- .commit(Instant.now()),
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(created.getProducingTransformInternal())
- .withState(stepContext.commitState())
- .build());
-
- InProcessExecutionContext secondFooContext =
- context.getExecutionContext(created.getProducingTransformInternal(),
- StructuralKey.of("foo", StringUtf8Coder.of()));
- assertThat(
- secondFooContext
- .getOrCreateStepContext("s1", "s1")
- .stateInternals()
- .state(StateNamespaces.global(), intBag)
- .read(),
- contains(1));
- }
-
-
- @Test
- public void getExecutionContextDifferentKeysIndependentState() {
- InProcessExecutionContext fooContext =
- context.getExecutionContext(created.getProducingTransformInternal(),
- StructuralKey.of("foo", StringUtf8Coder.of()));
-
- StateTag<Object, BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
-
- fooContext
- .getOrCreateStepContext("s1", "s1")
- .stateInternals()
- .state(StateNamespaces.global(), intBag)
- .add(1);
-
- InProcessExecutionContext barContext =
- context.getExecutionContext(created.getProducingTransformInternal(),
- StructuralKey.of("bar", StringUtf8Coder.of()));
- assertThat(barContext, not(equalTo(fooContext)));
- assertThat(
- barContext
- .getOrCreateStepContext("s1", "s1")
- .stateInternals()
- .state(StateNamespaces.global(), intBag)
- .read(),
- emptyIterable());
- }
-
- @Test
- public void getExecutionContextDifferentStepsIndependentState() {
- StructuralKey<?> myKey = StructuralKey.of("foo", StringUtf8Coder.of());
- InProcessExecutionContext fooContext =
- context.getExecutionContext(created.getProducingTransformInternal(), myKey);
-
- StateTag<Object, BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
-
- fooContext
- .getOrCreateStepContext("s1", "s1")
- .stateInternals()
- .state(StateNamespaces.global(), intBag)
- .add(1);
-
- InProcessExecutionContext barContext =
- context.getExecutionContext(downstream.getProducingTransformInternal(), myKey);
- assertThat(
- barContext
- .getOrCreateStepContext("s1", "s1")
- .stateInternals()
- .state(StateNamespaces.global(), intBag)
- .read(),
- emptyIterable());
- }
-
- @Test
- public void handleResultMergesCounters() {
- CounterSet counters = context.createCounterSet();
- Counter<Long> myCounter = Counter.longs("foo", AggregationKind.SUM);
- counters.addCounter(myCounter);
-
- myCounter.addValue(4L);
- InProcessTransformResult result =
- StepTransformResult.withoutHold(created.getProducingTransformInternal())
- .withCounters(counters)
- .build();
- context.handleResult(null, ImmutableList.<TimerData>of(), result);
- assertThat((Long) context.getCounters().getExistingCounter("foo").getAggregate(), equalTo(4L));
-
- CounterSet againCounters = context.createCounterSet();
- Counter<Long> myLongCounterAgain = Counter.longs("foo", AggregationKind.SUM);
- againCounters.add(myLongCounterAgain);
- myLongCounterAgain.addValue(8L);
-
- InProcessTransformResult secondResult =
- StepTransformResult.withoutHold(downstream.getProducingTransformInternal())
- .withCounters(againCounters)
- .build();
- context.handleResult(
- context.createRootBundle(created).commit(Instant.now()),
- ImmutableList.<TimerData>of(),
- secondResult);
- assertThat((Long) context.getCounters().getExistingCounter("foo").getAggregate(), equalTo(12L));
- }
-
- @Test
- public void handleResultStoresState() {
- StructuralKey<?> myKey = StructuralKey.of("foo".getBytes(), ByteArrayCoder.of());
- InProcessExecutionContext fooContext =
- context.getExecutionContext(downstream.getProducingTransformInternal(), myKey);
-
- StateTag<Object, BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
-
- CopyOnAccessInMemoryStateInternals<Object> state =
- fooContext.getOrCreateStepContext("s1", "s1").stateInternals();
- BagState<Integer> bag = state.state(StateNamespaces.global(), intBag);
- bag.add(1);
- bag.add(2);
- bag.add(4);
-
- InProcessTransformResult stateResult =
- StepTransformResult.withoutHold(downstream.getProducingTransformInternal())
- .withState(state)
- .build();
-
- context.handleResult(
- context.createKeyedBundle(null, myKey, created).commit(Instant.now()),
- ImmutableList.<TimerData>of(),
- stateResult);
-
- InProcessExecutionContext afterResultContext =
- context.getExecutionContext(downstream.getProducingTransformInternal(), myKey);
-
- CopyOnAccessInMemoryStateInternals<Object> afterResultState =
- afterResultContext.getOrCreateStepContext("s1", "s1").stateInternals();
- assertThat(afterResultState.state(StateNamespaces.global(), intBag).read(), contains(1, 2, 4));
- }
-
- @Test
- public void callAfterOutputMustHaveBeenProducedAfterEndOfWatermarkCallsback() throws Exception {
- final CountDownLatch callLatch = new CountDownLatch(1);
- Runnable callback =
- new Runnable() {
- @Override
- public void run() {
- callLatch.countDown();
- }
- };
-
- // Should call back after the end of the global window
- context.scheduleAfterOutputWouldBeProduced(
- downstream, GlobalWindow.INSTANCE, WindowingStrategy.globalDefault(), callback);
-
- InProcessTransformResult result =
- StepTransformResult.withHold(created.getProducingTransformInternal(), new Instant(0))
- .build();
-
- context.handleResult(null, ImmutableList.<TimerData>of(), result);
- // Difficult to demonstrate that we took no action in a multithreaded world; poll for a bit
- // will likely be flaky if this logic is broken
- assertThat(callLatch.await(500L, TimeUnit.MILLISECONDS), is(false));
-
- InProcessTransformResult finishedResult =
- StepTransformResult.withoutHold(created.getProducingTransformInternal()).build();
- context.handleResult(null, ImmutableList.<TimerData>of(), finishedResult);
- context.forceRefresh();
- // Obtain the value via blocking call
- assertThat(callLatch.await(1, TimeUnit.SECONDS), is(true));
- }
-
- @Test
- public void callAfterOutputMustHaveBeenProducedAlreadyAfterCallsImmediately() throws Exception {
- InProcessTransformResult finishedResult =
- StepTransformResult.withoutHold(created.getProducingTransformInternal()).build();
- context.handleResult(null, ImmutableList.<TimerData>of(), finishedResult);
-
- final CountDownLatch callLatch = new CountDownLatch(1);
- context.extractFiredTimers();
- Runnable callback =
- new Runnable() {
- @Override
- public void run() {
- callLatch.countDown();
- }
- };
- context.scheduleAfterOutputWouldBeProduced(
- downstream, GlobalWindow.INSTANCE, WindowingStrategy.globalDefault(), callback);
- assertThat(callLatch.await(1, TimeUnit.SECONDS), is(true));
- }
-
- @Test
- public void extractFiredTimersExtractsTimers() {
- InProcessTransformResult holdResult =
- StepTransformResult.withHold(created.getProducingTransformInternal(), new Instant(0))
- .build();
- context.handleResult(null, ImmutableList.<TimerData>of(), holdResult);
-
- StructuralKey<?> key = StructuralKey.of("foo".length(), VarIntCoder.of());
- TimerData toFire =
- TimerData.of(StateNamespaces.global(), new Instant(100L), TimeDomain.EVENT_TIME);
- InProcessTransformResult timerResult =
- StepTransformResult.withoutHold(downstream.getProducingTransformInternal())
- .withState(CopyOnAccessInMemoryStateInternals.withUnderlying(key, null))
- .withTimerUpdate(TimerUpdate.builder(key).setTimer(toFire).build())
- .build();
-
- // haven't added any timers, must be empty
- assertThat(context.extractFiredTimers().entrySet(), emptyIterable());
- context.handleResult(
- context.createKeyedBundle(null, key, created).commit(Instant.now()),
- ImmutableList.<TimerData>of(),
- timerResult);
-
- // timer hasn't fired
- assertThat(context.extractFiredTimers().entrySet(), emptyIterable());
-
- InProcessTransformResult advanceResult =
- StepTransformResult.withoutHold(created.getProducingTransformInternal()).build();
- // Should cause the downstream timer to fire
- context.handleResult(null, ImmutableList.<TimerData>of(), advanceResult);
-
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> fired =
- context.extractFiredTimers();
- assertThat(
- fired,
- Matchers.<AppliedPTransform<?, ?, ?>>hasKey(downstream.getProducingTransformInternal()));
- Map<StructuralKey<?>, FiredTimers> downstreamFired =
- fired.get(downstream.getProducingTransformInternal());
- assertThat(downstreamFired, Matchers.<Object>hasKey(key));
-
- FiredTimers firedForKey = downstreamFired.get(key);
- assertThat(firedForKey.getTimers(TimeDomain.PROCESSING_TIME), emptyIterable());
- assertThat(firedForKey.getTimers(TimeDomain.SYNCHRONIZED_PROCESSING_TIME), emptyIterable());
- assertThat(firedForKey.getTimers(TimeDomain.EVENT_TIME), contains(toFire));
-
- // Don't reextract timers
- assertThat(context.extractFiredTimers().entrySet(), emptyIterable());
- }
-
- @Test
- public void createBundleKeyedResultPropagatesKey() {
- StructuralKey<String> key = StructuralKey.of("foo", StringUtf8Coder.of());
- CommittedBundle<KV<String, Integer>> newBundle =
- context
- .createBundle(
- bundleFactory.createKeyedBundle(
- null, key,
- created).commit(Instant.now()),
- downstream).commit(Instant.now());
- assertThat(newBundle.getKey(), Matchers.<StructuralKey<?>>equalTo(key));
- }
-
- @Test
- public void createKeyedBundleKeyed() {
- StructuralKey<String> key = StructuralKey.of("foo", StringUtf8Coder.of());
- CommittedBundle<KV<String, Integer>> keyedBundle =
- context.createKeyedBundle(
- bundleFactory.createRootBundle(created).commit(Instant.now()),
- key,
- downstream).commit(Instant.now());
- assertThat(keyedBundle.getKey(),
- Matchers.<StructuralKey<?>>equalTo(key));
- }
-
- @Test
- public void isDoneWithUnboundedPCollectionAndShutdown() {
- context.getPipelineOptions().setShutdownUnboundedProducersWithMaxWatermark(true);
- assertThat(context.isDone(unbounded.getProducingTransformInternal()), is(false));
-
- context.handleResult(
- null,
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(unbounded.getProducingTransformInternal()).build());
- context.extractFiredTimers();
- assertThat(context.isDone(unbounded.getProducingTransformInternal()), is(true));
- }
-
- @Test
- public void isDoneWithUnboundedPCollectionAndNotShutdown() {
- context.getPipelineOptions().setShutdownUnboundedProducersWithMaxWatermark(false);
- assertThat(context.isDone(unbounded.getProducingTransformInternal()), is(false));
-
- context.handleResult(
- null,
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(unbounded.getProducingTransformInternal()).build());
- assertThat(context.isDone(unbounded.getProducingTransformInternal()), is(false));
- }
-
- @Test
- public void isDoneWithOnlyBoundedPCollections() {
- context.getPipelineOptions().setShutdownUnboundedProducersWithMaxWatermark(false);
- assertThat(context.isDone(created.getProducingTransformInternal()), is(false));
-
- context.handleResult(
- null,
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(created.getProducingTransformInternal()).build());
- context.extractFiredTimers();
- assertThat(context.isDone(created.getProducingTransformInternal()), is(true));
- }
-
- @Test
- public void isDoneWithPartiallyDone() {
- context.getPipelineOptions().setShutdownUnboundedProducersWithMaxWatermark(true);
- assertThat(context.isDone(), is(false));
-
- UncommittedBundle<Integer> rootBundle = context.createRootBundle(created);
- rootBundle.add(WindowedValue.valueInGlobalWindow(1));
- CommittedResult handleResult =
- context.handleResult(
- null,
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(created.getProducingTransformInternal())
- .addOutput(rootBundle)
- .build());
- @SuppressWarnings("unchecked")
- CommittedBundle<Integer> committedBundle =
- (CommittedBundle<Integer>) Iterables.getOnlyElement(handleResult.getOutputs());
- context.handleResult(
- null,
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(unbounded.getProducingTransformInternal()).build());
- assertThat(context.isDone(), is(false));
-
- for (AppliedPTransform<?, ?, ?> consumers : valueToConsumers.get(created)) {
- context.handleResult(
- committedBundle,
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(consumers).build());
- }
- context.extractFiredTimers();
- assertThat(context.isDone(), is(true));
- }
-
- @Test
- public void isDoneWithUnboundedAndNotShutdown() {
- context.getPipelineOptions().setShutdownUnboundedProducersWithMaxWatermark(false);
- assertThat(context.isDone(), is(false));
-
- context.handleResult(
- null,
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(created.getProducingTransformInternal()).build());
- context.handleResult(
- null,
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(unbounded.getProducingTransformInternal()).build());
- context.handleResult(
- context.createRootBundle(created).commit(Instant.now()),
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(downstream.getProducingTransformInternal()).build());
- context.extractFiredTimers();
- assertThat(context.isDone(), is(false));
-
- context.handleResult(
- context.createRootBundle(created).commit(Instant.now()),
- ImmutableList.<TimerData>of(),
- StepTransformResult.withoutHold(view.getProducingTransformInternal()).build());
- context.extractFiredTimers();
- assertThat(context.isDone(), is(false));
- }
-
- private static class TestBoundedWindow extends BoundedWindow {
- private final Instant ts;
-
- public TestBoundedWindow(Instant ts) {
- this.ts = ts;
- }
-
- @Override
- public Instant maxTimestamp() {
- return ts;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactoryTest.java
deleted file mode 100644
index e8d4711..0000000
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactoryTest.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static org.hamcrest.Matchers.contains;
-import static org.junit.Assert.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly.ReifyTimestampsAndWindows;
-import org.apache.beam.sdk.util.KeyedWorkItem;
-import org.apache.beam.sdk.util.KeyedWorkItems;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-
-import com.google.common.collect.HashMultiset;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Multiset;
-
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
-import org.joda.time.Instant;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for {@link InProcessGroupByKeyOnlyEvaluatorFactory}.
- */
-@RunWith(JUnit4.class)
-public class InProcessGroupByKeyOnlyEvaluatorFactoryTest {
- private BundleFactory bundleFactory = InProcessBundleFactory.create();
-
- @Test
- public void testInMemoryEvaluator() throws Exception {
- TestPipeline p = TestPipeline.create();
- KV<String, Integer> firstFoo = KV.of("foo", -1);
- KV<String, Integer> secondFoo = KV.of("foo", 1);
- KV<String, Integer> thirdFoo = KV.of("foo", 3);
- KV<String, Integer> firstBar = KV.of("bar", 22);
- KV<String, Integer> secondBar = KV.of("bar", 12);
- KV<String, Integer> firstBaz = KV.of("baz", Integer.MAX_VALUE);
- PCollection<KV<String, Integer>> values =
- p.apply(Create.of(firstFoo, firstBar, secondFoo, firstBaz, secondBar, thirdFoo));
- PCollection<KV<String, WindowedValue<Integer>>> kvs =
- values.apply(new ReifyTimestampsAndWindows<String, Integer>());
- PCollection<KeyedWorkItem<String, Integer>> groupedKvs =
- kvs.apply(new InProcessGroupByKey.InProcessGroupByKeyOnly<String, Integer>());
-
- CommittedBundle<KV<String, WindowedValue<Integer>>> inputBundle =
- bundleFactory.createRootBundle(kvs).commit(Instant.now());
- InProcessEvaluationContext evaluationContext = mock(InProcessEvaluationContext.class);
-
- StructuralKey<String> fooKey = StructuralKey.of("foo", StringUtf8Coder.of());
- UncommittedBundle<KeyedWorkItem<String, Integer>> fooBundle = bundleFactory.createKeyedBundle(
- null, fooKey,
- groupedKvs);
- StructuralKey<String> barKey = StructuralKey.of("bar", StringUtf8Coder.of());
- UncommittedBundle<KeyedWorkItem<String, Integer>> barBundle = bundleFactory.createKeyedBundle(
- null, barKey,
- groupedKvs);
- StructuralKey<String> bazKey = StructuralKey.of("baz", StringUtf8Coder.of());
- UncommittedBundle<KeyedWorkItem<String, Integer>> bazBundle = bundleFactory.createKeyedBundle(
- null, bazKey,
- groupedKvs);
-
- when(evaluationContext.createKeyedBundle(inputBundle,
- fooKey,
- groupedKvs)).thenReturn(fooBundle);
- when(evaluationContext.createKeyedBundle(inputBundle,
- barKey,
- groupedKvs)).thenReturn(barBundle);
- when(evaluationContext.createKeyedBundle(inputBundle,
- bazKey,
- groupedKvs)).thenReturn(bazBundle);
-
- // The input to a GroupByKey is assumed to be a KvCoder
- @SuppressWarnings("unchecked")
- Coder<String> keyCoder =
- ((KvCoder<String, WindowedValue<Integer>>) kvs.getCoder()).getKeyCoder();
- TransformEvaluator<KV<String, WindowedValue<Integer>>> evaluator =
- new InProcessGroupByKeyOnlyEvaluatorFactory()
- .forApplication(
- groupedKvs.getProducingTransformInternal(), inputBundle, evaluationContext);
-
- evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(firstFoo)));
- evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(secondFoo)));
- evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(thirdFoo)));
- evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(firstBar)));
- evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(secondBar)));
- evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(firstBaz)));
-
- evaluator.finishBundle();
-
- assertThat(
- fooBundle.commit(Instant.now()).getElements(),
- contains(
- new KeyedWorkItemMatcher<String, Integer>(
- KeyedWorkItems.elementsWorkItem(
- "foo",
- ImmutableSet.of(
- WindowedValue.valueInGlobalWindow(-1),
- WindowedValue.valueInGlobalWindow(1),
- WindowedValue.valueInGlobalWindow(3))),
- keyCoder)));
- assertThat(
- barBundle.commit(Instant.now()).getElements(),
- contains(
- new KeyedWorkItemMatcher<String, Integer>(
- KeyedWorkItems.elementsWorkItem(
- "bar",
- ImmutableSet.of(
- WindowedValue.valueInGlobalWindow(12),
- WindowedValue.valueInGlobalWindow(22))),
- keyCoder)));
- assertThat(
- bazBundle.commit(Instant.now()).getElements(),
- contains(
- new KeyedWorkItemMatcher<String, Integer>(
- KeyedWorkItems.elementsWorkItem(
- "baz",
- ImmutableSet.of(WindowedValue.valueInGlobalWindow(Integer.MAX_VALUE))),
- keyCoder)));
- }
-
- private <K, V> KV<K, WindowedValue<V>> gwValue(KV<K, V> kv) {
- return KV.of(kv.getKey(), WindowedValue.valueInGlobalWindow(kv.getValue()));
- }
-
- private static class KeyedWorkItemMatcher<K, V>
- extends BaseMatcher<WindowedValue<KeyedWorkItem<K, V>>> {
- private final KeyedWorkItem<K, V> myWorkItem;
- private final Coder<K> keyCoder;
-
- public KeyedWorkItemMatcher(KeyedWorkItem<K, V> myWorkItem, Coder<K> keyCoder) {
- this.myWorkItem = myWorkItem;
- this.keyCoder = keyCoder;
- }
-
- @Override
- public boolean matches(Object item) {
- if (item == null || !(item instanceof WindowedValue)) {
- return false;
- }
- WindowedValue<KeyedWorkItem<K, V>> that = (WindowedValue<KeyedWorkItem<K, V>>) item;
- Multiset<WindowedValue<V>> myValues = HashMultiset.create();
- Multiset<WindowedValue<V>> thatValues = HashMultiset.create();
- for (WindowedValue<V> value : myWorkItem.elementsIterable()) {
- myValues.add(value);
- }
- for (WindowedValue<V> value : that.getValue().elementsIterable()) {
- thatValues.add(value);
- }
- try {
- return myValues.equals(thatValues)
- && keyCoder
- .structuralValue(myWorkItem.key())
- .equals(keyCoder.structuralValue(that.getValue().key()));
- } catch (Exception e) {
- return false;
- }
- }
-
- @Override
- public void describeTo(Description description) {
- description
- .appendText("KeyedWorkItem<K, V> containing key ")
- .appendValue(myWorkItem.key())
- .appendText(" and values ")
- .appendValueList("[", ", ", "]", myWorkItem.elementsIterable());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessSideInputContainerTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessSideInputContainerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessSideInputContainerTest.java
deleted file mode 100644
index 746c0f8..0000000
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessSideInputContainerTest.java
+++ /dev/null
@@ -1,520 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasEntry;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.doAnswer;
-
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.Mean;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
-import org.apache.beam.sdk.util.PCollectionViews;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-
-import com.google.common.collect.ImmutableList;
-
-import org.joda.time.Instant;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.MockitoAnnotations;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Tests for {@link InProcessSideInputContainer}.
- */
-@RunWith(JUnit4.class)
-public class InProcessSideInputContainerTest {
- private static final BoundedWindow FIRST_WINDOW =
- new BoundedWindow() {
- @Override
- public Instant maxTimestamp() {
- return new Instant(789541L);
- }
-
- @Override
- public String toString() {
- return "firstWindow";
- }
- };
-
- private static final BoundedWindow SECOND_WINDOW =
- new BoundedWindow() {
- @Override
- public Instant maxTimestamp() {
- return new Instant(14564786L);
- }
-
- @Override
- public String toString() {
- return "secondWindow";
- }
- };
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @Mock
- private InProcessEvaluationContext context;
-
- private TestPipeline pipeline;
-
- private InProcessSideInputContainer container;
-
- private PCollectionView<Map<String, Integer>> mapView;
- private PCollectionView<Double> singletonView;
-
- // Not present in container.
- private PCollectionView<Iterable<Integer>> iterableView;
-
- @Before
- public void setup() {
- MockitoAnnotations.initMocks(this);
- pipeline = TestPipeline.create();
-
- PCollection<Integer> create =
- pipeline.apply("forBaseCollection", Create.<Integer>of(1, 2, 3, 4));
-
- mapView =
- create.apply("forKeyTypes", WithKeys.<String, Integer>of("foo"))
- .apply("asMapView", View.<String, Integer>asMap());
-
- singletonView = create.apply("forCombinedTypes", Mean.<Integer>globally().asSingletonView());
- iterableView = create.apply("asIterableView", View.<Integer>asIterable());
-
- container = InProcessSideInputContainer.create(
- context, ImmutableList.of(iterableView, mapView, singletonView));
- }
-
- @Test
- public void getAfterWriteReturnsPaneInWindow() throws Exception {
- WindowedValue<KV<String, Integer>> one =
- WindowedValue.of(
- KV.of("one", 1), new Instant(1L), FIRST_WINDOW, PaneInfo.ON_TIME_AND_ONLY_FIRING);
- WindowedValue<KV<String, Integer>> two =
- WindowedValue.of(
- KV.of("two", 2), new Instant(20L), FIRST_WINDOW, PaneInfo.ON_TIME_AND_ONLY_FIRING);
- container.write(mapView, ImmutableList.<WindowedValue<?>>of(one, two));
-
- Map<String, Integer> viewContents =
- container
- .createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
- .get(mapView, FIRST_WINDOW);
- assertThat(viewContents, hasEntry("one", 1));
- assertThat(viewContents, hasEntry("two", 2));
- assertThat(viewContents.size(), is(2));
- }
-
- @Test
- public void getReturnsLatestPaneInWindow() throws Exception {
- WindowedValue<KV<String, Integer>> one =
- WindowedValue.of(
- KV.of("one", 1),
- new Instant(1L),
- SECOND_WINDOW,
- PaneInfo.createPane(true, false, Timing.EARLY));
- WindowedValue<KV<String, Integer>> two =
- WindowedValue.of(
- KV.of("two", 2),
- new Instant(20L),
- SECOND_WINDOW,
- PaneInfo.createPane(true, false, Timing.EARLY));
- container.write(mapView, ImmutableList.<WindowedValue<?>>of(one, two));
-
- Map<String, Integer> viewContents =
- container
- .createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
- .get(mapView, SECOND_WINDOW);
- assertThat(viewContents, hasEntry("one", 1));
- assertThat(viewContents, hasEntry("two", 2));
- assertThat(viewContents.size(), is(2));
-
- WindowedValue<KV<String, Integer>> three =
- WindowedValue.of(
- KV.of("three", 3),
- new Instant(300L),
- SECOND_WINDOW,
- PaneInfo.createPane(false, false, Timing.EARLY, 1, -1));
- container.write(mapView, ImmutableList.<WindowedValue<?>>of(three));
-
- Map<String, Integer> overwrittenViewContents =
- container
- .createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
- .get(mapView, SECOND_WINDOW);
- assertThat(overwrittenViewContents, hasEntry("three", 3));
- assertThat(overwrittenViewContents.size(), is(1));
- }
-
- /**
- * Demonstrates that calling get() on a window that currently has no data does not return until
- * there is data in the pane.
- */
- @Test
- public void getNotReadyThrows() throws Exception {
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("not ready");
-
- container.createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
- .get(mapView, GlobalWindow.INSTANCE);
- }
-
- @Test
- public void withPCollectionViewsErrorsForContainsNotInViews() {
- PCollectionView<Map<String, Iterable<String>>> newView =
- PCollectionViews.multimapView(
- pipeline,
- WindowingStrategy.globalDefault(),
- KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()));
-
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("with unknown views " + ImmutableList.of(newView).toString());
-
- container.createReaderForViews(ImmutableList.<PCollectionView<?>>of(newView));
- }
-
- @Test
- public void withViewsForViewNotInContainerFails() {
- PCollectionView<Map<String, Iterable<String>>> newView =
- PCollectionViews.multimapView(
- pipeline,
- WindowingStrategy.globalDefault(),
- KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()));
-
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("unknown views");
- thrown.expectMessage(newView.toString());
-
- container.createReaderForViews(ImmutableList.<PCollectionView<?>>of(newView));
- }
-
- @Test
- public void getOnReaderForViewNotInReaderFails() {
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("unknown view: " + iterableView.toString());
-
- container.createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
- .get(iterableView, GlobalWindow.INSTANCE);
- }
-
- @Test
- public void writeForMultipleElementsInDifferentWindowsSucceeds() throws Exception {
- WindowedValue<Double> firstWindowedValue =
- WindowedValue.of(
- 2.875,
- FIRST_WINDOW.maxTimestamp().minus(200L),
- FIRST_WINDOW,
- PaneInfo.ON_TIME_AND_ONLY_FIRING);
- WindowedValue<Double> secondWindowedValue =
- WindowedValue.of(
- 4.125,
- SECOND_WINDOW.maxTimestamp().minus(2_000_000L),
- SECOND_WINDOW,
- PaneInfo.ON_TIME_AND_ONLY_FIRING);
- container.write(singletonView, ImmutableList.of(firstWindowedValue, secondWindowedValue));
- assertThat(
- container
- .createReaderForViews(ImmutableList.<PCollectionView<?>>of(singletonView))
- .get(singletonView, FIRST_WINDOW),
- equalTo(2.875));
- assertThat(
- container
- .createReaderForViews(ImmutableList.<PCollectionView<?>>of(singletonView))
- .get(singletonView, SECOND_WINDOW),
- equalTo(4.125));
- }
-
- @Test
- public void writeForMultipleIdenticalElementsInSameWindowSucceeds() throws Exception {
- WindowedValue<Integer> firstValue =
- WindowedValue.of(
- 44,
- FIRST_WINDOW.maxTimestamp().minus(200L),
- FIRST_WINDOW,
- PaneInfo.ON_TIME_AND_ONLY_FIRING);
- WindowedValue<Integer> secondValue =
- WindowedValue.of(
- 44,
- FIRST_WINDOW.maxTimestamp().minus(200L),
- FIRST_WINDOW,
- PaneInfo.ON_TIME_AND_ONLY_FIRING);
-
- container.write(iterableView, ImmutableList.of(firstValue, secondValue));
-
- assertThat(
- container
- .createReaderForViews(ImmutableList.<PCollectionView<?>>of(iterableView))
- .get(iterableView, FIRST_WINDOW),
- contains(44, 44));
- }
-
- @Test
- public void writeForElementInMultipleWindowsSucceeds() throws Exception {
- WindowedValue<Double> multiWindowedValue =
- WindowedValue.of(
- 2.875,
- FIRST_WINDOW.maxTimestamp().minus(200L),
- ImmutableList.of(FIRST_WINDOW, SECOND_WINDOW),
- PaneInfo.ON_TIME_AND_ONLY_FIRING);
- container.write(singletonView, ImmutableList.of(multiWindowedValue));
- assertThat(
- container
- .createReaderForViews(ImmutableList.<PCollectionView<?>>of(singletonView))
- .get(singletonView, FIRST_WINDOW),
- equalTo(2.875));
- assertThat(
- container
- .createReaderForViews(ImmutableList.<PCollectionView<?>>of(singletonView))
- .get(singletonView, SECOND_WINDOW),
- equalTo(2.875));
- }
-
- @Test
- public void finishDoesNotOverwriteWrittenElements() throws Exception {
- WindowedValue<KV<String, Integer>> one =
- WindowedValue.of(
- KV.of("one", 1),
- new Instant(1L),
- SECOND_WINDOW,
- PaneInfo.createPane(true, false, Timing.EARLY));
- WindowedValue<KV<String, Integer>> two =
- WindowedValue.of(
- KV.of("two", 2),
- new Instant(20L),
- SECOND_WINDOW,
- PaneInfo.createPane(true, false, Timing.EARLY));
- container.write(mapView, ImmutableList.<WindowedValue<?>>of(one, two));
-
- immediatelyInvokeCallback(mapView, SECOND_WINDOW);
-
- Map<String, Integer> viewContents =
- container
- .createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
- .get(mapView, SECOND_WINDOW);
-
- assertThat(viewContents, hasEntry("one", 1));
- assertThat(viewContents, hasEntry("two", 2));
- assertThat(viewContents.size(), is(2));
- }
-
- @Test
- public void finishOnPendingViewsSetsEmptyElements() throws Exception {
- immediatelyInvokeCallback(mapView, SECOND_WINDOW);
- Future<Map<String, Integer>> mapFuture =
- getFutureOfView(
- container.createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView)),
- mapView,
- SECOND_WINDOW);
-
- assertThat(mapFuture.get().isEmpty(), is(true));
- }
-
- /**
- * Demonstrates that calling isReady on an empty container throws an
- * {@link IllegalArgumentException}.
- */
- @Test
- public void isReadyInEmptyReaderThrows() {
- ReadyCheckingSideInputReader reader =
- container.createReaderForViews(ImmutableList.<PCollectionView<?>>of());
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("does not contain");
- thrown.expectMessage(ImmutableList.of().toString());
- reader.isReady(mapView, GlobalWindow.INSTANCE);
- }
-
- /**
- * Demonstrates that calling isReady returns false until elements are written to the
- * {@link PCollectionView}, {@link BoundedWindow} pair, at which point it returns true.
- */
- @Test
- public void isReadyForSomeNotReadyViewsFalseUntilElements() {
- container.write(
- mapView,
- ImmutableList.of(
- WindowedValue.of(
- KV.of("one", 1),
- SECOND_WINDOW.maxTimestamp().minus(100L),
- SECOND_WINDOW,
- PaneInfo.ON_TIME_AND_ONLY_FIRING)));
-
- ReadyCheckingSideInputReader reader =
- container.createReaderForViews(ImmutableList.of(mapView, singletonView));
- assertThat(reader.isReady(mapView, FIRST_WINDOW), is(false));
- assertThat(reader.isReady(mapView, SECOND_WINDOW), is(true));
-
- assertThat(reader.isReady(singletonView, SECOND_WINDOW), is(false));
-
- container.write(
- mapView,
- ImmutableList.of(
- WindowedValue.of(
- KV.of("too", 2),
- FIRST_WINDOW.maxTimestamp().minus(100L),
- FIRST_WINDOW,
- PaneInfo.ON_TIME_AND_ONLY_FIRING)));
- // Cached value is false
- assertThat(reader.isReady(mapView, FIRST_WINDOW), is(false));
-
- container.write(
- singletonView,
- ImmutableList.of(
- WindowedValue.of(
- 1.25,
- SECOND_WINDOW.maxTimestamp().minus(100L),
- SECOND_WINDOW,
- PaneInfo.ON_TIME_AND_ONLY_FIRING)));
- assertThat(reader.isReady(mapView, SECOND_WINDOW), is(true));
- assertThat(reader.isReady(singletonView, SECOND_WINDOW), is(false));
-
- assertThat(reader.isReady(mapView, GlobalWindow.INSTANCE), is(false));
- assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));
-
- reader = container.createReaderForViews(ImmutableList.of(mapView, singletonView));
- assertThat(reader.isReady(mapView, SECOND_WINDOW), is(true));
- assertThat(reader.isReady(singletonView, SECOND_WINDOW), is(true));
- assertThat(reader.isReady(mapView, FIRST_WINDOW), is(true));
- }
-
- @Test
- public void isReadyForEmptyWindowTrue() throws Exception {
- CountDownLatch onComplete = new CountDownLatch(1);
- immediatelyInvokeCallback(mapView, GlobalWindow.INSTANCE);
- CountDownLatch latch = invokeLatchedCallback(singletonView, GlobalWindow.INSTANCE, onComplete);
-
- ReadyCheckingSideInputReader reader =
- container.createReaderForViews(ImmutableList.of(mapView, singletonView));
- assertThat(reader.isReady(mapView, GlobalWindow.INSTANCE), is(true));
- assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));
-
- latch.countDown();
- if (!onComplete.await(1500L, TimeUnit.MILLISECONDS)) {
- fail("Callback to set empty values did not complete!");
- }
- // The cached value was false, so it continues to be true
- assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));
-
- // A new reader for the same container gets a fresh look
- reader = container.createReaderForViews(ImmutableList.of(mapView, singletonView));
- assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(true));
- }
-
- /**
- * When a callAfterWindowCloses with the specified view's producing transform, window, and
- * windowing strategy is invoked, immediately execute the callback.
- */
- private void immediatelyInvokeCallback(PCollectionView<?> view, BoundedWindow window) {
- doAnswer(
- new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- Object callback = invocation.getArguments()[3];
- Runnable callbackRunnable = (Runnable) callback;
- callbackRunnable.run();
- return null;
- }
- })
- .when(context)
- .scheduleAfterOutputWouldBeProduced(
- Mockito.eq(view),
- Mockito.eq(window),
- Mockito.eq(view.getWindowingStrategyInternal()),
- Mockito.any(Runnable.class));
- }
-
- /**
- * When a callAfterWindowCloses with the specified view's producing transform, window, and
- * windowing strategy is invoked, start a thread that will invoke the callback after the returned
- * {@link CountDownLatch} is counted down once.
- */
- private CountDownLatch invokeLatchedCallback(
- PCollectionView<?> view, BoundedWindow window, final CountDownLatch onComplete) {
- final CountDownLatch runLatch = new CountDownLatch(1);
- doAnswer(
- new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- Object callback = invocation.getArguments()[3];
- final Runnable callbackRunnable = (Runnable) callback;
- Executors.newSingleThreadExecutor().submit(new Runnable() {
- public void run() {
- try {
- if (!runLatch.await(1500L, TimeUnit.MILLISECONDS)) {
- fail("Run latch didn't count down within timeout");
- }
- callbackRunnable.run();
- onComplete.countDown();
- } catch (InterruptedException e) {
- fail("Unexpectedly interrupted while waiting for latch to be counted down");
- }
- }
- });
- return null;
- }
- })
- .when(context)
- .scheduleAfterOutputWouldBeProduced(
- Mockito.eq(view),
- Mockito.eq(window),
- Mockito.eq(view.getWindowingStrategyInternal()),
- Mockito.any(Runnable.class));
- return runLatch;
- }
-
- private <ValueT> Future<ValueT> getFutureOfView(final SideInputReader myReader,
- final PCollectionView<ValueT> view, final BoundedWindow window) {
- Callable<ValueT> callable = new Callable<ValueT>() {
- @Override
- public ValueT call() throws Exception {
- return myReader.get(view, window);
- }
- };
- return Executors.newSingleThreadExecutor().submit(callable);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessTimerInternalsTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessTimerInternalsTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessTimerInternalsTest.java
deleted file mode 100644
index 3e01f44..0000000
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessTimerInternalsTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertThat;
-import static org.mockito.Mockito.when;
-
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate.TimerUpdateBuilder;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TransformWatermarks;
-import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.TimerInternals.TimerData;
-import org.apache.beam.sdk.util.state.StateNamespaces;
-
-import org.joda.time.Duration;
-import org.joda.time.Instant;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-
-/**
- * Tests for {@link InProcessTimerInternals}.
- */
-@RunWith(JUnit4.class)
-public class InProcessTimerInternalsTest {
- private MockClock clock;
- @Mock private TransformWatermarks watermarks;
-
- private TimerUpdateBuilder timerUpdateBuilder;
-
- private InProcessTimerInternals internals;
-
- @Before
- public void setup() {
- MockitoAnnotations.initMocks(this);
- clock = MockClock.fromInstant(new Instant(0));
-
- timerUpdateBuilder = TimerUpdate.builder(StructuralKey.of(1234, VarIntCoder.of()));
-
- internals = InProcessTimerInternals.create(clock, watermarks, timerUpdateBuilder);
- }
-
- @Test
- public void setTimerAddsToBuilder() {
- TimerData eventTimer =
- TimerData.of(StateNamespaces.global(), new Instant(20145L), TimeDomain.EVENT_TIME);
- TimerData processingTimer =
- TimerData.of(StateNamespaces.global(), new Instant(125555555L), TimeDomain.PROCESSING_TIME);
- TimerData synchronizedProcessingTimer =
- TimerData.of(
- StateNamespaces.global(),
- new Instant(98745632189L),
- TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
- internals.setTimer(eventTimer);
- internals.setTimer(processingTimer);
- internals.setTimer(synchronizedProcessingTimer);
-
- assertThat(
- internals.getTimerUpdate().getSetTimers(),
- containsInAnyOrder(eventTimer, synchronizedProcessingTimer, processingTimer));
- }
-
- @Test
- public void deleteTimerDeletesOnBuilder() {
- TimerData eventTimer =
- TimerData.of(StateNamespaces.global(), new Instant(20145L), TimeDomain.EVENT_TIME);
- TimerData processingTimer =
- TimerData.of(StateNamespaces.global(), new Instant(125555555L), TimeDomain.PROCESSING_TIME);
- TimerData synchronizedProcessingTimer =
- TimerData.of(
- StateNamespaces.global(),
- new Instant(98745632189L),
- TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
- internals.deleteTimer(eventTimer);
- internals.deleteTimer(processingTimer);
- internals.deleteTimer(synchronizedProcessingTimer);
-
- assertThat(
- internals.getTimerUpdate().getDeletedTimers(),
- containsInAnyOrder(eventTimer, synchronizedProcessingTimer, processingTimer));
- }
-
- @Test
- public void getProcessingTimeIsClockNow() {
- assertThat(internals.currentProcessingTime(), equalTo(clock.now()));
- Instant oldProcessingTime = internals.currentProcessingTime();
-
- clock.advance(Duration.standardHours(12));
-
- assertThat(internals.currentProcessingTime(), equalTo(clock.now()));
- assertThat(
- internals.currentProcessingTime(),
- equalTo(oldProcessingTime.plus(Duration.standardHours(12))));
- }
-
- @Test
- public void getSynchronizedProcessingTimeIsWatermarkSynchronizedInputTime() {
- when(watermarks.getSynchronizedProcessingInputTime()).thenReturn(new Instant(12345L));
- assertThat(internals.currentSynchronizedProcessingTime(), equalTo(new Instant(12345L)));
- }
-
- @Test
- public void getInputWatermarkTimeUsesWatermarkTime() {
- when(watermarks.getInputWatermark()).thenReturn(new Instant(8765L));
- assertThat(internals.currentInputWatermarkTime(), equalTo(new Instant(8765L)));
- }
-
- @Test
- public void getOutputWatermarkTimeUsesWatermarkTime() {
- when(watermarks.getOutputWatermark()).thenReturn(new Instant(25525L));
- assertThat(internals.currentOutputWatermarkTime(), equalTo(new Instant(25525L)));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
new file mode 100644
index 0000000..3c9c9ee
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.util.IdentitySideInputWindowFn;
+import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.common.CounterSet;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+
+import org.hamcrest.Matchers;
+import org.joda.time.Instant;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tests for {@link ParDoEvaluator}.
+ */
+@RunWith(JUnit4.class)
+public class ParDoEvaluatorTest {
+ @Mock private EvaluationContext evaluationContext;
+ private PCollection<Integer> inputPc;
+ private TupleTag<Integer> mainOutputTag;
+ private List<TupleTag<?>> sideOutputTags;
+ private BundleFactory bundleFactory;
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ TestPipeline p = TestPipeline.create();
+ inputPc = p.apply(Create.of(1, 2, 3));
+ mainOutputTag = new TupleTag<Integer>() {};
+ sideOutputTags = TupleTagList.empty().getAll();
+
+ bundleFactory = ImmutableListBundleFactory.create();
+ }
+
+ @Test
+ public void sideInputsNotReadyResultHasUnprocessedElements() {
+ PCollectionView<Integer> singletonView =
+ inputPc
+ .apply(Window.into(new IdentitySideInputWindowFn()))
+ .apply(View.<Integer>asSingleton().withDefaultValue(0));
+ RecorderFn fn = new RecorderFn(singletonView);
+ PCollection<Integer> output = inputPc.apply(ParDo.of(fn).withSideInputs(singletonView));
+
+ CommittedBundle<Integer> inputBundle =
+ bundleFactory.createRootBundle(inputPc).commit(Instant.now());
+ UncommittedBundle<Integer> outputBundle = bundleFactory.createBundle(inputBundle, output);
+ when(evaluationContext.createBundle(inputBundle, output))
+ .thenReturn(outputBundle);
+
+ ParDoEvaluator<Integer> evaluator =
+ createEvaluator(singletonView, fn, inputBundle, output);
+
+ IntervalWindow nonGlobalWindow = new IntervalWindow(new Instant(0), new Instant(10_000L));
+ WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(3);
+ WindowedValue<Integer> second =
+ WindowedValue.of(2, new Instant(1234L), nonGlobalWindow, PaneInfo.NO_FIRING);
+ WindowedValue<Integer> third =
+ WindowedValue.of(
+ 1,
+ new Instant(2468L),
+ ImmutableList.of(nonGlobalWindow, GlobalWindow.INSTANCE),
+ PaneInfo.NO_FIRING);
+
+ evaluator.processElement(first);
+ evaluator.processElement(second);
+ evaluator.processElement(third);
+ TransformResult result = evaluator.finishBundle();
+
+ assertThat(
+ result.getUnprocessedElements(),
+ Matchers.<WindowedValue<?>>containsInAnyOrder(
+ second, WindowedValue.of(1, new Instant(2468L), nonGlobalWindow, PaneInfo.NO_FIRING)));
+ assertThat(result.getOutputBundles(), Matchers.<UncommittedBundle<?>>contains(outputBundle));
+ assertThat(fn.processed, containsInAnyOrder(1, 3));
+ assertThat(
+ Iterables.getOnlyElement(result.getOutputBundles()).commit(Instant.now()).getElements(),
+ Matchers.<WindowedValue<?>>containsInAnyOrder(
+ first.withValue(8),
+ WindowedValue.timestampedValueInGlobalWindow(6, new Instant(2468L))));
+ }
+
+ private ParDoEvaluator<Integer> createEvaluator(
+ PCollectionView<Integer> singletonView,
+ RecorderFn fn,
+ DirectRunner.CommittedBundle<Integer> inputBundle,
+ PCollection<Integer> output) {
+ when(
+ evaluationContext.createSideInputReader(
+ ImmutableList.<PCollectionView<?>>of(singletonView)))
+ .thenReturn(new ReadyInGlobalWindowReader());
+ DirectExecutionContext executionContext = mock(DirectExecutionContext.class);
+ DirectStepContext stepContext = mock(DirectStepContext.class);
+ when(
+ executionContext.getOrCreateStepContext(
+ Mockito.any(String.class), Mockito.any(String.class)))
+ .thenReturn(stepContext);
+ when(stepContext.getTimerUpdate()).thenReturn(TimerUpdate.empty());
+ when(
+ evaluationContext.getExecutionContext(
+ Mockito.any(AppliedPTransform.class), Mockito.any(StructuralKey.class)))
+ .thenReturn(executionContext);
+ when(evaluationContext.createCounterSet()).thenReturn(new CounterSet());
+
+ return ParDoEvaluator.create(
+ evaluationContext,
+ inputBundle,
+ (AppliedPTransform<PCollection<Integer>, ?, ?>) output.getProducingTransformInternal(),
+ fn,
+ ImmutableList.<PCollectionView<?>>of(singletonView),
+ mainOutputTag,
+ sideOutputTags,
+ ImmutableMap.<TupleTag<?>, PCollection<?>>of(mainOutputTag, output));
+ }
+
+ private static class RecorderFn extends DoFn<Integer, Integer> {
+ private Collection<Integer> processed;
+ private final PCollectionView<Integer> view;
+
+ public RecorderFn(PCollectionView<Integer> view) {
+ processed = new ArrayList<>();
+ this.view = view;
+ }
+
+ @Override
+ public void processElement(DoFn<Integer, Integer>.ProcessContext c) throws Exception {
+ processed.add(c.element());
+ c.output(c.element() + c.sideInput(view));
+ }
+ }
+
+ private static class ReadyInGlobalWindowReader implements ReadyCheckingSideInputReader {
+ @Override
+ @Nullable
+ public <T> T get(PCollectionView<T> view, BoundedWindow window) {
+ if (window.equals(GlobalWindow.INSTANCE)) {
+ return (T) (Integer) 5;
+ }
+ fail("Should only call get in the Global Window, others are not ready");
+ throw new AssertionError("Unreachable");
+ }
+
+ @Override
+ public <T> boolean contains(PCollectionView<T> view) {
+ return true;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return false;
+ }
+
+ @Override
+ public boolean isReady(PCollectionView<?> view, BoundedWindow window) {
+ return window.equals(GlobalWindow.INSTANCE);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoInProcessEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoInProcessEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoInProcessEvaluatorTest.java
deleted file mode 100644
index b78eb40..0000000
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoInProcessEvaluatorTest.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.InProcessExecutionContext.InProcessStepContext;
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.IdentitySideInputWindowFn;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.common.CounterSet;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TupleTagList;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-
-import org.hamcrest.Matchers;
-import org.joda.time.Instant;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.MockitoAnnotations;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import javax.annotation.Nullable;
-
-/**
- * Tests for {@link ParDoInProcessEvaluator}.
- */
-@RunWith(JUnit4.class)
-public class ParDoInProcessEvaluatorTest {
- @Mock private InProcessEvaluationContext evaluationContext;
- private PCollection<Integer> inputPc;
- private TupleTag<Integer> mainOutputTag;
- private List<TupleTag<?>> sideOutputTags;
- private BundleFactory bundleFactory;
-
- @Before
- public void setup() {
- MockitoAnnotations.initMocks(this);
- TestPipeline p = TestPipeline.create();
- inputPc = p.apply(Create.of(1, 2, 3));
- mainOutputTag = new TupleTag<Integer>() {};
- sideOutputTags = TupleTagList.empty().getAll();
-
- bundleFactory = InProcessBundleFactory.create();
- }
-
- @Test
- public void sideInputsNotReadyResultHasUnprocessedElements() {
- PCollectionView<Integer> singletonView =
- inputPc
- .apply(Window.into(new IdentitySideInputWindowFn()))
- .apply(View.<Integer>asSingleton().withDefaultValue(0));
- RecorderFn fn = new RecorderFn(singletonView);
- PCollection<Integer> output = inputPc.apply(ParDo.of(fn).withSideInputs(singletonView));
-
- CommittedBundle<Integer> inputBundle =
- bundleFactory.createRootBundle(inputPc).commit(Instant.now());
- UncommittedBundle<Integer> outputBundle = bundleFactory.createBundle(inputBundle, output);
- when(evaluationContext.createBundle(inputBundle, output))
- .thenReturn(outputBundle);
-
- ParDoInProcessEvaluator<Integer> evaluator =
- createEvaluator(singletonView, fn, inputBundle, output);
-
- IntervalWindow nonGlobalWindow = new IntervalWindow(new Instant(0), new Instant(10_000L));
- WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(3);
- WindowedValue<Integer> second =
- WindowedValue.of(2, new Instant(1234L), nonGlobalWindow, PaneInfo.NO_FIRING);
- WindowedValue<Integer> third =
- WindowedValue.of(
- 1,
- new Instant(2468L),
- ImmutableList.of(nonGlobalWindow, GlobalWindow.INSTANCE),
- PaneInfo.NO_FIRING);
-
- evaluator.processElement(first);
- evaluator.processElement(second);
- evaluator.processElement(third);
- InProcessTransformResult result = evaluator.finishBundle();
-
- assertThat(
- result.getUnprocessedElements(),
- Matchers.<WindowedValue<?>>containsInAnyOrder(
- second, WindowedValue.of(1, new Instant(2468L), nonGlobalWindow, PaneInfo.NO_FIRING)));
- assertThat(result.getOutputBundles(), Matchers.<UncommittedBundle<?>>contains(outputBundle));
- assertThat(fn.processed, containsInAnyOrder(1, 3));
- assertThat(
- Iterables.getOnlyElement(result.getOutputBundles()).commit(Instant.now()).getElements(),
- Matchers.<WindowedValue<?>>containsInAnyOrder(
- first.withValue(8),
- WindowedValue.timestampedValueInGlobalWindow(6, new Instant(2468L))));
- }
-
- private ParDoInProcessEvaluator<Integer> createEvaluator(
- PCollectionView<Integer> singletonView,
- RecorderFn fn,
- DirectRunner.CommittedBundle<Integer> inputBundle,
- PCollection<Integer> output) {
- when(
- evaluationContext.createSideInputReader(
- ImmutableList.<PCollectionView<?>>of(singletonView)))
- .thenReturn(new ReadyInGlobalWindowReader());
- InProcessExecutionContext executionContext = mock(InProcessExecutionContext.class);
- InProcessStepContext stepContext = mock(InProcessStepContext.class);
- when(
- executionContext.getOrCreateStepContext(
- Mockito.any(String.class), Mockito.any(String.class)))
- .thenReturn(stepContext);
- when(stepContext.getTimerUpdate()).thenReturn(TimerUpdate.empty());
- when(
- evaluationContext.getExecutionContext(
- Mockito.any(AppliedPTransform.class), Mockito.any(StructuralKey.class)))
- .thenReturn(executionContext);
- when(evaluationContext.createCounterSet()).thenReturn(new CounterSet());
-
- return ParDoInProcessEvaluator.create(
- evaluationContext,
- inputBundle,
- (AppliedPTransform<PCollection<Integer>, ?, ?>) output.getProducingTransformInternal(),
- fn,
- ImmutableList.<PCollectionView<?>>of(singletonView),
- mainOutputTag,
- sideOutputTags,
- ImmutableMap.<TupleTag<?>, PCollection<?>>of(mainOutputTag, output));
- }
-
- private static class RecorderFn extends DoFn<Integer, Integer> {
- private Collection<Integer> processed;
- private final PCollectionView<Integer> view;
-
- public RecorderFn(PCollectionView<Integer> view) {
- processed = new ArrayList<>();
- this.view = view;
- }
-
- @Override
- public void processElement(DoFn<Integer, Integer>.ProcessContext c) throws Exception {
- processed.add(c.element());
- c.output(c.element() + c.sideInput(view));
- }
- }
-
- private static class ReadyInGlobalWindowReader implements ReadyCheckingSideInputReader {
- @Override
- @Nullable
- public <T> T get(PCollectionView<T> view, BoundedWindow window) {
- if (window.equals(GlobalWindow.INSTANCE)) {
- return (T) (Integer) 5;
- }
- fail("Should only call get in the Global Window, others are not ready");
- throw new AssertionError("Unreachable");
- }
-
- @Override
- public <T> boolean contains(PCollectionView<T> view) {
- return true;
- }
-
- @Override
- public boolean isEmpty() {
- return false;
- }
-
- @Override
- public boolean isReady(PCollectionView<?> view, BoundedWindow window) {
- return window.equals(GlobalWindow.INSTANCE);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactoryTest.java
index e61881e..6206c22 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactoryTest.java
@@ -25,9 +25,9 @@ import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
@@ -69,7 +69,7 @@ import java.io.Serializable;
*/
@RunWith(JUnit4.class)
public class ParDoMultiEvaluatorFactoryTest implements Serializable {
- private transient BundleFactory bundleFactory = InProcessBundleFactory.create();
+ private transient BundleFactory bundleFactory = ImmutableListBundleFactory.create();
@Test
public void testParDoMultiInMemoryTransformEvaluator() throws Exception {
@@ -101,7 +101,7 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
PCollection<String> elementOutput = outputTuple.get(elementTag);
PCollection<Integer> lengthOutput = outputTuple.get(lengthTag);
- InProcessEvaluationContext evaluationContext = mock(InProcessEvaluationContext.class);
+ EvaluationContext evaluationContext = mock(EvaluationContext.class);
UncommittedBundle<KV<String, Integer>> mainOutputBundle =
bundleFactory.createRootBundle(mainOutput);
UncommittedBundle<String> elementOutputBundle = bundleFactory.createRootBundle(elementOutput);
@@ -112,8 +112,8 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
.thenReturn(elementOutputBundle);
when(evaluationContext.createBundle(inputBundle, lengthOutput)).thenReturn(lengthOutputBundle);
- InProcessExecutionContext executionContext =
- new InProcessExecutionContext(null, null, null, null);
+ DirectExecutionContext executionContext =
+ new DirectExecutionContext(null, null, null, null);
when(evaluationContext.getExecutionContext(mainOutput.getProducingTransformInternal(),
inputBundle.getKey())).thenReturn(executionContext);
CounterSet counters = new CounterSet();
@@ -130,7 +130,7 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
evaluator.processElement(
WindowedValue.valueInGlobalWindow("bazam", PaneInfo.ON_TIME_AND_ONLY_FIRING));
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(
result.getOutputBundles(),
Matchers.<UncommittedBundle<?>>containsInAnyOrder(
@@ -188,7 +188,7 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
PCollection<KV<String, Integer>> mainOutput = outputTuple.get(mainOutputTag);
PCollection<String> elementOutput = outputTuple.get(elementTag);
- InProcessEvaluationContext evaluationContext = mock(InProcessEvaluationContext.class);
+ EvaluationContext evaluationContext = mock(EvaluationContext.class);
UncommittedBundle<KV<String, Integer>> mainOutputBundle =
bundleFactory.createRootBundle(mainOutput);
UncommittedBundle<String> elementOutputBundle = bundleFactory.createRootBundle(elementOutput);
@@ -197,8 +197,8 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
when(evaluationContext.createBundle(inputBundle, elementOutput))
.thenReturn(elementOutputBundle);
- InProcessExecutionContext executionContext =
- new InProcessExecutionContext(null, null, null, null);
+ DirectExecutionContext executionContext =
+ new DirectExecutionContext(null, null, null, null);
when(evaluationContext.getExecutionContext(mainOutput.getProducingTransformInternal(),
inputBundle.getKey())).thenReturn(executionContext);
CounterSet counters = new CounterSet();
@@ -215,7 +215,7 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
evaluator.processElement(
WindowedValue.valueInGlobalWindow("bazam", PaneInfo.ON_TIME_AND_ONLY_FIRING));
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(
result.getOutputBundles(),
Matchers.<UncommittedBundle<?>>containsInAnyOrder(mainOutputBundle, elementOutputBundle));
@@ -278,7 +278,7 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
PCollection<KV<String, Integer>> mainOutput = outputTuple.get(mainOutputTag);
PCollection<String> elementOutput = outputTuple.get(elementTag);
- InProcessEvaluationContext evaluationContext = mock(InProcessEvaluationContext.class);
+ EvaluationContext evaluationContext = mock(EvaluationContext.class);
UncommittedBundle<KV<String, Integer>> mainOutputBundle =
bundleFactory.createRootBundle(mainOutput);
UncommittedBundle<String> elementOutputBundle = bundleFactory.createRootBundle(elementOutput);
@@ -287,7 +287,7 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
when(evaluationContext.createBundle(inputBundle, elementOutput))
.thenReturn(elementOutputBundle);
- InProcessExecutionContext executionContext = new InProcessExecutionContext(null,
+ DirectExecutionContext executionContext = new DirectExecutionContext(null,
StructuralKey.of("myKey", StringUtf8Coder.of()),
null,
null);
@@ -307,7 +307,7 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
evaluator.processElement(
WindowedValue.valueInGlobalWindow("bazam", PaneInfo.ON_TIME_AND_ONLY_FIRING));
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(
result.getOutputBundles(),
Matchers.<UncommittedBundle<?>>containsInAnyOrder(mainOutputBundle, elementOutputBundle));
@@ -390,7 +390,7 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
PCollection<KV<String, Integer>> mainOutput = outputTuple.get(mainOutputTag);
PCollection<String> elementOutput = outputTuple.get(elementTag);
- InProcessEvaluationContext evaluationContext = mock(InProcessEvaluationContext.class);
+ EvaluationContext evaluationContext = mock(EvaluationContext.class);
UncommittedBundle<KV<String, Integer>> mainOutputBundle =
bundleFactory.createRootBundle(mainOutput);
UncommittedBundle<String> elementOutputBundle = bundleFactory.createRootBundle(elementOutput);
@@ -399,7 +399,7 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
when(evaluationContext.createBundle(inputBundle, elementOutput))
.thenReturn(elementOutputBundle);
- InProcessExecutionContext executionContext = new InProcessExecutionContext(null,
+ DirectExecutionContext executionContext = new DirectExecutionContext(null,
StructuralKey.of("myKey", StringUtf8Coder.of()),
null, null);
when(evaluationContext.getExecutionContext(mainOutput.getProducingTransformInternal(),
@@ -418,7 +418,7 @@ public class ParDoMultiEvaluatorFactoryTest implements Serializable {
evaluator.processElement(
WindowedValue.valueInGlobalWindow("bazam", PaneInfo.ON_TIME_AND_ONLY_FIRING));
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(
result.getTimerUpdate(),
equalTo(
[12/12] incubator-beam git commit: This closes #474
Posted by ke...@apache.org.
This closes #474
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/518b23c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/518b23c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/518b23c2
Branch: refs/heads/master
Commit: 518b23c2d6dd6fa0e2aaa771b6f2b3de97d83ad1
Parents: 414bb4b 4649eeb
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed Jun 15 19:29:17 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed Jun 15 19:29:17 2016 -0700
----------------------------------------------------------------------
runners/direct-java/pom.xml | 2 +-
.../direct/AbstractModelEnforcement.java | 4 +-
.../direct/BoundedReadEvaluatorFactory.java | 16 +-
.../beam/runners/direct/BundleFactory.java | 8 +-
.../beam/runners/direct/CommittedResult.java | 6 +-
.../beam/runners/direct/CompletionCallback.java | 4 +-
.../runners/direct/DirectExecutionContext.java | 106 ++
.../beam/runners/direct/DirectGroupByKey.java | 132 ++
.../direct/DirectGroupByKeyOverrideFactory.java | 41 +
.../beam/runners/direct/DirectOptions.java | 101 ++
.../beam/runners/direct/DirectRegistrar.java | 57 +
.../beam/runners/direct/DirectRunner.java | 372 +++++
.../runners/direct/DirectTimerInternals.java | 84 ++
.../runners/direct/EmptyTransformEvaluator.java | 4 +-
.../direct/EncodabilityEnforcementFactory.java | 2 +-
.../beam/runners/direct/EvaluationContext.java | 429 ++++++
.../beam/runners/direct/EvaluatorKey.java | 4 +-
.../direct/ExecutorServiceParallelExecutor.java | 30 +-
.../runners/direct/FlattenEvaluatorFactory.java | 18 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 127 ++
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 186 +++
.../ImmutabilityCheckingBundleFactory.java | 4 +-
.../direct/ImmutabilityEnforcementFactory.java | 4 +-
.../direct/ImmutableListBundleFactory.java | 163 ++
.../direct/InMemoryWatermarkManager.java | 1420 -----------------
.../runners/direct/InProcessBundleFactory.java | 161 --
.../direct/InProcessBundleOutputManager.java | 51 -
.../direct/InProcessEvaluationContext.java | 429 ------
.../direct/InProcessExecutionContext.java | 105 --
.../beam/runners/direct/InProcessExecutor.java | 48 -
...rocessGroupAlsoByWindowEvaluatorFactory.java | 127 --
.../runners/direct/InProcessGroupByKey.java | 132 --
...InProcessGroupByKeyOnlyEvaluatorFactory.java | 185 ---
.../InProcessGroupByKeyOverrideFactory.java | 41 -
.../direct/InProcessPipelineOptions.java | 101 --
.../runners/direct/InProcessPipelineRunner.java | 370 -----
.../beam/runners/direct/InProcessRegistrar.java | 55 -
.../direct/InProcessSideInputContainer.java | 277 ----
.../runners/direct/InProcessTimerInternals.java | 84 --
.../direct/InProcessTransformResult.java | 84 --
.../beam/runners/direct/ModelEnforcement.java | 8 +-
.../runners/direct/ModelEnforcementFactory.java | 2 +-
.../beam/runners/direct/ParDoEvaluator.java | 186 +++
.../runners/direct/ParDoInProcessEvaluator.java | 186 ---
.../direct/ParDoMultiEvaluatorFactory.java | 10 +-
.../direct/ParDoSingleEvaluatorFactory.java | 10 +-
.../direct/PassthroughTransformEvaluator.java | 4 +-
.../beam/runners/direct/PipelineExecutor.java | 48 +
.../beam/runners/direct/SideInputContainer.java | 277 ++++
.../runners/direct/StepTransformResult.java | 8 +-
...readLocalInvalidatingTransformEvaluator.java | 2 +-
.../beam/runners/direct/TransformEvaluator.java | 6 +-
.../direct/TransformEvaluatorFactory.java | 4 +-
.../direct/TransformEvaluatorRegistry.java | 12 +-
.../beam/runners/direct/TransformExecutor.java | 16 +-
.../beam/runners/direct/TransformResult.java | 84 ++
.../direct/UnboundedReadEvaluatorFactory.java | 16 +-
.../direct/UncommittedBundleOutputManager.java | 51 +
.../runners/direct/ViewEvaluatorFactory.java | 22 +-
.../beam/runners/direct/WatermarkManager.java | 1420 +++++++++++++++++
.../runners/direct/WindowEvaluatorFactory.java | 18 +-
.../direct/AvroIOShardedWriteFactoryTest.java | 2 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 14 +-
.../runners/direct/CommittedResultTest.java | 14 +-
.../runners/direct/DirectRegistrarTest.java | 74 +
.../beam/runners/direct/DirectRunnerTest.java | 339 +++++
.../direct/DirectTimerInternalsTest.java | 134 ++
.../EncodabilityEnforcementFactoryTest.java | 4 +-
.../runners/direct/EvaluationContextTest.java | 545 +++++++
.../direct/FlattenEvaluatorFactoryTest.java | 16 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 13 +-
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 197 +++
.../ImmutabilityCheckingBundleFactoryTest.java | 6 +-
.../ImmutabilityEnforcementFactoryTest.java | 4 +-
.../direct/ImmutableListBundleFactoryTest.java | 231 +++
.../direct/InMemoryWatermarkManagerTest.java | 1428 ------------------
.../direct/InProcessBundleFactoryTest.java | 231 ---
.../direct/InProcessEvaluationContextTest.java | 545 -------
...ocessGroupByKeyOnlyEvaluatorFactoryTest.java | 196 ---
.../direct/InProcessPipelineRegistrarTest.java | 74 -
.../direct/InProcessPipelineRunnerTest.java | 339 -----
.../direct/InProcessSideInputContainerTest.java | 520 -------
.../direct/InProcessTimerInternalsTest.java | 134 --
.../beam/runners/direct/ParDoEvaluatorTest.java | 214 +++
.../direct/ParDoInProcessEvaluatorTest.java | 214 ---
.../direct/ParDoMultiEvaluatorFactoryTest.java | 36 +-
.../direct/ParDoSingleEvaluatorFactoryTest.java | 36 +-
.../runners/direct/SideInputContainerTest.java | 520 +++++++
.../direct/TextIOShardedWriteFactoryTest.java | 2 +-
...LocalInvalidatingTransformEvaluatorTest.java | 4 +-
.../runners/direct/TransformExecutorTest.java | 44 +-
.../UnboundedReadEvaluatorFactoryTest.java | 18 +-
.../direct/ViewEvaluatorFactoryTest.java | 8 +-
.../runners/direct/WatermarkManagerTest.java | 1428 ++++++++++++++++++
.../direct/WindowEvaluatorFactoryTest.java | 20 +-
.../translation/TransformTranslatorTest.java | 4 +-
.../beam/sdk/options/PipelineOptions.java | 2 +-
testing/travis/test_wordcount.sh | 4 +-
98 files changed, 7794 insertions(+), 7784 deletions(-)
----------------------------------------------------------------------
[10/12] incubator-beam git commit: Remove InProcess Prefixes
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
deleted file mode 100644
index fb8eb7c..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
+++ /dev/null
@@ -1,1420 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.TimerInternals;
-import org.apache.beam.sdk.util.TimerInternals.TimerData;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PValue;
-
-import com.google.auto.value.AutoValue;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ComparisonChain;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.SortedMultiset;
-import com.google.common.collect.TreeMultiset;
-
-import org.joda.time.Instant;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Objects;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.atomic.AtomicReference;
-
-import javax.annotation.Nullable;
-
-/**
- * Manages watermarks of {@link PCollection PCollections} and input and output watermarks of
- * {@link AppliedPTransform AppliedPTransforms} to provide event-time and completion tracking for
- * in-memory execution. {@link InMemoryWatermarkManager} is designed to update and return a
- * consistent view of watermarks in the presence of concurrent updates.
- *
- * <p>An {@link InMemoryWatermarkManager} is provided with the collection of root
- * {@link AppliedPTransform AppliedPTransforms} and a map of {@link PCollection PCollections} to
- * all the {@link AppliedPTransform AppliedPTransforms} that consume them at construction time.
- *
- * <p>Whenever a root {@link AppliedPTransform transform} produces elements, the
- * {@link InMemoryWatermarkManager} is provided with the produced elements and the output watermark
- * of the producing {@link AppliedPTransform transform}. The
- * {@link InMemoryWatermarkManager watermark manager} is responsible for computing the watermarks
- * of all {@link AppliedPTransform transforms} that consume one or more
- * {@link PCollection PCollections}.
- *
- * <p>Whenever a non-root {@link AppliedPTransform} finishes processing one or more in-flight
- * elements (referred to as the input {@link CommittedBundle bundle}), the following occurs
- * atomically:
- * <ul>
- * <li>All of the in-flight elements are removed from the collection of pending elements for the
- * {@link AppliedPTransform}.</li>
- * <li>All of the elements produced by the {@link AppliedPTransform} are added to the collection
- * of pending elements for each {@link AppliedPTransform} that consumes them.</li>
- * <li>The input watermark for the {@link AppliedPTransform} becomes the maximum value of
- * <ul>
- * <li>the previous input watermark</li>
- * <li>the minimum of
- * <ul>
- * <li>the timestamps of all currently pending elements</li>
- * <li>all input {@link PCollection} watermarks</li>
- * </ul>
- * </li>
- * </ul>
- * </li>
- * <li>The output watermark for the {@link AppliedPTransform} becomes the maximum of
- * <ul>
- * <li>the previous output watermark</li>
- * <li>the minimum of
- * <ul>
- * <li>the current input watermark</li>
- * <li>the current watermark holds</li>
- * </ul>
- * </li>
- * </ul>
- * </li>
- * <li>The watermark of the output {@link PCollection} can be advanced to the output watermark of
- * the {@link AppliedPTransform}</li>
- * <li>The watermark of all downstream {@link AppliedPTransform AppliedPTransforms} can be
- * advanced.</li>
- * </ul>
- *
- * <p>The watermark of a {@link PCollection} is equal to the output watermark of the
- * {@link AppliedPTransform} that produces it.
- *
- * <p>The watermarks for a {@link PTransform} are updated as follows when output is committed:<pre>
- * Watermark_In' = MAX(Watermark_In, MIN(U(TS_Pending), U(Watermark_InputPCollection)))
- * Watermark_Out' = MAX(Watermark_Out, MIN(Watermark_In', U(StateHold)))
- * Watermark_PCollection = Watermark_Out_ProducingPTransform
- * </pre>
- */
-public class InMemoryWatermarkManager {
- /**
- * The watermark of some {@link Pipeline} element, usually a {@link PTransform} or a
- * {@link PCollection}.
- *
- * <p>A watermark is a monotonically increasing value, which represents the point up to which the
- * system believes it has received all of the data. Data that arrives with a timestamp that is
- * before the watermark is considered late. {@link BoundedWindow#TIMESTAMP_MAX_VALUE} is a special
- * timestamp which indicates we have received all of the data and there will be no more on-time or
- * late data. This value is represented by {@link InMemoryWatermarkManager#THE_END_OF_TIME}.
- */
- private static interface Watermark {
- /**
- * Returns the current value of this watermark.
- */
- Instant get();
-
- /**
- * Refreshes the value of this watermark from its input watermarks and watermark holds.
- *
- * @return true if the value of the watermark has changed (and thus dependent watermark must
- * also be updated
- */
- WatermarkUpdate refresh();
- }
-
- /**
- * The result of computing a {@link Watermark}.
- */
- private static enum WatermarkUpdate {
- /** The watermark is later than the value at the previous time it was computed. */
- ADVANCED(true),
- /** The watermark is equal to the value at the previous time it was computed. */
- NO_CHANGE(false);
-
- private final boolean advanced;
-
- private WatermarkUpdate(boolean advanced) {
- this.advanced = advanced;
- }
-
- public boolean isAdvanced() {
- return advanced;
- }
-
- /**
- * Returns the {@link WatermarkUpdate} that is a result of combining the two watermark updates.
- *
- * If either of the input {@link WatermarkUpdate WatermarkUpdates} were advanced, the result
- * {@link WatermarkUpdate} has been advanced.
- */
- public WatermarkUpdate union(WatermarkUpdate that) {
- if (this.advanced) {
- return this;
- }
- return that;
- }
-
- /**
- * Returns the {@link WatermarkUpdate} based on the former and current
- * {@link Instant timestamps}.
- */
- public static WatermarkUpdate fromTimestamps(Instant oldTime, Instant currentTime) {
- if (currentTime.isAfter(oldTime)) {
- return ADVANCED;
- }
- return NO_CHANGE;
- }
- }
-
- /**
- * The input {@link Watermark} of an {@link AppliedPTransform}.
- *
- * <p>At any point, the value of an {@link AppliedPTransformInputWatermark} is equal to the
- * minimum watermark across all of its input {@link Watermark Watermarks}, and the minimum
- * timestamp of all of the pending elements, restricted to be monotonically increasing.
- *
- * <p>See {@link #refresh()} for more information.
- */
- private static class AppliedPTransformInputWatermark implements Watermark {
- private final Collection<? extends Watermark> inputWatermarks;
- private final SortedMultiset<WindowedValue<?>> pendingElements;
- private final Map<StructuralKey<?>, NavigableSet<TimerData>> objectTimers;
-
- private AtomicReference<Instant> currentWatermark;
-
- public AppliedPTransformInputWatermark(Collection<? extends Watermark> inputWatermarks) {
- this.inputWatermarks = inputWatermarks;
- this.pendingElements = TreeMultiset.create(new WindowedValueByTimestampComparator());
- this.objectTimers = new HashMap<>();
- currentWatermark = new AtomicReference<>(BoundedWindow.TIMESTAMP_MIN_VALUE);
- }
-
- @Override
- public Instant get() {
- return currentWatermark.get();
- }
-
- /**
- * {@inheritDoc}.
- *
- * <p>When refresh is called, the value of the {@link AppliedPTransformInputWatermark} becomes
- * equal to the maximum value of
- * <ul>
- * <li>the previous input watermark</li>
- * <li>the minimum of
- * <ul>
- * <li>the timestamps of all currently pending elements</li>
- * <li>all input {@link PCollection} watermarks</li>
- * </ul>
- * </li>
- * </ul>
- */
- @Override
- public synchronized WatermarkUpdate refresh() {
- Instant oldWatermark = currentWatermark.get();
- Instant minInputWatermark = BoundedWindow.TIMESTAMP_MAX_VALUE;
- for (Watermark inputWatermark : inputWatermarks) {
- minInputWatermark = INSTANT_ORDERING.min(minInputWatermark, inputWatermark.get());
- }
- if (!pendingElements.isEmpty()) {
- minInputWatermark = INSTANT_ORDERING.min(
- minInputWatermark, pendingElements.firstEntry().getElement().getTimestamp());
- }
- Instant newWatermark = INSTANT_ORDERING.max(oldWatermark, minInputWatermark);
- currentWatermark.set(newWatermark);
- return WatermarkUpdate.fromTimestamps(oldWatermark, newWatermark);
- }
-
- private synchronized void addPendingElements(Iterable<? extends WindowedValue<?>> newPending) {
- for (WindowedValue<?> pendingElement : newPending) {
- pendingElements.add(pendingElement);
- }
- }
-
- private synchronized void removePendingElements(
- Iterable<? extends WindowedValue<?>> finishedElements) {
- for (WindowedValue<?> finishedElement : finishedElements) {
- pendingElements.remove(finishedElement);
- }
- }
-
- private synchronized void updateTimers(TimerUpdate update) {
- NavigableSet<TimerData> keyTimers = objectTimers.get(update.key);
- if (keyTimers == null) {
- keyTimers = new TreeSet<>();
- objectTimers.put(update.key, keyTimers);
- }
- for (TimerData timer : update.setTimers) {
- if (TimeDomain.EVENT_TIME.equals(timer.getDomain())) {
- keyTimers.add(timer);
- }
- }
- for (TimerData timer : update.deletedTimers) {
- if (TimeDomain.EVENT_TIME.equals(timer.getDomain())) {
- keyTimers.remove(timer);
- }
- }
- // We don't keep references to timers that have been fired and delivered via #getFiredTimers()
- }
-
- private synchronized Map<StructuralKey<?>, List<TimerData>> extractFiredEventTimeTimers() {
- return extractFiredTimers(currentWatermark.get(), objectTimers);
- }
-
- @Override
- public synchronized String toString() {
- return MoreObjects.toStringHelper(AppliedPTransformInputWatermark.class)
- .add("pendingElements", pendingElements)
- .add("currentWatermark", currentWatermark)
- .toString();
- }
- }
-
- /**
- * The output {@link Watermark} of an {@link AppliedPTransform}.
- *
- * <p>The value of an {@link AppliedPTransformOutputWatermark} is equal to the minimum of the
- * current watermark hold and the {@link AppliedPTransformInputWatermark} for the same
- * {@link AppliedPTransform}, restricted to be monotonically increasing. See
- * {@link #refresh()} for more information.
- */
- private static class AppliedPTransformOutputWatermark implements Watermark {
- private final Watermark inputWatermark;
- private final PerKeyHolds holds;
- private AtomicReference<Instant> currentWatermark;
-
- public AppliedPTransformOutputWatermark(AppliedPTransformInputWatermark inputWatermark) {
- this.inputWatermark = inputWatermark;
- holds = new PerKeyHolds();
- currentWatermark = new AtomicReference<>(BoundedWindow.TIMESTAMP_MIN_VALUE);
- }
-
- public synchronized void updateHold(Object key, Instant newHold) {
- if (newHold == null) {
- holds.removeHold(key);
- } else {
- holds.updateHold(key, newHold);
- }
- }
-
- @Override
- public Instant get() {
- return currentWatermark.get();
- }
-
- /**
- * {@inheritDoc}.
- *
- * <p>When refresh is called, the value of the {@link AppliedPTransformOutputWatermark} becomes
- * equal to the maximum value of:
- * <ul>
- * <li>the previous output watermark</li>
- * <li>the minimum of
- * <ul>
- * <li>the current input watermark</li>
- * <li>the current watermark holds</li>
- * </ul>
- * </li>
- * </ul>
- */
- @Override
- public synchronized WatermarkUpdate refresh() {
- Instant oldWatermark = currentWatermark.get();
- Instant newWatermark = INSTANT_ORDERING.min(inputWatermark.get(), holds.getMinHold());
- newWatermark = INSTANT_ORDERING.max(oldWatermark, newWatermark);
- currentWatermark.set(newWatermark);
- return WatermarkUpdate.fromTimestamps(oldWatermark, newWatermark);
- }
-
- @Override
- public synchronized String toString() {
- return MoreObjects.toStringHelper(AppliedPTransformOutputWatermark.class)
- .add("holds", holds)
- .add("currentWatermark", currentWatermark)
- .toString();
- }
- }
-
- /**
- * The input {@link TimeDomain#SYNCHRONIZED_PROCESSING_TIME} hold for an
- * {@link AppliedPTransform}.
- *
- * <p>At any point, the hold value of an {@link SynchronizedProcessingTimeInputWatermark} is equal
- * to the minimum across all pending bundles at the {@link AppliedPTransform} and all upstream
- * {@link TimeDomain#SYNCHRONIZED_PROCESSING_TIME} watermarks. The value of the input
- * synchronized processing time at any step is equal to the maximum of:
- * <ul>
- * <li>The most recently returned synchronized processing input time
- * <li>The minimum of
- * <ul>
- * <li>The current processing time
- * <li>The current synchronized processing time input hold
- * </ul>
- * </ul>
- */
- private static class SynchronizedProcessingTimeInputWatermark implements Watermark {
- private final Collection<? extends Watermark> inputWms;
- private final Collection<CommittedBundle<?>> pendingBundles;
- private final Map<StructuralKey<?>, NavigableSet<TimerData>> processingTimers;
- private final Map<StructuralKey<?>, NavigableSet<TimerData>> synchronizedProcessingTimers;
-
- private final PriorityQueue<TimerData> pendingTimers;
-
- private AtomicReference<Instant> earliestHold;
-
- public SynchronizedProcessingTimeInputWatermark(Collection<? extends Watermark> inputWms) {
- this.inputWms = inputWms;
- this.pendingBundles = new HashSet<>();
- this.processingTimers = new HashMap<>();
- this.synchronizedProcessingTimers = new HashMap<>();
- this.pendingTimers = new PriorityQueue<>();
- Instant initialHold = BoundedWindow.TIMESTAMP_MAX_VALUE;
- for (Watermark wm : inputWms) {
- initialHold = INSTANT_ORDERING.min(initialHold, wm.get());
- }
- earliestHold = new AtomicReference<>(initialHold);
- }
-
- @Override
- public Instant get() {
- return earliestHold.get();
- }
-
- /**
- * {@inheritDoc}.
- *
- * <p>When refresh is called, the value of the {@link SynchronizedProcessingTimeInputWatermark}
- * becomes equal to the minimum value of
- * <ul>
- * <li>the timestamps of all currently pending bundles</li>
- * <li>all input {@link PCollection} synchronized processing time watermarks</li>
- * </ul>
- *
- * <p>Note that this value is not monotonic, but the returned value for the synchronized
- * processing time must be.
- */
- @Override
- public synchronized WatermarkUpdate refresh() {
- Instant oldHold = earliestHold.get();
- Instant minTime = THE_END_OF_TIME.get();
- for (Watermark input : inputWms) {
- minTime = INSTANT_ORDERING.min(minTime, input.get());
- }
- for (CommittedBundle<?> bundle : pendingBundles) {
- // TODO: Track elements in the bundle by the processing time they were output instead of
- // entire bundles. Requried to support arbitrarily splitting and merging bundles between
- // steps
- minTime = INSTANT_ORDERING.min(minTime, bundle.getSynchronizedProcessingOutputWatermark());
- }
- earliestHold.set(minTime);
- return WatermarkUpdate.fromTimestamps(oldHold, minTime);
- }
-
- public synchronized void addPending(CommittedBundle<?> bundle) {
- pendingBundles.add(bundle);
- }
-
- public synchronized void removePending(CommittedBundle<?> bundle) {
- pendingBundles.remove(bundle);
- }
-
- /**
- * Return the earliest timestamp of the earliest timer that has not been completed. This is
- * either the earliest timestamp across timers that have not been completed, or the earliest
- * timestamp across timers that have been delivered but have not been completed.
- */
- public synchronized Instant getEarliestTimerTimestamp() {
- Instant earliest = THE_END_OF_TIME.get();
- for (NavigableSet<TimerData> timers : processingTimers.values()) {
- if (!timers.isEmpty()) {
- earliest = INSTANT_ORDERING.min(timers.first().getTimestamp(), earliest);
- }
- }
- for (NavigableSet<TimerData> timers : synchronizedProcessingTimers.values()) {
- if (!timers.isEmpty()) {
- earliest = INSTANT_ORDERING.min(timers.first().getTimestamp(), earliest);
- }
- }
- if (!pendingTimers.isEmpty()) {
- earliest = INSTANT_ORDERING.min(pendingTimers.peek().getTimestamp(), earliest);
- }
- return earliest;
- }
-
- private synchronized void updateTimers(TimerUpdate update) {
- Map<TimeDomain, NavigableSet<TimerData>> timerMap = timerMap(update.key);
- for (TimerData addedTimer : update.setTimers) {
- NavigableSet<TimerData> timerQueue = timerMap.get(addedTimer.getDomain());
- if (timerQueue != null) {
- timerQueue.add(addedTimer);
- }
- }
-
- for (TimerData completedTimer : update.completedTimers) {
- pendingTimers.remove(completedTimer);
- }
- for (TimerData deletedTimer : update.deletedTimers) {
- NavigableSet<TimerData> timerQueue = timerMap.get(deletedTimer.getDomain());
- if (timerQueue != null) {
- timerQueue.remove(deletedTimer);
- }
- }
- }
-
- private synchronized Map<StructuralKey<?>, List<TimerData>> extractFiredDomainTimers(
- TimeDomain domain, Instant firingTime) {
- Map<StructuralKey<?>, List<TimerData>> firedTimers;
- switch (domain) {
- case PROCESSING_TIME:
- firedTimers = extractFiredTimers(firingTime, processingTimers);
- break;
- case SYNCHRONIZED_PROCESSING_TIME:
- firedTimers =
- extractFiredTimers(
- INSTANT_ORDERING.min(firingTime, earliestHold.get()),
- synchronizedProcessingTimers);
- break;
- default:
- throw new IllegalArgumentException(
- "Called getFiredTimers on a Synchronized Processing Time watermark"
- + " and gave a non-processing time domain "
- + domain);
- }
- for (Map.Entry<StructuralKey<?>, ? extends Collection<TimerData>> firedTimer :
- firedTimers.entrySet()) {
- pendingTimers.addAll(firedTimer.getValue());
- }
- return firedTimers;
- }
-
- private Map<TimeDomain, NavigableSet<TimerData>> timerMap(StructuralKey<?> key) {
- NavigableSet<TimerData> processingQueue = processingTimers.get(key);
- if (processingQueue == null) {
- processingQueue = new TreeSet<>();
- processingTimers.put(key, processingQueue);
- }
- NavigableSet<TimerData> synchronizedProcessingQueue =
- synchronizedProcessingTimers.get(key);
- if (synchronizedProcessingQueue == null) {
- synchronizedProcessingQueue = new TreeSet<>();
- synchronizedProcessingTimers.put(key, synchronizedProcessingQueue);
- }
- EnumMap<TimeDomain, NavigableSet<TimerData>> result = new EnumMap<>(TimeDomain.class);
- result.put(TimeDomain.PROCESSING_TIME, processingQueue);
- result.put(TimeDomain.SYNCHRONIZED_PROCESSING_TIME, synchronizedProcessingQueue);
- return result;
- }
-
- @Override
- public synchronized String toString() {
- return MoreObjects.toStringHelper(SynchronizedProcessingTimeInputWatermark.class)
- .add("earliestHold", earliestHold)
- .toString();
- }
- }
-
- /**
- * The output {@link TimeDomain#SYNCHRONIZED_PROCESSING_TIME} hold for an
- * {@link AppliedPTransform}.
- *
- * <p>At any point, the hold value of an {@link SynchronizedProcessingTimeOutputWatermark} is
- * equal to the minimum across all incomplete timers at the {@link AppliedPTransform} and all
- * upstream {@link TimeDomain#SYNCHRONIZED_PROCESSING_TIME} watermarks. The value of the output
- * synchronized processing time at any step is equal to the maximum of:
- * <ul>
- * <li>The most recently returned synchronized processing output time
- * <li>The minimum of
- * <ul>
- * <li>The current processing time
- * <li>The current synchronized processing time output hold
- * </ul>
- * </ul>
- */
- private static class SynchronizedProcessingTimeOutputWatermark implements Watermark {
- private final SynchronizedProcessingTimeInputWatermark inputWm;
- private AtomicReference<Instant> latestRefresh;
-
- public SynchronizedProcessingTimeOutputWatermark(
- SynchronizedProcessingTimeInputWatermark inputWm) {
- this.inputWm = inputWm;
- this.latestRefresh = new AtomicReference<>(BoundedWindow.TIMESTAMP_MIN_VALUE);
- }
-
- @Override
- public Instant get() {
- return latestRefresh.get();
- }
-
- /**
- * {@inheritDoc}.
- *
- * <p>When refresh is called, the value of the {@link SynchronizedProcessingTimeOutputWatermark}
- * becomes equal to the minimum value of:
- * <ul>
- * <li>the current input watermark.
- * <li>all {@link TimeDomain#SYNCHRONIZED_PROCESSING_TIME} timers that are based on the input
- * watermark.
- * <li>all {@link TimeDomain#PROCESSING_TIME} timers that are based on the input watermark.
- * </ul>
- *
- * <p>Note that this value is not monotonic, but the returned value for the synchronized
- * processing time must be.
- */
- @Override
- public synchronized WatermarkUpdate refresh() {
- // Hold the output synchronized processing time to the input watermark, which takes into
- // account buffered bundles, and the earliest pending timer, which determines what to hold
- // downstream timers to.
- Instant oldRefresh = latestRefresh.get();
- Instant newTimestamp =
- INSTANT_ORDERING.min(inputWm.get(), inputWm.getEarliestTimerTimestamp());
- latestRefresh.set(newTimestamp);
- return WatermarkUpdate.fromTimestamps(oldRefresh, newTimestamp);
- }
-
- @Override
- public synchronized String toString() {
- return MoreObjects.toStringHelper(SynchronizedProcessingTimeOutputWatermark.class)
- .add("latestRefresh", latestRefresh)
- .toString();
- }
- }
-
- /**
- * The {@code Watermark} that is after the latest time it is possible to represent in the global
- * window. This is a distinguished value representing a complete {@link PTransform}.
- */
- private static final Watermark THE_END_OF_TIME = new Watermark() {
- @Override
- public WatermarkUpdate refresh() {
- // THE_END_OF_TIME is a distinguished value that cannot be advanced.
- return WatermarkUpdate.NO_CHANGE;
- }
-
- @Override
- public Instant get() {
- return BoundedWindow.TIMESTAMP_MAX_VALUE;
- }
- };
-
- private static final Ordering<Instant> INSTANT_ORDERING = Ordering.natural();
-
- /**
- * A function that takes a WindowedValue and returns the exploded representation of that
- * {@link WindowedValue}.
- */
- private static final Function<WindowedValue<?>, ? extends Iterable<? extends WindowedValue<?>>>
- EXPLODE_WINDOWS_FN =
- new Function<WindowedValue<?>, Iterable<? extends WindowedValue<?>>>() {
- @Override
- public Iterable<? extends WindowedValue<?>> apply(WindowedValue<?> input) {
- return input.explodeWindows();
- }
- };
-
- /**
- * For each (Object, PriorityQueue) pair in the provided map, remove each Timer that is before the
- * latestTime argument and put in in the result with the same key, then remove all of the keys
- * which have no more pending timers.
- *
- * The result collection retains ordering of timers (from earliest to latest).
- */
- private static Map<StructuralKey<?>, List<TimerData>> extractFiredTimers(
- Instant latestTime, Map<StructuralKey<?>, NavigableSet<TimerData>> objectTimers) {
- Map<StructuralKey<?>, List<TimerData>> result = new HashMap<>();
- Set<StructuralKey<?>> emptyKeys = new HashSet<>();
- for (Map.Entry<StructuralKey<?>, NavigableSet<TimerData>> pendingTimers :
- objectTimers.entrySet()) {
- NavigableSet<TimerData> timers = pendingTimers.getValue();
- if (!timers.isEmpty() && timers.first().getTimestamp().isBefore(latestTime)) {
- ArrayList<TimerData> keyFiredTimers = new ArrayList<>();
- result.put(pendingTimers.getKey(), keyFiredTimers);
- while (!timers.isEmpty() && timers.first().getTimestamp().isBefore(latestTime)) {
- keyFiredTimers.add(timers.first());
- timers.remove(timers.first());
- }
- }
- if (timers.isEmpty()) {
- emptyKeys.add(pendingTimers.getKey());
- }
- }
- objectTimers.keySet().removeAll(emptyKeys);
- return result;
- }
-
- ////////////////////////////////////////////////////////////////////////////////////////////////
-
- /**
- * The {@link Clock} providing the current time in the {@link TimeDomain#PROCESSING_TIME} domain.
- */
- private final Clock clock;
-
- /**
- * A map from each {@link PCollection} to all {@link AppliedPTransform PTransform applications}
- * that consume that {@link PCollection}.
- */
- private final Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> consumers;
-
- /**
- * The input and output watermark of each {@link AppliedPTransform}.
- */
- private final Map<AppliedPTransform<?, ?, ?>, TransformWatermarks> transformToWatermarks;
-
- /**
- * A queue of pending updates to the state of this {@link InMemoryWatermarkManager}.
- */
- private final ConcurrentLinkedQueue<PendingWatermarkUpdate> pendingUpdates;
-
- /**
- * A queue of pending {@link AppliedPTransform AppliedPTransforms} that have potentially
- * stale data.
- */
- private final ConcurrentLinkedQueue<AppliedPTransform<?, ?, ?>> pendingRefreshes;
-
- /**
- * Creates a new {@link InMemoryWatermarkManager}. All watermarks within the newly created
- * {@link InMemoryWatermarkManager} start at {@link BoundedWindow#TIMESTAMP_MIN_VALUE}, the
- * minimum watermark, with no watermark holds or pending elements.
- *
- * @param rootTransforms the root-level transforms of the {@link Pipeline}
- * @param consumers a mapping between each {@link PCollection} in the {@link Pipeline} to the
- * transforms that consume it as a part of their input
- */
- public static InMemoryWatermarkManager create(
- Clock clock,
- Collection<AppliedPTransform<?, ?, ?>> rootTransforms,
- Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> consumers) {
- return new InMemoryWatermarkManager(clock, rootTransforms, consumers);
- }
-
- private InMemoryWatermarkManager(
- Clock clock,
- Collection<AppliedPTransform<?, ?, ?>> rootTransforms,
- Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> consumers) {
- this.clock = clock;
- this.consumers = consumers;
- this.pendingUpdates = new ConcurrentLinkedQueue<>();
- this.pendingRefreshes = new ConcurrentLinkedQueue<>();
-
- transformToWatermarks = new HashMap<>();
-
- for (AppliedPTransform<?, ?, ?> rootTransform : rootTransforms) {
- getTransformWatermark(rootTransform);
- }
- for (Collection<AppliedPTransform<?, ?, ?>> intermediateTransforms : consumers.values()) {
- for (AppliedPTransform<?, ?, ?> transform : intermediateTransforms) {
- getTransformWatermark(transform);
- }
- }
- }
-
- private TransformWatermarks getTransformWatermark(AppliedPTransform<?, ?, ?> transform) {
- TransformWatermarks wms = transformToWatermarks.get(transform);
- if (wms == null) {
- List<Watermark> inputCollectionWatermarks = getInputWatermarks(transform);
- AppliedPTransformInputWatermark inputWatermark =
- new AppliedPTransformInputWatermark(inputCollectionWatermarks);
- AppliedPTransformOutputWatermark outputWatermark =
- new AppliedPTransformOutputWatermark(inputWatermark);
-
- SynchronizedProcessingTimeInputWatermark inputProcessingWatermark =
- new SynchronizedProcessingTimeInputWatermark(getInputProcessingWatermarks(transform));
- SynchronizedProcessingTimeOutputWatermark outputProcessingWatermark =
- new SynchronizedProcessingTimeOutputWatermark(inputProcessingWatermark);
-
- wms =
- new TransformWatermarks(
- inputWatermark, outputWatermark, inputProcessingWatermark, outputProcessingWatermark);
- transformToWatermarks.put(transform, wms);
- }
- return wms;
- }
-
- private Collection<Watermark> getInputProcessingWatermarks(
- AppliedPTransform<?, ?, ?> transform) {
- ImmutableList.Builder<Watermark> inputWmsBuilder = ImmutableList.builder();
- Collection<? extends PValue> inputs = transform.getInput().expand();
- if (inputs.isEmpty()) {
- inputWmsBuilder.add(THE_END_OF_TIME);
- }
- for (PValue pvalue : inputs) {
- Watermark producerOutputWatermark =
- getTransformWatermark(pvalue.getProducingTransformInternal())
- .synchronizedProcessingOutputWatermark;
- inputWmsBuilder.add(producerOutputWatermark);
- }
- return inputWmsBuilder.build();
- }
-
- private List<Watermark> getInputWatermarks(AppliedPTransform<?, ?, ?> transform) {
- ImmutableList.Builder<Watermark> inputWatermarksBuilder = ImmutableList.builder();
- Collection<? extends PValue> inputs = transform.getInput().expand();
- if (inputs.isEmpty()) {
- inputWatermarksBuilder.add(THE_END_OF_TIME);
- }
- for (PValue pvalue : inputs) {
- Watermark producerOutputWatermark =
- getTransformWatermark(pvalue.getProducingTransformInternal()).outputWatermark;
- inputWatermarksBuilder.add(producerOutputWatermark);
- }
- List<Watermark> inputCollectionWatermarks = inputWatermarksBuilder.build();
- return inputCollectionWatermarks;
- }
-
- ////////////////////////////////////////////////////////////////////////////////////////////////
-
- /**
- * Gets the input and output watermarks for an {@link AppliedPTransform}. If the
- * {@link AppliedPTransform PTransform} has not processed any elements, return a watermark of
- * {@link BoundedWindow#TIMESTAMP_MIN_VALUE}.
- *
- * @return a snapshot of the input watermark and output watermark for the provided transform
- */
- public TransformWatermarks getWatermarks(AppliedPTransform<?, ?, ?> transform) {
- return transformToWatermarks.get(transform);
- }
-
- /**
- * Updates the watermarks of a transform with one or more inputs.
- *
- * <p>Each transform has two monotonically increasing watermarks: the input watermark, which can,
- * at any time, be updated to equal:
- * <pre>
- * MAX(CurrentInputWatermark, MIN(PendingElements, InputPCollectionWatermarks))
- * </pre>
- * and the output watermark, which can, at any time, be updated to equal:
- * <pre>
- * MAX(CurrentOutputWatermark, MIN(InputWatermark, WatermarkHolds))
- * </pre>.
- *
- * @param completed the input that has completed
- * @param timerUpdate the timers that were added, removed, and completed as part of producing
- * this update
- * @param result the result that was produced by processing the input
- * @param earliestHold the earliest watermark hold in the transform's state. {@code null} if there
- * is no hold
- */
- public void updateWatermarks(
- @Nullable CommittedBundle<?> completed,
- TimerUpdate timerUpdate,
- CommittedResult result,
- Instant earliestHold) {
- pendingUpdates.offer(PendingWatermarkUpdate.create(completed,
- timerUpdate,
- result,
- earliestHold));
- }
-
- /**
- * Applies all pending updates to this {@link InMemoryWatermarkManager}, causing the pending state
- * of all {@link TransformWatermarks} to be advanced as far as possible.
- */
- private void applyPendingUpdates() {
- Set<AppliedPTransform<?, ?, ?>> updatedTransforms = new HashSet<>();
- PendingWatermarkUpdate pending = pendingUpdates.poll();
- while (pending != null) {
- applyPendingUpdate(pending);
- updatedTransforms.add(pending.getTransform());
- pending = pendingUpdates.poll();
- }
- pendingRefreshes.addAll(updatedTransforms);
- }
-
- private void applyPendingUpdate(PendingWatermarkUpdate pending) {
- CommittedResult result = pending.getResult();
- AppliedPTransform transform = result.getTransform();
- CommittedBundle<?> inputBundle = pending.getInputBundle();
-
- updatePending(inputBundle, pending.getTimerUpdate(), result);
-
- TransformWatermarks transformWms = transformToWatermarks.get(transform);
- transformWms.setEventTimeHold(inputBundle == null ? null : inputBundle.getKey(),
- pending.getEarliestHold());
- }
-
- /**
- * First adds all produced elements to the queue of pending elements for each consumer, then adds
- * all pending timers to the collection of pending timers, then removes all completed and deleted
- * timers from the collection of pending timers, then removes all completed elements from the
- * pending queue of the transform.
- *
- * <p>It is required that all newly pending elements are added to the queue of pending elements
- * for each consumer prior to the completed elements being removed, as doing otherwise could cause
- * a Watermark to appear in a state in which the upstream (completed) element does not hold the
- * watermark but the element it produced is not yet pending. This can cause the watermark to
- * erroneously advance.
- */
- private void updatePending(
- CommittedBundle<?> input,
- TimerUpdate timerUpdate,
- CommittedResult result) {
- // Newly pending elements must be added before completed elements are removed, as the two
- // do not share a Mutex within this call and thus can be interleaved with external calls to
- // refresh.
- for (CommittedBundle<?> bundle : result.getOutputs()) {
- for (AppliedPTransform<?, ?, ?> consumer : consumers.get(bundle.getPCollection())) {
- TransformWatermarks watermarks = transformToWatermarks.get(consumer);
- watermarks.addPending(bundle);
- }
- }
-
- TransformWatermarks completedTransform = transformToWatermarks.get(result.getTransform());
- if (input != null) {
- // Add the unprocessed inputs
- completedTransform.addPending(result.getUnprocessedInputs());
- }
- completedTransform.updateTimers(timerUpdate);
- if (input != null) {
- completedTransform.removePending(input);
- }
- }
-
- /**
- * Refresh the watermarks contained within this {@link InMemoryWatermarkManager}, causing all
- * watermarks to be advanced as far as possible.
- */
- synchronized void refreshAll() {
- applyPendingUpdates();
- while (!pendingRefreshes.isEmpty()) {
- refreshWatermarks(pendingRefreshes.poll());
- }
- }
-
- private void refreshWatermarks(AppliedPTransform<?, ?, ?> toRefresh) {
- TransformWatermarks myWatermarks = transformToWatermarks.get(toRefresh);
- WatermarkUpdate updateResult = myWatermarks.refresh();
- Set<AppliedPTransform<?, ?, ?>> additionalRefreshes = new HashSet<>();
- if (updateResult.isAdvanced()) {
- for (PValue outputPValue : toRefresh.getOutput().expand()) {
- additionalRefreshes.addAll(consumers.get(outputPValue));
- }
- }
- pendingRefreshes.addAll(additionalRefreshes);
- }
-
- /**
- * Returns a map of each {@link PTransform} that has pending timers to those timers. All of the
- * pending timers will be removed from this {@link InMemoryWatermarkManager}.
- */
- public Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> extractFiredTimers() {
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> allTimers = new HashMap<>();
- for (Map.Entry<AppliedPTransform<?, ?, ?>, TransformWatermarks> watermarksEntry :
- transformToWatermarks.entrySet()) {
- Map<StructuralKey<?>, FiredTimers> keyFiredTimers =
- watermarksEntry.getValue().extractFiredTimers();
- if (!keyFiredTimers.isEmpty()) {
- allTimers.put(watermarksEntry.getKey(), keyFiredTimers);
- }
- }
- return allTimers;
- }
-
- /**
- * A (key, Instant) pair that holds the watermark. Holds are per-key, but the watermark is global,
- * and as such the watermark manager must track holds and the release of holds on a per-key basis.
- *
- * <p>The {@link #compareTo(KeyedHold)} method of {@link KeyedHold} is not consistent with equals,
- * as the key is arbitrarily ordered via identity, rather than object equality.
- */
- private static final class KeyedHold implements Comparable<KeyedHold> {
- private static final Ordering<Object> KEY_ORDERING = Ordering.arbitrary().nullsLast();
-
- private final Object key;
- private final Instant timestamp;
-
- /**
- * Create a new KeyedHold with the specified key and timestamp.
- */
- public static KeyedHold of(Object key, Instant timestamp) {
- return new KeyedHold(key, MoreObjects.firstNonNull(timestamp, THE_END_OF_TIME.get()));
- }
-
- private KeyedHold(Object key, Instant timestamp) {
- this.key = key;
- this.timestamp = timestamp;
- }
-
- @Override
- public int compareTo(KeyedHold that) {
- return ComparisonChain.start()
- .compare(this.timestamp, that.timestamp)
- .compare(this.key, that.key, KEY_ORDERING)
- .result();
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(timestamp, key);
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null || !(other instanceof KeyedHold)) {
- return false;
- }
- KeyedHold that = (KeyedHold) other;
- return Objects.equals(this.timestamp, that.timestamp) && Objects.equals(this.key, that.key);
- }
-
- /**
- * Get the value of this {@link KeyedHold}.
- */
- public Instant getTimestamp() {
- return timestamp;
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(KeyedHold.class)
- .add("key", key)
- .add("hold", timestamp)
- .toString();
- }
- }
-
- private static class PerKeyHolds {
- private final Map<Object, KeyedHold> keyedHolds;
- private final PriorityQueue<KeyedHold> allHolds;
-
- private PerKeyHolds() {
- this.keyedHolds = new HashMap<>();
- this.allHolds = new PriorityQueue<>();
- }
-
- /**
- * Gets the minimum hold across all keys in this {@link PerKeyHolds}, or THE_END_OF_TIME if
- * there are no holds within this {@link PerKeyHolds}.
- */
- public Instant getMinHold() {
- return allHolds.isEmpty() ? THE_END_OF_TIME.get() : allHolds.peek().getTimestamp();
- }
-
- /**
- * Updates the hold of the provided key to the provided value, removing any other holds for
- * the same key.
- */
- public void updateHold(@Nullable Object key, Instant newHold) {
- removeHold(key);
- KeyedHold newKeyedHold = KeyedHold.of(key, newHold);
- keyedHolds.put(key, newKeyedHold);
- allHolds.offer(newKeyedHold);
- }
-
- /**
- * Removes the hold of the provided key.
- */
- public void removeHold(Object key) {
- KeyedHold oldHold = keyedHolds.get(key);
- if (oldHold != null) {
- allHolds.remove(oldHold);
- }
- }
- }
-
- /**
- * A reference to the input and output watermarks of an {@link AppliedPTransform}.
- */
- public class TransformWatermarks {
- private final AppliedPTransformInputWatermark inputWatermark;
- private final AppliedPTransformOutputWatermark outputWatermark;
-
- private final SynchronizedProcessingTimeInputWatermark synchronizedProcessingInputWatermark;
- private final SynchronizedProcessingTimeOutputWatermark synchronizedProcessingOutputWatermark;
-
- private Instant latestSynchronizedInputWm;
- private Instant latestSynchronizedOutputWm;
-
- private TransformWatermarks(
- AppliedPTransformInputWatermark inputWatermark,
- AppliedPTransformOutputWatermark outputWatermark,
- SynchronizedProcessingTimeInputWatermark inputSynchProcessingWatermark,
- SynchronizedProcessingTimeOutputWatermark outputSynchProcessingWatermark) {
- this.inputWatermark = inputWatermark;
- this.outputWatermark = outputWatermark;
-
- this.synchronizedProcessingInputWatermark = inputSynchProcessingWatermark;
- this.synchronizedProcessingOutputWatermark = outputSynchProcessingWatermark;
- this.latestSynchronizedInputWm = BoundedWindow.TIMESTAMP_MIN_VALUE;
- this.latestSynchronizedOutputWm = BoundedWindow.TIMESTAMP_MIN_VALUE;
- }
-
- /**
- * Returns the input watermark of the {@link AppliedPTransform}.
- */
- public Instant getInputWatermark() {
- return Preconditions.checkNotNull(inputWatermark.get());
- }
-
- /**
- * Returns the output watermark of the {@link AppliedPTransform}.
- */
- public Instant getOutputWatermark() {
- return outputWatermark.get();
- }
-
- /**
- * Returns the synchronized processing input time of the {@link AppliedPTransform}.
- *
- * <p>The returned value is guaranteed to be monotonically increasing, and outside of the
- * presence of holds, will increase as the system time progresses.
- */
- public synchronized Instant getSynchronizedProcessingInputTime() {
- latestSynchronizedInputWm = INSTANT_ORDERING.max(
- latestSynchronizedInputWm,
- INSTANT_ORDERING.min(clock.now(), synchronizedProcessingInputWatermark.get()));
- return latestSynchronizedInputWm;
- }
-
- /**
- * Returns the synchronized processing output time of the {@link AppliedPTransform}.
- *
- * <p>The returned value is guaranteed to be monotonically increasing, and outside of the
- * presence of holds, will increase as the system time progresses.
- */
- public synchronized Instant getSynchronizedProcessingOutputTime() {
- latestSynchronizedOutputWm = INSTANT_ORDERING.max(
- latestSynchronizedOutputWm,
- INSTANT_ORDERING.min(clock.now(), synchronizedProcessingOutputWatermark.get()));
- return latestSynchronizedOutputWm;
- }
-
- private WatermarkUpdate refresh() {
- inputWatermark.refresh();
- synchronizedProcessingInputWatermark.refresh();
- WatermarkUpdate eventOutputUpdate = outputWatermark.refresh();
- WatermarkUpdate syncOutputUpdate = synchronizedProcessingOutputWatermark.refresh();
- return eventOutputUpdate.union(syncOutputUpdate);
- }
-
- private void setEventTimeHold(Object key, Instant newHold) {
- outputWatermark.updateHold(key, newHold);
- }
-
- private void removePending(CommittedBundle<?> bundle) {
- inputWatermark.removePendingElements(elementsFromBundle(bundle));
- synchronizedProcessingInputWatermark.removePending(bundle);
- }
-
- private void addPending(CommittedBundle<?> bundle) {
- inputWatermark.addPendingElements(elementsFromBundle(bundle));
- synchronizedProcessingInputWatermark.addPending(bundle);
- }
-
- private Iterable<? extends WindowedValue<?>> elementsFromBundle(CommittedBundle<?> bundle) {
- return FluentIterable.from(bundle.getElements()).transformAndConcat(EXPLODE_WINDOWS_FN);
- }
-
- private Map<StructuralKey<?>, FiredTimers> extractFiredTimers() {
- Map<StructuralKey<?>, List<TimerData>> eventTimeTimers =
- inputWatermark.extractFiredEventTimeTimers();
- Map<StructuralKey<?>, List<TimerData>> processingTimers;
- Map<StructuralKey<?>, List<TimerData>> synchronizedTimers;
- if (inputWatermark.get().equals(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
- processingTimers = synchronizedProcessingInputWatermark.extractFiredDomainTimers(
- TimeDomain.PROCESSING_TIME, BoundedWindow.TIMESTAMP_MAX_VALUE);
- synchronizedTimers = synchronizedProcessingInputWatermark.extractFiredDomainTimers(
- TimeDomain.PROCESSING_TIME, BoundedWindow.TIMESTAMP_MAX_VALUE);
- } else {
- processingTimers = synchronizedProcessingInputWatermark.extractFiredDomainTimers(
- TimeDomain.PROCESSING_TIME, clock.now());
- synchronizedTimers = synchronizedProcessingInputWatermark.extractFiredDomainTimers(
- TimeDomain.SYNCHRONIZED_PROCESSING_TIME, getSynchronizedProcessingInputTime());
- }
- Map<StructuralKey<?>, Map<TimeDomain, List<TimerData>>> groupedTimers = new HashMap<>();
- groupFiredTimers(groupedTimers, eventTimeTimers, processingTimers, synchronizedTimers);
-
- Map<StructuralKey<?>, FiredTimers> keyFiredTimers = new HashMap<>();
- for (Map.Entry<StructuralKey<?>, Map<TimeDomain, List<TimerData>>> firedTimers :
- groupedTimers.entrySet()) {
- keyFiredTimers.put(firedTimers.getKey(), new FiredTimers(firedTimers.getValue()));
- }
- return keyFiredTimers;
- }
-
- @SafeVarargs
- private final void groupFiredTimers(
- Map<StructuralKey<?>, Map<TimeDomain, List<TimerData>>> groupedToMutate,
- Map<StructuralKey<?>, List<TimerData>>... timersToGroup) {
- for (Map<StructuralKey<?>, List<TimerData>> subGroup : timersToGroup) {
- for (Map.Entry<StructuralKey<?>, List<TimerData>> newTimers : subGroup.entrySet()) {
- Map<TimeDomain, List<TimerData>> grouped = groupedToMutate.get(newTimers.getKey());
- if (grouped == null) {
- grouped = new HashMap<>();
- groupedToMutate.put(newTimers.getKey(), grouped);
- }
- grouped.put(newTimers.getValue().get(0).getDomain(), newTimers.getValue());
- }
- }
- }
-
- private void updateTimers(TimerUpdate update) {
- inputWatermark.updateTimers(update);
- synchronizedProcessingInputWatermark.updateTimers(update);
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(TransformWatermarks.class)
- .add("inputWatermark", inputWatermark)
- .add("outputWatermark", outputWatermark)
- .add("inputProcessingTime", synchronizedProcessingInputWatermark)
- .add("outputProcessingTime", synchronizedProcessingOutputWatermark)
- .toString();
- }
- }
-
- /**
- * A collection of newly set, deleted, and completed timers.
- *
- * <p>setTimers and deletedTimers are collections of {@link TimerData} that have been added to the
- * {@link TimerInternals} of an executed step. completedTimers are timers that were delivered as
- * the input to the executed step.
- */
- public static class TimerUpdate {
- private final StructuralKey<?> key;
- private final Iterable<? extends TimerData> completedTimers;
-
- private final Iterable<? extends TimerData> setTimers;
- private final Iterable<? extends TimerData> deletedTimers;
-
- /**
- * Returns a TimerUpdate for a null key with no timers.
- */
- public static TimerUpdate empty() {
- return new TimerUpdate(
- null,
- Collections.<TimerData>emptyList(),
- Collections.<TimerData>emptyList(),
- Collections.<TimerData>emptyList());
- }
-
- /**
- * Creates a new {@link TimerUpdate} builder with the provided completed timers that needs the
- * set and deleted timers to be added to it.
- */
- public static TimerUpdateBuilder builder(StructuralKey<?> key) {
- return new TimerUpdateBuilder(key);
- }
-
- /**
- * A {@link TimerUpdate} builder that needs to be provided with set timers and deleted timers.
- */
- public static final class TimerUpdateBuilder {
- private final StructuralKey<?> key;
- private final Collection<TimerData> completedTimers;
- private final Collection<TimerData> setTimers;
- private final Collection<TimerData> deletedTimers;
-
- private TimerUpdateBuilder(StructuralKey<?> key) {
- this.key = key;
- this.completedTimers = new HashSet<>();
- this.setTimers = new HashSet<>();
- this.deletedTimers = new HashSet<>();
- }
-
- /**
- * Adds all of the provided timers to the collection of completed timers, and returns this
- * {@link TimerUpdateBuilder}.
- */
- public TimerUpdateBuilder withCompletedTimers(Iterable<TimerData> completedTimers) {
- Iterables.addAll(this.completedTimers, completedTimers);
- return this;
- }
-
- /**
- * Adds the provided timer to the collection of set timers, removing it from deleted timers if
- * it has previously been deleted. Returns this {@link TimerUpdateBuilder}.
- */
- public TimerUpdateBuilder setTimer(TimerData setTimer) {
- deletedTimers.remove(setTimer);
- setTimers.add(setTimer);
- return this;
- }
-
- /**
- * Adds the provided timer to the collection of deleted timers, removing it from set timers if
- * it has previously been set. Returns this {@link TimerUpdateBuilder}.
- */
- public TimerUpdateBuilder deletedTimer(TimerData deletedTimer) {
- deletedTimers.add(deletedTimer);
- setTimers.remove(deletedTimer);
- return this;
- }
-
- /**
- * Returns a new {@link TimerUpdate} with the most recently set completedTimers, setTimers,
- * and deletedTimers.
- */
- public TimerUpdate build() {
- return new TimerUpdate(
- key,
- ImmutableSet.copyOf(completedTimers),
- ImmutableSet.copyOf(setTimers),
- ImmutableSet.copyOf(deletedTimers));
- }
- }
-
- private TimerUpdate(
- StructuralKey<?> key,
- Iterable<? extends TimerData> completedTimers,
- Iterable<? extends TimerData> setTimers,
- Iterable<? extends TimerData> deletedTimers) {
- this.key = key;
- this.completedTimers = completedTimers;
- this.setTimers = setTimers;
- this.deletedTimers = deletedTimers;
- }
-
- @VisibleForTesting
- StructuralKey<?> getKey() {
- return key;
- }
-
- @VisibleForTesting
- Iterable<? extends TimerData> getCompletedTimers() {
- return completedTimers;
- }
-
- @VisibleForTesting
- Iterable<? extends TimerData> getSetTimers() {
- return setTimers;
- }
-
- @VisibleForTesting
- Iterable<? extends TimerData> getDeletedTimers() {
- return deletedTimers;
- }
-
- /**
- * Returns a {@link TimerUpdate} that is like this one, but with the specified completed timers.
- */
- public TimerUpdate withCompletedTimers(Iterable<TimerData> completedTimers) {
- return new TimerUpdate(this.key, completedTimers, setTimers, deletedTimers);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(key, completedTimers, setTimers, deletedTimers);
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null || !(other instanceof TimerUpdate)) {
- return false;
- }
- TimerUpdate that = (TimerUpdate) other;
- return Objects.equals(this.key, that.key)
- && Objects.equals(this.completedTimers, that.completedTimers)
- && Objects.equals(this.setTimers, that.setTimers)
- && Objects.equals(this.deletedTimers, that.deletedTimers);
- }
- }
-
- /**
- * A pair of {@link TimerData} and key which can be delivered to the appropriate
- * {@link AppliedPTransform}. A timer fires at the transform that set it with a specific key when
- * the time domain in which it lives progresses past a specified time, as determined by the
- * {@link InMemoryWatermarkManager}.
- */
- public static class FiredTimers {
- private final Map<TimeDomain, ? extends Collection<TimerData>> timers;
-
- private FiredTimers(Map<TimeDomain, ? extends Collection<TimerData>> timers) {
- this.timers = timers;
- }
-
- /**
- * Gets all of the timers that have fired within the provided {@link TimeDomain}. If no timers
- * fired within the provided domain, return an empty collection.
- *
- * <p>Timers within a {@link TimeDomain} are guaranteed to be in order of increasing timestamp.
- */
- public Collection<TimerData> getTimers(TimeDomain domain) {
- Collection<TimerData> domainTimers = timers.get(domain);
- if (domainTimers == null) {
- return Collections.emptyList();
- }
- return domainTimers;
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(FiredTimers.class).add("timers", timers).toString();
- }
- }
-
- private static class WindowedValueByTimestampComparator extends Ordering<WindowedValue<?>> {
- @Override
- public int compare(WindowedValue<?> o1, WindowedValue<?> o2) {
- return ComparisonChain.start()
- .compare(o1.getTimestamp(), o2.getTimestamp())
- .result();
- }
- }
-
- public Set<AppliedPTransform<?, ?, ?>> getCompletedTransforms() {
- Set<AppliedPTransform<?, ?, ?>> result = new HashSet<>();
- for (Map.Entry<AppliedPTransform<?, ?, ?>, TransformWatermarks> wms :
- transformToWatermarks.entrySet()) {
- if (wms.getValue().getOutputWatermark().equals(THE_END_OF_TIME.get())) {
- result.add(wms.getKey());
- }
- }
- return result;
- }
-
- @AutoValue
- abstract static class PendingWatermarkUpdate {
- @Nullable
- public abstract CommittedBundle<?> getInputBundle();
- public abstract TimerUpdate getTimerUpdate();
- public abstract CommittedResult getResult();
- public abstract Instant getEarliestHold();
-
- /**
- * Gets the {@link AppliedPTransform} that generated this result.
- */
- public AppliedPTransform<?, ?, ?> getTransform() {
- return getResult().getTransform();
- }
-
- public static PendingWatermarkUpdate create(
- CommittedBundle<?> inputBundle,
- TimerUpdate timerUpdate,
- CommittedResult result, Instant earliestHold) {
- return new AutoValue_InMemoryWatermarkManager_PendingWatermarkUpdate(inputBundle,
- timerUpdate,
- result,
- earliestHold);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleFactory.java
deleted file mode 100644
index 0c7449c..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleFactory.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static com.google.common.base.Preconditions.checkState;
-
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.PCollection;
-
-import com.google.common.base.MoreObjects;
-import com.google.common.collect.ImmutableList;
-
-import org.joda.time.Instant;
-
-/**
- * A factory that produces bundles that perform no additional validation.
- */
-class InProcessBundleFactory implements BundleFactory {
- public static InProcessBundleFactory create() {
- return new InProcessBundleFactory();
- }
-
- private InProcessBundleFactory() {}
-
- @Override
- public <T> UncommittedBundle<T> createRootBundle(PCollection<T> output) {
- return InProcessBundle.create(output, StructuralKey.of(null, VoidCoder.of()));
- }
-
- @Override
- public <T> UncommittedBundle<T> createBundle(CommittedBundle<?> input, PCollection<T> output) {
- return InProcessBundle.create(output, input.getKey());
- }
-
- @Override
- public <K, T> UncommittedBundle<T> createKeyedBundle(
- CommittedBundle<?> input, StructuralKey<K> key, PCollection<T> output) {
- return InProcessBundle.create(output, key);
- }
-
- /**
- * A {@link UncommittedBundle} that buffers elements in memory.
- */
- private static final class InProcessBundle<T> implements UncommittedBundle<T> {
- private final PCollection<T> pcollection;
- private final StructuralKey<?> key;
- private boolean committed = false;
- private ImmutableList.Builder<WindowedValue<T>> elements;
-
- /**
- * Create a new {@link InProcessBundle} for the specified {@link PCollection}.
- */
- public static <T> InProcessBundle<T> create(PCollection<T> pcollection, StructuralKey<?> key) {
- return new InProcessBundle<>(pcollection, key);
- }
-
- private InProcessBundle(PCollection<T> pcollection, StructuralKey<?> key) {
- this.pcollection = pcollection;
- this.key = key;
- this.elements = ImmutableList.builder();
- }
-
- @Override
- public PCollection<T> getPCollection() {
- return pcollection;
- }
-
- @Override
- public InProcessBundle<T> add(WindowedValue<T> element) {
- checkState(
- !committed,
- "Can't add element %s to committed bundle in PCollection %s",
- element,
- pcollection);
- elements.add(element);
- return this;
- }
-
- @Override
- public CommittedBundle<T> commit(final Instant synchronizedCompletionTime) {
- checkState(!committed, "Can't commit already committed bundle %s", this);
- committed = true;
- final Iterable<WindowedValue<T>> committedElements = elements.build();
- return new CommittedInProcessBundle<>(
- pcollection, key, committedElements, synchronizedCompletionTime);
- }
- }
-
- private static class CommittedInProcessBundle<T> implements CommittedBundle<T> {
- public CommittedInProcessBundle(
- PCollection<T> pcollection,
- StructuralKey<?> key,
- Iterable<WindowedValue<T>> committedElements,
- Instant synchronizedCompletionTime) {
- this.pcollection = pcollection;
- this.key = key;
- this.committedElements = committedElements;
- this.synchronizedCompletionTime = synchronizedCompletionTime;
- }
-
- private final PCollection<T> pcollection;
- /** The structural value key of the Bundle, as specified by the coder that created it. */
- private final StructuralKey<?> key;
- private final Iterable<WindowedValue<T>> committedElements;
- private final Instant synchronizedCompletionTime;
-
- @Override
- public StructuralKey<?> getKey() {
- return key;
- }
-
- @Override
- public Iterable<WindowedValue<T>> getElements() {
- return committedElements;
- }
-
- @Override
- public PCollection<T> getPCollection() {
- return pcollection;
- }
-
- @Override
- public Instant getSynchronizedProcessingOutputWatermark() {
- return synchronizedCompletionTime;
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .omitNullValues()
- .add("pcollection", pcollection)
- .add("key", key)
- .add("elements", committedElements)
- .toString();
- }
-
- @Override
- public CommittedBundle<T> withElements(Iterable<WindowedValue<T>> elements) {
- return new CommittedInProcessBundle<>(
- pcollection, key, ImmutableList.copyOf(elements), synchronizedCompletionTime);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleOutputManager.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleOutputManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleOutputManager.java
deleted file mode 100644
index bd07040..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleOutputManager.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.sdk.util.DoFnRunners.OutputManager;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.TupleTag;
-
-import java.util.Map;
-
-/**
- * An {@link OutputManager} that outputs to {@link CommittedBundle Bundles} used by the
- * {@link DirectRunner}.
- */
-public class InProcessBundleOutputManager implements OutputManager {
- private final Map<TupleTag<?>, UncommittedBundle<?>> bundles;
-
- public static InProcessBundleOutputManager create(
- Map<TupleTag<?>, UncommittedBundle<?>> outputBundles) {
- return new InProcessBundleOutputManager(outputBundles);
- }
-
- public InProcessBundleOutputManager(Map<TupleTag<?>, UncommittedBundle<?>> bundles) {
- this.bundles = bundles;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <T> void output(TupleTag<T> tag, WindowedValue<T> output) {
- @SuppressWarnings("rawtypes")
- UncommittedBundle bundle = bundles.get(tag);
- bundle.add(output);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessEvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessEvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessEvaluationContext.java
deleted file mode 100644
index 220ff83..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessEvaluationContext.java
+++ /dev/null
@@ -1,429 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.FiredTimers;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TransformWatermarks;
-import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.runners.direct.DirectRunner.PCollectionViewWriter;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.Trigger;
-import org.apache.beam.sdk.util.ExecutionContext;
-import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
-import org.apache.beam.sdk.util.SideInputReader;
-import org.apache.beam.sdk.util.TimerInternals.TimerData;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.WindowingStrategy;
-import org.apache.beam.sdk.util.common.CounterSet;
-import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollection.IsBounded;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PValue;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.util.concurrent.MoreExecutors;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import javax.annotation.Nullable;
-
-/**
- * The evaluation context for a specific pipeline being executed by the
- * {@link DirectRunner}. Contains state shared within the execution across all
- * transforms.
- *
- * <p>{@link InProcessEvaluationContext} contains shared state for an execution of the
- * {@link DirectRunner} that can be used while evaluating a {@link PTransform}. This
- * consists of views into underlying state and watermark implementations, access to read and write
- * {@link PCollectionView PCollectionViews}, and constructing {@link CounterSet CounterSets} and
- * {@link ExecutionContext ExecutionContexts}. This includes executing callbacks asynchronously when
- * state changes to the appropriate point (e.g. when a {@link PCollectionView} is requested and
- * known to be empty).
- *
- * <p>{@link InProcessEvaluationContext} also handles results by committing finalizing bundles based
- * on the current global state and updating the global state appropriately. This includes updating
- * the per-{@link StepAndKey} state, updating global watermarks, and executing any callbacks that
- * can be executed.
- */
-class InProcessEvaluationContext {
- /** The step name for each {@link AppliedPTransform} in the {@link Pipeline}. */
- private final Map<AppliedPTransform<?, ?, ?>, String> stepNames;
-
- /** The options that were used to create this {@link Pipeline}. */
- private final DirectOptions options;
-
- private final BundleFactory bundleFactory;
- /** The current processing time and event time watermarks and timers. */
- private final InMemoryWatermarkManager watermarkManager;
-
- /** Executes callbacks based on the progression of the watermark. */
- private final WatermarkCallbackExecutor callbackExecutor;
-
- /** The stateInternals of the world, by applied PTransform and key. */
- private final ConcurrentMap<StepAndKey, CopyOnAccessInMemoryStateInternals<?>>
- applicationStateInternals;
-
- private final InProcessSideInputContainer sideInputContainer;
-
- private final CounterSet mergedCounters;
-
- public static InProcessEvaluationContext create(
- DirectOptions options,
- BundleFactory bundleFactory,
- Collection<AppliedPTransform<?, ?, ?>> rootTransforms,
- Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> valueToConsumers,
- Map<AppliedPTransform<?, ?, ?>, String> stepNames,
- Collection<PCollectionView<?>> views) {
- return new InProcessEvaluationContext(
- options, bundleFactory, rootTransforms, valueToConsumers, stepNames, views);
- }
-
- private InProcessEvaluationContext(
- DirectOptions options,
- BundleFactory bundleFactory,
- Collection<AppliedPTransform<?, ?, ?>> rootTransforms,
- Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> valueToConsumers,
- Map<AppliedPTransform<?, ?, ?>, String> stepNames,
- Collection<PCollectionView<?>> views) {
- this.options = checkNotNull(options);
- this.bundleFactory = checkNotNull(bundleFactory);
- checkNotNull(rootTransforms);
- checkNotNull(valueToConsumers);
- checkNotNull(stepNames);
- checkNotNull(views);
- this.stepNames = stepNames;
-
- this.watermarkManager =
- InMemoryWatermarkManager.create(
- NanosOffsetClock.create(), rootTransforms, valueToConsumers);
- this.sideInputContainer = InProcessSideInputContainer.create(this, views);
-
- this.applicationStateInternals = new ConcurrentHashMap<>();
- this.mergedCounters = new CounterSet();
-
- this.callbackExecutor =
- WatermarkCallbackExecutor.create(MoreExecutors.directExecutor());
- }
-
- /**
- * Handle the provided {@link InProcessTransformResult}, produced after evaluating the provided
- * {@link CommittedBundle} (potentially null, if the result of a root {@link PTransform}).
- *
- * <p>The result is the output of running the transform contained in the
- * {@link InProcessTransformResult} on the contents of the provided bundle.
- *
- * @param completedBundle the bundle that was processed to produce the result. Potentially
- * {@code null} if the transform that produced the result is a root
- * transform
- * @param completedTimers the timers that were delivered to produce the {@code completedBundle},
- * or an empty iterable if no timers were delivered
- * @param result the result of evaluating the input bundle
- * @return the committed bundles contained within the handled {@code result}
- */
- public CommittedResult handleResult(
- @Nullable CommittedBundle<?> completedBundle,
- Iterable<TimerData> completedTimers,
- InProcessTransformResult result) {
- Iterable<? extends CommittedBundle<?>> committedBundles =
- commitBundles(result.getOutputBundles());
- // Update watermarks and timers
- CommittedResult committedResult = CommittedResult.create(result,
- completedBundle == null
- ? null
- : completedBundle.withElements((Iterable) result.getUnprocessedElements()),
- committedBundles);
- watermarkManager.updateWatermarks(
- completedBundle,
- result.getTimerUpdate().withCompletedTimers(completedTimers),
- committedResult,
- result.getWatermarkHold());
- // Update counters
- if (result.getCounters() != null) {
- mergedCounters.merge(result.getCounters());
- }
- // Update state internals
- CopyOnAccessInMemoryStateInternals<?> theirState = result.getState();
- if (theirState != null) {
- CopyOnAccessInMemoryStateInternals<?> committedState = theirState.commit();
- StepAndKey stepAndKey =
- StepAndKey.of(
- result.getTransform(), completedBundle == null ? null : completedBundle.getKey());
- if (!committedState.isEmpty()) {
- applicationStateInternals.put(stepAndKey, committedState);
- } else {
- applicationStateInternals.remove(stepAndKey);
- }
- }
- return committedResult;
- }
-
- private Iterable<? extends CommittedBundle<?>> commitBundles(
- Iterable<? extends UncommittedBundle<?>> bundles) {
- ImmutableList.Builder<CommittedBundle<?>> completed = ImmutableList.builder();
- for (UncommittedBundle<?> inProgress : bundles) {
- AppliedPTransform<?, ?, ?> producing =
- inProgress.getPCollection().getProducingTransformInternal();
- TransformWatermarks watermarks = watermarkManager.getWatermarks(producing);
- CommittedBundle<?> committed =
- inProgress.commit(watermarks.getSynchronizedProcessingOutputTime());
- // Empty bundles don't impact watermarks and shouldn't trigger downstream execution, so
- // filter them out
- if (!Iterables.isEmpty(committed.getElements())) {
- completed.add(committed);
- }
- }
- return completed.build();
- }
-
- private void fireAllAvailableCallbacks() {
- for (AppliedPTransform<?, ?, ?> transform : stepNames.keySet()) {
- fireAvailableCallbacks(transform);
- }
- }
-
- private void fireAvailableCallbacks(AppliedPTransform<?, ?, ?> producingTransform) {
- TransformWatermarks watermarks = watermarkManager.getWatermarks(producingTransform);
- callbackExecutor.fireForWatermark(producingTransform, watermarks.getOutputWatermark());
- }
-
- /**
- * Create a {@link UncommittedBundle} for use by a source.
- */
- public <T> UncommittedBundle<T> createRootBundle(PCollection<T> output) {
- return bundleFactory.createRootBundle(output);
- }
-
- /**
- * Create a {@link UncommittedBundle} whose elements belong to the specified {@link
- * PCollection}.
- */
- public <T> UncommittedBundle<T> createBundle(CommittedBundle<?> input, PCollection<T> output) {
- return bundleFactory.createBundle(input, output);
- }
-
- /**
- * Create a {@link UncommittedBundle} with the specified keys at the specified step. For use by
- * {@link InProcessGroupByKeyOnly} {@link PTransform PTransforms}.
- */
- public <K, T> UncommittedBundle<T> createKeyedBundle(
- CommittedBundle<?> input, StructuralKey<K> key, PCollection<T> output) {
- return bundleFactory.createKeyedBundle(input, key, output);
- }
-
- /**
- * Create a {@link PCollectionViewWriter}, whose elements will be used in the provided
- * {@link PCollectionView}.
- */
- public <ElemT, ViewT> PCollectionViewWriter<ElemT, ViewT> createPCollectionViewWriter(
- PCollection<Iterable<ElemT>> input, final PCollectionView<ViewT> output) {
- return new PCollectionViewWriter<ElemT, ViewT>() {
- @Override
- public void add(Iterable<WindowedValue<ElemT>> values) {
- sideInputContainer.write(output, values);
- }
- };
- }
-
- /**
- * Schedule a callback to be executed after output would be produced for the given window
- * if there had been input.
- *
- * <p>Output would be produced when the watermark for a {@link PValue} passes the point at
- * which the trigger for the specified window (with the specified windowing strategy) must have
- * fired from the perspective of that {@link PValue}, as specified by the value of
- * {@link Trigger#getWatermarkThatGuaranteesFiring(BoundedWindow)} for the trigger of the
- * {@link WindowingStrategy}. When the callback has fired, either values will have been produced
- * for a key in that window, the window is empty, or all elements in the window are late. The
- * callback will be executed regardless of whether values have been produced.
- */
- public void scheduleAfterOutputWouldBeProduced(
- PValue value,
- BoundedWindow window,
- WindowingStrategy<?, ?> windowingStrategy,
- Runnable runnable) {
- AppliedPTransform<?, ?, ?> producing = getProducing(value);
- callbackExecutor.callOnGuaranteedFiring(producing, window, windowingStrategy, runnable);
-
- fireAvailableCallbacks(lookupProducing(value));
- }
-
- private AppliedPTransform<?, ?, ?> getProducing(PValue value) {
- if (value.getProducingTransformInternal() != null) {
- return value.getProducingTransformInternal();
- }
- return lookupProducing(value);
- }
-
- private AppliedPTransform<?, ?, ?> lookupProducing(PValue value) {
- for (AppliedPTransform<?, ?, ?> transform : stepNames.keySet()) {
- if (transform.getOutput().equals(value) || transform.getOutput().expand().contains(value)) {
- return transform;
- }
- }
- return null;
- }
-
- /**
- * Get the options used by this {@link Pipeline}.
- */
- public DirectOptions getPipelineOptions() {
- return options;
- }
-
- /**
- * Get an {@link ExecutionContext} for the provided {@link AppliedPTransform} and key.
- */
- public InProcessExecutionContext getExecutionContext(
- AppliedPTransform<?, ?, ?> application, StructuralKey<?> key) {
- StepAndKey stepAndKey = StepAndKey.of(application, key);
- return new InProcessExecutionContext(
- options.getClock(),
- key,
- (CopyOnAccessInMemoryStateInternals<Object>) applicationStateInternals.get(stepAndKey),
- watermarkManager.getWatermarks(application));
- }
-
- /**
- * Get all of the steps used in this {@link Pipeline}.
- */
- public Collection<AppliedPTransform<?, ?, ?>> getSteps() {
- return stepNames.keySet();
- }
-
- /**
- * Get the Step Name for the provided application.
- */
- public String getStepName(AppliedPTransform<?, ?, ?> application) {
- return stepNames.get(application);
- }
-
- /**
- * Returns a {@link ReadyCheckingSideInputReader} capable of reading the provided
- * {@link PCollectionView PCollectionViews}.
- *
- * @param sideInputs the {@link PCollectionView PCollectionViews} the result should be able to
- * read
- * @return a {@link SideInputReader} that can read all of the provided {@link PCollectionView
- * PCollectionViews}
- */
- public ReadyCheckingSideInputReader createSideInputReader(
- final List<PCollectionView<?>> sideInputs) {
- return sideInputContainer.createReaderForViews(sideInputs);
- }
-
-
- /**
- * Create a {@link CounterSet} for this {@link Pipeline}. The {@link CounterSet} is independent
- * of all other {@link CounterSet CounterSets} created by this call.
- *
- * The {@link InProcessEvaluationContext} is responsible for unifying the counters present in
- * all created {@link CounterSet CounterSets} when the transforms that call this method
- * complete.
- */
- public CounterSet createCounterSet() {
- return new CounterSet();
- }
-
- /**
- * Returns all of the counters that have been merged into this context via calls to
- * {@link CounterSet#merge(CounterSet)}.
- */
- public CounterSet getCounters() {
- return mergedCounters;
- }
-
- @VisibleForTesting
- void forceRefresh() {
- watermarkManager.refreshAll();
- fireAllAvailableCallbacks();
- }
-
- /**
- * Extracts all timers that have been fired and have not already been extracted.
- *
- * <p>This is a destructive operation. Timers will only appear in the result of this method once
- * for each time they are set.
- */
- public Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> extractFiredTimers() {
- forceRefresh();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> fired =
- watermarkManager.extractFiredTimers();
- return fired;
- }
-
- /**
- * Returns true if the step will not produce additional output.
- *
- * <p>If the provided transform produces only {@link IsBounded#BOUNDED}
- * {@link PCollection PCollections}, returns true if the watermark is at
- * {@link BoundedWindow#TIMESTAMP_MAX_VALUE positive infinity}.
- *
- * <p>If the provided transform produces any {@link IsBounded#UNBOUNDED}
- * {@link PCollection PCollections}, returns the value of
- * {@link DirectOptions#isShutdownUnboundedProducersWithMaxWatermark()}.
- */
- public boolean isDone(AppliedPTransform<?, ?, ?> transform) {
- // if the PTransform's watermark isn't at the max value, it isn't done
- if (watermarkManager
- .getWatermarks(transform)
- .getOutputWatermark()
- .isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
- return false;
- }
- // If the PTransform has any unbounded outputs, and unbounded producers should not be shut down,
- // the PTransform may produce additional output. It is not done.
- for (PValue output : transform.getOutput().expand()) {
- if (output instanceof PCollection) {
- IsBounded bounded = ((PCollection<?>) output).isBounded();
- if (bounded.equals(IsBounded.UNBOUNDED)
- && !options.isShutdownUnboundedProducersWithMaxWatermark()) {
- return false;
- }
- }
- }
- // The PTransform's watermark was at positive infinity and all of its outputs are known to be
- // done. It is done.
- return true;
- }
-
- /**
- * Returns true if all steps are done.
- */
- public boolean isDone() {
- for (AppliedPTransform<?, ?, ?> transform : stepNames.keySet()) {
- if (!isDone(transform)) {
- return false;
- }
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutionContext.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutionContext.java
deleted file mode 100644
index d2558ce..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutionContext.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TransformWatermarks;
-import org.apache.beam.sdk.util.BaseExecutionContext;
-import org.apache.beam.sdk.util.ExecutionContext;
-import org.apache.beam.sdk.util.TimerInternals;
-import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
-
-/**
- * Execution Context for the {@link DirectRunner}.
- *
- * This implementation is not thread safe. A new {@link InProcessExecutionContext} must be created
- * for each thread that requires it.
- */
-class InProcessExecutionContext
- extends BaseExecutionContext<InProcessExecutionContext.InProcessStepContext> {
- private final Clock clock;
- private final StructuralKey<?> key;
- private final CopyOnAccessInMemoryStateInternals<Object> existingState;
- private final TransformWatermarks watermarks;
-
- public InProcessExecutionContext(Clock clock, StructuralKey<?> key,
- CopyOnAccessInMemoryStateInternals<Object> existingState, TransformWatermarks watermarks) {
- this.clock = clock;
- this.key = key;
- this.existingState = existingState;
- this.watermarks = watermarks;
- }
-
- @Override
- protected InProcessStepContext createStepContext(String stepName, String transformName) {
- return new InProcessStepContext(this, stepName, transformName);
- }
-
- /**
- * Step Context for the {@link DirectRunner}.
- */
- public class InProcessStepContext
- extends org.apache.beam.sdk.util.BaseExecutionContext.StepContext {
- private CopyOnAccessInMemoryStateInternals<Object> stateInternals;
- private InProcessTimerInternals timerInternals;
-
- public InProcessStepContext(
- ExecutionContext executionContext, String stepName, String transformName) {
- super(executionContext, stepName, transformName);
- }
-
- @Override
- public CopyOnAccessInMemoryStateInternals<Object> stateInternals() {
- if (stateInternals == null) {
- stateInternals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, existingState);
- }
- return stateInternals;
- }
-
- @Override
- public InProcessTimerInternals timerInternals() {
- if (timerInternals == null) {
- timerInternals =
- InProcessTimerInternals.create(clock, watermarks, TimerUpdate.builder(key));
- }
- return timerInternals;
- }
-
- /**
- * Commits the state of this step, and returns the committed state. If the step has not
- * accessed any state, return null.
- */
- public CopyOnAccessInMemoryStateInternals<?> commitState() {
- if (stateInternals != null) {
- return stateInternals.commit();
- }
- return null;
- }
-
- /**
- * Gets the timer update of the {@link TimerInternals} of this {@link InProcessStepContext},
- * which is empty if the {@link TimerInternals} were never accessed.
- */
- public TimerUpdate getTimerUpdate() {
- if (timerInternals == null) {
- return TimerUpdate.empty();
- }
- return timerInternals.getTimerUpdate();
- }
- }
-}
[04/12] incubator-beam git commit: Remove InProcess Prefixes
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactoryTest.java
index 8b8d44f..c378cf4 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactoryTest.java
@@ -25,9 +25,9 @@ import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
@@ -66,7 +66,7 @@ import java.io.Serializable;
*/
@RunWith(JUnit4.class)
public class ParDoSingleEvaluatorFactoryTest implements Serializable {
- private transient BundleFactory bundleFactory = InProcessBundleFactory.create();
+ private transient BundleFactory bundleFactory = ImmutableListBundleFactory.create();
@Test
public void testParDoInMemoryTransformEvaluator() throws Exception {
@@ -85,11 +85,11 @@ public class ParDoSingleEvaluatorFactoryTest implements Serializable {
CommittedBundle<String> inputBundle =
bundleFactory.createRootBundle(input).commit(Instant.now());
- InProcessEvaluationContext evaluationContext = mock(InProcessEvaluationContext.class);
+ EvaluationContext evaluationContext = mock(EvaluationContext.class);
UncommittedBundle<Integer> outputBundle = bundleFactory.createRootBundle(collection);
when(evaluationContext.createBundle(inputBundle, collection)).thenReturn(outputBundle);
- InProcessExecutionContext executionContext =
- new InProcessExecutionContext(null, null, null, null);
+ DirectExecutionContext executionContext =
+ new DirectExecutionContext(null, null, null, null);
when(evaluationContext.getExecutionContext(collection.getProducingTransformInternal(),
inputBundle.getKey())).thenReturn(executionContext);
CounterSet counters = new CounterSet();
@@ -106,7 +106,7 @@ public class ParDoSingleEvaluatorFactoryTest implements Serializable {
evaluator.processElement(
WindowedValue.valueInGlobalWindow("bazam", PaneInfo.ON_TIME_AND_ONLY_FIRING));
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(result.getOutputBundles(), Matchers.<UncommittedBundle<?>>contains(outputBundle));
assertThat(result.getWatermarkHold(), equalTo(BoundedWindow.TIMESTAMP_MAX_VALUE));
assertThat(result.getCounters(), equalTo(counters));
@@ -137,11 +137,11 @@ public class ParDoSingleEvaluatorFactoryTest implements Serializable {
CommittedBundle<String> inputBundle =
bundleFactory.createRootBundle(input).commit(Instant.now());
- InProcessEvaluationContext evaluationContext = mock(InProcessEvaluationContext.class);
+ EvaluationContext evaluationContext = mock(EvaluationContext.class);
UncommittedBundle<Integer> outputBundle = bundleFactory.createRootBundle(collection);
when(evaluationContext.createBundle(inputBundle, collection)).thenReturn(outputBundle);
- InProcessExecutionContext executionContext =
- new InProcessExecutionContext(null, null, null, null);
+ DirectExecutionContext executionContext =
+ new DirectExecutionContext(null, null, null, null);
when(evaluationContext.getExecutionContext(collection.getProducingTransformInternal(),
inputBundle.getKey())).thenReturn(executionContext);
CounterSet counters = new CounterSet();
@@ -158,7 +158,7 @@ public class ParDoSingleEvaluatorFactoryTest implements Serializable {
evaluator.processElement(
WindowedValue.valueInGlobalWindow("bazam", PaneInfo.ON_TIME_AND_ONLY_FIRING));
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(
result.getOutputBundles(), Matchers.<UncommittedBundle<?>>containsInAnyOrder(outputBundle));
assertThat(result.getWatermarkHold(), equalTo(BoundedWindow.TIMESTAMP_MAX_VALUE));
@@ -198,13 +198,13 @@ public class ParDoSingleEvaluatorFactoryTest implements Serializable {
CommittedBundle<String> inputBundle =
bundleFactory.createRootBundle(input).commit(Instant.now());
- InProcessEvaluationContext evaluationContext = mock(InProcessEvaluationContext.class);
+ EvaluationContext evaluationContext = mock(EvaluationContext.class);
UncommittedBundle<KV<String, Integer>> mainOutputBundle =
bundleFactory.createRootBundle(mainOutput);
when(evaluationContext.createBundle(inputBundle, mainOutput)).thenReturn(mainOutputBundle);
- InProcessExecutionContext executionContext = new InProcessExecutionContext(null,
+ DirectExecutionContext executionContext = new DirectExecutionContext(null,
StructuralKey.of("myKey", StringUtf8Coder.of()),
null, null);
when(evaluationContext.getExecutionContext(mainOutput.getProducingTransformInternal(),
@@ -224,7 +224,7 @@ public class ParDoSingleEvaluatorFactoryTest implements Serializable {
evaluator.processElement(
WindowedValue.valueInGlobalWindow("bazam", PaneInfo.ON_TIME_AND_ONLY_FIRING));
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(result.getWatermarkHold(), equalTo(new Instant(124438L)));
assertThat(result.getState(), not(nullValue()));
assertThat(
@@ -298,13 +298,13 @@ public class ParDoSingleEvaluatorFactoryTest implements Serializable {
CommittedBundle<String> inputBundle =
bundleFactory.createRootBundle(input).commit(Instant.now());
- InProcessEvaluationContext evaluationContext = mock(InProcessEvaluationContext.class);
+ EvaluationContext evaluationContext = mock(EvaluationContext.class);
UncommittedBundle<KV<String, Integer>> mainOutputBundle =
bundleFactory.createRootBundle(mainOutput);
when(evaluationContext.createBundle(inputBundle, mainOutput)).thenReturn(mainOutputBundle);
- InProcessExecutionContext executionContext = new InProcessExecutionContext(null,
+ DirectExecutionContext executionContext = new DirectExecutionContext(null,
key,
null,
null);
@@ -321,7 +321,7 @@ public class ParDoSingleEvaluatorFactoryTest implements Serializable {
evaluator.processElement(WindowedValue.valueInGlobalWindow("foo"));
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(result.getTimerUpdate(),
equalTo(TimerUpdate.builder(StructuralKey.of("myKey", StringUtf8Coder.of()))
.setTimer(addedTimer)
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
new file mode 100644
index 0000000..c0242ed
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/SideInputContainerTest.java
@@ -0,0 +1,520 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.doAnswer;
+
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.Mean;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
+import org.apache.beam.sdk.util.PCollectionViews;
+import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
+import org.apache.beam.sdk.util.SideInputReader;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+
+import com.google.common.collect.ImmutableList;
+
+import org.joda.time.Instant;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tests for {@link SideInputContainer}.
+ */
+@RunWith(JUnit4.class)
+public class SideInputContainerTest {
+ private static final BoundedWindow FIRST_WINDOW =
+ new BoundedWindow() {
+ @Override
+ public Instant maxTimestamp() {
+ return new Instant(789541L);
+ }
+
+ @Override
+ public String toString() {
+ return "firstWindow";
+ }
+ };
+
+ private static final BoundedWindow SECOND_WINDOW =
+ new BoundedWindow() {
+ @Override
+ public Instant maxTimestamp() {
+ return new Instant(14564786L);
+ }
+
+ @Override
+ public String toString() {
+ return "secondWindow";
+ }
+ };
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Mock
+ private EvaluationContext context;
+
+ private TestPipeline pipeline;
+
+ private SideInputContainer container;
+
+ private PCollectionView<Map<String, Integer>> mapView;
+ private PCollectionView<Double> singletonView;
+
+ // Not present in container.
+ private PCollectionView<Iterable<Integer>> iterableView;
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ pipeline = TestPipeline.create();
+
+ PCollection<Integer> create =
+ pipeline.apply("forBaseCollection", Create.<Integer>of(1, 2, 3, 4));
+
+ mapView =
+ create.apply("forKeyTypes", WithKeys.<String, Integer>of("foo"))
+ .apply("asMapView", View.<String, Integer>asMap());
+
+ singletonView = create.apply("forCombinedTypes", Mean.<Integer>globally().asSingletonView());
+ iterableView = create.apply("asIterableView", View.<Integer>asIterable());
+
+ container = SideInputContainer.create(
+ context, ImmutableList.of(iterableView, mapView, singletonView));
+ }
+
+ @Test
+ public void getAfterWriteReturnsPaneInWindow() throws Exception {
+ WindowedValue<KV<String, Integer>> one =
+ WindowedValue.of(
+ KV.of("one", 1), new Instant(1L), FIRST_WINDOW, PaneInfo.ON_TIME_AND_ONLY_FIRING);
+ WindowedValue<KV<String, Integer>> two =
+ WindowedValue.of(
+ KV.of("two", 2), new Instant(20L), FIRST_WINDOW, PaneInfo.ON_TIME_AND_ONLY_FIRING);
+ container.write(mapView, ImmutableList.<WindowedValue<?>>of(one, two));
+
+ Map<String, Integer> viewContents =
+ container
+ .createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
+ .get(mapView, FIRST_WINDOW);
+ assertThat(viewContents, hasEntry("one", 1));
+ assertThat(viewContents, hasEntry("two", 2));
+ assertThat(viewContents.size(), is(2));
+ }
+
+ @Test
+ public void getReturnsLatestPaneInWindow() throws Exception {
+ WindowedValue<KV<String, Integer>> one =
+ WindowedValue.of(
+ KV.of("one", 1),
+ new Instant(1L),
+ SECOND_WINDOW,
+ PaneInfo.createPane(true, false, Timing.EARLY));
+ WindowedValue<KV<String, Integer>> two =
+ WindowedValue.of(
+ KV.of("two", 2),
+ new Instant(20L),
+ SECOND_WINDOW,
+ PaneInfo.createPane(true, false, Timing.EARLY));
+ container.write(mapView, ImmutableList.<WindowedValue<?>>of(one, two));
+
+ Map<String, Integer> viewContents =
+ container
+ .createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
+ .get(mapView, SECOND_WINDOW);
+ assertThat(viewContents, hasEntry("one", 1));
+ assertThat(viewContents, hasEntry("two", 2));
+ assertThat(viewContents.size(), is(2));
+
+ WindowedValue<KV<String, Integer>> three =
+ WindowedValue.of(
+ KV.of("three", 3),
+ new Instant(300L),
+ SECOND_WINDOW,
+ PaneInfo.createPane(false, false, Timing.EARLY, 1, -1));
+ container.write(mapView, ImmutableList.<WindowedValue<?>>of(three));
+
+ Map<String, Integer> overwrittenViewContents =
+ container
+ .createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
+ .get(mapView, SECOND_WINDOW);
+ assertThat(overwrittenViewContents, hasEntry("three", 3));
+ assertThat(overwrittenViewContents.size(), is(1));
+ }
+
+ /**
+ * Demonstrates that calling get() on a window that currently has no data does not return until
+ * there is data in the pane.
+ */
+ @Test
+ public void getNotReadyThrows() throws Exception {
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("not ready");
+
+ container.createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
+ .get(mapView, GlobalWindow.INSTANCE);
+ }
+
+ @Test
+ public void withPCollectionViewsErrorsForContainsNotInViews() {
+ PCollectionView<Map<String, Iterable<String>>> newView =
+ PCollectionViews.multimapView(
+ pipeline,
+ WindowingStrategy.globalDefault(),
+ KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()));
+
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("with unknown views " + ImmutableList.of(newView).toString());
+
+ container.createReaderForViews(ImmutableList.<PCollectionView<?>>of(newView));
+ }
+
+ @Test
+ public void withViewsForViewNotInContainerFails() {
+ PCollectionView<Map<String, Iterable<String>>> newView =
+ PCollectionViews.multimapView(
+ pipeline,
+ WindowingStrategy.globalDefault(),
+ KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()));
+
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("unknown views");
+ thrown.expectMessage(newView.toString());
+
+ container.createReaderForViews(ImmutableList.<PCollectionView<?>>of(newView));
+ }
+
+ @Test
+ public void getOnReaderForViewNotInReaderFails() {
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("unknown view: " + iterableView.toString());
+
+ container.createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
+ .get(iterableView, GlobalWindow.INSTANCE);
+ }
+
+ @Test
+ public void writeForMultipleElementsInDifferentWindowsSucceeds() throws Exception {
+ WindowedValue<Double> firstWindowedValue =
+ WindowedValue.of(
+ 2.875,
+ FIRST_WINDOW.maxTimestamp().minus(200L),
+ FIRST_WINDOW,
+ PaneInfo.ON_TIME_AND_ONLY_FIRING);
+ WindowedValue<Double> secondWindowedValue =
+ WindowedValue.of(
+ 4.125,
+ SECOND_WINDOW.maxTimestamp().minus(2_000_000L),
+ SECOND_WINDOW,
+ PaneInfo.ON_TIME_AND_ONLY_FIRING);
+ container.write(singletonView, ImmutableList.of(firstWindowedValue, secondWindowedValue));
+ assertThat(
+ container
+ .createReaderForViews(ImmutableList.<PCollectionView<?>>of(singletonView))
+ .get(singletonView, FIRST_WINDOW),
+ equalTo(2.875));
+ assertThat(
+ container
+ .createReaderForViews(ImmutableList.<PCollectionView<?>>of(singletonView))
+ .get(singletonView, SECOND_WINDOW),
+ equalTo(4.125));
+ }
+
+ @Test
+ public void writeForMultipleIdenticalElementsInSameWindowSucceeds() throws Exception {
+ WindowedValue<Integer> firstValue =
+ WindowedValue.of(
+ 44,
+ FIRST_WINDOW.maxTimestamp().minus(200L),
+ FIRST_WINDOW,
+ PaneInfo.ON_TIME_AND_ONLY_FIRING);
+ WindowedValue<Integer> secondValue =
+ WindowedValue.of(
+ 44,
+ FIRST_WINDOW.maxTimestamp().minus(200L),
+ FIRST_WINDOW,
+ PaneInfo.ON_TIME_AND_ONLY_FIRING);
+
+ container.write(iterableView, ImmutableList.of(firstValue, secondValue));
+
+ assertThat(
+ container
+ .createReaderForViews(ImmutableList.<PCollectionView<?>>of(iterableView))
+ .get(iterableView, FIRST_WINDOW),
+ contains(44, 44));
+ }
+
+ @Test
+ public void writeForElementInMultipleWindowsSucceeds() throws Exception {
+ WindowedValue<Double> multiWindowedValue =
+ WindowedValue.of(
+ 2.875,
+ FIRST_WINDOW.maxTimestamp().minus(200L),
+ ImmutableList.of(FIRST_WINDOW, SECOND_WINDOW),
+ PaneInfo.ON_TIME_AND_ONLY_FIRING);
+ container.write(singletonView, ImmutableList.of(multiWindowedValue));
+ assertThat(
+ container
+ .createReaderForViews(ImmutableList.<PCollectionView<?>>of(singletonView))
+ .get(singletonView, FIRST_WINDOW),
+ equalTo(2.875));
+ assertThat(
+ container
+ .createReaderForViews(ImmutableList.<PCollectionView<?>>of(singletonView))
+ .get(singletonView, SECOND_WINDOW),
+ equalTo(2.875));
+ }
+
+ @Test
+ public void finishDoesNotOverwriteWrittenElements() throws Exception {
+ WindowedValue<KV<String, Integer>> one =
+ WindowedValue.of(
+ KV.of("one", 1),
+ new Instant(1L),
+ SECOND_WINDOW,
+ PaneInfo.createPane(true, false, Timing.EARLY));
+ WindowedValue<KV<String, Integer>> two =
+ WindowedValue.of(
+ KV.of("two", 2),
+ new Instant(20L),
+ SECOND_WINDOW,
+ PaneInfo.createPane(true, false, Timing.EARLY));
+ container.write(mapView, ImmutableList.<WindowedValue<?>>of(one, two));
+
+ immediatelyInvokeCallback(mapView, SECOND_WINDOW);
+
+ Map<String, Integer> viewContents =
+ container
+ .createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView))
+ .get(mapView, SECOND_WINDOW);
+
+ assertThat(viewContents, hasEntry("one", 1));
+ assertThat(viewContents, hasEntry("two", 2));
+ assertThat(viewContents.size(), is(2));
+ }
+
+ @Test
+ public void finishOnPendingViewsSetsEmptyElements() throws Exception {
+ immediatelyInvokeCallback(mapView, SECOND_WINDOW);
+ Future<Map<String, Integer>> mapFuture =
+ getFutureOfView(
+ container.createReaderForViews(ImmutableList.<PCollectionView<?>>of(mapView)),
+ mapView,
+ SECOND_WINDOW);
+
+ assertThat(mapFuture.get().isEmpty(), is(true));
+ }
+
+ /**
+ * Demonstrates that calling isReady on an empty container throws an
+ * {@link IllegalArgumentException}.
+ */
+ @Test
+ public void isReadyInEmptyReaderThrows() {
+ ReadyCheckingSideInputReader reader =
+ container.createReaderForViews(ImmutableList.<PCollectionView<?>>of());
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("does not contain");
+ thrown.expectMessage(ImmutableList.of().toString());
+ reader.isReady(mapView, GlobalWindow.INSTANCE);
+ }
+
+ /**
+ * Demonstrates that calling isReady returns false until elements are written to the
+ * {@link PCollectionView}, {@link BoundedWindow} pair, at which point it returns true.
+ */
+ @Test
+ public void isReadyForSomeNotReadyViewsFalseUntilElements() {
+ container.write(
+ mapView,
+ ImmutableList.of(
+ WindowedValue.of(
+ KV.of("one", 1),
+ SECOND_WINDOW.maxTimestamp().minus(100L),
+ SECOND_WINDOW,
+ PaneInfo.ON_TIME_AND_ONLY_FIRING)));
+
+ ReadyCheckingSideInputReader reader =
+ container.createReaderForViews(ImmutableList.of(mapView, singletonView));
+ assertThat(reader.isReady(mapView, FIRST_WINDOW), is(false));
+ assertThat(reader.isReady(mapView, SECOND_WINDOW), is(true));
+
+ assertThat(reader.isReady(singletonView, SECOND_WINDOW), is(false));
+
+ container.write(
+ mapView,
+ ImmutableList.of(
+ WindowedValue.of(
+ KV.of("too", 2),
+ FIRST_WINDOW.maxTimestamp().minus(100L),
+ FIRST_WINDOW,
+ PaneInfo.ON_TIME_AND_ONLY_FIRING)));
+ // Cached value is false
+ assertThat(reader.isReady(mapView, FIRST_WINDOW), is(false));
+
+ container.write(
+ singletonView,
+ ImmutableList.of(
+ WindowedValue.of(
+ 1.25,
+ SECOND_WINDOW.maxTimestamp().minus(100L),
+ SECOND_WINDOW,
+ PaneInfo.ON_TIME_AND_ONLY_FIRING)));
+ assertThat(reader.isReady(mapView, SECOND_WINDOW), is(true));
+ assertThat(reader.isReady(singletonView, SECOND_WINDOW), is(false));
+
+ assertThat(reader.isReady(mapView, GlobalWindow.INSTANCE), is(false));
+ assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));
+
+ reader = container.createReaderForViews(ImmutableList.of(mapView, singletonView));
+ assertThat(reader.isReady(mapView, SECOND_WINDOW), is(true));
+ assertThat(reader.isReady(singletonView, SECOND_WINDOW), is(true));
+ assertThat(reader.isReady(mapView, FIRST_WINDOW), is(true));
+ }
+
+ @Test
+ public void isReadyForEmptyWindowTrue() throws Exception {
+ CountDownLatch onComplete = new CountDownLatch(1);
+ immediatelyInvokeCallback(mapView, GlobalWindow.INSTANCE);
+ CountDownLatch latch = invokeLatchedCallback(singletonView, GlobalWindow.INSTANCE, onComplete);
+
+ ReadyCheckingSideInputReader reader =
+ container.createReaderForViews(ImmutableList.of(mapView, singletonView));
+ assertThat(reader.isReady(mapView, GlobalWindow.INSTANCE), is(true));
+ assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));
+
+ latch.countDown();
+ if (!onComplete.await(1500L, TimeUnit.MILLISECONDS)) {
+ fail("Callback to set empty values did not complete!");
+ }
+ // The cached value was false, so it continues to be true
+ assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));
+
+ // A new reader for the same container gets a fresh look
+ reader = container.createReaderForViews(ImmutableList.of(mapView, singletonView));
+ assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(true));
+ }
+
+ /**
+ * When a callAfterWindowCloses with the specified view's producing transform, window, and
+ * windowing strategy is invoked, immediately execute the callback.
+ */
+ private void immediatelyInvokeCallback(PCollectionView<?> view, BoundedWindow window) {
+ doAnswer(
+ new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object callback = invocation.getArguments()[3];
+ Runnable callbackRunnable = (Runnable) callback;
+ callbackRunnable.run();
+ return null;
+ }
+ })
+ .when(context)
+ .scheduleAfterOutputWouldBeProduced(
+ Mockito.eq(view),
+ Mockito.eq(window),
+ Mockito.eq(view.getWindowingStrategyInternal()),
+ Mockito.any(Runnable.class));
+ }
+
+ /**
+ * When a callAfterWindowCloses with the specified view's producing transform, window, and
+ * windowing strategy is invoked, start a thread that will invoke the callback after the returned
+ * {@link CountDownLatch} is counted down once.
+ */
+ private CountDownLatch invokeLatchedCallback(
+ PCollectionView<?> view, BoundedWindow window, final CountDownLatch onComplete) {
+ final CountDownLatch runLatch = new CountDownLatch(1);
+ doAnswer(
+ new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object callback = invocation.getArguments()[3];
+ final Runnable callbackRunnable = (Runnable) callback;
+ Executors.newSingleThreadExecutor().submit(new Runnable() {
+ public void run() {
+ try {
+ if (!runLatch.await(1500L, TimeUnit.MILLISECONDS)) {
+ fail("Run latch didn't count down within timeout");
+ }
+ callbackRunnable.run();
+ onComplete.countDown();
+ } catch (InterruptedException e) {
+ fail("Unexpectedly interrupted while waiting for latch to be counted down");
+ }
+ }
+ });
+ return null;
+ }
+ })
+ .when(context)
+ .scheduleAfterOutputWouldBeProduced(
+ Mockito.eq(view),
+ Mockito.eq(window),
+ Mockito.eq(view.getWindowingStrategyInternal()),
+ Mockito.any(Runnable.class));
+ return runLatch;
+ }
+
+ private <ValueT> Future<ValueT> getFutureOfView(final SideInputReader myReader,
+ final PCollectionView<ValueT> view, final BoundedWindow window) {
+ Callable<ValueT> callable = new Callable<ValueT>() {
+ @Override
+ public ValueT call() throws Exception {
+ return myReader.get(view, window);
+ }
+ };
+ return Executors.newSingleThreadExecutor().submit(callable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ThreadLocalInvalidatingTransformEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ThreadLocalInvalidatingTransformEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ThreadLocalInvalidatingTransformEvaluatorTest.java
index b8d9a76..6e477d3 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ThreadLocalInvalidatingTransformEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ThreadLocalInvalidatingTransformEvaluatorTest.java
@@ -115,7 +115,7 @@ public class ThreadLocalInvalidatingTransformEvaluatorTest {
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
finishBundleCalled = true;
return null;
}
@@ -128,7 +128,7 @@ public class ThreadLocalInvalidatingTransformEvaluatorTest {
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
throw new Exception();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
index a5e6cee..cb5cd46 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
@@ -75,14 +75,14 @@ public class TransformExecutorTest {
private RegisteringCompletionCallback completionCallback;
private TransformExecutorService transformEvaluationState;
private BundleFactory bundleFactory;
- @Mock private InProcessEvaluationContext evaluationContext;
+ @Mock private EvaluationContext evaluationContext;
@Mock private TransformEvaluatorRegistry registry;
@Before
public void setup() {
MockitoAnnotations.initMocks(this);
- bundleFactory = InProcessBundleFactory.create();
+ bundleFactory = ImmutableListBundleFactory.create();
transformEvaluationState =
TransformExecutorServices.parallel(MoreExecutors.newDirectExecutorService());
@@ -97,7 +97,7 @@ public class TransformExecutorTest {
@Test
public void callWithNullInputBundleFinishesBundleAndCompletes() throws Exception {
- final InProcessTransformResult result =
+ final TransformResult result =
StepTransformResult.withoutHold(created.getProducingTransformInternal()).build();
final AtomicBoolean finishCalled = new AtomicBoolean(false);
TransformEvaluator<Object> evaluator =
@@ -108,7 +108,7 @@ public class TransformExecutorTest {
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
finishCalled.set(true);
return result;
}
@@ -135,7 +135,7 @@ public class TransformExecutorTest {
@Test
public void inputBundleProcessesEachElementFinishesAndCompletes() throws Exception {
- final InProcessTransformResult result =
+ final TransformResult result =
StepTransformResult.withoutHold(downstream.getProducingTransformInternal()).build();
final Collection<WindowedValue<String>> elementsProcessed = new ArrayList<>();
TransformEvaluator<String> evaluator =
@@ -147,7 +147,7 @@ public class TransformExecutorTest {
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
return result;
}
};
@@ -183,7 +183,7 @@ public class TransformExecutorTest {
@Test
public void processElementThrowsExceptionCallsback() throws Exception {
- final InProcessTransformResult result =
+ final TransformResult result =
StepTransformResult.withoutHold(downstream.getProducingTransformInternal()).build();
final Exception exception = new Exception();
TransformEvaluator<String> evaluator =
@@ -194,7 +194,7 @@ public class TransformExecutorTest {
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
return result;
}
};
@@ -233,7 +233,7 @@ public class TransformExecutorTest {
public void processElement(WindowedValue<String> element) throws Exception {}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
throw exception;
}
};
@@ -264,7 +264,7 @@ public class TransformExecutorTest {
@Test
public void duringCallGetThreadIsNonNull() throws Exception {
- final InProcessTransformResult result =
+ final TransformResult result =
StepTransformResult.withoutHold(downstream.getProducingTransformInternal()).build();
final CountDownLatch testLatch = new CountDownLatch(1);
final CountDownLatch evaluatorLatch = new CountDownLatch(1);
@@ -276,7 +276,7 @@ public class TransformExecutorTest {
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
testLatch.countDown();
evaluatorLatch.await();
return result;
@@ -306,7 +306,7 @@ public class TransformExecutorTest {
@Test
public void callWithEnforcementAppliesEnforcement() throws Exception {
- final InProcessTransformResult result =
+ final TransformResult result =
StepTransformResult.withoutHold(downstream.getProducingTransformInternal()).build();
TransformEvaluator<Object> evaluator =
@@ -316,7 +316,7 @@ public class TransformExecutorTest {
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
return result;
}
};
@@ -365,7 +365,7 @@ public class TransformExecutorTest {
}
});
- final InProcessTransformResult result =
+ final TransformResult result =
StepTransformResult.withoutHold(pcBytes.getProducingTransformInternal()).build();
final CountDownLatch testLatch = new CountDownLatch(1);
final CountDownLatch evaluatorLatch = new CountDownLatch(1);
@@ -376,7 +376,7 @@ public class TransformExecutorTest {
public void processElement(WindowedValue<Object> element) throws Exception {}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
testLatch.countDown();
evaluatorLatch.await();
return result;
@@ -423,7 +423,7 @@ public class TransformExecutorTest {
}
});
- final InProcessTransformResult result =
+ final TransformResult result =
StepTransformResult.withoutHold(pcBytes.getProducingTransformInternal()).build();
final CountDownLatch testLatch = new CountDownLatch(1);
final CountDownLatch evaluatorLatch = new CountDownLatch(1);
@@ -437,7 +437,7 @@ public class TransformExecutorTest {
}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
return result;
}
};
@@ -470,7 +470,7 @@ public class TransformExecutorTest {
}
private static class RegisteringCompletionCallback implements CompletionCallback {
- private InProcessTransformResult handledResult = null;
+ private TransformResult handledResult = null;
private Throwable handledThrowable = null;
private final CountDownLatch onMethod;
@@ -480,7 +480,7 @@ public class TransformExecutorTest {
@Override
public CommittedResult handleResult(
- CommittedBundle<?> inputBundle, InProcessTransformResult result) {
+ CommittedBundle<?> inputBundle, TransformResult result) {
handledResult = result;
onMethod.countDown();
@SuppressWarnings("rawtypes") Iterable unprocessedElements =
@@ -516,7 +516,7 @@ public class TransformExecutorTest {
private static class TestEnforcement<T> implements ModelEnforcement<T> {
private final List<WindowedValue<T>> beforeElements = new ArrayList<>();
private final List<WindowedValue<T>> afterElements = new ArrayList<>();
- private final List<InProcessTransformResult> finishedBundles = new ArrayList<>();
+ private final List<TransformResult> finishedBundles = new ArrayList<>();
@Override
public void beforeElement(WindowedValue<T> element) {
@@ -531,7 +531,7 @@ public class TransformExecutorTest {
@Override
public void afterFinish(
CommittedBundle<T> input,
- InProcessTransformResult result,
+ TransformResult result,
Iterable<? extends CommittedBundle<?>> outputs) {
finishedBundles.add(result);
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
index be5c489..e182e8d 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
@@ -72,10 +72,10 @@ import javax.annotation.Nullable;
public class UnboundedReadEvaluatorFactoryTest {
private PCollection<Long> longs;
private TransformEvaluatorFactory factory;
- private InProcessEvaluationContext context;
+ private EvaluationContext context;
private UncommittedBundle<Long> output;
- private BundleFactory bundleFactory = InProcessBundleFactory.create();
+ private BundleFactory bundleFactory = ImmutableListBundleFactory.create();
@Before
public void setup() {
@@ -85,7 +85,7 @@ public class UnboundedReadEvaluatorFactoryTest {
longs = p.apply(Read.from(source));
factory = new UnboundedReadEvaluatorFactory();
- context = mock(InProcessEvaluationContext.class);
+ context = mock(EvaluationContext.class);
output = bundleFactory.createRootBundle(longs);
when(context.createRootBundle(longs)).thenReturn(output);
}
@@ -95,7 +95,7 @@ public class UnboundedReadEvaluatorFactoryTest {
TransformEvaluator<?> evaluator =
factory.forApplication(longs.getProducingTransformInternal(), null, context);
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(
result.getWatermarkHold(), Matchers.<ReadableInstant>lessThan(DateTime.now().toInstant()));
assertThat(
@@ -114,7 +114,7 @@ public class UnboundedReadEvaluatorFactoryTest {
TransformEvaluator<?> evaluator =
factory.forApplication(longs.getProducingTransformInternal(), null, context);
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(
result.getWatermarkHold(), Matchers.<ReadableInstant>lessThan(DateTime.now().toInstant()));
assertThat(
@@ -127,7 +127,7 @@ public class UnboundedReadEvaluatorFactoryTest {
when(context.createRootBundle(longs)).thenReturn(secondOutput);
TransformEvaluator<?> secondEvaluator =
factory.forApplication(longs.getProducingTransformInternal(), null, context);
- InProcessTransformResult secondResult = secondEvaluator.finishBundle();
+ TransformResult secondResult = secondEvaluator.finishBundle();
assertThat(
secondResult.getWatermarkHold(),
Matchers.<ReadableInstant>lessThan(DateTime.now().toInstant()));
@@ -220,7 +220,7 @@ public class UnboundedReadEvaluatorFactoryTest {
factory.forApplication(longs.getProducingTransformInternal(), null, context);
assertThat(secondEvaluator, nullValue());
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
assertThat(
result.getWatermarkHold(), Matchers.<ReadableInstant>lessThan(DateTime.now().toInstant()));
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
index 714e9c9..6820792 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
@@ -51,7 +51,7 @@ import org.junit.runners.JUnit4;
*/
@RunWith(JUnit4.class)
public class ViewEvaluatorFactoryTest {
- private BundleFactory bundleFactory = InProcessBundleFactory.create();
+ private BundleFactory bundleFactory = ImmutableListBundleFactory.create();
@Test
public void testInMemoryEvaluator() throws Exception {
@@ -69,7 +69,7 @@ public class ViewEvaluatorFactoryTest {
PCollectionView<Iterable<String>> view =
concat.apply(new ViewEvaluatorFactory.WriteView<>(createView));
- InProcessEvaluationContext context = mock(InProcessEvaluationContext.class);
+ EvaluationContext context = mock(EvaluationContext.class);
TestViewWriter<String, Iterable<String>> viewWriter = new TestViewWriter<>();
when(context.createPCollectionViewWriter(concat, view)).thenReturn(viewWriter);
[11/12] incubator-beam git commit: Remove InProcess Prefixes
Posted by ke...@apache.org.
Remove InProcess Prefixes
These prefixes are out of date with the rename of the runner. Most of
the prefixes are be droped in their entirety, as the classes are scoped
to the direct runner module.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/4649eebe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/4649eebe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/4649eebe
Branch: refs/heads/master
Commit: 4649eebe276d9593862fd180a73676f959ec53ea
Parents: 2bafda1
Author: Thomas Groh <tg...@google.com>
Authored: Wed Jun 15 11:21:41 2016 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Wed Jun 15 16:14:20 2016 -0700
----------------------------------------------------------------------
.../direct/AbstractModelEnforcement.java | 2 +-
.../direct/BoundedReadEvaluatorFactory.java | 12 +-
.../beam/runners/direct/BundleFactory.java | 4 +-
.../beam/runners/direct/CommittedResult.java | 4 +-
.../beam/runners/direct/CompletionCallback.java | 2 +-
.../runners/direct/DirectExecutionContext.java | 106 ++
.../beam/runners/direct/DirectGroupByKey.java | 132 ++
.../direct/DirectGroupByKeyOverrideFactory.java | 41 +
.../beam/runners/direct/DirectRegistrar.java | 16 +-
.../beam/runners/direct/DirectRunner.java | 29 +-
.../runners/direct/DirectTimerInternals.java | 84 ++
.../runners/direct/EmptyTransformEvaluator.java | 4 +-
.../beam/runners/direct/EvaluationContext.java | 429 ++++++
.../beam/runners/direct/EvaluatorKey.java | 4 +-
.../direct/ExecutorServiceParallelExecutor.java | 28 +-
.../runners/direct/FlattenEvaluatorFactory.java | 12 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 127 ++
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 186 +++
.../direct/ImmutabilityEnforcementFactory.java | 2 +-
.../direct/ImmutableListBundleFactory.java | 163 ++
.../direct/InMemoryWatermarkManager.java | 1420 -----------------
.../runners/direct/InProcessBundleFactory.java | 161 --
.../direct/InProcessBundleOutputManager.java | 51 -
.../direct/InProcessEvaluationContext.java | 429 ------
.../direct/InProcessExecutionContext.java | 105 --
.../beam/runners/direct/InProcessExecutor.java | 48 -
...rocessGroupAlsoByWindowEvaluatorFactory.java | 127 --
.../runners/direct/InProcessGroupByKey.java | 132 --
...InProcessGroupByKeyOnlyEvaluatorFactory.java | 185 ---
.../InProcessGroupByKeyOverrideFactory.java | 41 -
.../direct/InProcessSideInputContainer.java | 277 ----
.../runners/direct/InProcessTimerInternals.java | 84 --
.../direct/InProcessTransformResult.java | 84 --
.../beam/runners/direct/ModelEnforcement.java | 6 +-
.../beam/runners/direct/ParDoEvaluator.java | 186 +++
.../runners/direct/ParDoInProcessEvaluator.java | 186 ---
.../direct/ParDoMultiEvaluatorFactory.java | 6 +-
.../direct/ParDoSingleEvaluatorFactory.java | 6 +-
.../direct/PassthroughTransformEvaluator.java | 2 +-
.../beam/runners/direct/PipelineExecutor.java | 48 +
.../beam/runners/direct/SideInputContainer.java | 277 ++++
.../runners/direct/StepTransformResult.java | 6 +-
...readLocalInvalidatingTransformEvaluator.java | 2 +-
.../beam/runners/direct/TransformEvaluator.java | 4 +-
.../direct/TransformEvaluatorFactory.java | 2 +-
.../direct/TransformEvaluatorRegistry.java | 10 +-
.../beam/runners/direct/TransformExecutor.java | 14 +-
.../beam/runners/direct/TransformResult.java | 84 ++
.../direct/UnboundedReadEvaluatorFactory.java | 12 +-
.../direct/UncommittedBundleOutputManager.java | 51 +
.../runners/direct/ViewEvaluatorFactory.java | 16 +-
.../beam/runners/direct/WatermarkManager.java | 1420 +++++++++++++++++
.../runners/direct/WindowEvaluatorFactory.java | 12 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 10 +-
.../runners/direct/CommittedResultTest.java | 2 +-
.../runners/direct/DirectRegistrarTest.java | 18 +-
.../direct/DirectTimerInternalsTest.java | 134 ++
.../EncodabilityEnforcementFactoryTest.java | 2 +-
.../runners/direct/EvaluationContextTest.java | 545 +++++++
.../direct/FlattenEvaluatorFactoryTest.java | 12 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 9 +-
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 197 +++
.../ImmutabilityCheckingBundleFactoryTest.java | 2 +-
.../ImmutabilityEnforcementFactoryTest.java | 2 +-
.../direct/ImmutableListBundleFactoryTest.java | 231 +++
.../direct/InMemoryWatermarkManagerTest.java | 1428 ------------------
.../direct/InProcessBundleFactoryTest.java | 231 ---
.../direct/InProcessEvaluationContextTest.java | 545 -------
...ocessGroupByKeyOnlyEvaluatorFactoryTest.java | 196 ---
.../direct/InProcessSideInputContainerTest.java | 520 -------
.../direct/InProcessTimerInternalsTest.java | 134 --
.../beam/runners/direct/ParDoEvaluatorTest.java | 214 +++
.../direct/ParDoInProcessEvaluatorTest.java | 214 ---
.../direct/ParDoMultiEvaluatorFactoryTest.java | 32 +-
.../direct/ParDoSingleEvaluatorFactoryTest.java | 32 +-
.../runners/direct/SideInputContainerTest.java | 520 +++++++
...LocalInvalidatingTransformEvaluatorTest.java | 4 +-
.../runners/direct/TransformExecutorTest.java | 42 +-
.../UnboundedReadEvaluatorFactoryTest.java | 14 +-
.../direct/ViewEvaluatorFactoryTest.java | 4 +-
.../runners/direct/WatermarkManagerTest.java | 1428 ++++++++++++++++++
.../direct/WindowEvaluatorFactoryTest.java | 16 +-
82 files changed, 6814 insertions(+), 6805 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/AbstractModelEnforcement.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/AbstractModelEnforcement.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/AbstractModelEnforcement.java
index 2ae0275..81f0f5f 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/AbstractModelEnforcement.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/AbstractModelEnforcement.java
@@ -33,6 +33,6 @@ abstract class AbstractModelEnforcement<T> implements ModelEnforcement<T> {
@Override
public void afterFinish(
CommittedBundle<T> input,
- InProcessTransformResult result,
+ TransformResult result,
Iterable<? extends CommittedBundle<?>> outputs) {}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
index 63d248a..e550f54 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
@@ -55,14 +55,14 @@ final class BoundedReadEvaluatorFactory implements TransformEvaluatorFactory {
public <InputT> TransformEvaluator<InputT> forApplication(
AppliedPTransform<?, ?, ?> application,
@Nullable CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext)
+ EvaluationContext evaluationContext)
throws IOException {
return getTransformEvaluator((AppliedPTransform) application, evaluationContext);
}
private <OutputT> TransformEvaluator<?> getTransformEvaluator(
final AppliedPTransform<?, PCollection<OutputT>, Bounded<OutputT>> transform,
- final InProcessEvaluationContext evaluationContext) {
+ final EvaluationContext evaluationContext) {
return getTransformEvaluatorQueue(transform, evaluationContext).poll();
}
@@ -76,7 +76,7 @@ final class BoundedReadEvaluatorFactory implements TransformEvaluatorFactory {
@SuppressWarnings("unchecked")
private <OutputT> Queue<BoundedReadEvaluator<OutputT>> getTransformEvaluatorQueue(
final AppliedPTransform<?, PCollection<OutputT>, Bounded<OutputT>> transform,
- final InProcessEvaluationContext evaluationContext) {
+ final EvaluationContext evaluationContext) {
// Key by the application and the context the evaluation is occurring in (which call to
// Pipeline#run).
EvaluatorKey key = new EvaluatorKey(transform, evaluationContext);
@@ -110,7 +110,7 @@ final class BoundedReadEvaluatorFactory implements TransformEvaluatorFactory {
*/
private static class BoundedReadEvaluator<OutputT> implements TransformEvaluator<Object> {
private final AppliedPTransform<?, PCollection<OutputT>, Bounded<OutputT>> transform;
- private final InProcessEvaluationContext evaluationContext;
+ private final EvaluationContext evaluationContext;
/**
* The source being read from by this {@link BoundedReadEvaluator}. This may not be the same
* as the source derived from {@link #transform} due to splitting.
@@ -119,7 +119,7 @@ final class BoundedReadEvaluatorFactory implements TransformEvaluatorFactory {
public BoundedReadEvaluator(
AppliedPTransform<?, PCollection<OutputT>, Bounded<OutputT>> transform,
- InProcessEvaluationContext evaluationContext,
+ EvaluationContext evaluationContext,
BoundedSource<OutputT> source) {
this.transform = transform;
this.evaluationContext = evaluationContext;
@@ -130,7 +130,7 @@ final class BoundedReadEvaluatorFactory implements TransformEvaluatorFactory {
public void processElement(WindowedValue<Object> element) {}
@Override
- public InProcessTransformResult finishBundle() throws IOException {
+ public TransformResult finishBundle() throws IOException {
try (final BoundedReader<OutputT> reader =
source.createReader(evaluationContext.getPipelineOptions());) {
boolean contentsRemaining = reader.start();
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BundleFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BundleFactory.java
index a546cfb..0241d87 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BundleFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BundleFactory.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
+import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.transforms.PTransform;
@@ -41,7 +41,7 @@ public interface BundleFactory {
/**
* Create an {@link UncommittedBundle} with the specified keys at the specified step. For use by
- * {@link InProcessGroupByKeyOnly} {@link PTransform PTransforms}. Elements added to the bundle
+ * {@link DirectGroupByKeyOnly} {@link PTransform PTransforms}. Elements added to the bundle
* belong to the {@code output} {@link PCollection}.
*/
public <K, T> UncommittedBundle<T> createKeyedBundle(
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
index b241493..e86f07d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
@@ -26,7 +26,7 @@ import com.google.auto.value.AutoValue;
import javax.annotation.Nullable;
/**
- * A {@link InProcessTransformResult} that has been committed.
+ * A {@link TransformResult} that has been committed.
*/
@AutoValue
abstract class CommittedResult {
@@ -50,7 +50,7 @@ abstract class CommittedResult {
public abstract Iterable<? extends CommittedBundle<?>> getOutputs();
public static CommittedResult create(
- InProcessTransformResult original,
+ TransformResult original,
CommittedBundle<?> unprocessedElements,
Iterable<? extends CommittedBundle<?>> outputs) {
return new AutoValue_CommittedResult(original.getTransform(),
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
index 8ee4b44..0c5fe24 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
@@ -27,7 +27,7 @@ interface CompletionCallback {
* Handle a successful result, returning the committed outputs of the result.
*/
CommittedResult handleResult(
- CommittedBundle<?> inputBundle, InProcessTransformResult result);
+ CommittedBundle<?> inputBundle, TransformResult result);
/**
* Handle a result that terminated abnormally due to the provided {@link Throwable}.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
new file mode 100644
index 0000000..2d2b87d
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectExecutionContext.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
+import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
+import org.apache.beam.sdk.util.BaseExecutionContext;
+import org.apache.beam.sdk.util.ExecutionContext;
+import org.apache.beam.sdk.util.TimerInternals;
+import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
+
+/**
+ * Execution Context for the {@link DirectRunner}.
+ *
+ * This implementation is not thread safe. A new {@link DirectExecutionContext} must be created
+ * for each thread that requires it.
+ */
+class DirectExecutionContext
+ extends BaseExecutionContext<DirectStepContext> {
+ private final Clock clock;
+ private final StructuralKey<?> key;
+ private final CopyOnAccessInMemoryStateInternals<Object> existingState;
+ private final TransformWatermarks watermarks;
+
+ public DirectExecutionContext(Clock clock, StructuralKey<?> key,
+ CopyOnAccessInMemoryStateInternals<Object> existingState, TransformWatermarks watermarks) {
+ this.clock = clock;
+ this.key = key;
+ this.existingState = existingState;
+ this.watermarks = watermarks;
+ }
+
+ @Override
+ protected DirectStepContext createStepContext(String stepName, String transformName) {
+ return new DirectStepContext(this, stepName, transformName);
+ }
+
+ /**
+ * Step Context for the {@link DirectRunner}.
+ */
+ public class DirectStepContext
+ extends org.apache.beam.sdk.util.BaseExecutionContext.StepContext {
+ private CopyOnAccessInMemoryStateInternals<Object> stateInternals;
+ private DirectTimerInternals timerInternals;
+
+ public DirectStepContext(
+ ExecutionContext executionContext, String stepName, String transformName) {
+ super(executionContext, stepName, transformName);
+ }
+
+ @Override
+ public CopyOnAccessInMemoryStateInternals<Object> stateInternals() {
+ if (stateInternals == null) {
+ stateInternals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, existingState);
+ }
+ return stateInternals;
+ }
+
+ @Override
+ public DirectTimerInternals timerInternals() {
+ if (timerInternals == null) {
+ timerInternals =
+ DirectTimerInternals.create(clock, watermarks, TimerUpdate.builder(key));
+ }
+ return timerInternals;
+ }
+
+ /**
+ * Commits the state of this step, and returns the committed state. If the step has not
+ * accessed any state, return null.
+ */
+ public CopyOnAccessInMemoryStateInternals<?> commitState() {
+ if (stateInternals != null) {
+ return stateInternals.commit();
+ }
+ return null;
+ }
+
+ /**
+ * Gets the timer update of the {@link TimerInternals} of this {@link DirectStepContext},
+ * which is empty if the {@link TimerInternals} were never accessed.
+ */
+ public TimerUpdate getTimerUpdate() {
+ if (timerInternals == null) {
+ return TimerUpdate.empty();
+ }
+ return timerInternals.getTimerUpdate();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
new file mode 100644
index 0000000..0200676
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.direct;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly.ReifyTimestampsAndWindows;
+import org.apache.beam.sdk.util.KeyedWorkItem;
+import org.apache.beam.sdk.util.KeyedWorkItemCoder;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+class DirectGroupByKey<K, V>
+ extends ForwardingPTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> {
+ private final GroupByKey<K, V> original;
+
+ DirectGroupByKey(GroupByKey<K, V> from) {
+ this.original = from;
+ }
+
+ @Override
+ public PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> delegate() {
+ return original;
+ }
+
+ @Override
+ public PCollection<KV<K, Iterable<V>>> apply(PCollection<KV<K, V>> input) {
+ @SuppressWarnings("unchecked")
+ KvCoder<K, V> inputCoder = (KvCoder<K, V>) input.getCoder();
+
+ // This operation groups by the combination of key and window,
+ // merging windows as needed, using the windows assigned to the
+ // key/value input elements and the window merge operation of the
+ // window function associated with the input PCollection.
+ WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
+
+ // By default, implement GroupByKey via a series of lower-level operations.
+ return input
+ // Make each input element's timestamp and assigned windows
+ // explicit, in the value part.
+ .apply(new ReifyTimestampsAndWindows<K, V>())
+ .apply(new DirectGroupByKeyOnly<K, V>())
+ .setCoder(
+ KeyedWorkItemCoder.of(
+ inputCoder.getKeyCoder(),
+ inputCoder.getValueCoder(),
+ input.getWindowingStrategy().getWindowFn().windowCoder()))
+
+ // Group each key's values by window, merging windows as needed.
+ .apply("GroupAlsoByWindow", new DirectGroupAlsoByWindow<K, V>(windowingStrategy))
+
+ // And update the windowing strategy as appropriate.
+ .setWindowingStrategyInternal(original.updateWindowingStrategy(windowingStrategy))
+ .setCoder(
+ KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getValueCoder())));
+ }
+
+ static final class DirectGroupByKeyOnly<K, V>
+ extends PTransform<PCollection<KV<K, WindowedValue<V>>>, PCollection<KeyedWorkItem<K, V>>> {
+ @Override
+ public PCollection<KeyedWorkItem<K, V>> apply(PCollection<KV<K, WindowedValue<V>>> input) {
+ return PCollection.<KeyedWorkItem<K, V>>createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded());
+ }
+
+ DirectGroupByKeyOnly() {}
+ }
+
+ static final class DirectGroupAlsoByWindow<K, V>
+ extends PTransform<PCollection<KeyedWorkItem<K, V>>, PCollection<KV<K, Iterable<V>>>> {
+
+ private final WindowingStrategy<?, ?> windowingStrategy;
+
+ public DirectGroupAlsoByWindow(WindowingStrategy<?, ?> windowingStrategy) {
+ this.windowingStrategy = windowingStrategy;
+ }
+
+ public WindowingStrategy<?, ?> getWindowingStrategy() {
+ return windowingStrategy;
+ }
+
+ private KeyedWorkItemCoder<K, V> getKeyedWorkItemCoder(Coder<KeyedWorkItem<K, V>> inputCoder) {
+ // Coder<KV<...>> --> KvCoder<...>
+ checkArgument(
+ inputCoder instanceof KeyedWorkItemCoder,
+ "%s requires a %s<...> but got %s",
+ getClass().getSimpleName(),
+ KvCoder.class.getSimpleName(),
+ inputCoder);
+ @SuppressWarnings("unchecked")
+ KeyedWorkItemCoder<K, V> kvCoder = (KeyedWorkItemCoder<K, V>) inputCoder;
+ return kvCoder;
+ }
+
+ public Coder<K> getKeyCoder(Coder<KeyedWorkItem<K, V>> inputCoder) {
+ return getKeyedWorkItemCoder(inputCoder).getKeyCoder();
+ }
+
+ public Coder<V> getValueCoder(Coder<KeyedWorkItem<K, V>> inputCoder) {
+ return getKeyedWorkItemCoder(inputCoder).getElementCoder();
+ }
+
+ @Override
+ public PCollection<KV<K, Iterable<V>>> apply(PCollection<KeyedWorkItem<K, V>> input) {
+ return PCollection.<KV<K, Iterable<V>>>createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
new file mode 100644
index 0000000..c64f3f0
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.POutput;
+
+/**
+ * A {@link PTransformOverrideFactory} for {@link GroupByKey} PTransforms.
+ */
+final class DirectGroupByKeyOverrideFactory
+ implements PTransformOverrideFactory {
+ @Override
+ public <InputT extends PInput, OutputT extends POutput> PTransform<InputT, OutputT> override(
+ PTransform<InputT, OutputT> transform) {
+ if (transform instanceof GroupByKey) {
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ PTransform<InputT, OutputT> override =
+ (PTransform) new DirectGroupByKey((GroupByKey) transform);
+ return override;
+ }
+ return transform;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
index eb027fa..7c094ae 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
@@ -27,29 +27,31 @@ import com.google.common.collect.ImmutableList;
/**
* Contains the {@link PipelineRunnerRegistrar} and {@link PipelineOptionsRegistrar} for the
- * {@link DirectRunner}.
+ * {@link org.apache.beam.runners.direct.DirectRunner}.
*/
public class DirectRegistrar {
private DirectRegistrar() {}
/**
- * Registers the {@link DirectRunner}.
+ * Registers the {@link org.apache.beam.runners.direct.DirectRunner}.
*/
@AutoService(PipelineRunnerRegistrar.class)
- public static class InProcessRunner implements PipelineRunnerRegistrar {
+ public static class DirectRunner implements PipelineRunnerRegistrar {
@Override
public Iterable<Class<? extends PipelineRunner<?>>> getPipelineRunners() {
- return ImmutableList.<Class<? extends PipelineRunner<?>>>of(DirectRunner.class);
+ return ImmutableList.<Class<? extends PipelineRunner<?>>>of(
+ org.apache.beam.runners.direct.DirectRunner.class);
}
}
/**
- * Registers the {@link DirectOptions}.
+ * Registers the {@link org.apache.beam.runners.direct.DirectOptions}.
*/
@AutoService(PipelineOptionsRegistrar.class)
- public static class InProcessOptions implements PipelineOptionsRegistrar {
+ public static class DirectOptions implements PipelineOptionsRegistrar {
@Override
public Iterable<Class<? extends PipelineOptions>> getPipelineOptions() {
- return ImmutableList.<Class<? extends PipelineOptions>>of(DirectOptions.class);
+ return ImmutableList.<Class<? extends PipelineOptions>>of(
+ org.apache.beam.runners.direct.DirectOptions.class);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index 2f5a0bc..2584739 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -17,9 +17,9 @@
*/
package org.apache.beam.runners.direct;
+import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
import org.apache.beam.runners.direct.DirectRunner.DirectPipelineResult;
-import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
-import org.apache.beam.runners.direct.ViewEvaluatorFactory.InProcessViewOverrideFactory;
+import org.apache.beam.runners.direct.ViewEvaluatorFactory.ViewOverrideFactory;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
import org.apache.beam.sdk.PipelineResult;
@@ -78,8 +78,8 @@ public class DirectRunner
private static Map<Class<? extends PTransform>, PTransformOverrideFactory>
defaultTransformOverrides =
ImmutableMap.<Class<? extends PTransform>, PTransformOverrideFactory>builder()
- .put(GroupByKey.class, new InProcessGroupByKeyOverrideFactory())
- .put(CreatePCollectionView.class, new InProcessViewOverrideFactory())
+ .put(GroupByKey.class, new DirectGroupByKeyOverrideFactory())
+ .put(CreatePCollectionView.class, new ViewOverrideFactory())
.put(AvroIO.Write.Bound.class, new AvroIOShardedWriteFactory())
.put(TextIO.Write.Bound.class, new TextIOShardedWriteFactory())
.build();
@@ -97,6 +97,7 @@ public class DirectRunner
*/
PCollection<T> getPCollection();
+
/**
* Outputs an element to this bundle.
*
@@ -217,13 +218,13 @@ public class DirectRunner
KeyedPValueTrackingVisitor keyedPValueVisitor =
KeyedPValueTrackingVisitor.create(
ImmutableSet.<Class<? extends PTransform>>of(
- GroupByKey.class, InProcessGroupByKeyOnly.class));
+ GroupByKey.class, DirectGroupByKeyOnly.class));
pipeline.traverseTopologically(keyedPValueVisitor);
DisplayDataValidator.validatePipeline(pipeline);
- InProcessEvaluationContext context =
- InProcessEvaluationContext.create(
+ EvaluationContext context =
+ EvaluationContext.create(
getPipelineOptions(),
createBundleFactory(getPipelineOptions()),
consumerTrackingVisitor.getRootTransforms(),
@@ -234,7 +235,7 @@ public class DirectRunner
// independent executor service for each run
ExecutorService executorService =
context.getPipelineOptions().getExecutorServiceFactory().create();
- InProcessExecutor executor =
+ PipelineExecutor executor =
ExecutorServiceParallelExecutor.create(
executorService,
consumerTrackingVisitor.getValueToConsumers(),
@@ -283,7 +284,7 @@ public class DirectRunner
}
private BundleFactory createBundleFactory(DirectOptions pipelineOptions) {
- BundleFactory bundleFactory = InProcessBundleFactory.create();
+ BundleFactory bundleFactory = ImmutableListBundleFactory.create();
if (pipelineOptions.isTestImmutability()) {
bundleFactory = ImmutabilityCheckingBundleFactory.create(bundleFactory);
}
@@ -296,14 +297,14 @@ public class DirectRunner
* Throws {@link UnsupportedOperationException} for all methods.
*/
public static class DirectPipelineResult implements PipelineResult {
- private final InProcessExecutor executor;
- private final InProcessEvaluationContext evaluationContext;
+ private final PipelineExecutor executor;
+ private final EvaluationContext evaluationContext;
private final Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps;
private State state;
private DirectPipelineResult(
- InProcessExecutor executor,
- InProcessEvaluationContext evaluationContext,
+ PipelineExecutor executor,
+ EvaluationContext evaluationContext,
Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps) {
this.executor = executor;
this.evaluationContext = evaluationContext;
@@ -350,7 +351,7 @@ public class DirectRunner
* {@link DirectOptions#isShutdownUnboundedProducersWithMaxWatermark()} set to false,
* this method will never return.
*
- * See also {@link InProcessExecutor#awaitCompletion()}.
+ * See also {@link PipelineExecutor#awaitCompletion()}.
*/
public State awaitCompletion() throws Throwable {
if (!state.isTerminal()) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java
new file mode 100644
index 0000000..a4705dd
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectTimerInternals.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate.TimerUpdateBuilder;
+import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
+import org.apache.beam.sdk.util.TimerInternals;
+
+import org.joda.time.Instant;
+
+import javax.annotation.Nullable;
+
+/**
+ * An implementation of {@link TimerInternals} where all relevant data exists in memory.
+ */
+public class DirectTimerInternals implements TimerInternals {
+ private final Clock processingTimeClock;
+ private final TransformWatermarks watermarks;
+ private final TimerUpdateBuilder timerUpdateBuilder;
+
+ public static DirectTimerInternals create(
+ Clock clock, TransformWatermarks watermarks, TimerUpdateBuilder timerUpdateBuilder) {
+ return new DirectTimerInternals(clock, watermarks, timerUpdateBuilder);
+ }
+
+ private DirectTimerInternals(
+ Clock clock, TransformWatermarks watermarks, TimerUpdateBuilder timerUpdateBuilder) {
+ this.processingTimeClock = clock;
+ this.watermarks = watermarks;
+ this.timerUpdateBuilder = timerUpdateBuilder;
+ }
+
+ @Override
+ public void setTimer(TimerData timerKey) {
+ timerUpdateBuilder.setTimer(timerKey);
+ }
+
+ @Override
+ public void deleteTimer(TimerData timerKey) {
+ timerUpdateBuilder.deletedTimer(timerKey);
+ }
+
+ public TimerUpdate getTimerUpdate() {
+ return timerUpdateBuilder.build();
+ }
+
+ @Override
+ public Instant currentProcessingTime() {
+ return processingTimeClock.now();
+ }
+
+ @Override
+ @Nullable
+ public Instant currentSynchronizedProcessingTime() {
+ return watermarks.getSynchronizedProcessingInputTime();
+ }
+
+ @Override
+ public Instant currentInputWatermarkTime() {
+ return watermarks.getInputWatermark();
+ }
+
+ @Override
+ @Nullable
+ public Instant currentOutputWatermarkTime() {
+ return watermarks.getOutputWatermark();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyTransformEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyTransformEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyTransformEvaluator.java
index 5379038..778c5aa 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyTransformEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EmptyTransformEvaluator.java
@@ -24,7 +24,7 @@ import org.apache.beam.sdk.util.WindowedValue;
/**
* A {@link TransformEvaluator} that ignores all input and produces no output. The result of
* invoking {@link #finishBundle()} on this evaluator is to return an
- * {@link InProcessTransformResult} with no elements and a timestamp hold equal to
+ * {@link TransformResult} with no elements and a timestamp hold equal to
* {@link BoundedWindow#TIMESTAMP_MIN_VALUE}. Because the result contains no elements, this hold
* will not affect the watermark.
*/
@@ -43,7 +43,7 @@ final class EmptyTransformEvaluator<T> implements TransformEvaluator<T> {
public void processElement(WindowedValue<T> element) throws Exception {}
@Override
- public InProcessTransformResult finishBundle() throws Exception {
+ public TransformResult finishBundle() throws Exception {
return StepTransformResult.withHold(transform, BoundedWindow.TIMESTAMP_MIN_VALUE)
.build();
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
new file mode 100644
index 0000000..bee878b
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
@@ -0,0 +1,429 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.PCollectionViewWriter;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.WatermarkManager.FiredTimers;
+import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.Trigger;
+import org.apache.beam.sdk.util.ExecutionContext;
+import org.apache.beam.sdk.util.ReadyCheckingSideInputReader;
+import org.apache.beam.sdk.util.SideInputReader;
+import org.apache.beam.sdk.util.TimerInternals.TimerData;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.util.common.CounterSet;
+import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.annotation.Nullable;
+
+/**
+ * The evaluation context for a specific pipeline being executed by the
+ * {@link DirectRunner}. Contains state shared within the execution across all
+ * transforms.
+ *
+ * <p>{@link EvaluationContext} contains shared state for an execution of the
+ * {@link DirectRunner} that can be used while evaluating a {@link PTransform}. This
+ * consists of views into underlying state and watermark implementations, access to read and write
+ * {@link PCollectionView PCollectionViews}, and constructing {@link CounterSet CounterSets} and
+ * {@link ExecutionContext ExecutionContexts}. This includes executing callbacks asynchronously when
+ * state changes to the appropriate point (e.g. when a {@link PCollectionView} is requested and
+ * known to be empty).
+ *
+ * <p>{@link EvaluationContext} also handles results by committing finalizing bundles based
+ * on the current global state and updating the global state appropriately. This includes updating
+ * the per-{@link StepAndKey} state, updating global watermarks, and executing any callbacks that
+ * can be executed.
+ */
+class EvaluationContext {
+ /** The step name for each {@link AppliedPTransform} in the {@link Pipeline}. */
+ private final Map<AppliedPTransform<?, ?, ?>, String> stepNames;
+
+ /** The options that were used to create this {@link Pipeline}. */
+ private final DirectOptions options;
+
+ private final BundleFactory bundleFactory;
+ /** The current processing time and event time watermarks and timers. */
+ private final WatermarkManager watermarkManager;
+
+ /** Executes callbacks based on the progression of the watermark. */
+ private final WatermarkCallbackExecutor callbackExecutor;
+
+ /** The stateInternals of the world, by applied PTransform and key. */
+ private final ConcurrentMap<StepAndKey, CopyOnAccessInMemoryStateInternals<?>>
+ applicationStateInternals;
+
+ private final SideInputContainer sideInputContainer;
+
+ private final CounterSet mergedCounters;
+
+ public static EvaluationContext create(
+ DirectOptions options,
+ BundleFactory bundleFactory,
+ Collection<AppliedPTransform<?, ?, ?>> rootTransforms,
+ Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> valueToConsumers,
+ Map<AppliedPTransform<?, ?, ?>, String> stepNames,
+ Collection<PCollectionView<?>> views) {
+ return new EvaluationContext(
+ options, bundleFactory, rootTransforms, valueToConsumers, stepNames, views);
+ }
+
+ private EvaluationContext(
+ DirectOptions options,
+ BundleFactory bundleFactory,
+ Collection<AppliedPTransform<?, ?, ?>> rootTransforms,
+ Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> valueToConsumers,
+ Map<AppliedPTransform<?, ?, ?>, String> stepNames,
+ Collection<PCollectionView<?>> views) {
+ this.options = checkNotNull(options);
+ this.bundleFactory = checkNotNull(bundleFactory);
+ checkNotNull(rootTransforms);
+ checkNotNull(valueToConsumers);
+ checkNotNull(stepNames);
+ checkNotNull(views);
+ this.stepNames = stepNames;
+
+ this.watermarkManager =
+ WatermarkManager.create(
+ NanosOffsetClock.create(), rootTransforms, valueToConsumers);
+ this.sideInputContainer = SideInputContainer.create(this, views);
+
+ this.applicationStateInternals = new ConcurrentHashMap<>();
+ this.mergedCounters = new CounterSet();
+
+ this.callbackExecutor =
+ WatermarkCallbackExecutor.create(MoreExecutors.directExecutor());
+ }
+
+ /**
+ * Handle the provided {@link TransformResult}, produced after evaluating the provided
+ * {@link CommittedBundle} (potentially null, if the result of a root {@link PTransform}).
+ *
+ * <p>The result is the output of running the transform contained in the
+ * {@link TransformResult} on the contents of the provided bundle.
+ *
+ * @param completedBundle the bundle that was processed to produce the result. Potentially
+ * {@code null} if the transform that produced the result is a root
+ * transform
+ * @param completedTimers the timers that were delivered to produce the {@code completedBundle},
+ * or an empty iterable if no timers were delivered
+ * @param result the result of evaluating the input bundle
+ * @return the committed bundles contained within the handled {@code result}
+ */
+ public CommittedResult handleResult(
+ @Nullable CommittedBundle<?> completedBundle,
+ Iterable<TimerData> completedTimers,
+ TransformResult result) {
+ Iterable<? extends CommittedBundle<?>> committedBundles =
+ commitBundles(result.getOutputBundles());
+ // Update watermarks and timers
+ CommittedResult committedResult = CommittedResult.create(result,
+ completedBundle == null
+ ? null
+ : completedBundle.withElements((Iterable) result.getUnprocessedElements()),
+ committedBundles);
+ watermarkManager.updateWatermarks(
+ completedBundle,
+ result.getTimerUpdate().withCompletedTimers(completedTimers),
+ committedResult,
+ result.getWatermarkHold());
+ // Update counters
+ if (result.getCounters() != null) {
+ mergedCounters.merge(result.getCounters());
+ }
+ // Update state internals
+ CopyOnAccessInMemoryStateInternals<?> theirState = result.getState();
+ if (theirState != null) {
+ CopyOnAccessInMemoryStateInternals<?> committedState = theirState.commit();
+ StepAndKey stepAndKey =
+ StepAndKey.of(
+ result.getTransform(), completedBundle == null ? null : completedBundle.getKey());
+ if (!committedState.isEmpty()) {
+ applicationStateInternals.put(stepAndKey, committedState);
+ } else {
+ applicationStateInternals.remove(stepAndKey);
+ }
+ }
+ return committedResult;
+ }
+
+ private Iterable<? extends CommittedBundle<?>> commitBundles(
+ Iterable<? extends UncommittedBundle<?>> bundles) {
+ ImmutableList.Builder<CommittedBundle<?>> completed = ImmutableList.builder();
+ for (UncommittedBundle<?> inProgress : bundles) {
+ AppliedPTransform<?, ?, ?> producing =
+ inProgress.getPCollection().getProducingTransformInternal();
+ TransformWatermarks watermarks = watermarkManager.getWatermarks(producing);
+ CommittedBundle<?> committed =
+ inProgress.commit(watermarks.getSynchronizedProcessingOutputTime());
+ // Empty bundles don't impact watermarks and shouldn't trigger downstream execution, so
+ // filter them out
+ if (!Iterables.isEmpty(committed.getElements())) {
+ completed.add(committed);
+ }
+ }
+ return completed.build();
+ }
+
+ private void fireAllAvailableCallbacks() {
+ for (AppliedPTransform<?, ?, ?> transform : stepNames.keySet()) {
+ fireAvailableCallbacks(transform);
+ }
+ }
+
+ private void fireAvailableCallbacks(AppliedPTransform<?, ?, ?> producingTransform) {
+ TransformWatermarks watermarks = watermarkManager.getWatermarks(producingTransform);
+ callbackExecutor.fireForWatermark(producingTransform, watermarks.getOutputWatermark());
+ }
+
+ /**
+ * Create a {@link UncommittedBundle} for use by a source.
+ */
+ public <T> UncommittedBundle<T> createRootBundle(PCollection<T> output) {
+ return bundleFactory.createRootBundle(output);
+ }
+
+ /**
+ * Create a {@link UncommittedBundle} whose elements belong to the specified {@link
+ * PCollection}.
+ */
+ public <T> UncommittedBundle<T> createBundle(CommittedBundle<?> input, PCollection<T> output) {
+ return bundleFactory.createBundle(input, output);
+ }
+
+ /**
+ * Create a {@link UncommittedBundle} with the specified keys at the specified step. For use by
+ * {@link DirectGroupByKeyOnly} {@link PTransform PTransforms}.
+ */
+ public <K, T> UncommittedBundle<T> createKeyedBundle(
+ CommittedBundle<?> input, StructuralKey<K> key, PCollection<T> output) {
+ return bundleFactory.createKeyedBundle(input, key, output);
+ }
+
+ /**
+ * Create a {@link PCollectionViewWriter}, whose elements will be used in the provided
+ * {@link PCollectionView}.
+ */
+ public <ElemT, ViewT> PCollectionViewWriter<ElemT, ViewT> createPCollectionViewWriter(
+ PCollection<Iterable<ElemT>> input, final PCollectionView<ViewT> output) {
+ return new PCollectionViewWriter<ElemT, ViewT>() {
+ @Override
+ public void add(Iterable<WindowedValue<ElemT>> values) {
+ sideInputContainer.write(output, values);
+ }
+ };
+ }
+
+ /**
+ * Schedule a callback to be executed after output would be produced for the given window
+ * if there had been input.
+ *
+ * <p>Output would be produced when the watermark for a {@link PValue} passes the point at
+ * which the trigger for the specified window (with the specified windowing strategy) must have
+ * fired from the perspective of that {@link PValue}, as specified by the value of
+ * {@link Trigger#getWatermarkThatGuaranteesFiring(BoundedWindow)} for the trigger of the
+ * {@link WindowingStrategy}. When the callback has fired, either values will have been produced
+ * for a key in that window, the window is empty, or all elements in the window are late. The
+ * callback will be executed regardless of whether values have been produced.
+ */
+ public void scheduleAfterOutputWouldBeProduced(
+ PValue value,
+ BoundedWindow window,
+ WindowingStrategy<?, ?> windowingStrategy,
+ Runnable runnable) {
+ AppliedPTransform<?, ?, ?> producing = getProducing(value);
+ callbackExecutor.callOnGuaranteedFiring(producing, window, windowingStrategy, runnable);
+
+ fireAvailableCallbacks(lookupProducing(value));
+ }
+
+ private AppliedPTransform<?, ?, ?> getProducing(PValue value) {
+ if (value.getProducingTransformInternal() != null) {
+ return value.getProducingTransformInternal();
+ }
+ return lookupProducing(value);
+ }
+
+ private AppliedPTransform<?, ?, ?> lookupProducing(PValue value) {
+ for (AppliedPTransform<?, ?, ?> transform : stepNames.keySet()) {
+ if (transform.getOutput().equals(value) || transform.getOutput().expand().contains(value)) {
+ return transform;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get the options used by this {@link Pipeline}.
+ */
+ public DirectOptions getPipelineOptions() {
+ return options;
+ }
+
+ /**
+ * Get an {@link ExecutionContext} for the provided {@link AppliedPTransform} and key.
+ */
+ public DirectExecutionContext getExecutionContext(
+ AppliedPTransform<?, ?, ?> application, StructuralKey<?> key) {
+ StepAndKey stepAndKey = StepAndKey.of(application, key);
+ return new DirectExecutionContext(
+ options.getClock(),
+ key,
+ (CopyOnAccessInMemoryStateInternals<Object>) applicationStateInternals.get(stepAndKey),
+ watermarkManager.getWatermarks(application));
+ }
+
+ /**
+ * Get all of the steps used in this {@link Pipeline}.
+ */
+ public Collection<AppliedPTransform<?, ?, ?>> getSteps() {
+ return stepNames.keySet();
+ }
+
+ /**
+ * Get the Step Name for the provided application.
+ */
+ public String getStepName(AppliedPTransform<?, ?, ?> application) {
+ return stepNames.get(application);
+ }
+
+ /**
+ * Returns a {@link ReadyCheckingSideInputReader} capable of reading the provided
+ * {@link PCollectionView PCollectionViews}.
+ *
+ * @param sideInputs the {@link PCollectionView PCollectionViews} the result should be able to
+ * read
+ * @return a {@link SideInputReader} that can read all of the provided {@link PCollectionView
+ * PCollectionViews}
+ */
+ public ReadyCheckingSideInputReader createSideInputReader(
+ final List<PCollectionView<?>> sideInputs) {
+ return sideInputContainer.createReaderForViews(sideInputs);
+ }
+
+
+ /**
+ * Create a {@link CounterSet} for this {@link Pipeline}. The {@link CounterSet} is independent
+ * of all other {@link CounterSet CounterSets} created by this call.
+ *
+ * The {@link EvaluationContext} is responsible for unifying the counters present in
+ * all created {@link CounterSet CounterSets} when the transforms that call this method
+ * complete.
+ */
+ public CounterSet createCounterSet() {
+ return new CounterSet();
+ }
+
+ /**
+ * Returns all of the counters that have been merged into this context via calls to
+ * {@link CounterSet#merge(CounterSet)}.
+ */
+ public CounterSet getCounters() {
+ return mergedCounters;
+ }
+
+ @VisibleForTesting
+ void forceRefresh() {
+ watermarkManager.refreshAll();
+ fireAllAvailableCallbacks();
+ }
+
+ /**
+ * Extracts all timers that have been fired and have not already been extracted.
+ *
+ * <p>This is a destructive operation. Timers will only appear in the result of this method once
+ * for each time they are set.
+ */
+ public Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> extractFiredTimers() {
+ forceRefresh();
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> fired =
+ watermarkManager.extractFiredTimers();
+ return fired;
+ }
+
+ /**
+ * Returns true if the step will not produce additional output.
+ *
+ * <p>If the provided transform produces only {@link IsBounded#BOUNDED}
+ * {@link PCollection PCollections}, returns true if the watermark is at
+ * {@link BoundedWindow#TIMESTAMP_MAX_VALUE positive infinity}.
+ *
+ * <p>If the provided transform produces any {@link IsBounded#UNBOUNDED}
+ * {@link PCollection PCollections}, returns the value of
+ * {@link DirectOptions#isShutdownUnboundedProducersWithMaxWatermark()}.
+ */
+ public boolean isDone(AppliedPTransform<?, ?, ?> transform) {
+ // if the PTransform's watermark isn't at the max value, it isn't done
+ if (watermarkManager
+ .getWatermarks(transform)
+ .getOutputWatermark()
+ .isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
+ return false;
+ }
+ // If the PTransform has any unbounded outputs, and unbounded producers should not be shut down,
+ // the PTransform may produce additional output. It is not done.
+ for (PValue output : transform.getOutput().expand()) {
+ if (output instanceof PCollection) {
+ IsBounded bounded = ((PCollection<?>) output).isBounded();
+ if (bounded.equals(IsBounded.UNBOUNDED)
+ && !options.isShutdownUnboundedProducersWithMaxWatermark()) {
+ return false;
+ }
+ }
+ }
+ // The PTransform's watermark was at positive infinity and all of its outputs are known to be
+ // done. It is done.
+ return true;
+ }
+
+ /**
+ * Returns true if all steps are done.
+ */
+ public boolean isDone() {
+ for (AppliedPTransform<?, ?, ?> transform : stepNames.keySet()) {
+ if (!isDone(transform)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluatorKey.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluatorKey.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluatorKey.java
index 1c36751..164e05a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluatorKey.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluatorKey.java
@@ -31,9 +31,9 @@ import java.util.Objects;
*/
final class EvaluatorKey {
private final AppliedPTransform<?, ?, ?> transform;
- private final InProcessEvaluationContext context;
+ private final EvaluationContext context;
- public EvaluatorKey(AppliedPTransform<?, ?, ?> transform, InProcessEvaluationContext context) {
+ public EvaluatorKey(AppliedPTransform<?, ?, ?> transform, EvaluationContext context) {
this.transform = transform;
this.context = context;
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
index 14570a5..4bb5021 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
@@ -17,8 +17,8 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.FiredTimers;
import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.WatermarkManager.FiredTimers;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
@@ -57,10 +57,10 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
/**
- * An {@link InProcessExecutor} that uses an underlying {@link ExecutorService} and
- * {@link InProcessEvaluationContext} to execute a {@link Pipeline}.
+ * An {@link PipelineExecutor} that uses an underlying {@link ExecutorService} and
+ * {@link EvaluationContext} to execute a {@link Pipeline}.
*/
-final class ExecutorServiceParallelExecutor implements InProcessExecutor {
+final class ExecutorServiceParallelExecutor implements PipelineExecutor {
private static final Logger LOG = LoggerFactory.getLogger(ExecutorServiceParallelExecutor.class);
private final ExecutorService executorService;
@@ -72,7 +72,7 @@ final class ExecutorServiceParallelExecutor implements InProcessExecutor {
private final Map<Class<? extends PTransform>, Collection<ModelEnforcementFactory>>
transformEnforcements;
- private final InProcessEvaluationContext evaluationContext;
+ private final EvaluationContext evaluationContext;
private final LoadingCache<StepAndKey, TransformExecutorService> executorServices;
@@ -91,7 +91,7 @@ final class ExecutorServiceParallelExecutor implements InProcessExecutor {
TransformEvaluatorRegistry registry,
@SuppressWarnings("rawtypes")
Map<Class<? extends PTransform>, Collection<ModelEnforcementFactory>> transformEnforcements,
- InProcessEvaluationContext context) {
+ EvaluationContext context) {
return new ExecutorServiceParallelExecutor(
executorService, valueToConsumers, keyedPValues, registry, transformEnforcements, context);
}
@@ -103,7 +103,7 @@ final class ExecutorServiceParallelExecutor implements InProcessExecutor {
TransformEvaluatorRegistry registry,
@SuppressWarnings("rawtypes")
Map<Class<? extends PTransform>, Collection<ModelEnforcementFactory>> transformEnforcements,
- InProcessEvaluationContext context) {
+ EvaluationContext context) {
this.executorService = executorService;
this.valueToConsumers = valueToConsumers;
this.keyedPValues = keyedPValues;
@@ -215,18 +215,18 @@ final class ExecutorServiceParallelExecutor implements InProcessExecutor {
/**
* The base implementation of {@link CompletionCallback} that provides implementations for
- * {@link #handleResult(CommittedBundle, InProcessTransformResult)} and
+ * {@link #handleResult(CommittedBundle, TransformResult)} and
* {@link #handleThrowable(CommittedBundle, Throwable)}, given an implementation of
- * {@link #getCommittedResult(CommittedBundle, InProcessTransformResult)}.
+ * {@link #getCommittedResult(CommittedBundle, TransformResult)}.
*/
private abstract class CompletionCallbackBase implements CompletionCallback {
protected abstract CommittedResult getCommittedResult(
CommittedBundle<?> inputBundle,
- InProcessTransformResult result);
+ TransformResult result);
@Override
public final CommittedResult handleResult(
- CommittedBundle<?> inputBundle, InProcessTransformResult result) {
+ CommittedBundle<?> inputBundle, TransformResult result) {
CommittedResult committedResult = getCommittedResult(inputBundle, result);
for (CommittedBundle<?> outputBundle : committedResult.getOutputs()) {
allUpdates.offer(ExecutorUpdate.fromBundle(outputBundle,
@@ -254,7 +254,7 @@ final class ExecutorServiceParallelExecutor implements InProcessExecutor {
private class DefaultCompletionCallback extends CompletionCallbackBase {
@Override
public CommittedResult getCommittedResult(
- CommittedBundle<?> inputBundle, InProcessTransformResult result) {
+ CommittedBundle<?> inputBundle, TransformResult result) {
return evaluationContext.handleResult(inputBundle,
Collections.<TimerData>emptyList(),
result);
@@ -264,7 +264,7 @@ final class ExecutorServiceParallelExecutor implements InProcessExecutor {
/**
* A {@link CompletionCallback} where the completed bundle was produced to deliver some collection
* of {@link TimerData timers}. When the evaluator completes successfully, reports all of the
- * timers used to create the input to the {@link InProcessEvaluationContext evaluation context}
+ * timers used to create the input to the {@link EvaluationContext evaluation context}
* as part of the result.
*/
private class TimerCompletionCallback extends CompletionCallbackBase {
@@ -276,7 +276,7 @@ final class ExecutorServiceParallelExecutor implements InProcessExecutor {
@Override
public CommittedResult getCommittedResult(
- CommittedBundle<?> inputBundle, InProcessTransformResult result) {
+ CommittedBundle<?> inputBundle, TransformResult result) {
return evaluationContext.handleResult(inputBundle, timers, result);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
index bbe8787..c84f620 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
@@ -36,7 +36,7 @@ class FlattenEvaluatorFactory implements TransformEvaluatorFactory {
public <InputT> TransformEvaluator<InputT> forApplication(
AppliedPTransform<?, ?, ?> application,
CommittedBundle<?> inputBundle,
- InProcessEvaluationContext evaluationContext) {
+ EvaluationContext evaluationContext) {
@SuppressWarnings({"cast", "unchecked", "rawtypes"})
TransformEvaluator<InputT> evaluator = (TransformEvaluator<InputT>) createInMemoryEvaluator(
(AppliedPTransform) application, inputBundle, evaluationContext);
@@ -48,7 +48,7 @@ class FlattenEvaluatorFactory implements TransformEvaluatorFactory {
PCollectionList<InputT>, PCollection<InputT>, FlattenPCollectionList<InputT>>
application,
final CommittedBundle<InputT> inputBundle,
- final InProcessEvaluationContext evaluationContext) {
+ final EvaluationContext evaluationContext) {
if (inputBundle == null) {
// it is impossible to call processElement on a flatten with no input bundle. A Flatten with
// no input bundle occurs as an output of Flatten.pcollections(PCollectionList.empty())
@@ -57,17 +57,17 @@ class FlattenEvaluatorFactory implements TransformEvaluatorFactory {
}
final UncommittedBundle<InputT> outputBundle =
evaluationContext.createBundle(inputBundle, application.getOutput());
- final InProcessTransformResult result =
+ final TransformResult result =
StepTransformResult.withoutHold(application).addOutput(outputBundle).build();
return new FlattenEvaluator<>(outputBundle, result);
}
private static class FlattenEvaluator<InputT> implements TransformEvaluator<InputT> {
private final UncommittedBundle<InputT> outputBundle;
- private final InProcessTransformResult result;
+ private final TransformResult result;
public FlattenEvaluator(
- UncommittedBundle<InputT> outputBundle, InProcessTransformResult result) {
+ UncommittedBundle<InputT> outputBundle, TransformResult result) {
this.outputBundle = outputBundle;
this.result = result;
}
@@ -78,7 +78,7 @@ class FlattenEvaluatorFactory implements TransformEvaluatorFactory {
}
@Override
- public InProcessTransformResult finishBundle() {
+ public TransformResult finishBundle() {
return result;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
new file mode 100644
index 0000000..9782ab1
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupAlsoByWindowEvaluatorFactory.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.runners.core.GroupAlsoByWindowViaWindowSetDoFn;
+import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupAlsoByWindow;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly;
+import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly.GroupByKeyOnly;
+import org.apache.beam.sdk.util.KeyedWorkItem;
+import org.apache.beam.sdk.util.SystemReduceFn;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Collections;
+
+/**
+ * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
+ * {@link GroupByKeyOnly} {@link PTransform}.
+ */
+class GroupAlsoByWindowEvaluatorFactory implements TransformEvaluatorFactory {
+ @Override
+ public <InputT> TransformEvaluator<InputT> forApplication(
+ AppliedPTransform<?, ?, ?> application,
+ CommittedBundle<?> inputBundle,
+ EvaluationContext evaluationContext) {
+ @SuppressWarnings({"cast", "unchecked", "rawtypes"})
+ TransformEvaluator<InputT> evaluator =
+ createEvaluator(
+ (AppliedPTransform) application, (CommittedBundle) inputBundle, evaluationContext);
+ return evaluator;
+ }
+
+ private <K, V> TransformEvaluator<KeyedWorkItem<K, V>> createEvaluator(
+ AppliedPTransform<
+ PCollection<KeyedWorkItem<K, V>>,
+ PCollection<KV<K, Iterable<V>>>,
+ DirectGroupAlsoByWindow<K, V>> application,
+ CommittedBundle<KeyedWorkItem<K, V>> inputBundle,
+ EvaluationContext evaluationContext) {
+ return new GroupAlsoByWindowEvaluator<>(
+ evaluationContext, inputBundle, application);
+ }
+
+ /**
+ * A transform evaluator for the pseudo-primitive {@link GroupAlsoByWindow}. Windowing is ignored;
+ * all input should be in the global window since all output will be as well.
+ *
+ * @see GroupByKeyViaGroupByKeyOnly
+ */
+ private static class GroupAlsoByWindowEvaluator<K, V>
+ implements TransformEvaluator<KeyedWorkItem<K, V>> {
+
+ private final TransformEvaluator<KeyedWorkItem<K, V>> gabwParDoEvaluator;
+
+ public GroupAlsoByWindowEvaluator(
+ final EvaluationContext evaluationContext,
+ CommittedBundle<KeyedWorkItem<K, V>> inputBundle,
+ final AppliedPTransform<
+ PCollection<KeyedWorkItem<K, V>>,
+ PCollection<KV<K, Iterable<V>>>,
+ DirectGroupAlsoByWindow<K, V>> application) {
+
+ Coder<V> valueCoder =
+ application.getTransform().getValueCoder(inputBundle.getPCollection().getCoder());
+
+ @SuppressWarnings("unchecked")
+ WindowingStrategy<?, BoundedWindow> windowingStrategy =
+ (WindowingStrategy<?, BoundedWindow>) application.getTransform().getWindowingStrategy();
+
+ DoFn<KeyedWorkItem<K, V>, KV<K, Iterable<V>>> gabwDoFn =
+ GroupAlsoByWindowViaWindowSetDoFn.create(
+ windowingStrategy,
+ SystemReduceFn.<K, V, BoundedWindow>buffering(valueCoder));
+
+ TupleTag<KV<K, Iterable<V>>> mainOutputTag = new TupleTag<KV<K, Iterable<V>>>() {};
+
+ // Not technically legit, as the application is not a ParDo
+ this.gabwParDoEvaluator =
+ ParDoEvaluator.create(
+ evaluationContext,
+ inputBundle,
+ application,
+ gabwDoFn,
+ Collections.<PCollectionView<?>>emptyList(),
+ mainOutputTag,
+ Collections.<TupleTag<?>>emptyList(),
+ ImmutableMap.<TupleTag<?>, PCollection<?>>of(mainOutputTag, application.getOutput()));
+ }
+
+ @Override
+ public void processElement(WindowedValue<KeyedWorkItem<K, V>> element) throws Exception {
+ gabwParDoEvaluator.processElement(element);
+ }
+
+ @Override
+ public TransformResult finishBundle() throws Exception {
+ return gabwParDoEvaluator.finishBundle();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java
new file mode 100644
index 0000000..0e419c3
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactory.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static org.apache.beam.sdk.util.CoderUtils.encodeToByteArray;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.StepTransformResult.Builder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly;
+import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly.GroupByKeyOnly;
+import org.apache.beam.sdk.util.KeyedWorkItem;
+import org.apache.beam.sdk.util.KeyedWorkItems;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
+ * {@link GroupByKeyOnly} {@link PTransform}.
+ */
+class GroupByKeyOnlyEvaluatorFactory implements TransformEvaluatorFactory {
+ @Override
+ public <InputT> TransformEvaluator<InputT> forApplication(
+ AppliedPTransform<?, ?, ?> application,
+ CommittedBundle<?> inputBundle,
+ EvaluationContext evaluationContext) {
+ @SuppressWarnings({"cast", "unchecked", "rawtypes"})
+ TransformEvaluator<InputT> evaluator =
+ createEvaluator(
+ (AppliedPTransform) application, (CommittedBundle) inputBundle, evaluationContext);
+ return evaluator;
+ }
+
+ private <K, V> TransformEvaluator<KV<K, WindowedValue<V>>> createEvaluator(
+ final AppliedPTransform<
+ PCollection<KV<K, WindowedValue<V>>>,
+ PCollection<KeyedWorkItem<K, V>>,
+ DirectGroupByKeyOnly<K, V>>
+ application,
+ final CommittedBundle<KV<K, WindowedValue<V>>> inputBundle,
+ final EvaluationContext evaluationContext) {
+ return new GroupByKeyOnlyEvaluator<>(evaluationContext, inputBundle, application);
+ }
+
+ /**
+ * A transform evaluator for the pseudo-primitive {@link GroupByKeyOnly}. Windowing is ignored;
+ * all input should be in the global window since all output will be as well.
+ *
+ * @see GroupByKeyViaGroupByKeyOnly
+ */
+ private static class GroupByKeyOnlyEvaluator<K, V>
+ implements TransformEvaluator<KV<K, WindowedValue<V>>> {
+ private final EvaluationContext evaluationContext;
+
+ private final CommittedBundle<KV<K, WindowedValue<V>>> inputBundle;
+ private final AppliedPTransform<
+ PCollection<KV<K, WindowedValue<V>>>,
+ PCollection<KeyedWorkItem<K, V>>,
+ DirectGroupByKeyOnly<K, V>> application;
+ private final Coder<K> keyCoder;
+ private Map<GroupingKey<K>, List<WindowedValue<V>>> groupingMap;
+
+ public GroupByKeyOnlyEvaluator(
+ EvaluationContext evaluationContext,
+ CommittedBundle<KV<K, WindowedValue<V>>> inputBundle,
+ AppliedPTransform<
+ PCollection<KV<K, WindowedValue<V>>>,
+ PCollection<KeyedWorkItem<K, V>>,
+ DirectGroupByKeyOnly<K, V>> application) {
+ this.evaluationContext = evaluationContext;
+ this.inputBundle = inputBundle;
+ this.application = application;
+ this.keyCoder = getKeyCoder(application.getInput().getCoder());
+ this.groupingMap = new HashMap<>();
+ }
+
+ private Coder<K> getKeyCoder(Coder<KV<K, WindowedValue<V>>> coder) {
+ checkState(
+ coder instanceof KvCoder,
+ "%s requires a coder of class %s."
+ + " This is an internal error; this is checked during pipeline construction"
+ + " but became corrupted.",
+ getClass().getSimpleName(),
+ KvCoder.class.getSimpleName());
+ @SuppressWarnings("unchecked")
+ Coder<K> keyCoder = ((KvCoder<K, WindowedValue<V>>) coder).getKeyCoder();
+ return keyCoder;
+ }
+
+ @Override
+ public void processElement(WindowedValue<KV<K, WindowedValue<V>>> element) {
+ KV<K, WindowedValue<V>> kv = element.getValue();
+ K key = kv.getKey();
+ byte[] encodedKey;
+ try {
+ encodedKey = encodeToByteArray(keyCoder, key);
+ } catch (CoderException exn) {
+ // TODO: Put in better element printing:
+ // truncate if too long.
+ throw new IllegalArgumentException(
+ String.format("unable to encode key %s of input to %s using %s", key, this, keyCoder),
+ exn);
+ }
+ GroupingKey<K> groupingKey = new GroupingKey<>(key, encodedKey);
+ List<WindowedValue<V>> values = groupingMap.get(groupingKey);
+ if (values == null) {
+ values = new ArrayList<WindowedValue<V>>();
+ groupingMap.put(groupingKey, values);
+ }
+ values.add(kv.getValue());
+ }
+
+ @Override
+ public TransformResult finishBundle() {
+ Builder resultBuilder = StepTransformResult.withoutHold(application);
+ for (Map.Entry<GroupingKey<K>, List<WindowedValue<V>>> groupedEntry :
+ groupingMap.entrySet()) {
+ K key = groupedEntry.getKey().key;
+ KeyedWorkItem<K, V> groupedKv =
+ KeyedWorkItems.elementsWorkItem(key, groupedEntry.getValue());
+ UncommittedBundle<KeyedWorkItem<K, V>> bundle = evaluationContext.createKeyedBundle(
+ inputBundle,
+ StructuralKey.of(key, keyCoder),
+ application.getOutput());
+ bundle.add(WindowedValue.valueInGlobalWindow(groupedKv));
+ resultBuilder.addOutput(bundle);
+ }
+ return resultBuilder.build();
+ }
+
+ private static class GroupingKey<K> {
+ private K key;
+ private byte[] encodedKey;
+
+ public GroupingKey(K key, byte[] encodedKey) {
+ this.key = key;
+ this.encodedKey = encodedKey;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof GroupingKey) {
+ GroupingKey<?> that = (GroupingKey<?>) o;
+ return Arrays.equals(this.encodedKey, that.encodedKey);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(encodedKey);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
index d121442..b0eb38f 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
@@ -77,7 +77,7 @@ class ImmutabilityEnforcementFactory implements ModelEnforcementFactory {
@Override
public void afterFinish(
CommittedBundle<T> input,
- InProcessTransformResult result,
+ TransformResult result,
Iterable<? extends CommittedBundle<?>> outputs) {
for (MutationDetector detector : mutationElements.values()) {
verifyUnmodified(detector);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java
new file mode 100644
index 0000000..25a0d05
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollection;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableList;
+
+import org.joda.time.Instant;
+
+/**
+ * A factory that produces bundles that perform no additional validation.
+ */
+class ImmutableListBundleFactory implements BundleFactory {
+ public static ImmutableListBundleFactory create() {
+ return new ImmutableListBundleFactory();
+ }
+
+ private ImmutableListBundleFactory() {}
+
+ @Override
+ public <T> UncommittedBundle<T> createRootBundle(PCollection<T> output) {
+ return UncommittedImmutableListBundle.create(output, StructuralKey.of(null, VoidCoder.of()));
+ }
+
+ @Override
+ public <T> UncommittedBundle<T> createBundle(CommittedBundle<?> input, PCollection<T> output) {
+ return UncommittedImmutableListBundle.create(output, input.getKey());
+ }
+
+ @Override
+ public <K, T> UncommittedBundle<T> createKeyedBundle(
+ CommittedBundle<?> input, StructuralKey<K> key, PCollection<T> output) {
+ return UncommittedImmutableListBundle.create(output, key);
+ }
+
+ /**
+ * A {@link UncommittedBundle} that buffers elements in memory.
+ */
+ private static final class UncommittedImmutableListBundle<T> implements UncommittedBundle<T> {
+ private final PCollection<T> pcollection;
+ private final StructuralKey<?> key;
+ private boolean committed = false;
+ private ImmutableList.Builder<WindowedValue<T>> elements;
+
+ /**
+ * Create a new {@link UncommittedImmutableListBundle} for the specified {@link PCollection}.
+ */
+ public static <T> UncommittedImmutableListBundle<T> create(
+ PCollection<T> pcollection,
+ StructuralKey<?> key) {
+ return new UncommittedImmutableListBundle<>(pcollection, key);
+ }
+
+ private UncommittedImmutableListBundle(PCollection<T> pcollection, StructuralKey<?> key) {
+ this.pcollection = pcollection;
+ this.key = key;
+ this.elements = ImmutableList.builder();
+ }
+
+ @Override
+ public PCollection<T> getPCollection() {
+ return pcollection;
+ }
+
+ @Override
+ public UncommittedImmutableListBundle<T> add(WindowedValue<T> element) {
+ checkState(
+ !committed,
+ "Can't add element %s to committed bundle in PCollection %s",
+ element,
+ pcollection);
+ elements.add(element);
+ return this;
+ }
+
+ @Override
+ public CommittedBundle<T> commit(final Instant synchronizedCompletionTime) {
+ checkState(!committed, "Can't commit already committed bundle %s", this);
+ committed = true;
+ final Iterable<WindowedValue<T>> committedElements = elements.build();
+ return new CommittedImmutableListBundle<>(
+ pcollection, key, committedElements, synchronizedCompletionTime);
+ }
+ }
+
+ private static class CommittedImmutableListBundle<T> implements CommittedBundle<T> {
+ public CommittedImmutableListBundle(
+ PCollection<T> pcollection,
+ StructuralKey<?> key,
+ Iterable<WindowedValue<T>> committedElements,
+ Instant synchronizedCompletionTime) {
+ this.pcollection = pcollection;
+ this.key = key;
+ this.committedElements = committedElements;
+ this.synchronizedCompletionTime = synchronizedCompletionTime;
+ }
+
+ private final PCollection<T> pcollection;
+ /** The structural value key of the Bundle, as specified by the coder that created it. */
+ private final StructuralKey<?> key;
+ private final Iterable<WindowedValue<T>> committedElements;
+ private final Instant synchronizedCompletionTime;
+
+ @Override
+ public StructuralKey<?> getKey() {
+ return key;
+ }
+
+ @Override
+ public Iterable<WindowedValue<T>> getElements() {
+ return committedElements;
+ }
+
+ @Override
+ public PCollection<T> getPCollection() {
+ return pcollection;
+ }
+
+ @Override
+ public Instant getSynchronizedProcessingOutputWatermark() {
+ return synchronizedCompletionTime;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .omitNullValues()
+ .add("pcollection", pcollection)
+ .add("key", key)
+ .add("elements", committedElements)
+ .toString();
+ }
+
+ @Override
+ public CommittedBundle<T> withElements(Iterable<WindowedValue<T>> elements) {
+ return new CommittedImmutableListBundle<>(
+ pcollection, key, ImmutableList.copyOf(elements), synchronizedCompletionTime);
+ }
+ }
+}
[07/12] incubator-beam git commit: Remove InProcess Prefixes
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTimerInternalsTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTimerInternalsTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTimerInternalsTest.java
new file mode 100644
index 0000000..c038910
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectTimerInternalsTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate.TimerUpdateBuilder;
+import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.util.TimerInternals.TimerData;
+import org.apache.beam.sdk.util.state.StateNamespaces;
+
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/**
+ * Tests for {@link DirectTimerInternals}.
+ */
+@RunWith(JUnit4.class)
+public class DirectTimerInternalsTest {
+ private MockClock clock;
+ @Mock private TransformWatermarks watermarks;
+
+ private TimerUpdateBuilder timerUpdateBuilder;
+
+ private DirectTimerInternals internals;
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ clock = MockClock.fromInstant(new Instant(0));
+
+ timerUpdateBuilder = TimerUpdate.builder(StructuralKey.of(1234, VarIntCoder.of()));
+
+ internals = DirectTimerInternals.create(clock, watermarks, timerUpdateBuilder);
+ }
+
+ @Test
+ public void setTimerAddsToBuilder() {
+ TimerData eventTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(20145L), TimeDomain.EVENT_TIME);
+ TimerData processingTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(125555555L), TimeDomain.PROCESSING_TIME);
+ TimerData synchronizedProcessingTimer =
+ TimerData.of(
+ StateNamespaces.global(),
+ new Instant(98745632189L),
+ TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
+ internals.setTimer(eventTimer);
+ internals.setTimer(processingTimer);
+ internals.setTimer(synchronizedProcessingTimer);
+
+ assertThat(
+ internals.getTimerUpdate().getSetTimers(),
+ containsInAnyOrder(eventTimer, synchronizedProcessingTimer, processingTimer));
+ }
+
+ @Test
+ public void deleteTimerDeletesOnBuilder() {
+ TimerData eventTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(20145L), TimeDomain.EVENT_TIME);
+ TimerData processingTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(125555555L), TimeDomain.PROCESSING_TIME);
+ TimerData synchronizedProcessingTimer =
+ TimerData.of(
+ StateNamespaces.global(),
+ new Instant(98745632189L),
+ TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
+ internals.deleteTimer(eventTimer);
+ internals.deleteTimer(processingTimer);
+ internals.deleteTimer(synchronizedProcessingTimer);
+
+ assertThat(
+ internals.getTimerUpdate().getDeletedTimers(),
+ containsInAnyOrder(eventTimer, synchronizedProcessingTimer, processingTimer));
+ }
+
+ @Test
+ public void getProcessingTimeIsClockNow() {
+ assertThat(internals.currentProcessingTime(), equalTo(clock.now()));
+ Instant oldProcessingTime = internals.currentProcessingTime();
+
+ clock.advance(Duration.standardHours(12));
+
+ assertThat(internals.currentProcessingTime(), equalTo(clock.now()));
+ assertThat(
+ internals.currentProcessingTime(),
+ equalTo(oldProcessingTime.plus(Duration.standardHours(12))));
+ }
+
+ @Test
+ public void getSynchronizedProcessingTimeIsWatermarkSynchronizedInputTime() {
+ when(watermarks.getSynchronizedProcessingInputTime()).thenReturn(new Instant(12345L));
+ assertThat(internals.currentSynchronizedProcessingTime(), equalTo(new Instant(12345L)));
+ }
+
+ @Test
+ public void getInputWatermarkTimeUsesWatermarkTime() {
+ when(watermarks.getInputWatermark()).thenReturn(new Instant(8765L));
+ assertThat(internals.currentInputWatermarkTime(), equalTo(new Instant(8765L)));
+ }
+
+ @Test
+ public void getOutputWatermarkTimeUsesWatermarkTime() {
+ when(watermarks.getOutputWatermark()).thenReturn(new Instant(25525L));
+ assertThat(internals.currentOutputWatermarkTime(), equalTo(new Instant(25525L)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactoryTest.java
index e129489..b903ef1 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactoryTest.java
@@ -51,7 +51,7 @@ import java.util.Collections;
public class EncodabilityEnforcementFactoryTest {
@Rule public ExpectedException thrown = ExpectedException.none();
private EncodabilityEnforcementFactory factory = EncodabilityEnforcementFactory.create();
- private BundleFactory bundleFactory = InProcessBundleFactory.create();
+ private BundleFactory bundleFactory = ImmutableListBundleFactory.create();
@Test
public void encodeFailsThrows() {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
new file mode 100644
index 0000000..1726866
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -0,0 +1,545 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.runners.direct.DirectExecutionContext.DirectStepContext;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.PCollectionViewWriter;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.WatermarkManager.FiredTimers;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.io.CountingInput;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo.Timing;
+import org.apache.beam.sdk.util.SideInputReader;
+import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.util.TimerInternals.TimerData;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.util.common.Counter;
+import org.apache.beam.sdk.util.common.Counter.AggregationKind;
+import org.apache.beam.sdk.util.common.CounterSet;
+import org.apache.beam.sdk.util.state.BagState;
+import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
+import org.apache.beam.sdk.util.state.StateNamespaces;
+import org.apache.beam.sdk.util.state.StateTag;
+import org.apache.beam.sdk.util.state.StateTags;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+import org.hamcrest.Matchers;
+import org.joda.time.Instant;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tests for {@link EvaluationContext}.
+ */
+@RunWith(JUnit4.class)
+public class EvaluationContextTest {
+ private TestPipeline p;
+ private EvaluationContext context;
+
+ private PCollection<Integer> created;
+ private PCollection<KV<String, Integer>> downstream;
+ private PCollectionView<Iterable<Integer>> view;
+ private PCollection<Long> unbounded;
+ private Collection<AppliedPTransform<?, ?, ?>> rootTransforms;
+ private Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> valueToConsumers;
+
+ private BundleFactory bundleFactory;
+
+ @Before
+ public void setup() {
+ DirectRunner runner =
+ DirectRunner.fromOptions(PipelineOptionsFactory.create());
+
+ p = TestPipeline.create();
+
+ created = p.apply(Create.of(1, 2, 3));
+ downstream = created.apply(WithKeys.<String, Integer>of("foo"));
+ view = created.apply(View.<Integer>asIterable());
+ unbounded = p.apply(CountingInput.unbounded());
+
+ ConsumerTrackingPipelineVisitor cVis = new ConsumerTrackingPipelineVisitor();
+ p.traverseTopologically(cVis);
+ rootTransforms = cVis.getRootTransforms();
+ valueToConsumers = cVis.getValueToConsumers();
+
+ bundleFactory = ImmutableListBundleFactory.create();
+
+ context =
+ EvaluationContext.create(
+ runner.getPipelineOptions(),
+ ImmutableListBundleFactory.create(),
+ rootTransforms,
+ valueToConsumers,
+ cVis.getStepNames(),
+ cVis.getViews());
+ }
+
+ @Test
+ public void writeToViewWriterThenReadReads() {
+ PCollectionViewWriter<Integer, Iterable<Integer>> viewWriter =
+ context.createPCollectionViewWriter(
+ PCollection.<Iterable<Integer>>createPrimitiveOutputInternal(
+ p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED),
+ view);
+ BoundedWindow window = new TestBoundedWindow(new Instant(1024L));
+ BoundedWindow second = new TestBoundedWindow(new Instant(899999L));
+ WindowedValue<Integer> firstValue =
+ WindowedValue.of(1, new Instant(1222), window, PaneInfo.ON_TIME_AND_ONLY_FIRING);
+ WindowedValue<Integer> secondValue =
+ WindowedValue.of(
+ 2, new Instant(8766L), second, PaneInfo.createPane(true, false, Timing.ON_TIME, 0, 0));
+ Iterable<WindowedValue<Integer>> values = ImmutableList.of(firstValue, secondValue);
+ viewWriter.add(values);
+
+ SideInputReader reader =
+ context.createSideInputReader(ImmutableList.<PCollectionView<?>>of(view));
+ assertThat(reader.get(view, window), containsInAnyOrder(1));
+ assertThat(reader.get(view, second), containsInAnyOrder(2));
+
+ WindowedValue<Integer> overrittenSecondValue =
+ WindowedValue.of(
+ 4444, new Instant(8677L), second, PaneInfo.createPane(false, true, Timing.LATE, 1, 1));
+ viewWriter.add(Collections.singleton(overrittenSecondValue));
+ assertThat(reader.get(view, second), containsInAnyOrder(2));
+ // The cached value is served in the earlier reader
+ reader = context.createSideInputReader(ImmutableList.<PCollectionView<?>>of(view));
+ assertThat(reader.get(view, second), containsInAnyOrder(4444));
+ }
+
+ @Test
+ public void getExecutionContextSameStepSameKeyState() {
+ DirectExecutionContext fooContext =
+ context.getExecutionContext(created.getProducingTransformInternal(),
+ StructuralKey.of("foo", StringUtf8Coder.of()));
+
+ StateTag<Object, BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
+
+ DirectStepContext stepContext = fooContext.getOrCreateStepContext("s1", "s1");
+ stepContext.stateInternals().state(StateNamespaces.global(), intBag).add(1);
+
+ context.handleResult(ImmutableListBundleFactory.create()
+ .createKeyedBundle(null, StructuralKey.of("foo", StringUtf8Coder.of()), created)
+ .commit(Instant.now()),
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(created.getProducingTransformInternal())
+ .withState(stepContext.commitState())
+ .build());
+
+ DirectExecutionContext secondFooContext =
+ context.getExecutionContext(created.getProducingTransformInternal(),
+ StructuralKey.of("foo", StringUtf8Coder.of()));
+ assertThat(
+ secondFooContext
+ .getOrCreateStepContext("s1", "s1")
+ .stateInternals()
+ .state(StateNamespaces.global(), intBag)
+ .read(),
+ contains(1));
+ }
+
+
+ @Test
+ public void getExecutionContextDifferentKeysIndependentState() {
+ DirectExecutionContext fooContext =
+ context.getExecutionContext(created.getProducingTransformInternal(),
+ StructuralKey.of("foo", StringUtf8Coder.of()));
+
+ StateTag<Object, BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
+
+ fooContext
+ .getOrCreateStepContext("s1", "s1")
+ .stateInternals()
+ .state(StateNamespaces.global(), intBag)
+ .add(1);
+
+ DirectExecutionContext barContext =
+ context.getExecutionContext(created.getProducingTransformInternal(),
+ StructuralKey.of("bar", StringUtf8Coder.of()));
+ assertThat(barContext, not(equalTo(fooContext)));
+ assertThat(
+ barContext
+ .getOrCreateStepContext("s1", "s1")
+ .stateInternals()
+ .state(StateNamespaces.global(), intBag)
+ .read(),
+ emptyIterable());
+ }
+
+ @Test
+ public void getExecutionContextDifferentStepsIndependentState() {
+ StructuralKey<?> myKey = StructuralKey.of("foo", StringUtf8Coder.of());
+ DirectExecutionContext fooContext =
+ context.getExecutionContext(created.getProducingTransformInternal(), myKey);
+
+ StateTag<Object, BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
+
+ fooContext
+ .getOrCreateStepContext("s1", "s1")
+ .stateInternals()
+ .state(StateNamespaces.global(), intBag)
+ .add(1);
+
+ DirectExecutionContext barContext =
+ context.getExecutionContext(downstream.getProducingTransformInternal(), myKey);
+ assertThat(
+ barContext
+ .getOrCreateStepContext("s1", "s1")
+ .stateInternals()
+ .state(StateNamespaces.global(), intBag)
+ .read(),
+ emptyIterable());
+ }
+
+ @Test
+ public void handleResultMergesCounters() {
+ CounterSet counters = context.createCounterSet();
+ Counter<Long> myCounter = Counter.longs("foo", AggregationKind.SUM);
+ counters.addCounter(myCounter);
+
+ myCounter.addValue(4L);
+ TransformResult result =
+ StepTransformResult.withoutHold(created.getProducingTransformInternal())
+ .withCounters(counters)
+ .build();
+ context.handleResult(null, ImmutableList.<TimerData>of(), result);
+ assertThat((Long) context.getCounters().getExistingCounter("foo").getAggregate(), equalTo(4L));
+
+ CounterSet againCounters = context.createCounterSet();
+ Counter<Long> myLongCounterAgain = Counter.longs("foo", AggregationKind.SUM);
+ againCounters.add(myLongCounterAgain);
+ myLongCounterAgain.addValue(8L);
+
+ TransformResult secondResult =
+ StepTransformResult.withoutHold(downstream.getProducingTransformInternal())
+ .withCounters(againCounters)
+ .build();
+ context.handleResult(
+ context.createRootBundle(created).commit(Instant.now()),
+ ImmutableList.<TimerData>of(),
+ secondResult);
+ assertThat((Long) context.getCounters().getExistingCounter("foo").getAggregate(), equalTo(12L));
+ }
+
+ @Test
+ public void handleResultStoresState() {
+ StructuralKey<?> myKey = StructuralKey.of("foo".getBytes(), ByteArrayCoder.of());
+ DirectExecutionContext fooContext =
+ context.getExecutionContext(downstream.getProducingTransformInternal(), myKey);
+
+ StateTag<Object, BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());
+
+ CopyOnAccessInMemoryStateInternals<Object> state =
+ fooContext.getOrCreateStepContext("s1", "s1").stateInternals();
+ BagState<Integer> bag = state.state(StateNamespaces.global(), intBag);
+ bag.add(1);
+ bag.add(2);
+ bag.add(4);
+
+ TransformResult stateResult =
+ StepTransformResult.withoutHold(downstream.getProducingTransformInternal())
+ .withState(state)
+ .build();
+
+ context.handleResult(
+ context.createKeyedBundle(null, myKey, created).commit(Instant.now()),
+ ImmutableList.<TimerData>of(),
+ stateResult);
+
+ DirectExecutionContext afterResultContext =
+ context.getExecutionContext(downstream.getProducingTransformInternal(), myKey);
+
+ CopyOnAccessInMemoryStateInternals<Object> afterResultState =
+ afterResultContext.getOrCreateStepContext("s1", "s1").stateInternals();
+ assertThat(afterResultState.state(StateNamespaces.global(), intBag).read(), contains(1, 2, 4));
+ }
+
+ @Test
+ public void callAfterOutputMustHaveBeenProducedAfterEndOfWatermarkCallsback() throws Exception {
+ final CountDownLatch callLatch = new CountDownLatch(1);
+ Runnable callback =
+ new Runnable() {
+ @Override
+ public void run() {
+ callLatch.countDown();
+ }
+ };
+
+ // Should call back after the end of the global window
+ context.scheduleAfterOutputWouldBeProduced(
+ downstream, GlobalWindow.INSTANCE, WindowingStrategy.globalDefault(), callback);
+
+ TransformResult result =
+ StepTransformResult.withHold(created.getProducingTransformInternal(), new Instant(0))
+ .build();
+
+ context.handleResult(null, ImmutableList.<TimerData>of(), result);
+ // Difficult to demonstrate that we took no action in a multithreaded world; poll for a bit
+ // will likely be flaky if this logic is broken
+ assertThat(callLatch.await(500L, TimeUnit.MILLISECONDS), is(false));
+
+ TransformResult finishedResult =
+ StepTransformResult.withoutHold(created.getProducingTransformInternal()).build();
+ context.handleResult(null, ImmutableList.<TimerData>of(), finishedResult);
+ context.forceRefresh();
+ // Obtain the value via blocking call
+ assertThat(callLatch.await(1, TimeUnit.SECONDS), is(true));
+ }
+
+ @Test
+ public void callAfterOutputMustHaveBeenProducedAlreadyAfterCallsImmediately() throws Exception {
+ TransformResult finishedResult =
+ StepTransformResult.withoutHold(created.getProducingTransformInternal()).build();
+ context.handleResult(null, ImmutableList.<TimerData>of(), finishedResult);
+
+ final CountDownLatch callLatch = new CountDownLatch(1);
+ context.extractFiredTimers();
+ Runnable callback =
+ new Runnable() {
+ @Override
+ public void run() {
+ callLatch.countDown();
+ }
+ };
+ context.scheduleAfterOutputWouldBeProduced(
+ downstream, GlobalWindow.INSTANCE, WindowingStrategy.globalDefault(), callback);
+ assertThat(callLatch.await(1, TimeUnit.SECONDS), is(true));
+ }
+
+ @Test
+ public void extractFiredTimersExtractsTimers() {
+ TransformResult holdResult =
+ StepTransformResult.withHold(created.getProducingTransformInternal(), new Instant(0))
+ .build();
+ context.handleResult(null, ImmutableList.<TimerData>of(), holdResult);
+
+ StructuralKey<?> key = StructuralKey.of("foo".length(), VarIntCoder.of());
+ TimerData toFire =
+ TimerData.of(StateNamespaces.global(), new Instant(100L), TimeDomain.EVENT_TIME);
+ TransformResult timerResult =
+ StepTransformResult.withoutHold(downstream.getProducingTransformInternal())
+ .withState(CopyOnAccessInMemoryStateInternals.withUnderlying(key, null))
+ .withTimerUpdate(TimerUpdate.builder(key).setTimer(toFire).build())
+ .build();
+
+ // haven't added any timers, must be empty
+ assertThat(context.extractFiredTimers().entrySet(), emptyIterable());
+ context.handleResult(
+ context.createKeyedBundle(null, key, created).commit(Instant.now()),
+ ImmutableList.<TimerData>of(),
+ timerResult);
+
+ // timer hasn't fired
+ assertThat(context.extractFiredTimers().entrySet(), emptyIterable());
+
+ TransformResult advanceResult =
+ StepTransformResult.withoutHold(created.getProducingTransformInternal()).build();
+ // Should cause the downstream timer to fire
+ context.handleResult(null, ImmutableList.<TimerData>of(), advanceResult);
+
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> fired =
+ context.extractFiredTimers();
+ assertThat(
+ fired,
+ Matchers.<AppliedPTransform<?, ?, ?>>hasKey(downstream.getProducingTransformInternal()));
+ Map<StructuralKey<?>, FiredTimers> downstreamFired =
+ fired.get(downstream.getProducingTransformInternal());
+ assertThat(downstreamFired, Matchers.<Object>hasKey(key));
+
+ FiredTimers firedForKey = downstreamFired.get(key);
+ assertThat(firedForKey.getTimers(TimeDomain.PROCESSING_TIME), emptyIterable());
+ assertThat(firedForKey.getTimers(TimeDomain.SYNCHRONIZED_PROCESSING_TIME), emptyIterable());
+ assertThat(firedForKey.getTimers(TimeDomain.EVENT_TIME), contains(toFire));
+
+ // Don't reextract timers
+ assertThat(context.extractFiredTimers().entrySet(), emptyIterable());
+ }
+
+ @Test
+ public void createBundleKeyedResultPropagatesKey() {
+ StructuralKey<String> key = StructuralKey.of("foo", StringUtf8Coder.of());
+ CommittedBundle<KV<String, Integer>> newBundle =
+ context
+ .createBundle(
+ bundleFactory.createKeyedBundle(
+ null, key,
+ created).commit(Instant.now()),
+ downstream).commit(Instant.now());
+ assertThat(newBundle.getKey(), Matchers.<StructuralKey<?>>equalTo(key));
+ }
+
+ @Test
+ public void createKeyedBundleKeyed() {
+ StructuralKey<String> key = StructuralKey.of("foo", StringUtf8Coder.of());
+ CommittedBundle<KV<String, Integer>> keyedBundle =
+ context.createKeyedBundle(
+ bundleFactory.createRootBundle(created).commit(Instant.now()),
+ key,
+ downstream).commit(Instant.now());
+ assertThat(keyedBundle.getKey(),
+ Matchers.<StructuralKey<?>>equalTo(key));
+ }
+
+ @Test
+ public void isDoneWithUnboundedPCollectionAndShutdown() {
+ context.getPipelineOptions().setShutdownUnboundedProducersWithMaxWatermark(true);
+ assertThat(context.isDone(unbounded.getProducingTransformInternal()), is(false));
+
+ context.handleResult(
+ null,
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(unbounded.getProducingTransformInternal()).build());
+ context.extractFiredTimers();
+ assertThat(context.isDone(unbounded.getProducingTransformInternal()), is(true));
+ }
+
+ @Test
+ public void isDoneWithUnboundedPCollectionAndNotShutdown() {
+ context.getPipelineOptions().setShutdownUnboundedProducersWithMaxWatermark(false);
+ assertThat(context.isDone(unbounded.getProducingTransformInternal()), is(false));
+
+ context.handleResult(
+ null,
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(unbounded.getProducingTransformInternal()).build());
+ assertThat(context.isDone(unbounded.getProducingTransformInternal()), is(false));
+ }
+
+ @Test
+ public void isDoneWithOnlyBoundedPCollections() {
+ context.getPipelineOptions().setShutdownUnboundedProducersWithMaxWatermark(false);
+ assertThat(context.isDone(created.getProducingTransformInternal()), is(false));
+
+ context.handleResult(
+ null,
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(created.getProducingTransformInternal()).build());
+ context.extractFiredTimers();
+ assertThat(context.isDone(created.getProducingTransformInternal()), is(true));
+ }
+
+ @Test
+ public void isDoneWithPartiallyDone() {
+ context.getPipelineOptions().setShutdownUnboundedProducersWithMaxWatermark(true);
+ assertThat(context.isDone(), is(false));
+
+ UncommittedBundle<Integer> rootBundle = context.createRootBundle(created);
+ rootBundle.add(WindowedValue.valueInGlobalWindow(1));
+ CommittedResult handleResult =
+ context.handleResult(
+ null,
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(created.getProducingTransformInternal())
+ .addOutput(rootBundle)
+ .build());
+ @SuppressWarnings("unchecked")
+ CommittedBundle<Integer> committedBundle =
+ (CommittedBundle<Integer>) Iterables.getOnlyElement(handleResult.getOutputs());
+ context.handleResult(
+ null,
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(unbounded.getProducingTransformInternal()).build());
+ assertThat(context.isDone(), is(false));
+
+ for (AppliedPTransform<?, ?, ?> consumers : valueToConsumers.get(created)) {
+ context.handleResult(
+ committedBundle,
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(consumers).build());
+ }
+ context.extractFiredTimers();
+ assertThat(context.isDone(), is(true));
+ }
+
+ @Test
+ public void isDoneWithUnboundedAndNotShutdown() {
+ context.getPipelineOptions().setShutdownUnboundedProducersWithMaxWatermark(false);
+ assertThat(context.isDone(), is(false));
+
+ context.handleResult(
+ null,
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(created.getProducingTransformInternal()).build());
+ context.handleResult(
+ null,
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(unbounded.getProducingTransformInternal()).build());
+ context.handleResult(
+ context.createRootBundle(created).commit(Instant.now()),
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(downstream.getProducingTransformInternal()).build());
+ context.extractFiredTimers();
+ assertThat(context.isDone(), is(false));
+
+ context.handleResult(
+ context.createRootBundle(created).commit(Instant.now()),
+ ImmutableList.<TimerData>of(),
+ StepTransformResult.withoutHold(view.getProducingTransformInternal()).build());
+ context.extractFiredTimers();
+ assertThat(context.isDone(), is(false));
+ }
+
+ private static class TestBoundedWindow extends BoundedWindow {
+ private final Instant ts;
+
+ public TestBoundedWindow(Instant ts) {
+ this.ts = ts;
+ }
+
+ @Override
+ public Instant maxTimestamp() {
+ return ts;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
index 5efb090..0bc3036 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/FlattenEvaluatorFactoryTest.java
@@ -45,7 +45,7 @@ import org.junit.runners.JUnit4;
*/
@RunWith(JUnit4.class)
public class FlattenEvaluatorFactoryTest {
- private BundleFactory bundleFactory = InProcessBundleFactory.create();
+ private BundleFactory bundleFactory = ImmutableListBundleFactory.create();
@Test
public void testFlattenInMemoryEvaluator() throws Exception {
TestPipeline p = TestPipeline.create();
@@ -60,7 +60,7 @@ public class FlattenEvaluatorFactoryTest {
CommittedBundle<Integer> rightBundle =
bundleFactory.createRootBundle(right).commit(Instant.now());
- InProcessEvaluationContext context = mock(InProcessEvaluationContext.class);
+ EvaluationContext context = mock(EvaluationContext.class);
UncommittedBundle<Integer> flattenedLeftBundle = bundleFactory.createRootBundle(flattened);
UncommittedBundle<Integer> flattenedRightBundle = bundleFactory.createRootBundle(flattened);
@@ -87,8 +87,8 @@ public class FlattenEvaluatorFactoryTest {
rightSideEvaluator.processElement(
WindowedValue.timestampedValueInGlobalWindow(-4, new Instant(-4096)));
- InProcessTransformResult rightSideResult = rightSideEvaluator.finishBundle();
- InProcessTransformResult leftSideResult = leftSideEvaluator.finishBundle();
+ TransformResult rightSideResult = rightSideEvaluator.finishBundle();
+ TransformResult leftSideResult = leftSideEvaluator.finishBundle();
assertThat(
rightSideResult.getOutputBundles(),
@@ -124,13 +124,13 @@ public class FlattenEvaluatorFactoryTest {
PCollection<Integer> flattened = list.apply(Flatten.<Integer>pCollections());
- InProcessEvaluationContext context = mock(InProcessEvaluationContext.class);
+ EvaluationContext context = mock(EvaluationContext.class);
FlattenEvaluatorFactory factory = new FlattenEvaluatorFactory();
TransformEvaluator<Integer> emptyEvaluator =
factory.forApplication(flattened.getProducingTransformInternal(), null, context);
- InProcessTransformResult leftSideResult = emptyEvaluator.finishBundle();
+ TransformResult leftSideResult = emptyEvaluator.finishBundle();
assertThat(leftSideResult.getOutputBundles(), emptyIterable());
assertThat(
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java
index b589db0..a529043 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyEvaluatorFactoryTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.Coder;
@@ -52,7 +53,7 @@ import org.junit.runners.JUnit4;
*/
@RunWith(JUnit4.class)
public class GroupByKeyEvaluatorFactoryTest {
- private BundleFactory bundleFactory = InProcessBundleFactory.create();
+ private BundleFactory bundleFactory = ImmutableListBundleFactory.create();
@Test
public void testInMemoryEvaluator() throws Exception {
@@ -68,11 +69,11 @@ public class GroupByKeyEvaluatorFactoryTest {
PCollection<KV<String, WindowedValue<Integer>>> kvs =
values.apply(new ReifyTimestampsAndWindows<String, Integer>());
PCollection<KeyedWorkItem<String, Integer>> groupedKvs =
- kvs.apply(new InProcessGroupByKey.InProcessGroupByKeyOnly<String, Integer>());
+ kvs.apply(new DirectGroupByKeyOnly<String, Integer>());
CommittedBundle<KV<String, WindowedValue<Integer>>> inputBundle =
bundleFactory.createRootBundle(kvs).commit(Instant.now());
- InProcessEvaluationContext evaluationContext = mock(InProcessEvaluationContext.class);
+ EvaluationContext evaluationContext = mock(EvaluationContext.class);
StructuralKey<String> fooKey = StructuralKey.of("foo", StringUtf8Coder.of());
UncommittedBundle<KeyedWorkItem<String, Integer>> fooBundle =
bundleFactory.createKeyedBundle(null, fooKey, groupedKvs);
@@ -100,7 +101,7 @@ public class GroupByKeyEvaluatorFactoryTest {
Coder<String> keyCoder =
((KvCoder<String, WindowedValue<Integer>>) kvs.getCoder()).getKeyCoder();
TransformEvaluator<KV<String, WindowedValue<Integer>>> evaluator =
- new InProcessGroupByKeyOnlyEvaluatorFactory()
+ new GroupByKeyOnlyEvaluatorFactory()
.forApplication(
groupedKvs.getProducingTransformInternal(), inputBundle, evaluationContext);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactoryTest.java
new file mode 100644
index 0000000..5fb5a76
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/GroupByKeyOnlyEvaluatorFactoryTest.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static org.hamcrest.Matchers.contains;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.beam.runners.direct.DirectGroupByKey.DirectGroupByKeyOnly;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly.ReifyTimestampsAndWindows;
+import org.apache.beam.sdk.util.KeyedWorkItem;
+import org.apache.beam.sdk.util.KeyedWorkItems;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+import com.google.common.collect.HashMultiset;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multiset;
+
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.joda.time.Instant;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link GroupByKeyOnlyEvaluatorFactory}.
+ */
+@RunWith(JUnit4.class)
+public class GroupByKeyOnlyEvaluatorFactoryTest {
+ private BundleFactory bundleFactory = ImmutableListBundleFactory.create();
+
+ @Test
+ public void testInMemoryEvaluator() throws Exception {
+ TestPipeline p = TestPipeline.create();
+ KV<String, Integer> firstFoo = KV.of("foo", -1);
+ KV<String, Integer> secondFoo = KV.of("foo", 1);
+ KV<String, Integer> thirdFoo = KV.of("foo", 3);
+ KV<String, Integer> firstBar = KV.of("bar", 22);
+ KV<String, Integer> secondBar = KV.of("bar", 12);
+ KV<String, Integer> firstBaz = KV.of("baz", Integer.MAX_VALUE);
+ PCollection<KV<String, Integer>> values =
+ p.apply(Create.of(firstFoo, firstBar, secondFoo, firstBaz, secondBar, thirdFoo));
+ PCollection<KV<String, WindowedValue<Integer>>> kvs =
+ values.apply(new ReifyTimestampsAndWindows<String, Integer>());
+ PCollection<KeyedWorkItem<String, Integer>> groupedKvs =
+ kvs.apply(new DirectGroupByKeyOnly<String, Integer>());
+
+ CommittedBundle<KV<String, WindowedValue<Integer>>> inputBundle =
+ bundleFactory.createRootBundle(kvs).commit(Instant.now());
+ EvaluationContext evaluationContext = mock(EvaluationContext.class);
+
+ StructuralKey<String> fooKey = StructuralKey.of("foo", StringUtf8Coder.of());
+ UncommittedBundle<KeyedWorkItem<String, Integer>> fooBundle = bundleFactory.createKeyedBundle(
+ null, fooKey,
+ groupedKvs);
+ StructuralKey<String> barKey = StructuralKey.of("bar", StringUtf8Coder.of());
+ UncommittedBundle<KeyedWorkItem<String, Integer>> barBundle = bundleFactory.createKeyedBundle(
+ null, barKey,
+ groupedKvs);
+ StructuralKey<String> bazKey = StructuralKey.of("baz", StringUtf8Coder.of());
+ UncommittedBundle<KeyedWorkItem<String, Integer>> bazBundle = bundleFactory.createKeyedBundle(
+ null, bazKey,
+ groupedKvs);
+
+ when(evaluationContext.createKeyedBundle(inputBundle,
+ fooKey,
+ groupedKvs)).thenReturn(fooBundle);
+ when(evaluationContext.createKeyedBundle(inputBundle,
+ barKey,
+ groupedKvs)).thenReturn(barBundle);
+ when(evaluationContext.createKeyedBundle(inputBundle,
+ bazKey,
+ groupedKvs)).thenReturn(bazBundle);
+
+ // The input to a GroupByKey is assumed to be a KvCoder
+ @SuppressWarnings("unchecked")
+ Coder<String> keyCoder =
+ ((KvCoder<String, WindowedValue<Integer>>) kvs.getCoder()).getKeyCoder();
+ TransformEvaluator<KV<String, WindowedValue<Integer>>> evaluator =
+ new GroupByKeyOnlyEvaluatorFactory()
+ .forApplication(
+ groupedKvs.getProducingTransformInternal(), inputBundle, evaluationContext);
+
+ evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(firstFoo)));
+ evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(secondFoo)));
+ evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(thirdFoo)));
+ evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(firstBar)));
+ evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(secondBar)));
+ evaluator.processElement(WindowedValue.valueInEmptyWindows(gwValue(firstBaz)));
+
+ evaluator.finishBundle();
+
+ assertThat(
+ fooBundle.commit(Instant.now()).getElements(),
+ contains(
+ new KeyedWorkItemMatcher<String, Integer>(
+ KeyedWorkItems.elementsWorkItem(
+ "foo",
+ ImmutableSet.of(
+ WindowedValue.valueInGlobalWindow(-1),
+ WindowedValue.valueInGlobalWindow(1),
+ WindowedValue.valueInGlobalWindow(3))),
+ keyCoder)));
+ assertThat(
+ barBundle.commit(Instant.now()).getElements(),
+ contains(
+ new KeyedWorkItemMatcher<String, Integer>(
+ KeyedWorkItems.elementsWorkItem(
+ "bar",
+ ImmutableSet.of(
+ WindowedValue.valueInGlobalWindow(12),
+ WindowedValue.valueInGlobalWindow(22))),
+ keyCoder)));
+ assertThat(
+ bazBundle.commit(Instant.now()).getElements(),
+ contains(
+ new KeyedWorkItemMatcher<String, Integer>(
+ KeyedWorkItems.elementsWorkItem(
+ "baz",
+ ImmutableSet.of(WindowedValue.valueInGlobalWindow(Integer.MAX_VALUE))),
+ keyCoder)));
+ }
+
+ private <K, V> KV<K, WindowedValue<V>> gwValue(KV<K, V> kv) {
+ return KV.of(kv.getKey(), WindowedValue.valueInGlobalWindow(kv.getValue()));
+ }
+
+ private static class KeyedWorkItemMatcher<K, V>
+ extends BaseMatcher<WindowedValue<KeyedWorkItem<K, V>>> {
+ private final KeyedWorkItem<K, V> myWorkItem;
+ private final Coder<K> keyCoder;
+
+ public KeyedWorkItemMatcher(KeyedWorkItem<K, V> myWorkItem, Coder<K> keyCoder) {
+ this.myWorkItem = myWorkItem;
+ this.keyCoder = keyCoder;
+ }
+
+ @Override
+ public boolean matches(Object item) {
+ if (item == null || !(item instanceof WindowedValue)) {
+ return false;
+ }
+ WindowedValue<KeyedWorkItem<K, V>> that = (WindowedValue<KeyedWorkItem<K, V>>) item;
+ Multiset<WindowedValue<V>> myValues = HashMultiset.create();
+ Multiset<WindowedValue<V>> thatValues = HashMultiset.create();
+ for (WindowedValue<V> value : myWorkItem.elementsIterable()) {
+ myValues.add(value);
+ }
+ for (WindowedValue<V> value : that.getValue().elementsIterable()) {
+ thatValues.add(value);
+ }
+ try {
+ return myValues.equals(thatValues)
+ && keyCoder
+ .structuralValue(myWorkItem.key())
+ .equals(keyCoder.structuralValue(that.getValue().key()));
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description
+ .appendText("KeyedWorkItem<K, V> containing key ")
+ .appendValue(myWorkItem.key())
+ .appendText(" and values ")
+ .appendValueList("[", ", ", "]", myWorkItem.elementsIterable());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactoryTest.java
index c4da86c..d40cf93 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactoryTest.java
@@ -57,7 +57,7 @@ public class ImmutabilityCheckingBundleFactoryTest {
TestPipeline p = TestPipeline.create();
created = p.apply(Create.<byte[]>of().withCoder(ByteArrayCoder.of()));
transformed = created.apply(ParDo.of(new IdentityDoFn<byte[]>()));
- factory = ImmutabilityCheckingBundleFactory.create(InProcessBundleFactory.create());
+ factory = ImmutabilityCheckingBundleFactory.create(ImmutableListBundleFactory.create());
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
index ead9c9e..890e06d 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
@@ -53,7 +53,7 @@ public class ImmutabilityEnforcementFactoryTest implements Serializable {
@Before
public void setup() {
factory = new ImmutabilityEnforcementFactory();
- bundleFactory = InProcessBundleFactory.create();
+ bundleFactory = ImmutableListBundleFactory.create();
TestPipeline p = TestPipeline.create();
pcollection =
p.apply(Create.of("foo".getBytes(), "spamhameggs".getBytes()))
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java
new file mode 100644
index 0000000..21e4bcb
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+import com.google.common.collect.ImmutableList;
+
+import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
+import org.joda.time.Instant;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Tests for {@link ImmutableListBundleFactory}.
+ */
+@RunWith(JUnit4.class)
+public class ImmutableListBundleFactoryTest {
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ private ImmutableListBundleFactory bundleFactory = ImmutableListBundleFactory.create();
+
+ private PCollection<Integer> created;
+ private PCollection<KV<String, Integer>> downstream;
+
+ @Before
+ public void setup() {
+ TestPipeline p = TestPipeline.create();
+ created = p.apply(Create.of(1, 2, 3));
+ downstream = created.apply(WithKeys.<String, Integer>of("foo"));
+ }
+
+ @Test
+ public void createRootBundleShouldCreateWithEmptyKey() {
+ PCollection<Integer> pcollection = TestPipeline.create().apply(Create.of(1));
+
+ UncommittedBundle<Integer> inFlightBundle = bundleFactory.createRootBundle(pcollection);
+
+ CommittedBundle<Integer> bundle = inFlightBundle.commit(Instant.now());
+
+ assertThat(bundle.getKey(),
+ Matchers.<StructuralKey<?>>equalTo(StructuralKey.of(null, VoidCoder.of())));
+ }
+
+ private <T> void createKeyedBundle(Coder<T> coder, T key) throws Exception {
+ PCollection<Integer> pcollection = TestPipeline.create().apply(Create.of(1));
+ StructuralKey skey = StructuralKey.of(key, coder);
+
+ UncommittedBundle<Integer> inFlightBundle =
+ bundleFactory.createKeyedBundle(null, skey, pcollection);
+
+ CommittedBundle<Integer> bundle = inFlightBundle.commit(Instant.now());
+ assertThat(bundle.getKey(), equalTo(skey));
+ }
+
+ @Test
+ public void keyedWithNullKeyShouldCreateKeyedBundle() throws Exception {
+ createKeyedBundle(VoidCoder.of(), null);
+ }
+
+ @Test
+ public void keyedWithKeyShouldCreateKeyedBundle() throws Exception {
+ createKeyedBundle(StringUtf8Coder.of(), "foo");
+ createKeyedBundle(VarIntCoder.of(), 1234);
+ createKeyedBundle(ByteArrayCoder.of(), new byte[] {0, 2, 4, 99});
+ }
+
+ private <T> CommittedBundle<T>
+ afterCommitGetElementsShouldHaveAddedElements(Iterable<WindowedValue<T>> elems) {
+ PCollection<T> pcollection = TestPipeline.create().apply(Create.<T>of());
+
+ UncommittedBundle<T> bundle = bundleFactory.createRootBundle(pcollection);
+ Collection<Matcher<? super WindowedValue<T>>> expectations = new ArrayList<>();
+ for (WindowedValue<T> elem : elems) {
+ bundle.add(elem);
+ expectations.add(equalTo(elem));
+ }
+ Matcher<Iterable<? extends WindowedValue<T>>> containsMatcher =
+ Matchers.<WindowedValue<T>>containsInAnyOrder(expectations);
+ CommittedBundle<T> committed = bundle.commit(Instant.now());
+ assertThat(committed.getElements(), containsMatcher);
+
+ return committed;
+ }
+
+ @Test
+ public void getElementsBeforeAddShouldReturnEmptyIterable() {
+ afterCommitGetElementsShouldHaveAddedElements(Collections.<WindowedValue<Integer>>emptyList());
+ }
+
+ @Test
+ public void getElementsAfterAddShouldReturnAddedElements() {
+ WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1);
+ WindowedValue<Integer> secondValue =
+ WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L));
+
+ afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue));
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void withElementsShouldReturnIndependentBundle() {
+ WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1);
+ WindowedValue<Integer> secondValue =
+ WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L));
+
+ CommittedBundle<Integer> committed =
+ afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue));
+
+ WindowedValue<Integer> firstReplacement =
+ WindowedValue.of(
+ 9,
+ new Instant(2048L),
+ new IntervalWindow(new Instant(2044L), Instant.now()),
+ PaneInfo.NO_FIRING);
+ WindowedValue<Integer> secondReplacement =
+ WindowedValue.timestampedValueInGlobalWindow(-1, Instant.now());
+ CommittedBundle<Integer> withed =
+ committed.withElements(ImmutableList.of(firstReplacement, secondReplacement));
+
+ assertThat(withed.getElements(), containsInAnyOrder(firstReplacement, secondReplacement));
+ assertThat(committed.getElements(), containsInAnyOrder(firstValue, secondValue));
+ assertThat(withed.getKey(), Matchers.<StructuralKey<?>>equalTo(committed.getKey()));
+ assertThat(withed.getPCollection(), equalTo(committed.getPCollection()));
+ assertThat(
+ withed.getSynchronizedProcessingOutputWatermark(),
+ equalTo(committed.getSynchronizedProcessingOutputWatermark()));
+ }
+
+ @Test
+ public void addAfterCommitShouldThrowException() {
+ PCollection<Integer> pcollection = TestPipeline.create().apply(Create.<Integer>of());
+
+ UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(pcollection);
+ bundle.add(WindowedValue.valueInGlobalWindow(1));
+ CommittedBundle<Integer> firstCommit = bundle.commit(Instant.now());
+ assertThat(firstCommit.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(1)));
+
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("3");
+ thrown.expectMessage("committed");
+
+ bundle.add(WindowedValue.valueInGlobalWindow(3));
+ }
+
+ @Test
+ public void commitAfterCommitShouldThrowException() {
+ PCollection<Integer> pcollection = TestPipeline.create().apply(Create.<Integer>of());
+
+ UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(pcollection);
+ bundle.add(WindowedValue.valueInGlobalWindow(1));
+ CommittedBundle<Integer> firstCommit = bundle.commit(Instant.now());
+ assertThat(firstCommit.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(1)));
+
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("committed");
+
+ bundle.commit(Instant.now());
+ }
+
+ @Test
+ public void createBundleUnkeyedResultUnkeyed() {
+ CommittedBundle<KV<String, Integer>> newBundle =
+ bundleFactory
+ .createBundle(bundleFactory.createRootBundle(created).commit(Instant.now()), downstream)
+ .commit(Instant.now());
+ }
+
+ @Test
+ public void createBundleKeyedResultPropagatesKey() {
+ CommittedBundle<KV<String, Integer>> newBundle =
+ bundleFactory.createBundle(
+ bundleFactory.createKeyedBundle(
+ null,
+ StructuralKey.of("foo", StringUtf8Coder.of()),
+ created).commit(Instant.now()),
+ downstream).commit(Instant.now());
+ assertThat(newBundle.getKey().getKey(), Matchers.<Object>equalTo("foo"));
+ }
+
+ @Test
+ public void createKeyedBundleKeyed() {
+ CommittedBundle<KV<String, Integer>> keyedBundle = bundleFactory.createKeyedBundle(
+ bundleFactory.createRootBundle(created).commit(Instant.now()),
+ StructuralKey.of("foo", StringUtf8Coder.of()),
+ downstream).commit(Instant.now());
+ assertThat(keyedBundle.getKey().getKey(), Matchers.<Object>equalTo("foo"));
+ }
+}
[03/12] incubator-beam git commit: Remove InProcess Prefixes
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
new file mode 100644
index 0000000..7173129
--- /dev/null
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
@@ -0,0 +1,1428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.WatermarkManager.FiredTimers;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate;
+import org.apache.beam.runners.direct.WatermarkManager.TimerUpdate.TimerUpdateBuilder;
+import org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.util.TimerInternals.TimerData;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.state.StateNamespaces;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TimestampedValue;
+
+import com.google.common.collect.ImmutableList;
+
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.joda.time.Instant;
+import org.joda.time.ReadableInstant;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tests for {@link WatermarkManager}.
+ */
+@RunWith(JUnit4.class)
+public class WatermarkManagerTest implements Serializable {
+ private transient MockClock clock;
+
+ private transient PCollection<Integer> createdInts;
+
+ private transient PCollection<Integer> filtered;
+ private transient PCollection<Integer> filteredTimesTwo;
+ private transient PCollection<KV<String, Integer>> keyed;
+
+ private transient PCollection<Integer> intsToFlatten;
+ private transient PCollection<Integer> flattened;
+
+ private transient WatermarkManager manager;
+ private transient BundleFactory bundleFactory;
+
+ @Before
+ public void setup() {
+ TestPipeline p = TestPipeline.create();
+
+ createdInts = p.apply("createdInts", Create.of(1, 2, 3));
+
+ filtered = createdInts.apply("filtered", Filter.greaterThan(1));
+ filteredTimesTwo = filtered.apply("timesTwo", ParDo.of(new DoFn<Integer, Integer>() {
+ @Override
+ public void processElement(DoFn<Integer, Integer>.ProcessContext c) throws Exception {
+ c.output(c.element() * 2);
+ }
+ }));
+
+ keyed = createdInts.apply("keyed", WithKeys.<String, Integer>of("MyKey"));
+
+ intsToFlatten = p.apply("intsToFlatten", Create.of(-1, 256, 65535));
+ PCollectionList<Integer> preFlatten = PCollectionList.of(createdInts).and(intsToFlatten);
+ flattened = preFlatten.apply("flattened", Flatten.<Integer>pCollections());
+
+ Collection<AppliedPTransform<?, ?, ?>> rootTransforms =
+ ImmutableList.<AppliedPTransform<?, ?, ?>>of(
+ createdInts.getProducingTransformInternal(),
+ intsToFlatten.getProducingTransformInternal());
+
+ Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> consumers = new HashMap<>();
+ consumers.put(
+ createdInts,
+ ImmutableList.<AppliedPTransform<?, ?, ?>>of(filtered.getProducingTransformInternal(),
+ keyed.getProducingTransformInternal(), flattened.getProducingTransformInternal()));
+ consumers.put(
+ filtered,
+ Collections.<AppliedPTransform<?, ?, ?>>singleton(
+ filteredTimesTwo.getProducingTransformInternal()));
+ consumers.put(filteredTimesTwo, Collections.<AppliedPTransform<?, ?, ?>>emptyList());
+ consumers.put(keyed, Collections.<AppliedPTransform<?, ?, ?>>emptyList());
+
+ consumers.put(
+ intsToFlatten,
+ Collections.<AppliedPTransform<?, ?, ?>>singleton(
+ flattened.getProducingTransformInternal()));
+ consumers.put(flattened, Collections.<AppliedPTransform<?, ?, ?>>emptyList());
+
+ clock = MockClock.fromInstant(new Instant(1000));
+
+ manager = WatermarkManager.create(clock, rootTransforms, consumers);
+ bundleFactory = ImmutableListBundleFactory.create();
+ }
+
+ /**
+ * Demonstrates that getWatermark, when called on an {@link AppliedPTransform} that has not
+ * processed any elements, returns the {@link BoundedWindow#TIMESTAMP_MIN_VALUE}.
+ */
+ @Test
+ public void getWatermarkForUntouchedTransform() {
+ TransformWatermarks watermarks =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+
+ assertThat(watermarks.getInputWatermark(), equalTo(BoundedWindow.TIMESTAMP_MIN_VALUE));
+ assertThat(watermarks.getOutputWatermark(), equalTo(BoundedWindow.TIMESTAMP_MIN_VALUE));
+ }
+
+ /**
+ * Demonstrates that getWatermark for a transform that consumes no input uses the Watermark
+ * Hold value provided to it as the output watermark.
+ */
+ @Test
+ public void getWatermarkForUpdatedSourceTransform() {
+ CommittedBundle<Integer> output = multiWindowedBundle(createdInts, 1);
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(output)),
+ new Instant(8000L));
+ manager.refreshAll();
+ TransformWatermarks updatedSourceWatermark =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+
+ assertThat(updatedSourceWatermark.getOutputWatermark(), equalTo(new Instant(8000L)));
+ }
+
+ /**
+ * Demonstrates that getWatermark for a transform that takes multiple inputs is held to the
+ * minimum watermark across all of its inputs.
+ */
+ @Test
+ public void getWatermarkForMultiInputTransform() {
+ CommittedBundle<Integer> secondPcollectionBundle = multiWindowedBundle(intsToFlatten, -1);
+
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(intsToFlatten.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(secondPcollectionBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+
+ // We didn't do anything for the first source, so we shouldn't have progressed the watermark
+ TransformWatermarks firstSourceWatermark =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+ assertThat(
+ firstSourceWatermark.getOutputWatermark(),
+ not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
+
+ // the Second Source output all of the elements so it should be done (with a watermark at the
+ // end of time).
+ TransformWatermarks secondSourceWatermark =
+ manager.getWatermarks(intsToFlatten.getProducingTransformInternal());
+ assertThat(
+ secondSourceWatermark.getOutputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+
+ // We haven't consumed anything yet, so our watermark should be at the beginning of time
+ TransformWatermarks transformWatermark =
+ manager.getWatermarks(flattened.getProducingTransformInternal());
+ assertThat(
+ transformWatermark.getInputWatermark(), not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
+ assertThat(
+ transformWatermark.getOutputWatermark(), not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
+
+ CommittedBundle<Integer> flattenedBundleSecondCreate = multiWindowedBundle(flattened, -1);
+ // We have finished processing the bundle from the second PCollection, but we haven't consumed
+ // anything from the first PCollection yet; so our watermark shouldn't advance
+ manager.updateWatermarks(secondPcollectionBundle,
+ TimerUpdate.empty(),
+ result(flattened.getProducingTransformInternal(),
+ secondPcollectionBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(flattenedBundleSecondCreate)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ TransformWatermarks transformAfterProcessing =
+ manager.getWatermarks(flattened.getProducingTransformInternal());
+ manager.updateWatermarks(secondPcollectionBundle,
+ TimerUpdate.empty(),
+ result(flattened.getProducingTransformInternal(),
+ secondPcollectionBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(flattenedBundleSecondCreate)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ assertThat(
+ transformAfterProcessing.getInputWatermark(),
+ not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
+ assertThat(
+ transformAfterProcessing.getOutputWatermark(),
+ not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
+
+ Instant firstCollectionTimestamp = new Instant(10000);
+ CommittedBundle<Integer> firstPcollectionBundle =
+ timestampedBundle(createdInts, TimestampedValue.<Integer>of(5, firstCollectionTimestamp));
+ // the source is done, but elements are still buffered. The source output watermark should be
+ // past the end of the global window
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(firstPcollectionBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ TransformWatermarks firstSourceWatermarks =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+ assertThat(
+ firstSourceWatermarks.getOutputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+
+ // We still haven't consumed any of the first source's input, so the watermark should still not
+ // progress
+ TransformWatermarks flattenAfterSourcesProduced =
+ manager.getWatermarks(flattened.getProducingTransformInternal());
+ assertThat(
+ flattenAfterSourcesProduced.getInputWatermark(), not(laterThan(firstCollectionTimestamp)));
+ assertThat(
+ flattenAfterSourcesProduced.getOutputWatermark(), not(laterThan(firstCollectionTimestamp)));
+
+ // We have buffered inputs, but since the PCollection has all of the elements (has a WM past the
+ // end of the global window), we should have a watermark equal to the min among buffered
+ // elements
+ TransformWatermarks withBufferedElements =
+ manager.getWatermarks(flattened.getProducingTransformInternal());
+ assertThat(withBufferedElements.getInputWatermark(), equalTo(firstCollectionTimestamp));
+ assertThat(withBufferedElements.getOutputWatermark(), equalTo(firstCollectionTimestamp));
+
+ CommittedBundle<?> completedFlattenBundle =
+ bundleFactory.createRootBundle(flattened).commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.updateWatermarks(firstPcollectionBundle,
+ TimerUpdate.empty(),
+ result(flattened.getProducingTransformInternal(),
+ firstPcollectionBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(completedFlattenBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ TransformWatermarks afterConsumingAllInput =
+ manager.getWatermarks(flattened.getProducingTransformInternal());
+ assertThat(
+ afterConsumingAllInput.getInputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ assertThat(
+ afterConsumingAllInput.getOutputWatermark(),
+ not(laterThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ }
+
+ /**
+ * Demonstrates that pending elements are independent among
+ * {@link AppliedPTransform AppliedPTransforms} that consume the same input {@link PCollection}.
+ */
+ @Test
+ public void getWatermarkForMultiConsumedCollection() {
+ CommittedBundle<Integer> createdBundle = timestampedBundle(createdInts,
+ TimestampedValue.of(1, new Instant(1_000_000L)), TimestampedValue.of(2, new Instant(1234L)),
+ TimestampedValue.of(3, new Instant(-1000L)));
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(createdBundle)),
+ new Instant(Long.MAX_VALUE));
+ manager.refreshAll();
+ TransformWatermarks createdAfterProducing =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+ assertThat(
+ createdAfterProducing.getOutputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+
+ CommittedBundle<KV<String, Integer>> keyBundle =
+ timestampedBundle(keyed, TimestampedValue.of(KV.of("MyKey", 1), new Instant(1_000_000L)),
+ TimestampedValue.of(KV.of("MyKey", 2), new Instant(1234L)),
+ TimestampedValue.of(KV.of("MyKey", 3), new Instant(-1000L)));
+ manager.updateWatermarks(createdBundle,
+ TimerUpdate.empty(),
+ result(keyed.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(keyBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ TransformWatermarks keyedWatermarks =
+ manager.getWatermarks(keyed.getProducingTransformInternal());
+ assertThat(
+ keyedWatermarks.getInputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ assertThat(
+ keyedWatermarks.getOutputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+
+ TransformWatermarks filteredWatermarks =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ assertThat(filteredWatermarks.getInputWatermark(), not(laterThan(new Instant(-1000L))));
+ assertThat(filteredWatermarks.getOutputWatermark(), not(laterThan(new Instant(-1000L))));
+
+ CommittedBundle<Integer> filteredBundle =
+ timestampedBundle(filtered, TimestampedValue.of(2, new Instant(1234L)));
+ manager.updateWatermarks(createdBundle,
+ TimerUpdate.empty(),
+ result(filtered.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(filteredBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ TransformWatermarks filteredProcessedWatermarks =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ assertThat(
+ filteredProcessedWatermarks.getInputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ assertThat(
+ filteredProcessedWatermarks.getOutputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ }
+
+ /**
+ * Demonstrates that the watermark of an {@link AppliedPTransform} is held to the provided
+ * watermark hold.
+ */
+ @Test
+ public void updateWatermarkWithWatermarkHolds() {
+ CommittedBundle<Integer> createdBundle = timestampedBundle(createdInts,
+ TimestampedValue.of(1, new Instant(1_000_000L)),
+ TimestampedValue.of(2, new Instant(1234L)),
+ TimestampedValue.of(3, new Instant(-1000L)));
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(createdBundle)),
+ new Instant(Long.MAX_VALUE));
+
+ CommittedBundle<KV<String, Integer>> keyBundle = timestampedBundle(keyed,
+ TimestampedValue.of(KV.of("MyKey", 1), new Instant(1_000_000L)),
+ TimestampedValue.of(KV.of("MyKey", 2), new Instant(1234L)),
+ TimestampedValue.of(KV.of("MyKey", 3), new Instant(-1000L)));
+ manager.updateWatermarks(createdBundle,
+ TimerUpdate.empty(),
+ result(keyed.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(keyBundle)),
+ new Instant(500L));
+ manager.refreshAll();
+ TransformWatermarks keyedWatermarks =
+ manager.getWatermarks(keyed.getProducingTransformInternal());
+ assertThat(
+ keyedWatermarks.getInputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ assertThat(keyedWatermarks.getOutputWatermark(), not(laterThan(new Instant(500L))));
+ }
+
+ /**
+ * Demonstrates that the watermark of an {@link AppliedPTransform} is held to the provided
+ * watermark hold.
+ */
+ @Test
+ public void updateWatermarkWithKeyedWatermarkHolds() {
+ CommittedBundle<Integer> firstKeyBundle = bundleFactory.createKeyedBundle(null,
+ StructuralKey.of("Odd", StringUtf8Coder.of()),
+ createdInts)
+ .add(WindowedValue.timestampedValueInGlobalWindow(1, new Instant(1_000_000L)))
+ .add(WindowedValue.timestampedValueInGlobalWindow(3, new Instant(-1000L)))
+ .commit(clock.now());
+
+ CommittedBundle<Integer> secondKeyBundle = bundleFactory.createKeyedBundle(null,
+ StructuralKey.of("Even", StringUtf8Coder.of()),
+ createdInts)
+ .add(WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1234L)))
+ .commit(clock.now());
+
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ ImmutableList.of(firstKeyBundle, secondKeyBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+
+ manager.updateWatermarks(firstKeyBundle,
+ TimerUpdate.empty(),
+ result(filtered.getProducingTransformInternal(),
+ firstKeyBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>emptyList()),
+ new Instant(-1000L));
+ manager.updateWatermarks(secondKeyBundle,
+ TimerUpdate.empty(),
+ result(filtered.getProducingTransformInternal(),
+ secondKeyBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>emptyList()),
+ new Instant(1234L));
+ manager.refreshAll();
+
+ TransformWatermarks filteredWatermarks =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ assertThat(filteredWatermarks.getInputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ assertThat(filteredWatermarks.getOutputWatermark(), not(laterThan(new Instant(-1000L))));
+
+ CommittedBundle<Integer> fauxFirstKeyTimerBundle = bundleFactory.createKeyedBundle(null,
+ StructuralKey.of("Odd", StringUtf8Coder.of()),
+ createdInts).commit(clock.now());
+ manager.updateWatermarks(fauxFirstKeyTimerBundle,
+ TimerUpdate.empty(),
+ result(filtered.getProducingTransformInternal(),
+ fauxFirstKeyTimerBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>emptyList()),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+
+ assertThat(filteredWatermarks.getOutputWatermark(), equalTo(new Instant(1234L)));
+
+ CommittedBundle<Integer> fauxSecondKeyTimerBundle = bundleFactory.createKeyedBundle(null,
+ StructuralKey.of("Even", StringUtf8Coder.of()),
+ createdInts).commit(clock.now());
+ manager.updateWatermarks(fauxSecondKeyTimerBundle,
+ TimerUpdate.empty(),
+ result(filtered.getProducingTransformInternal(),
+ fauxSecondKeyTimerBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>emptyList()),
+ new Instant(5678L));
+ manager.refreshAll();
+ assertThat(filteredWatermarks.getOutputWatermark(), equalTo(new Instant(5678L)));
+
+ manager.updateWatermarks(fauxSecondKeyTimerBundle,
+ TimerUpdate.empty(),
+ result(filtered.getProducingTransformInternal(),
+ fauxSecondKeyTimerBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>emptyList()),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ assertThat(filteredWatermarks.getOutputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ }
+
+ /**
+ * Demonstrates that updated output watermarks are monotonic in the presence of late data, when
+ * called on an {@link AppliedPTransform} that consumes no input.
+ */
+ @Test
+ public void updateOutputWatermarkShouldBeMonotonic() {
+ CommittedBundle<?> firstInput =
+ bundleFactory.createRootBundle(createdInts).commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.updateWatermarks(null, TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(firstInput)),
+ new Instant(0L));
+ manager.refreshAll();
+ TransformWatermarks firstWatermarks =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+ assertThat(firstWatermarks.getOutputWatermark(), equalTo(new Instant(0L)));
+
+ CommittedBundle<?> secondInput =
+ bundleFactory.createRootBundle(createdInts).commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(secondInput)),
+ new Instant(-250L));
+ manager.refreshAll();
+ TransformWatermarks secondWatermarks =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+ assertThat(secondWatermarks.getOutputWatermark(), not(earlierThan(new Instant(0L))));
+ }
+
+ /**
+ * Demonstrates that updated output watermarks are monotonic in the presence of watermark holds
+ * that become earlier than a previous watermark hold.
+ */
+ @Test
+ public void updateWatermarkWithHoldsShouldBeMonotonic() {
+ CommittedBundle<Integer> createdBundle = timestampedBundle(createdInts,
+ TimestampedValue.of(1, new Instant(1_000_000L)),
+ TimestampedValue.of(2, new Instant(1234L)),
+ TimestampedValue.of(3, new Instant(-1000L)));
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(createdBundle)),
+ new Instant(Long.MAX_VALUE));
+
+ CommittedBundle<KV<String, Integer>> keyBundle =
+ timestampedBundle(keyed, TimestampedValue.of(KV.of("MyKey", 1), new Instant(1_000_000L)),
+ TimestampedValue.of(KV.of("MyKey", 2), new Instant(1234L)),
+ TimestampedValue.of(KV.of("MyKey", 3), new Instant(-1000L)));
+ manager.updateWatermarks(createdBundle,
+ TimerUpdate.empty(),
+ result(keyed.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(keyBundle)),
+ new Instant(500L));
+ manager.refreshAll();
+ TransformWatermarks keyedWatermarks =
+ manager.getWatermarks(keyed.getProducingTransformInternal());
+ assertThat(
+ keyedWatermarks.getInputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ assertThat(keyedWatermarks.getOutputWatermark(), not(laterThan(new Instant(500L))));
+ Instant oldOutputWatermark = keyedWatermarks.getOutputWatermark();
+
+ TransformWatermarks updatedWatermarks =
+ manager.getWatermarks(keyed.getProducingTransformInternal());
+ assertThat(
+ updatedWatermarks.getInputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ // We added a hold prior to the old watermark; we shouldn't progress (due to the earlier hold)
+ // but the watermark is monotonic and should not backslide to the new, earlier hold
+ assertThat(updatedWatermarks.getOutputWatermark(), equalTo(oldOutputWatermark));
+ }
+
+ @Test
+ public void updateWatermarkWithUnprocessedElements() {
+ WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(1);
+ WindowedValue<Integer> second =
+ WindowedValue.timestampedValueInGlobalWindow(2, new Instant(-1000L));
+ WindowedValue<Integer> third =
+ WindowedValue.timestampedValueInGlobalWindow(3, new Instant(1234L));
+ CommittedBundle<Integer> createdBundle = bundleFactory.createRootBundle(createdInts)
+ .add(first)
+ .add(second)
+ .add(third)
+ .commit(clock.now());
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(createdBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+
+ CommittedBundle<KV<String, Integer>> keyBundle = timestampedBundle(keyed,
+ TimestampedValue.of(KV.of("MyKey", 1), BoundedWindow.TIMESTAMP_MIN_VALUE));
+ manager.updateWatermarks(createdBundle,
+ TimerUpdate.empty(),
+ result(keyed.getProducingTransformInternal(),
+ createdBundle.withElements(ImmutableList.of(second, third)),
+ Collections.<CommittedBundle<?>>singleton(keyBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ TransformWatermarks keyedWatermarks =
+ manager.getWatermarks(keyed.getProducingTransformInternal());
+ // the unprocessed second and third are readded to pending
+ assertThat(
+ keyedWatermarks.getInputWatermark(), not(laterThan(new Instant(-1000L))));
+ }
+
+ /**
+ * Demonstrates that updateWatermarks in the presence of late data is monotonic.
+ */
+ @Test
+ public void updateWatermarkWithLateData() {
+ Instant sourceWatermark = new Instant(1_000_000L);
+ CommittedBundle<Integer> createdBundle = timestampedBundle(createdInts,
+ TimestampedValue.of(1, sourceWatermark), TimestampedValue.of(2, new Instant(1234L)));
+
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(createdBundle)),
+ sourceWatermark);
+
+ CommittedBundle<KV<String, Integer>> keyBundle =
+ timestampedBundle(keyed, TimestampedValue.of(KV.of("MyKey", 1), sourceWatermark),
+ TimestampedValue.of(KV.of("MyKey", 2), new Instant(1234L)));
+
+ // Finish processing the on-time data. The watermarks should progress to be equal to the source
+ manager.updateWatermarks(createdBundle,
+ TimerUpdate.empty(),
+ result(keyed.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(keyBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ TransformWatermarks onTimeWatermarks =
+ manager.getWatermarks(keyed.getProducingTransformInternal());
+ assertThat(onTimeWatermarks.getInputWatermark(), equalTo(sourceWatermark));
+ assertThat(onTimeWatermarks.getOutputWatermark(), equalTo(sourceWatermark));
+
+ CommittedBundle<Integer> lateDataBundle =
+ timestampedBundle(createdInts, TimestampedValue.of(3, new Instant(-1000L)));
+ // the late data arrives in a downstream PCollection after its watermark has advanced past it;
+ // we don't advance the watermark past the current watermark until we've consumed the late data
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(lateDataBundle)),
+ new Instant(2_000_000L));
+ manager.refreshAll();
+ TransformWatermarks bufferedLateWm =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+ assertThat(bufferedLateWm.getOutputWatermark(), equalTo(new Instant(2_000_000L)));
+
+ // The input watermark should be held to its previous value (not advanced due to late data; not
+ // moved backwards in the presence of watermarks due to monotonicity).
+ TransformWatermarks lateDataBufferedWatermark =
+ manager.getWatermarks(keyed.getProducingTransformInternal());
+ assertThat(lateDataBufferedWatermark.getInputWatermark(), not(earlierThan(sourceWatermark)));
+ assertThat(lateDataBufferedWatermark.getOutputWatermark(), not(earlierThan(sourceWatermark)));
+
+ CommittedBundle<KV<String, Integer>> lateKeyedBundle =
+ timestampedBundle(keyed, TimestampedValue.of(KV.of("MyKey", 3), new Instant(-1000L)));
+ manager.updateWatermarks(lateDataBundle,
+ TimerUpdate.empty(),
+ result(keyed.getProducingTransformInternal(),
+ lateDataBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(lateKeyedBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ }
+
+ public void updateWatermarkWithDifferentWindowedValueInstances() {
+ manager.updateWatermarks(
+ null,
+ TimerUpdate.empty(), result(createdInts.getProducingTransformInternal(), null,
+ Collections.<CommittedBundle<?>>singleton(
+ bundleFactory
+ .createRootBundle(createdInts)
+ .add(WindowedValue.valueInGlobalWindow(1))
+ .commit(Instant.now()))),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+
+ CommittedBundle<Integer> createdBundle = bundleFactory.createRootBundle(createdInts)
+ .add(WindowedValue.valueInGlobalWindow(1))
+ .commit(Instant.now());
+ manager.updateWatermarks(createdBundle,
+ TimerUpdate.empty(),
+ result(keyed.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>emptyList()),
+ null);
+ manager.refreshAll();
+ TransformWatermarks onTimeWatermarks =
+ manager.getWatermarks(keyed.getProducingTransformInternal());
+ assertThat(onTimeWatermarks.getInputWatermark(), equalTo(BoundedWindow.TIMESTAMP_MAX_VALUE));
+ }
+
+ /**
+ * Demonstrates that after watermarks of an upstream transform are updated, but no output has been
+ * produced, the watermarks of a downstream process are advanced.
+ */
+ @Test
+ public void getWatermarksAfterOnlyEmptyOutput() {
+ CommittedBundle<Integer> emptyCreateOutput = multiWindowedBundle(createdInts);
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(emptyCreateOutput)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ TransformWatermarks updatedSourceWatermarks =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+
+ assertThat(
+ updatedSourceWatermarks.getOutputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+
+ TransformWatermarks finishedFilterWatermarks =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ assertThat(
+ finishedFilterWatermarks.getInputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ assertThat(
+ finishedFilterWatermarks.getOutputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ }
+
+ /**
+ * Demonstrates that after watermarks of an upstream transform are updated, but no output has been
+ * produced, and the downstream transform has a watermark hold, the watermark is held to the hold.
+ */
+ @Test
+ public void getWatermarksAfterHoldAndEmptyOutput() {
+ CommittedBundle<Integer> firstCreateOutput = multiWindowedBundle(createdInts, 1, 2);
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(firstCreateOutput)),
+ new Instant(12_000L));
+
+ CommittedBundle<Integer> firstFilterOutput = multiWindowedBundle(filtered);
+ manager.updateWatermarks(firstCreateOutput,
+ TimerUpdate.empty(),
+ result(filtered.getProducingTransformInternal(),
+ firstCreateOutput.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(firstFilterOutput)),
+ new Instant(10_000L));
+ manager.refreshAll();
+ TransformWatermarks firstFilterWatermarks =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ assertThat(firstFilterWatermarks.getInputWatermark(), not(earlierThan(new Instant(12_000L))));
+ assertThat(firstFilterWatermarks.getOutputWatermark(), not(laterThan(new Instant(10_000L))));
+
+ CommittedBundle<Integer> emptyCreateOutput = multiWindowedBundle(createdInts);
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(emptyCreateOutput)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ TransformWatermarks updatedSourceWatermarks =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+
+ assertThat(
+ updatedSourceWatermarks.getOutputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+
+ TransformWatermarks finishedFilterWatermarks =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ assertThat(
+ finishedFilterWatermarks.getInputWatermark(),
+ not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
+ assertThat(finishedFilterWatermarks.getOutputWatermark(), not(laterThan(new Instant(10_000L))));
+ }
+
+ @Test
+ public void getSynchronizedProcessingTimeInputWatermarksHeldToPendingBundles() {
+ TransformWatermarks watermarks =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+ assertThat(watermarks.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
+ assertThat(
+ watermarks.getSynchronizedProcessingOutputTime(),
+ equalTo(BoundedWindow.TIMESTAMP_MIN_VALUE));
+
+ TransformWatermarks filteredWatermarks =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ // Non-root processing watermarks don't progress until data has been processed
+ assertThat(
+ filteredWatermarks.getSynchronizedProcessingInputTime(),
+ not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
+ assertThat(
+ filteredWatermarks.getSynchronizedProcessingOutputTime(),
+ not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
+
+ CommittedBundle<Integer> createOutput =
+ bundleFactory.createRootBundle(createdInts).commit(new Instant(1250L));
+
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(createOutput)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ TransformWatermarks createAfterUpdate =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+ assertThat(createAfterUpdate.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
+ assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
+
+ TransformWatermarks filterAfterProduced =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ assertThat(
+ filterAfterProduced.getSynchronizedProcessingInputTime(), not(laterThan(clock.now())));
+ assertThat(
+ filterAfterProduced.getSynchronizedProcessingOutputTime(), not(laterThan(clock.now())));
+
+ clock.set(new Instant(1500L));
+ assertThat(createAfterUpdate.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
+ assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
+ assertThat(
+ filterAfterProduced.getSynchronizedProcessingInputTime(),
+ not(laterThan(new Instant(1250L))));
+ assertThat(
+ filterAfterProduced.getSynchronizedProcessingOutputTime(),
+ not(laterThan(new Instant(1250L))));
+
+ CommittedBundle<?> filterOutputBundle =
+ bundleFactory.createRootBundle(intsToFlatten).commit(new Instant(1250L));
+ manager.updateWatermarks(createOutput,
+ TimerUpdate.empty(),
+ result(filtered.getProducingTransformInternal(),
+ createOutput.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(filterOutputBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ TransformWatermarks filterAfterConsumed =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ assertThat(
+ filterAfterConsumed.getSynchronizedProcessingInputTime(),
+ not(laterThan(createAfterUpdate.getSynchronizedProcessingOutputTime())));
+ assertThat(
+ filterAfterConsumed.getSynchronizedProcessingOutputTime(),
+ not(laterThan(filterAfterConsumed.getSynchronizedProcessingInputTime())));
+ }
+
+ /**
+ * Demonstrates that the Synchronized Processing Time output watermark cannot progress past
+ * pending timers in the same set. This propagates to all downstream SynchronizedProcessingTimes.
+ *
+ * <p>Also demonstrate that the result is monotonic.
+ */
+ // @Test
+ public void getSynchronizedProcessingTimeOutputHeldToPendingTimers() {
+ CommittedBundle<Integer> createdBundle = multiWindowedBundle(createdInts, 1, 2, 4, 8);
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(createdBundle)),
+ new Instant(1248L));
+ manager.refreshAll();
+
+ TransformWatermarks filteredWms =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ TransformWatermarks filteredDoubledWms =
+ manager.getWatermarks(filteredTimesTwo.getProducingTransformInternal());
+ Instant initialFilteredWm = filteredWms.getSynchronizedProcessingOutputTime();
+ Instant initialFilteredDoubledWm = filteredDoubledWms.getSynchronizedProcessingOutputTime();
+
+ StructuralKey<String> key = StructuralKey.of("key", StringUtf8Coder.of());
+ CommittedBundle<Integer> filteredBundle = multiWindowedBundle(filtered, 2, 8);
+ TimerData pastTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(250L), TimeDomain.PROCESSING_TIME);
+ TimerData futureTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(4096L), TimeDomain.PROCESSING_TIME);
+ TimerUpdate timers =
+ TimerUpdate.builder(key).setTimer(pastTimer).setTimer(futureTimer).build();
+ manager.updateWatermarks(createdBundle,
+ timers,
+ result(filtered.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(filteredBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ Instant startTime = clock.now();
+ clock.set(startTime.plus(250L));
+ // We're held based on the past timer
+ assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
+ assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
+ // And we're monotonic
+ assertThat(
+ filteredWms.getSynchronizedProcessingOutputTime(), not(earlierThan(initialFilteredWm)));
+ assertThat(
+ filteredDoubledWms.getSynchronizedProcessingOutputTime(),
+ not(earlierThan(initialFilteredDoubledWm)));
+
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firedTimers =
+ manager.extractFiredTimers();
+ assertThat(
+ firedTimers.get(filtered.getProducingTransformInternal())
+ .get(key)
+ .getTimers(TimeDomain.PROCESSING_TIME),
+ contains(pastTimer));
+ // Our timer has fired, but has not been completed, so it holds our synchronized processing WM
+ assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
+ assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
+
+ CommittedBundle<Integer> filteredTimerBundle =
+ bundleFactory
+ .createKeyedBundle(null, key, filtered)
+ .commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ CommittedBundle<Integer> filteredTimerResult =
+ bundleFactory.createKeyedBundle(null, key, filteredTimesTwo)
+ .commit(filteredWms.getSynchronizedProcessingOutputTime());
+ // Complete the processing time timer
+ manager.updateWatermarks(filteredTimerBundle,
+ TimerUpdate.builder(key)
+ .withCompletedTimers(Collections.<TimerData>singleton(pastTimer)).build(),
+ result(filtered.getProducingTransformInternal(),
+ filteredTimerBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(filteredTimerResult)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+
+ clock.set(startTime.plus(500L));
+ assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(laterThan(clock.now())));
+ // filtered should be held to the time at which the filteredTimerResult fired
+ assertThat(
+ filteredDoubledWms.getSynchronizedProcessingOutputTime(),
+ not(earlierThan(filteredTimerResult.getSynchronizedProcessingOutputWatermark())));
+
+ manager.updateWatermarks(filteredTimerResult,
+ TimerUpdate.empty(),
+ result(filteredTimesTwo.getProducingTransformInternal(),
+ filteredTimerResult.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>emptyList()),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
+
+ clock.set(new Instant(Long.MAX_VALUE));
+ assertThat(filteredWms.getSynchronizedProcessingOutputTime(), equalTo(new Instant(4096)));
+ assertThat(
+ filteredDoubledWms.getSynchronizedProcessingOutputTime(), equalTo(new Instant(4096)));
+ }
+
+ /**
+ * Demonstrates that if any earlier processing holds appear in the synchronized processing time
+ * output hold the result is monotonic.
+ */
+ @Test
+ public void getSynchronizedProcessingTimeOutputTimeIsMonotonic() {
+ Instant startTime = clock.now();
+ TransformWatermarks watermarks =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+ assertThat(watermarks.getSynchronizedProcessingInputTime(), equalTo(startTime));
+
+ TransformWatermarks filteredWatermarks =
+ manager.getWatermarks(filtered.getProducingTransformInternal());
+ // Non-root processing watermarks don't progress until data has been processed
+ assertThat(
+ filteredWatermarks.getSynchronizedProcessingInputTime(),
+ not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
+ assertThat(
+ filteredWatermarks.getSynchronizedProcessingOutputTime(),
+ not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
+
+ CommittedBundle<Integer> createOutput =
+ bundleFactory.createRootBundle(createdInts).commit(new Instant(1250L));
+
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(createOutput)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+ TransformWatermarks createAfterUpdate =
+ manager.getWatermarks(createdInts.getProducingTransformInternal());
+ assertThat(createAfterUpdate.getSynchronizedProcessingInputTime(), not(laterThan(clock.now())));
+ assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(),
+ not(laterThan(clock.now())));
+
+ CommittedBundle<Integer> createSecondOutput =
+ bundleFactory.createRootBundle(createdInts).commit(new Instant(750L));
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(createSecondOutput)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+
+ assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
+ }
+
+ @Test
+ public void synchronizedProcessingInputTimeIsHeldToUpstreamProcessingTimeTimers() {
+ CommittedBundle<Integer> created = multiWindowedBundle(createdInts, 1, 2, 3);
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(created)),
+ new Instant(40_900L));
+ manager.refreshAll();
+
+ CommittedBundle<Integer> filteredBundle = multiWindowedBundle(filtered, 2, 4);
+ Instant upstreamHold = new Instant(2048L);
+ TimerData upstreamProcessingTimer =
+ TimerData.of(StateNamespaces.global(), upstreamHold, TimeDomain.PROCESSING_TIME);
+ manager.updateWatermarks(created,
+ TimerUpdate.builder(StructuralKey.of("key", StringUtf8Coder.of()))
+ .setTimer(upstreamProcessingTimer)
+ .build(),
+ result(filtered.getProducingTransformInternal(),
+ created.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(filteredBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+
+ TransformWatermarks downstreamWms =
+ manager.getWatermarks(filteredTimesTwo.getProducingTransformInternal());
+ assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
+
+ clock.set(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(upstreamHold));
+
+ manager.extractFiredTimers();
+ // Pending processing time timers that have been fired but aren't completed hold the
+ // synchronized processing time
+ assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(upstreamHold));
+
+ CommittedBundle<Integer> otherCreated = multiWindowedBundle(createdInts, 4, 8, 12);
+ manager.updateWatermarks(otherCreated,
+ TimerUpdate.builder(StructuralKey.of("key", StringUtf8Coder.of()))
+ .withCompletedTimers(Collections.singleton(upstreamProcessingTimer)).build(),
+ result(filtered.getProducingTransformInternal(),
+ otherCreated.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>emptyList()),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+
+ assertThat(downstreamWms.getSynchronizedProcessingInputTime(), not(earlierThan(clock.now())));
+ }
+
+ @Test
+ public void synchronizedProcessingInputTimeIsHeldToPendingBundleTimes() {
+ CommittedBundle<Integer> created = multiWindowedBundle(createdInts, 1, 2, 3);
+ manager.updateWatermarks(
+ null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>singleton(created)),
+ new Instant(29_919_235L));
+
+ Instant upstreamHold = new Instant(2048L);
+ CommittedBundle<Integer> filteredBundle = bundleFactory.createKeyedBundle(created,
+ StructuralKey.of("key", StringUtf8Coder.of()),
+ filtered).commit(upstreamHold);
+ manager.updateWatermarks(
+ created,
+ TimerUpdate.empty(),
+ result(filtered.getProducingTransformInternal(),
+ created.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(filteredBundle)),
+ BoundedWindow.TIMESTAMP_MAX_VALUE);
+ manager.refreshAll();
+
+ TransformWatermarks downstreamWms =
+ manager.getWatermarks(filteredTimesTwo.getProducingTransformInternal());
+ assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
+
+ clock.set(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(upstreamHold));
+ }
+
+ @Test
+ public void extractFiredTimersReturnsFiredEventTimeTimers() {
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> initialTimers =
+ manager.extractFiredTimers();
+ // Watermarks haven't advanced
+ assertThat(initialTimers.entrySet(), emptyIterable());
+
+ // Advance WM of keyed past the first timer, but ahead of the second and third
+ CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.singleton(createdBundle)),
+ new Instant(1500L));
+ manager.refreshAll();
+
+ TimerData earliestTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(1000), TimeDomain.EVENT_TIME);
+ TimerData middleTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(5000L), TimeDomain.EVENT_TIME);
+ TimerData lastTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(10000L), TimeDomain.EVENT_TIME);
+ StructuralKey<byte[]> key = StructuralKey.of(new byte[] {1, 4, 9}, ByteArrayCoder.of());
+ TimerUpdate update =
+ TimerUpdate.builder(key)
+ .setTimer(earliestTimer)
+ .setTimer(middleTimer)
+ .setTimer(lastTimer)
+ .build();
+
+ manager.updateWatermarks(createdBundle,
+ update,
+ result(filtered.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(intsToFlatten))),
+ new Instant(1000L));
+ manager.refreshAll();
+
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firstTransformFiredTimers =
+ manager.extractFiredTimers();
+ assertThat(
+ firstTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
+ Map<StructuralKey<?>, FiredTimers> firstFilteredTimers =
+ firstTransformFiredTimers.get(filtered.getProducingTransformInternal());
+ assertThat(firstFilteredTimers.get(key), not(nullValue()));
+ FiredTimers firstFired = firstFilteredTimers.get(key);
+ assertThat(firstFired.getTimers(TimeDomain.EVENT_TIME), contains(earliestTimer));
+
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>emptyList()),
+ new Instant(50_000L));
+ manager.refreshAll();
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> secondTransformFiredTimers =
+ manager.extractFiredTimers();
+ assertThat(
+ secondTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
+ Map<StructuralKey<?>, FiredTimers> secondFilteredTimers =
+ secondTransformFiredTimers.get(filtered.getProducingTransformInternal());
+ assertThat(secondFilteredTimers.get(key), not(nullValue()));
+ FiredTimers secondFired = secondFilteredTimers.get(key);
+ // Contains, in order, middleTimer and then lastTimer
+ assertThat(secondFired.getTimers(TimeDomain.EVENT_TIME), contains(middleTimer, lastTimer));
+ }
+
+ @Test
+ public void extractFiredTimersReturnsFiredProcessingTimeTimers() {
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> initialTimers =
+ manager.extractFiredTimers();
+ // Watermarks haven't advanced
+ assertThat(initialTimers.entrySet(), emptyIterable());
+
+ // Advance WM of keyed past the first timer, but ahead of the second and third
+ CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.singleton(createdBundle)),
+ new Instant(1500L));
+
+ TimerData earliestTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(999L), TimeDomain.PROCESSING_TIME);
+ TimerData middleTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(5000L), TimeDomain.PROCESSING_TIME);
+ TimerData lastTimer =
+ TimerData.of(StateNamespaces.global(), new Instant(10000L), TimeDomain.PROCESSING_TIME);
+ StructuralKey<?> key = StructuralKey.of(-12L, VarLongCoder.of());
+ TimerUpdate update =
+ TimerUpdate.builder(key)
+ .setTimer(lastTimer)
+ .setTimer(earliestTimer)
+ .setTimer(middleTimer)
+ .build();
+
+ manager.updateWatermarks(
+ createdBundle,
+ update,
+ result(filtered.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(intsToFlatten))),
+ new Instant(1000L));
+ manager.refreshAll();
+
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firstTransformFiredTimers =
+ manager.extractFiredTimers();
+ assertThat(
+ firstTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
+ Map<StructuralKey<?>, FiredTimers> firstFilteredTimers =
+ firstTransformFiredTimers.get(filtered.getProducingTransformInternal());
+ assertThat(firstFilteredTimers.get(key), not(nullValue()));
+ FiredTimers firstFired = firstFilteredTimers.get(key);
+ assertThat(firstFired.getTimers(TimeDomain.PROCESSING_TIME), contains(earliestTimer));
+
+ clock.set(new Instant(50_000L));
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>emptyList()),
+ new Instant(50_000L));
+ manager.refreshAll();
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> secondTransformFiredTimers =
+ manager.extractFiredTimers();
+ assertThat(
+ secondTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
+ Map<StructuralKey<?>, FiredTimers> secondFilteredTimers =
+ secondTransformFiredTimers.get(filtered.getProducingTransformInternal());
+ assertThat(secondFilteredTimers.get(key), not(nullValue()));
+ FiredTimers secondFired = secondFilteredTimers.get(key);
+ // Contains, in order, middleTimer and then lastTimer
+ assertThat(secondFired.getTimers(TimeDomain.PROCESSING_TIME), contains(middleTimer, lastTimer));
+ }
+
+ @Test
+ public void extractFiredTimersReturnsFiredSynchronizedProcessingTimeTimers() {
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> initialTimers =
+ manager.extractFiredTimers();
+ // Watermarks haven't advanced
+ assertThat(initialTimers.entrySet(), emptyIterable());
+
+ // Advance WM of keyed past the first timer, but ahead of the second and third
+ CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.singleton(createdBundle)),
+ new Instant(1500L));
+
+ TimerData earliestTimer = TimerData.of(
+ StateNamespaces.global(), new Instant(999L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
+ TimerData middleTimer = TimerData.of(
+ StateNamespaces.global(), new Instant(5000L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
+ TimerData lastTimer = TimerData.of(
+ StateNamespaces.global(), new Instant(10000L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
+ StructuralKey<byte[]> key = StructuralKey.of(new byte[] {2, -2, 22}, ByteArrayCoder.of());
+ TimerUpdate update =
+ TimerUpdate.builder(key)
+ .setTimer(lastTimer)
+ .setTimer(earliestTimer)
+ .setTimer(middleTimer)
+ .build();
+
+ manager.updateWatermarks(
+ createdBundle,
+ update,
+ result(filtered.getProducingTransformInternal(),
+ createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
+ Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(intsToFlatten))),
+ new Instant(1000L));
+ manager.refreshAll();
+
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firstTransformFiredTimers =
+ manager.extractFiredTimers();
+ assertThat(
+ firstTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
+ Map<StructuralKey<?>, FiredTimers> firstFilteredTimers =
+ firstTransformFiredTimers.get(filtered.getProducingTransformInternal());
+ assertThat(firstFilteredTimers.get(key), not(nullValue()));
+ FiredTimers firstFired = firstFilteredTimers.get(key);
+ assertThat(
+ firstFired.getTimers(TimeDomain.SYNCHRONIZED_PROCESSING_TIME), contains(earliestTimer));
+
+ clock.set(new Instant(50_000L));
+ manager.updateWatermarks(null,
+ TimerUpdate.empty(),
+ result(createdInts.getProducingTransformInternal(),
+ null,
+ Collections.<CommittedBundle<?>>emptyList()),
+ new Instant(50_000L));
+ manager.refreshAll();
+ Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> secondTransformFiredTimers =
+ manager.extractFiredTimers();
+ assertThat(
+ secondTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
+ Map<StructuralKey<?>, FiredTimers> secondFilteredTimers =
+ secondTransformFiredTimers.get(filtered.getProducingTransformInternal());
+ assertThat(secondFilteredTimers.get(key), not(nullValue()));
+ FiredTimers secondFired = secondFilteredTimers.get(key);
+ // Contains, in order, middleTimer and then lastTimer
+ assertThat(
+ secondFired.getTimers(TimeDomain.SYNCHRONIZED_PROCESSING_TIME),
+ contains(middleTimer, lastTimer));
+ }
+
+ @Test
+ public void timerUpdateBuilderBuildAddsAllAddedTimers() {
+ TimerData set = TimerData.of(StateNamespaces.global(), new Instant(10L), TimeDomain.EVENT_TIME);
+ TimerData deleted =
+ TimerData.of(StateNamespaces.global(), new Instant(24L), TimeDomain.PROCESSING_TIME);
+ TimerData completedOne = TimerData.of(
+ StateNamespaces.global(), new Instant(1024L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
+ TimerData completedTwo =
+ TimerData.of(StateNamespaces.global(), new Instant(2048L), TimeDomain.EVENT_TIME);
+
+ TimerUpdate update =
+ TimerUpdate.builder(StructuralKey.of("foo", StringUtf8Coder.of()))
+ .withCompletedTimers(ImmutableList.of(completedOne, completedTwo))
+ .setTimer(set)
+ .deletedTimer(deleted)
+ .build();
+
+ assertThat(update.getCompletedTimers(), containsInAnyOrder(completedOne, completedTwo));
+ assertThat(update.getSetTimers(), contains(set));
+ assertThat(update.getDeletedTimers(), contains(deleted));
+ }
+
+ @Test
+ public void timerUpdateBuilderWithSetThenDeleteHasOnlyDeleted() {
+ TimerUpdateBuilder builder = TimerUpdate.builder(null);
+ TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
+
+ TimerUpdate built = builder.setTimer(timer).deletedTimer(timer).build();
+
+ assertThat(built.getSetTimers(), emptyIterable());
+ assertThat(built.getDeletedTimers(), contains(timer));
+ }
+
+ @Test
+ public void timerUpdateBuilderWithDeleteThenSetHasOnlySet() {
+ TimerUpdateBuilder builder = TimerUpdate.builder(null);
+ TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
+
+ TimerUpdate built = builder.deletedTimer(timer).setTimer(timer).build();
+
+ assertThat(built.getSetTimers(), contains(timer));
+ assertThat(built.getDeletedTimers(), emptyIterable());
+ }
+
+ @Test
+ public void timerUpdateBuilderWithSetAfterBuildNotAddedToBuilt() {
+ TimerUpdateBuilder builder = TimerUpdate.builder(null);
+ TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
+
+ TimerUpdate built = builder.build();
+ builder.setTimer(timer);
+ assertThat(built.getSetTimers(), emptyIterable());
+ builder.build();
+ assertThat(built.getSetTimers(), emptyIterable());
+ }
+
+ @Test
+ public void timerUpdateBuilderWithDeleteAfterBuildNotAddedToBuilt() {
+ TimerUpdateBuilder builder = TimerUpdate.builder(null);
+ TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
+
+ TimerUpdate built = builder.build();
+ builder.deletedTimer(timer);
+ assertThat(built.getDeletedTimers(), emptyIterable());
+ builder.build();
+ assertThat(built.getDeletedTimers(), emptyIterable());
+ }
+
+ @Test
+ public void timerUpdateBuilderWithCompletedAfterBuildNotAddedToBuilt() {
+ TimerUpdateBuilder builder = TimerUpdate.builder(null);
+ TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
+
+ TimerUpdate built = builder.build();
+ builder.withCompletedTimers(ImmutableList.of(timer));
+ assertThat(built.getCompletedTimers(), emptyIterable());
+ builder.build();
+ assertThat(built.getCompletedTimers(), emptyIterable());
+ }
+
+ @Test
+ public void timerUpdateWithCompletedTimersNotAddedToExisting() {
+ TimerUpdateBuilder builder = TimerUpdate.builder(null);
+ TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
+
+ TimerUpdate built = builder.build();
+ assertThat(built.getCompletedTimers(), emptyIterable());
+ assertThat(
+ built.withCompletedTimers(ImmutableList.of(timer)).getCompletedTimers(), contains(timer));
+ assertThat(built.getCompletedTimers(), emptyIterable());
+ }
+
+ private static Matcher<Instant> earlierThan(final Instant laterInstant) {
+ return new BaseMatcher<Instant>() {
+ @Override
+ public boolean matches(Object item) {
+ ReadableInstant instant = (ReadableInstant) item;
+ return instant.isBefore(laterInstant);
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("earlier than ").appendValue(laterInstant);
+ }
+ };
+ }
+
+ private static Matcher<Instant> laterThan(final Instant shouldBeEarlier) {
+ return new BaseMatcher<Instant>() {
+ @Override
+ public boolean matches(Object item) {
+ ReadableInstant instant = (ReadableInstant) item;
+ return instant.isAfter(shouldBeEarlier);
+ }
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("later than ").appendValue(shouldBeEarlier);
+ }
+ };
+ }
+
+ @SafeVarargs
+ private final <T> CommittedBundle<T> timestampedBundle(
+ PCollection<T> pc, TimestampedValue<T>... values) {
+ UncommittedBundle<T> bundle = bundleFactory.createRootBundle(pc);
+ for (TimestampedValue<T> value : values) {
+ bundle.add(
+ WindowedValue.timestampedValueInGlobalWindow(value.getValue(), value.getTimestamp()));
+ }
+ return bundle.commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ }
+
+ @SafeVarargs
+ private final <T> CommittedBundle<T> multiWindowedBundle(PCollection<T> pc, T... values) {
+ UncommittedBundle<T> bundle = bundleFactory.createRootBundle(pc);
+ Collection<BoundedWindow> windows =
+ ImmutableList.of(
+ GlobalWindow.INSTANCE,
+ new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(0)));
+ for (T value : values) {
+ bundle.add(
+ WindowedValue.of(value, BoundedWindow.TIMESTAMP_MIN_VALUE, windows, PaneInfo.NO_FIRING));
+ }
+ return bundle.commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
+ }
+
+ private final CommittedResult result(
+ AppliedPTransform<?, ?, ?> transform,
+ @Nullable CommittedBundle<?> unprocessedBundle,
+ Iterable<? extends CommittedBundle<?>> bundles) {
+ return CommittedResult.create(StepTransformResult.withoutHold(transform).build(),
+ unprocessedBundle,
+ bundles);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
index 8a3591b..71abcca 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WindowEvaluatorFactoryTest.java
@@ -69,7 +69,7 @@ public class WindowEvaluatorFactoryTest {
private PCollection<Long> input;
private WindowEvaluatorFactory factory;
- @Mock private InProcessEvaluationContext evaluationContext;
+ @Mock private EvaluationContext evaluationContext;
private BundleFactory bundleFactory;
@@ -96,7 +96,7 @@ public class WindowEvaluatorFactoryTest {
TestPipeline p = TestPipeline.create();
input = p.apply(Create.of(1L, 2L, 3L));
- bundleFactory = InProcessBundleFactory.create();
+ bundleFactory = ImmutableListBundleFactory.create();
factory = new WindowEvaluatorFactory();
}
@@ -112,7 +112,7 @@ public class WindowEvaluatorFactoryTest {
UncommittedBundle<Long> outputBundle = createOutputBundle(triggering, inputBundle);
- InProcessTransformResult result = runEvaluator(triggering, inputBundle, transform);
+ TransformResult result = runEvaluator(triggering, inputBundle, transform);
assertThat(
Iterables.getOnlyElement(result.getOutputBundles()),
@@ -134,7 +134,7 @@ public class WindowEvaluatorFactoryTest {
BoundedWindow firstSecondWindow = new IntervalWindow(EPOCH, EPOCH.plus(windowDuration));
BoundedWindow thirdWindow = new IntervalWindow(EPOCH.minus(windowDuration), EPOCH);
- InProcessTransformResult result = runEvaluator(windowed, inputBundle, transform);
+ TransformResult result = runEvaluator(windowed, inputBundle, transform);
assertThat(
Iterables.getOnlyElement(result.getOutputBundles()),
@@ -163,7 +163,7 @@ public class WindowEvaluatorFactoryTest {
CommittedBundle<Long> inputBundle = createInputBundle();
UncommittedBundle<Long> outputBundle = createOutputBundle(windowed, inputBundle);
- InProcessTransformResult result = runEvaluator(windowed, inputBundle, transform);
+ TransformResult result = runEvaluator(windowed, inputBundle, transform);
assertThat(
Iterables.getOnlyElement(result.getOutputBundles()),
@@ -205,7 +205,7 @@ public class WindowEvaluatorFactoryTest {
CommittedBundle<Long> inputBundle = createInputBundle();
UncommittedBundle<Long> outputBundle = createOutputBundle(windowed, inputBundle);
- InProcessTransformResult result = runEvaluator(windowed, inputBundle, transform);
+ TransformResult result = runEvaluator(windowed, inputBundle, transform);
assertThat(
Iterables.getOnlyElement(result.getOutputBundles()),
@@ -251,7 +251,7 @@ public class WindowEvaluatorFactoryTest {
return outputBundle;
}
- private InProcessTransformResult runEvaluator(
+ private TransformResult runEvaluator(
PCollection<Long> windowed,
CommittedBundle<Long> inputBundle,
Window.Bound<Long> windowTransform /* Required while Window.Bound is a composite */)
@@ -265,7 +265,7 @@ public class WindowEvaluatorFactoryTest {
evaluator.processElement(first);
evaluator.processElement(second);
evaluator.processElement(third);
- InProcessTransformResult result = evaluator.finishBundle();
+ TransformResult result = evaluator.finishBundle();
return result;
}
[06/12] incubator-beam git commit: Remove InProcess Prefixes
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InMemoryWatermarkManagerTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InMemoryWatermarkManagerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InMemoryWatermarkManagerTest.java
deleted file mode 100644
index 21c941a..0000000
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InMemoryWatermarkManagerTest.java
+++ /dev/null
@@ -1,1428 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.emptyIterable;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
-import static org.junit.Assert.assertThat;
-
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.FiredTimers;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate.TimerUpdateBuilder;
-import org.apache.beam.runners.direct.InMemoryWatermarkManager.TransformWatermarks;
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.sdk.coders.ByteArrayCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.Filter;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.TimeDomain;
-import org.apache.beam.sdk.util.TimerInternals.TimerData;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.state.StateNamespaces;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionList;
-import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TimestampedValue;
-
-import com.google.common.collect.ImmutableList;
-
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
-import org.hamcrest.Matcher;
-import org.joda.time.Instant;
-import org.joda.time.ReadableInstant;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.annotation.Nullable;
-
-/**
- * Tests for {@link InMemoryWatermarkManager}.
- */
-@RunWith(JUnit4.class)
-public class InMemoryWatermarkManagerTest implements Serializable {
- private transient MockClock clock;
-
- private transient PCollection<Integer> createdInts;
-
- private transient PCollection<Integer> filtered;
- private transient PCollection<Integer> filteredTimesTwo;
- private transient PCollection<KV<String, Integer>> keyed;
-
- private transient PCollection<Integer> intsToFlatten;
- private transient PCollection<Integer> flattened;
-
- private transient InMemoryWatermarkManager manager;
- private transient BundleFactory bundleFactory;
-
- @Before
- public void setup() {
- TestPipeline p = TestPipeline.create();
-
- createdInts = p.apply("createdInts", Create.of(1, 2, 3));
-
- filtered = createdInts.apply("filtered", Filter.greaterThan(1));
- filteredTimesTwo = filtered.apply("timesTwo", ParDo.of(new DoFn<Integer, Integer>() {
- @Override
- public void processElement(DoFn<Integer, Integer>.ProcessContext c) throws Exception {
- c.output(c.element() * 2);
- }
- }));
-
- keyed = createdInts.apply("keyed", WithKeys.<String, Integer>of("MyKey"));
-
- intsToFlatten = p.apply("intsToFlatten", Create.of(-1, 256, 65535));
- PCollectionList<Integer> preFlatten = PCollectionList.of(createdInts).and(intsToFlatten);
- flattened = preFlatten.apply("flattened", Flatten.<Integer>pCollections());
-
- Collection<AppliedPTransform<?, ?, ?>> rootTransforms =
- ImmutableList.<AppliedPTransform<?, ?, ?>>of(
- createdInts.getProducingTransformInternal(),
- intsToFlatten.getProducingTransformInternal());
-
- Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> consumers = new HashMap<>();
- consumers.put(
- createdInts,
- ImmutableList.<AppliedPTransform<?, ?, ?>>of(filtered.getProducingTransformInternal(),
- keyed.getProducingTransformInternal(), flattened.getProducingTransformInternal()));
- consumers.put(
- filtered,
- Collections.<AppliedPTransform<?, ?, ?>>singleton(
- filteredTimesTwo.getProducingTransformInternal()));
- consumers.put(filteredTimesTwo, Collections.<AppliedPTransform<?, ?, ?>>emptyList());
- consumers.put(keyed, Collections.<AppliedPTransform<?, ?, ?>>emptyList());
-
- consumers.put(
- intsToFlatten,
- Collections.<AppliedPTransform<?, ?, ?>>singleton(
- flattened.getProducingTransformInternal()));
- consumers.put(flattened, Collections.<AppliedPTransform<?, ?, ?>>emptyList());
-
- clock = MockClock.fromInstant(new Instant(1000));
-
- manager = InMemoryWatermarkManager.create(clock, rootTransforms, consumers);
- bundleFactory = InProcessBundleFactory.create();
- }
-
- /**
- * Demonstrates that getWatermark, when called on an {@link AppliedPTransform} that has not
- * processed any elements, returns the {@link BoundedWindow#TIMESTAMP_MIN_VALUE}.
- */
- @Test
- public void getWatermarkForUntouchedTransform() {
- TransformWatermarks watermarks =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
-
- assertThat(watermarks.getInputWatermark(), equalTo(BoundedWindow.TIMESTAMP_MIN_VALUE));
- assertThat(watermarks.getOutputWatermark(), equalTo(BoundedWindow.TIMESTAMP_MIN_VALUE));
- }
-
- /**
- * Demonstrates that getWatermark for a transform that consumes no input uses the Watermark
- * Hold value provided to it as the output watermark.
- */
- @Test
- public void getWatermarkForUpdatedSourceTransform() {
- CommittedBundle<Integer> output = multiWindowedBundle(createdInts, 1);
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(output)),
- new Instant(8000L));
- manager.refreshAll();
- TransformWatermarks updatedSourceWatermark =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
-
- assertThat(updatedSourceWatermark.getOutputWatermark(), equalTo(new Instant(8000L)));
- }
-
- /**
- * Demonstrates that getWatermark for a transform that takes multiple inputs is held to the
- * minimum watermark across all of its inputs.
- */
- @Test
- public void getWatermarkForMultiInputTransform() {
- CommittedBundle<Integer> secondPcollectionBundle = multiWindowedBundle(intsToFlatten, -1);
-
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(intsToFlatten.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(secondPcollectionBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
-
- // We didn't do anything for the first source, so we shouldn't have progressed the watermark
- TransformWatermarks firstSourceWatermark =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
- assertThat(
- firstSourceWatermark.getOutputWatermark(),
- not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
-
- // the Second Source output all of the elements so it should be done (with a watermark at the
- // end of time).
- TransformWatermarks secondSourceWatermark =
- manager.getWatermarks(intsToFlatten.getProducingTransformInternal());
- assertThat(
- secondSourceWatermark.getOutputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
-
- // We haven't consumed anything yet, so our watermark should be at the beginning of time
- TransformWatermarks transformWatermark =
- manager.getWatermarks(flattened.getProducingTransformInternal());
- assertThat(
- transformWatermark.getInputWatermark(), not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
- assertThat(
- transformWatermark.getOutputWatermark(), not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
-
- CommittedBundle<Integer> flattenedBundleSecondCreate = multiWindowedBundle(flattened, -1);
- // We have finished processing the bundle from the second PCollection, but we haven't consumed
- // anything from the first PCollection yet; so our watermark shouldn't advance
- manager.updateWatermarks(secondPcollectionBundle,
- TimerUpdate.empty(),
- result(flattened.getProducingTransformInternal(),
- secondPcollectionBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(flattenedBundleSecondCreate)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- TransformWatermarks transformAfterProcessing =
- manager.getWatermarks(flattened.getProducingTransformInternal());
- manager.updateWatermarks(secondPcollectionBundle,
- TimerUpdate.empty(),
- result(flattened.getProducingTransformInternal(),
- secondPcollectionBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(flattenedBundleSecondCreate)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- assertThat(
- transformAfterProcessing.getInputWatermark(),
- not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
- assertThat(
- transformAfterProcessing.getOutputWatermark(),
- not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
-
- Instant firstCollectionTimestamp = new Instant(10000);
- CommittedBundle<Integer> firstPcollectionBundle =
- timestampedBundle(createdInts, TimestampedValue.<Integer>of(5, firstCollectionTimestamp));
- // the source is done, but elements are still buffered. The source output watermark should be
- // past the end of the global window
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(firstPcollectionBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- TransformWatermarks firstSourceWatermarks =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
- assertThat(
- firstSourceWatermarks.getOutputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
-
- // We still haven't consumed any of the first source's input, so the watermark should still not
- // progress
- TransformWatermarks flattenAfterSourcesProduced =
- manager.getWatermarks(flattened.getProducingTransformInternal());
- assertThat(
- flattenAfterSourcesProduced.getInputWatermark(), not(laterThan(firstCollectionTimestamp)));
- assertThat(
- flattenAfterSourcesProduced.getOutputWatermark(), not(laterThan(firstCollectionTimestamp)));
-
- // We have buffered inputs, but since the PCollection has all of the elements (has a WM past the
- // end of the global window), we should have a watermark equal to the min among buffered
- // elements
- TransformWatermarks withBufferedElements =
- manager.getWatermarks(flattened.getProducingTransformInternal());
- assertThat(withBufferedElements.getInputWatermark(), equalTo(firstCollectionTimestamp));
- assertThat(withBufferedElements.getOutputWatermark(), equalTo(firstCollectionTimestamp));
-
- CommittedBundle<?> completedFlattenBundle =
- bundleFactory.createRootBundle(flattened).commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.updateWatermarks(firstPcollectionBundle,
- TimerUpdate.empty(),
- result(flattened.getProducingTransformInternal(),
- firstPcollectionBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(completedFlattenBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- TransformWatermarks afterConsumingAllInput =
- manager.getWatermarks(flattened.getProducingTransformInternal());
- assertThat(
- afterConsumingAllInput.getInputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- assertThat(
- afterConsumingAllInput.getOutputWatermark(),
- not(laterThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- }
-
- /**
- * Demonstrates that pending elements are independent among
- * {@link AppliedPTransform AppliedPTransforms} that consume the same input {@link PCollection}.
- */
- @Test
- public void getWatermarkForMultiConsumedCollection() {
- CommittedBundle<Integer> createdBundle = timestampedBundle(createdInts,
- TimestampedValue.of(1, new Instant(1_000_000L)), TimestampedValue.of(2, new Instant(1234L)),
- TimestampedValue.of(3, new Instant(-1000L)));
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(createdBundle)),
- new Instant(Long.MAX_VALUE));
- manager.refreshAll();
- TransformWatermarks createdAfterProducing =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
- assertThat(
- createdAfterProducing.getOutputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
-
- CommittedBundle<KV<String, Integer>> keyBundle =
- timestampedBundle(keyed, TimestampedValue.of(KV.of("MyKey", 1), new Instant(1_000_000L)),
- TimestampedValue.of(KV.of("MyKey", 2), new Instant(1234L)),
- TimestampedValue.of(KV.of("MyKey", 3), new Instant(-1000L)));
- manager.updateWatermarks(createdBundle,
- TimerUpdate.empty(),
- result(keyed.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(keyBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- TransformWatermarks keyedWatermarks =
- manager.getWatermarks(keyed.getProducingTransformInternal());
- assertThat(
- keyedWatermarks.getInputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- assertThat(
- keyedWatermarks.getOutputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
-
- TransformWatermarks filteredWatermarks =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- assertThat(filteredWatermarks.getInputWatermark(), not(laterThan(new Instant(-1000L))));
- assertThat(filteredWatermarks.getOutputWatermark(), not(laterThan(new Instant(-1000L))));
-
- CommittedBundle<Integer> filteredBundle =
- timestampedBundle(filtered, TimestampedValue.of(2, new Instant(1234L)));
- manager.updateWatermarks(createdBundle,
- TimerUpdate.empty(),
- result(filtered.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(filteredBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- TransformWatermarks filteredProcessedWatermarks =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- assertThat(
- filteredProcessedWatermarks.getInputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- assertThat(
- filteredProcessedWatermarks.getOutputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- }
-
- /**
- * Demonstrates that the watermark of an {@link AppliedPTransform} is held to the provided
- * watermark hold.
- */
- @Test
- public void updateWatermarkWithWatermarkHolds() {
- CommittedBundle<Integer> createdBundle = timestampedBundle(createdInts,
- TimestampedValue.of(1, new Instant(1_000_000L)),
- TimestampedValue.of(2, new Instant(1234L)),
- TimestampedValue.of(3, new Instant(-1000L)));
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(createdBundle)),
- new Instant(Long.MAX_VALUE));
-
- CommittedBundle<KV<String, Integer>> keyBundle = timestampedBundle(keyed,
- TimestampedValue.of(KV.of("MyKey", 1), new Instant(1_000_000L)),
- TimestampedValue.of(KV.of("MyKey", 2), new Instant(1234L)),
- TimestampedValue.of(KV.of("MyKey", 3), new Instant(-1000L)));
- manager.updateWatermarks(createdBundle,
- TimerUpdate.empty(),
- result(keyed.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(keyBundle)),
- new Instant(500L));
- manager.refreshAll();
- TransformWatermarks keyedWatermarks =
- manager.getWatermarks(keyed.getProducingTransformInternal());
- assertThat(
- keyedWatermarks.getInputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- assertThat(keyedWatermarks.getOutputWatermark(), not(laterThan(new Instant(500L))));
- }
-
- /**
- * Demonstrates that the watermark of an {@link AppliedPTransform} is held to the provided
- * watermark hold.
- */
- @Test
- public void updateWatermarkWithKeyedWatermarkHolds() {
- CommittedBundle<Integer> firstKeyBundle = bundleFactory.createKeyedBundle(null,
- StructuralKey.of("Odd", StringUtf8Coder.of()),
- createdInts)
- .add(WindowedValue.timestampedValueInGlobalWindow(1, new Instant(1_000_000L)))
- .add(WindowedValue.timestampedValueInGlobalWindow(3, new Instant(-1000L)))
- .commit(clock.now());
-
- CommittedBundle<Integer> secondKeyBundle = bundleFactory.createKeyedBundle(null,
- StructuralKey.of("Even", StringUtf8Coder.of()),
- createdInts)
- .add(WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1234L)))
- .commit(clock.now());
-
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- ImmutableList.of(firstKeyBundle, secondKeyBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
-
- manager.updateWatermarks(firstKeyBundle,
- TimerUpdate.empty(),
- result(filtered.getProducingTransformInternal(),
- firstKeyBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>emptyList()),
- new Instant(-1000L));
- manager.updateWatermarks(secondKeyBundle,
- TimerUpdate.empty(),
- result(filtered.getProducingTransformInternal(),
- secondKeyBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>emptyList()),
- new Instant(1234L));
- manager.refreshAll();
-
- TransformWatermarks filteredWatermarks =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- assertThat(filteredWatermarks.getInputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- assertThat(filteredWatermarks.getOutputWatermark(), not(laterThan(new Instant(-1000L))));
-
- CommittedBundle<Integer> fauxFirstKeyTimerBundle = bundleFactory.createKeyedBundle(null,
- StructuralKey.of("Odd", StringUtf8Coder.of()),
- createdInts).commit(clock.now());
- manager.updateWatermarks(fauxFirstKeyTimerBundle,
- TimerUpdate.empty(),
- result(filtered.getProducingTransformInternal(),
- fauxFirstKeyTimerBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>emptyList()),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
-
- assertThat(filteredWatermarks.getOutputWatermark(), equalTo(new Instant(1234L)));
-
- CommittedBundle<Integer> fauxSecondKeyTimerBundle = bundleFactory.createKeyedBundle(null,
- StructuralKey.of("Even", StringUtf8Coder.of()),
- createdInts).commit(clock.now());
- manager.updateWatermarks(fauxSecondKeyTimerBundle,
- TimerUpdate.empty(),
- result(filtered.getProducingTransformInternal(),
- fauxSecondKeyTimerBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>emptyList()),
- new Instant(5678L));
- manager.refreshAll();
- assertThat(filteredWatermarks.getOutputWatermark(), equalTo(new Instant(5678L)));
-
- manager.updateWatermarks(fauxSecondKeyTimerBundle,
- TimerUpdate.empty(),
- result(filtered.getProducingTransformInternal(),
- fauxSecondKeyTimerBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>emptyList()),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- assertThat(filteredWatermarks.getOutputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- }
-
- /**
- * Demonstrates that updated output watermarks are monotonic in the presence of late data, when
- * called on an {@link AppliedPTransform} that consumes no input.
- */
- @Test
- public void updateOutputWatermarkShouldBeMonotonic() {
- CommittedBundle<?> firstInput =
- bundleFactory.createRootBundle(createdInts).commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.updateWatermarks(null, TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(firstInput)),
- new Instant(0L));
- manager.refreshAll();
- TransformWatermarks firstWatermarks =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
- assertThat(firstWatermarks.getOutputWatermark(), equalTo(new Instant(0L)));
-
- CommittedBundle<?> secondInput =
- bundleFactory.createRootBundle(createdInts).commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(secondInput)),
- new Instant(-250L));
- manager.refreshAll();
- TransformWatermarks secondWatermarks =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
- assertThat(secondWatermarks.getOutputWatermark(), not(earlierThan(new Instant(0L))));
- }
-
- /**
- * Demonstrates that updated output watermarks are monotonic in the presence of watermark holds
- * that become earlier than a previous watermark hold.
- */
- @Test
- public void updateWatermarkWithHoldsShouldBeMonotonic() {
- CommittedBundle<Integer> createdBundle = timestampedBundle(createdInts,
- TimestampedValue.of(1, new Instant(1_000_000L)),
- TimestampedValue.of(2, new Instant(1234L)),
- TimestampedValue.of(3, new Instant(-1000L)));
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(createdBundle)),
- new Instant(Long.MAX_VALUE));
-
- CommittedBundle<KV<String, Integer>> keyBundle =
- timestampedBundle(keyed, TimestampedValue.of(KV.of("MyKey", 1), new Instant(1_000_000L)),
- TimestampedValue.of(KV.of("MyKey", 2), new Instant(1234L)),
- TimestampedValue.of(KV.of("MyKey", 3), new Instant(-1000L)));
- manager.updateWatermarks(createdBundle,
- TimerUpdate.empty(),
- result(keyed.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(keyBundle)),
- new Instant(500L));
- manager.refreshAll();
- TransformWatermarks keyedWatermarks =
- manager.getWatermarks(keyed.getProducingTransformInternal());
- assertThat(
- keyedWatermarks.getInputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- assertThat(keyedWatermarks.getOutputWatermark(), not(laterThan(new Instant(500L))));
- Instant oldOutputWatermark = keyedWatermarks.getOutputWatermark();
-
- TransformWatermarks updatedWatermarks =
- manager.getWatermarks(keyed.getProducingTransformInternal());
- assertThat(
- updatedWatermarks.getInputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- // We added a hold prior to the old watermark; we shouldn't progress (due to the earlier hold)
- // but the watermark is monotonic and should not backslide to the new, earlier hold
- assertThat(updatedWatermarks.getOutputWatermark(), equalTo(oldOutputWatermark));
- }
-
- @Test
- public void updateWatermarkWithUnprocessedElements() {
- WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(1);
- WindowedValue<Integer> second =
- WindowedValue.timestampedValueInGlobalWindow(2, new Instant(-1000L));
- WindowedValue<Integer> third =
- WindowedValue.timestampedValueInGlobalWindow(3, new Instant(1234L));
- CommittedBundle<Integer> createdBundle = bundleFactory.createRootBundle(createdInts)
- .add(first)
- .add(second)
- .add(third)
- .commit(clock.now());
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(createdBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
-
- CommittedBundle<KV<String, Integer>> keyBundle = timestampedBundle(keyed,
- TimestampedValue.of(KV.of("MyKey", 1), BoundedWindow.TIMESTAMP_MIN_VALUE));
- manager.updateWatermarks(createdBundle,
- TimerUpdate.empty(),
- result(keyed.getProducingTransformInternal(),
- createdBundle.withElements(ImmutableList.of(second, third)),
- Collections.<CommittedBundle<?>>singleton(keyBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- TransformWatermarks keyedWatermarks =
- manager.getWatermarks(keyed.getProducingTransformInternal());
- // the unprocessed second and third are readded to pending
- assertThat(
- keyedWatermarks.getInputWatermark(), not(laterThan(new Instant(-1000L))));
- }
-
- /**
- * Demonstrates that updateWatermarks in the presence of late data is monotonic.
- */
- @Test
- public void updateWatermarkWithLateData() {
- Instant sourceWatermark = new Instant(1_000_000L);
- CommittedBundle<Integer> createdBundle = timestampedBundle(createdInts,
- TimestampedValue.of(1, sourceWatermark), TimestampedValue.of(2, new Instant(1234L)));
-
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(createdBundle)),
- sourceWatermark);
-
- CommittedBundle<KV<String, Integer>> keyBundle =
- timestampedBundle(keyed, TimestampedValue.of(KV.of("MyKey", 1), sourceWatermark),
- TimestampedValue.of(KV.of("MyKey", 2), new Instant(1234L)));
-
- // Finish processing the on-time data. The watermarks should progress to be equal to the source
- manager.updateWatermarks(createdBundle,
- TimerUpdate.empty(),
- result(keyed.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(keyBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- TransformWatermarks onTimeWatermarks =
- manager.getWatermarks(keyed.getProducingTransformInternal());
- assertThat(onTimeWatermarks.getInputWatermark(), equalTo(sourceWatermark));
- assertThat(onTimeWatermarks.getOutputWatermark(), equalTo(sourceWatermark));
-
- CommittedBundle<Integer> lateDataBundle =
- timestampedBundle(createdInts, TimestampedValue.of(3, new Instant(-1000L)));
- // the late data arrives in a downstream PCollection after its watermark has advanced past it;
- // we don't advance the watermark past the current watermark until we've consumed the late data
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(lateDataBundle)),
- new Instant(2_000_000L));
- manager.refreshAll();
- TransformWatermarks bufferedLateWm =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
- assertThat(bufferedLateWm.getOutputWatermark(), equalTo(new Instant(2_000_000L)));
-
- // The input watermark should be held to its previous value (not advanced due to late data; not
- // moved backwards in the presence of watermarks due to monotonicity).
- TransformWatermarks lateDataBufferedWatermark =
- manager.getWatermarks(keyed.getProducingTransformInternal());
- assertThat(lateDataBufferedWatermark.getInputWatermark(), not(earlierThan(sourceWatermark)));
- assertThat(lateDataBufferedWatermark.getOutputWatermark(), not(earlierThan(sourceWatermark)));
-
- CommittedBundle<KV<String, Integer>> lateKeyedBundle =
- timestampedBundle(keyed, TimestampedValue.of(KV.of("MyKey", 3), new Instant(-1000L)));
- manager.updateWatermarks(lateDataBundle,
- TimerUpdate.empty(),
- result(keyed.getProducingTransformInternal(),
- lateDataBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(lateKeyedBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- }
-
- public void updateWatermarkWithDifferentWindowedValueInstances() {
- manager.updateWatermarks(
- null,
- TimerUpdate.empty(), result(createdInts.getProducingTransformInternal(), null,
- Collections.<CommittedBundle<?>>singleton(
- bundleFactory
- .createRootBundle(createdInts)
- .add(WindowedValue.valueInGlobalWindow(1))
- .commit(Instant.now()))),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
-
- CommittedBundle<Integer> createdBundle = bundleFactory.createRootBundle(createdInts)
- .add(WindowedValue.valueInGlobalWindow(1))
- .commit(Instant.now());
- manager.updateWatermarks(createdBundle,
- TimerUpdate.empty(),
- result(keyed.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>emptyList()),
- null);
- manager.refreshAll();
- TransformWatermarks onTimeWatermarks =
- manager.getWatermarks(keyed.getProducingTransformInternal());
- assertThat(onTimeWatermarks.getInputWatermark(), equalTo(BoundedWindow.TIMESTAMP_MAX_VALUE));
- }
-
- /**
- * Demonstrates that after watermarks of an upstream transform are updated, but no output has been
- * produced, the watermarks of a downstream process are advanced.
- */
- @Test
- public void getWatermarksAfterOnlyEmptyOutput() {
- CommittedBundle<Integer> emptyCreateOutput = multiWindowedBundle(createdInts);
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(emptyCreateOutput)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- TransformWatermarks updatedSourceWatermarks =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
-
- assertThat(
- updatedSourceWatermarks.getOutputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
-
- TransformWatermarks finishedFilterWatermarks =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- assertThat(
- finishedFilterWatermarks.getInputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- assertThat(
- finishedFilterWatermarks.getOutputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- }
-
- /**
- * Demonstrates that after watermarks of an upstream transform are updated, but no output has been
- * produced, and the downstream transform has a watermark hold, the watermark is held to the hold.
- */
- @Test
- public void getWatermarksAfterHoldAndEmptyOutput() {
- CommittedBundle<Integer> firstCreateOutput = multiWindowedBundle(createdInts, 1, 2);
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(firstCreateOutput)),
- new Instant(12_000L));
-
- CommittedBundle<Integer> firstFilterOutput = multiWindowedBundle(filtered);
- manager.updateWatermarks(firstCreateOutput,
- TimerUpdate.empty(),
- result(filtered.getProducingTransformInternal(),
- firstCreateOutput.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(firstFilterOutput)),
- new Instant(10_000L));
- manager.refreshAll();
- TransformWatermarks firstFilterWatermarks =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- assertThat(firstFilterWatermarks.getInputWatermark(), not(earlierThan(new Instant(12_000L))));
- assertThat(firstFilterWatermarks.getOutputWatermark(), not(laterThan(new Instant(10_000L))));
-
- CommittedBundle<Integer> emptyCreateOutput = multiWindowedBundle(createdInts);
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(emptyCreateOutput)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- TransformWatermarks updatedSourceWatermarks =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
-
- assertThat(
- updatedSourceWatermarks.getOutputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
-
- TransformWatermarks finishedFilterWatermarks =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- assertThat(
- finishedFilterWatermarks.getInputWatermark(),
- not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
- assertThat(finishedFilterWatermarks.getOutputWatermark(), not(laterThan(new Instant(10_000L))));
- }
-
- @Test
- public void getSynchronizedProcessingTimeInputWatermarksHeldToPendingBundles() {
- TransformWatermarks watermarks =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
- assertThat(watermarks.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
- assertThat(
- watermarks.getSynchronizedProcessingOutputTime(),
- equalTo(BoundedWindow.TIMESTAMP_MIN_VALUE));
-
- TransformWatermarks filteredWatermarks =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- // Non-root processing watermarks don't progress until data has been processed
- assertThat(
- filteredWatermarks.getSynchronizedProcessingInputTime(),
- not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
- assertThat(
- filteredWatermarks.getSynchronizedProcessingOutputTime(),
- not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
-
- CommittedBundle<Integer> createOutput =
- bundleFactory.createRootBundle(createdInts).commit(new Instant(1250L));
-
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(createOutput)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- TransformWatermarks createAfterUpdate =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
- assertThat(createAfterUpdate.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
- assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
-
- TransformWatermarks filterAfterProduced =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- assertThat(
- filterAfterProduced.getSynchronizedProcessingInputTime(), not(laterThan(clock.now())));
- assertThat(
- filterAfterProduced.getSynchronizedProcessingOutputTime(), not(laterThan(clock.now())));
-
- clock.set(new Instant(1500L));
- assertThat(createAfterUpdate.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
- assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
- assertThat(
- filterAfterProduced.getSynchronizedProcessingInputTime(),
- not(laterThan(new Instant(1250L))));
- assertThat(
- filterAfterProduced.getSynchronizedProcessingOutputTime(),
- not(laterThan(new Instant(1250L))));
-
- CommittedBundle<?> filterOutputBundle =
- bundleFactory.createRootBundle(intsToFlatten).commit(new Instant(1250L));
- manager.updateWatermarks(createOutput,
- TimerUpdate.empty(),
- result(filtered.getProducingTransformInternal(),
- createOutput.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(filterOutputBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- TransformWatermarks filterAfterConsumed =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- assertThat(
- filterAfterConsumed.getSynchronizedProcessingInputTime(),
- not(laterThan(createAfterUpdate.getSynchronizedProcessingOutputTime())));
- assertThat(
- filterAfterConsumed.getSynchronizedProcessingOutputTime(),
- not(laterThan(filterAfterConsumed.getSynchronizedProcessingInputTime())));
- }
-
- /**
- * Demonstrates that the Synchronized Processing Time output watermark cannot progress past
- * pending timers in the same set. This propagates to all downstream SynchronizedProcessingTimes.
- *
- * <p>Also demonstrate that the result is monotonic.
- */
- // @Test
- public void getSynchronizedProcessingTimeOutputHeldToPendingTimers() {
- CommittedBundle<Integer> createdBundle = multiWindowedBundle(createdInts, 1, 2, 4, 8);
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(createdBundle)),
- new Instant(1248L));
- manager.refreshAll();
-
- TransformWatermarks filteredWms =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- TransformWatermarks filteredDoubledWms =
- manager.getWatermarks(filteredTimesTwo.getProducingTransformInternal());
- Instant initialFilteredWm = filteredWms.getSynchronizedProcessingOutputTime();
- Instant initialFilteredDoubledWm = filteredDoubledWms.getSynchronizedProcessingOutputTime();
-
- StructuralKey<String> key = StructuralKey.of("key", StringUtf8Coder.of());
- CommittedBundle<Integer> filteredBundle = multiWindowedBundle(filtered, 2, 8);
- TimerData pastTimer =
- TimerData.of(StateNamespaces.global(), new Instant(250L), TimeDomain.PROCESSING_TIME);
- TimerData futureTimer =
- TimerData.of(StateNamespaces.global(), new Instant(4096L), TimeDomain.PROCESSING_TIME);
- TimerUpdate timers =
- TimerUpdate.builder(key).setTimer(pastTimer).setTimer(futureTimer).build();
- manager.updateWatermarks(createdBundle,
- timers,
- result(filtered.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(filteredBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- Instant startTime = clock.now();
- clock.set(startTime.plus(250L));
- // We're held based on the past timer
- assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
- assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
- // And we're monotonic
- assertThat(
- filteredWms.getSynchronizedProcessingOutputTime(), not(earlierThan(initialFilteredWm)));
- assertThat(
- filteredDoubledWms.getSynchronizedProcessingOutputTime(),
- not(earlierThan(initialFilteredDoubledWm)));
-
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firedTimers =
- manager.extractFiredTimers();
- assertThat(
- firedTimers.get(filtered.getProducingTransformInternal())
- .get(key)
- .getTimers(TimeDomain.PROCESSING_TIME),
- contains(pastTimer));
- // Our timer has fired, but has not been completed, so it holds our synchronized processing WM
- assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
- assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
-
- CommittedBundle<Integer> filteredTimerBundle =
- bundleFactory
- .createKeyedBundle(null, key, filtered)
- .commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
- CommittedBundle<Integer> filteredTimerResult =
- bundleFactory.createKeyedBundle(null, key, filteredTimesTwo)
- .commit(filteredWms.getSynchronizedProcessingOutputTime());
- // Complete the processing time timer
- manager.updateWatermarks(filteredTimerBundle,
- TimerUpdate.builder(key)
- .withCompletedTimers(Collections.<TimerData>singleton(pastTimer)).build(),
- result(filtered.getProducingTransformInternal(),
- filteredTimerBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(filteredTimerResult)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
-
- clock.set(startTime.plus(500L));
- assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(laterThan(clock.now())));
- // filtered should be held to the time at which the filteredTimerResult fired
- assertThat(
- filteredDoubledWms.getSynchronizedProcessingOutputTime(),
- not(earlierThan(filteredTimerResult.getSynchronizedProcessingOutputWatermark())));
-
- manager.updateWatermarks(filteredTimerResult,
- TimerUpdate.empty(),
- result(filteredTimesTwo.getProducingTransformInternal(),
- filteredTimerResult.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>emptyList()),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
-
- clock.set(new Instant(Long.MAX_VALUE));
- assertThat(filteredWms.getSynchronizedProcessingOutputTime(), equalTo(new Instant(4096)));
- assertThat(
- filteredDoubledWms.getSynchronizedProcessingOutputTime(), equalTo(new Instant(4096)));
- }
-
- /**
- * Demonstrates that if any earlier processing holds appear in the synchronized processing time
- * output hold the result is monotonic.
- */
- @Test
- public void getSynchronizedProcessingTimeOutputTimeIsMonotonic() {
- Instant startTime = clock.now();
- TransformWatermarks watermarks =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
- assertThat(watermarks.getSynchronizedProcessingInputTime(), equalTo(startTime));
-
- TransformWatermarks filteredWatermarks =
- manager.getWatermarks(filtered.getProducingTransformInternal());
- // Non-root processing watermarks don't progress until data has been processed
- assertThat(
- filteredWatermarks.getSynchronizedProcessingInputTime(),
- not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
- assertThat(
- filteredWatermarks.getSynchronizedProcessingOutputTime(),
- not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
-
- CommittedBundle<Integer> createOutput =
- bundleFactory.createRootBundle(createdInts).commit(new Instant(1250L));
-
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(createOutput)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
- TransformWatermarks createAfterUpdate =
- manager.getWatermarks(createdInts.getProducingTransformInternal());
- assertThat(createAfterUpdate.getSynchronizedProcessingInputTime(), not(laterThan(clock.now())));
- assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(),
- not(laterThan(clock.now())));
-
- CommittedBundle<Integer> createSecondOutput =
- bundleFactory.createRootBundle(createdInts).commit(new Instant(750L));
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(createSecondOutput)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
-
- assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
- }
-
- @Test
- public void synchronizedProcessingInputTimeIsHeldToUpstreamProcessingTimeTimers() {
- CommittedBundle<Integer> created = multiWindowedBundle(createdInts, 1, 2, 3);
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(created)),
- new Instant(40_900L));
- manager.refreshAll();
-
- CommittedBundle<Integer> filteredBundle = multiWindowedBundle(filtered, 2, 4);
- Instant upstreamHold = new Instant(2048L);
- TimerData upstreamProcessingTimer =
- TimerData.of(StateNamespaces.global(), upstreamHold, TimeDomain.PROCESSING_TIME);
- manager.updateWatermarks(created,
- TimerUpdate.builder(StructuralKey.of("key", StringUtf8Coder.of()))
- .setTimer(upstreamProcessingTimer)
- .build(),
- result(filtered.getProducingTransformInternal(),
- created.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(filteredBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
-
- TransformWatermarks downstreamWms =
- manager.getWatermarks(filteredTimesTwo.getProducingTransformInternal());
- assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
-
- clock.set(BoundedWindow.TIMESTAMP_MAX_VALUE);
- assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(upstreamHold));
-
- manager.extractFiredTimers();
- // Pending processing time timers that have been fired but aren't completed hold the
- // synchronized processing time
- assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(upstreamHold));
-
- CommittedBundle<Integer> otherCreated = multiWindowedBundle(createdInts, 4, 8, 12);
- manager.updateWatermarks(otherCreated,
- TimerUpdate.builder(StructuralKey.of("key", StringUtf8Coder.of()))
- .withCompletedTimers(Collections.singleton(upstreamProcessingTimer)).build(),
- result(filtered.getProducingTransformInternal(),
- otherCreated.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>emptyList()),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
-
- assertThat(downstreamWms.getSynchronizedProcessingInputTime(), not(earlierThan(clock.now())));
- }
-
- @Test
- public void synchronizedProcessingInputTimeIsHeldToPendingBundleTimes() {
- CommittedBundle<Integer> created = multiWindowedBundle(createdInts, 1, 2, 3);
- manager.updateWatermarks(
- null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>singleton(created)),
- new Instant(29_919_235L));
-
- Instant upstreamHold = new Instant(2048L);
- CommittedBundle<Integer> filteredBundle = bundleFactory.createKeyedBundle(created,
- StructuralKey.of("key", StringUtf8Coder.of()),
- filtered).commit(upstreamHold);
- manager.updateWatermarks(
- created,
- TimerUpdate.empty(),
- result(filtered.getProducingTransformInternal(),
- created.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(filteredBundle)),
- BoundedWindow.TIMESTAMP_MAX_VALUE);
- manager.refreshAll();
-
- TransformWatermarks downstreamWms =
- manager.getWatermarks(filteredTimesTwo.getProducingTransformInternal());
- assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
-
- clock.set(BoundedWindow.TIMESTAMP_MAX_VALUE);
- assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(upstreamHold));
- }
-
- @Test
- public void extractFiredTimersReturnsFiredEventTimeTimers() {
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> initialTimers =
- manager.extractFiredTimers();
- // Watermarks haven't advanced
- assertThat(initialTimers.entrySet(), emptyIterable());
-
- // Advance WM of keyed past the first timer, but ahead of the second and third
- CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.singleton(createdBundle)),
- new Instant(1500L));
- manager.refreshAll();
-
- TimerData earliestTimer =
- TimerData.of(StateNamespaces.global(), new Instant(1000), TimeDomain.EVENT_TIME);
- TimerData middleTimer =
- TimerData.of(StateNamespaces.global(), new Instant(5000L), TimeDomain.EVENT_TIME);
- TimerData lastTimer =
- TimerData.of(StateNamespaces.global(), new Instant(10000L), TimeDomain.EVENT_TIME);
- StructuralKey<byte[]> key = StructuralKey.of(new byte[] {1, 4, 9}, ByteArrayCoder.of());
- TimerUpdate update =
- TimerUpdate.builder(key)
- .setTimer(earliestTimer)
- .setTimer(middleTimer)
- .setTimer(lastTimer)
- .build();
-
- manager.updateWatermarks(createdBundle,
- update,
- result(filtered.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(intsToFlatten))),
- new Instant(1000L));
- manager.refreshAll();
-
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firstTransformFiredTimers =
- manager.extractFiredTimers();
- assertThat(
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> firstFilteredTimers =
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(firstFilteredTimers.get(key), not(nullValue()));
- FiredTimers firstFired = firstFilteredTimers.get(key);
- assertThat(firstFired.getTimers(TimeDomain.EVENT_TIME), contains(earliestTimer));
-
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>emptyList()),
- new Instant(50_000L));
- manager.refreshAll();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> secondTransformFiredTimers =
- manager.extractFiredTimers();
- assertThat(
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> secondFilteredTimers =
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(secondFilteredTimers.get(key), not(nullValue()));
- FiredTimers secondFired = secondFilteredTimers.get(key);
- // Contains, in order, middleTimer and then lastTimer
- assertThat(secondFired.getTimers(TimeDomain.EVENT_TIME), contains(middleTimer, lastTimer));
- }
-
- @Test
- public void extractFiredTimersReturnsFiredProcessingTimeTimers() {
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> initialTimers =
- manager.extractFiredTimers();
- // Watermarks haven't advanced
- assertThat(initialTimers.entrySet(), emptyIterable());
-
- // Advance WM of keyed past the first timer, but ahead of the second and third
- CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.singleton(createdBundle)),
- new Instant(1500L));
-
- TimerData earliestTimer =
- TimerData.of(StateNamespaces.global(), new Instant(999L), TimeDomain.PROCESSING_TIME);
- TimerData middleTimer =
- TimerData.of(StateNamespaces.global(), new Instant(5000L), TimeDomain.PROCESSING_TIME);
- TimerData lastTimer =
- TimerData.of(StateNamespaces.global(), new Instant(10000L), TimeDomain.PROCESSING_TIME);
- StructuralKey<?> key = StructuralKey.of(-12L, VarLongCoder.of());
- TimerUpdate update =
- TimerUpdate.builder(key)
- .setTimer(lastTimer)
- .setTimer(earliestTimer)
- .setTimer(middleTimer)
- .build();
-
- manager.updateWatermarks(
- createdBundle,
- update,
- result(filtered.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(intsToFlatten))),
- new Instant(1000L));
- manager.refreshAll();
-
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firstTransformFiredTimers =
- manager.extractFiredTimers();
- assertThat(
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> firstFilteredTimers =
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(firstFilteredTimers.get(key), not(nullValue()));
- FiredTimers firstFired = firstFilteredTimers.get(key);
- assertThat(firstFired.getTimers(TimeDomain.PROCESSING_TIME), contains(earliestTimer));
-
- clock.set(new Instant(50_000L));
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>emptyList()),
- new Instant(50_000L));
- manager.refreshAll();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> secondTransformFiredTimers =
- manager.extractFiredTimers();
- assertThat(
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> secondFilteredTimers =
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(secondFilteredTimers.get(key), not(nullValue()));
- FiredTimers secondFired = secondFilteredTimers.get(key);
- // Contains, in order, middleTimer and then lastTimer
- assertThat(secondFired.getTimers(TimeDomain.PROCESSING_TIME), contains(middleTimer, lastTimer));
- }
-
- @Test
- public void extractFiredTimersReturnsFiredSynchronizedProcessingTimeTimers() {
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> initialTimers =
- manager.extractFiredTimers();
- // Watermarks haven't advanced
- assertThat(initialTimers.entrySet(), emptyIterable());
-
- // Advance WM of keyed past the first timer, but ahead of the second and third
- CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.singleton(createdBundle)),
- new Instant(1500L));
-
- TimerData earliestTimer = TimerData.of(
- StateNamespaces.global(), new Instant(999L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
- TimerData middleTimer = TimerData.of(
- StateNamespaces.global(), new Instant(5000L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
- TimerData lastTimer = TimerData.of(
- StateNamespaces.global(), new Instant(10000L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
- StructuralKey<byte[]> key = StructuralKey.of(new byte[] {2, -2, 22}, ByteArrayCoder.of());
- TimerUpdate update =
- TimerUpdate.builder(key)
- .setTimer(lastTimer)
- .setTimer(earliestTimer)
- .setTimer(middleTimer)
- .build();
-
- manager.updateWatermarks(
- createdBundle,
- update,
- result(filtered.getProducingTransformInternal(),
- createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()),
- Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(intsToFlatten))),
- new Instant(1000L));
- manager.refreshAll();
-
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firstTransformFiredTimers =
- manager.extractFiredTimers();
- assertThat(
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> firstFilteredTimers =
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(firstFilteredTimers.get(key), not(nullValue()));
- FiredTimers firstFired = firstFilteredTimers.get(key);
- assertThat(
- firstFired.getTimers(TimeDomain.SYNCHRONIZED_PROCESSING_TIME), contains(earliestTimer));
-
- clock.set(new Instant(50_000L));
- manager.updateWatermarks(null,
- TimerUpdate.empty(),
- result(createdInts.getProducingTransformInternal(),
- null,
- Collections.<CommittedBundle<?>>emptyList()),
- new Instant(50_000L));
- manager.refreshAll();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> secondTransformFiredTimers =
- manager.extractFiredTimers();
- assertThat(
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> secondFilteredTimers =
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(secondFilteredTimers.get(key), not(nullValue()));
- FiredTimers secondFired = secondFilteredTimers.get(key);
- // Contains, in order, middleTimer and then lastTimer
- assertThat(
- secondFired.getTimers(TimeDomain.SYNCHRONIZED_PROCESSING_TIME),
- contains(middleTimer, lastTimer));
- }
-
- @Test
- public void timerUpdateBuilderBuildAddsAllAddedTimers() {
- TimerData set = TimerData.of(StateNamespaces.global(), new Instant(10L), TimeDomain.EVENT_TIME);
- TimerData deleted =
- TimerData.of(StateNamespaces.global(), new Instant(24L), TimeDomain.PROCESSING_TIME);
- TimerData completedOne = TimerData.of(
- StateNamespaces.global(), new Instant(1024L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
- TimerData completedTwo =
- TimerData.of(StateNamespaces.global(), new Instant(2048L), TimeDomain.EVENT_TIME);
-
- TimerUpdate update =
- TimerUpdate.builder(StructuralKey.of("foo", StringUtf8Coder.of()))
- .withCompletedTimers(ImmutableList.of(completedOne, completedTwo))
- .setTimer(set)
- .deletedTimer(deleted)
- .build();
-
- assertThat(update.getCompletedTimers(), containsInAnyOrder(completedOne, completedTwo));
- assertThat(update.getSetTimers(), contains(set));
- assertThat(update.getDeletedTimers(), contains(deleted));
- }
-
- @Test
- public void timerUpdateBuilderWithSetThenDeleteHasOnlyDeleted() {
- TimerUpdateBuilder builder = TimerUpdate.builder(null);
- TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
-
- TimerUpdate built = builder.setTimer(timer).deletedTimer(timer).build();
-
- assertThat(built.getSetTimers(), emptyIterable());
- assertThat(built.getDeletedTimers(), contains(timer));
- }
-
- @Test
- public void timerUpdateBuilderWithDeleteThenSetHasOnlySet() {
- TimerUpdateBuilder builder = TimerUpdate.builder(null);
- TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
-
- TimerUpdate built = builder.deletedTimer(timer).setTimer(timer).build();
-
- assertThat(built.getSetTimers(), contains(timer));
- assertThat(built.getDeletedTimers(), emptyIterable());
- }
-
- @Test
- public void timerUpdateBuilderWithSetAfterBuildNotAddedToBuilt() {
- TimerUpdateBuilder builder = TimerUpdate.builder(null);
- TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
-
- TimerUpdate built = builder.build();
- builder.setTimer(timer);
- assertThat(built.getSetTimers(), emptyIterable());
- builder.build();
- assertThat(built.getSetTimers(), emptyIterable());
- }
-
- @Test
- public void timerUpdateBuilderWithDeleteAfterBuildNotAddedToBuilt() {
- TimerUpdateBuilder builder = TimerUpdate.builder(null);
- TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
-
- TimerUpdate built = builder.build();
- builder.deletedTimer(timer);
- assertThat(built.getDeletedTimers(), emptyIterable());
- builder.build();
- assertThat(built.getDeletedTimers(), emptyIterable());
- }
-
- @Test
- public void timerUpdateBuilderWithCompletedAfterBuildNotAddedToBuilt() {
- TimerUpdateBuilder builder = TimerUpdate.builder(null);
- TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
-
- TimerUpdate built = builder.build();
- builder.withCompletedTimers(ImmutableList.of(timer));
- assertThat(built.getCompletedTimers(), emptyIterable());
- builder.build();
- assertThat(built.getCompletedTimers(), emptyIterable());
- }
-
- @Test
- public void timerUpdateWithCompletedTimersNotAddedToExisting() {
- TimerUpdateBuilder builder = TimerUpdate.builder(null);
- TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
-
- TimerUpdate built = builder.build();
- assertThat(built.getCompletedTimers(), emptyIterable());
- assertThat(
- built.withCompletedTimers(ImmutableList.of(timer)).getCompletedTimers(), contains(timer));
- assertThat(built.getCompletedTimers(), emptyIterable());
- }
-
- private static Matcher<Instant> earlierThan(final Instant laterInstant) {
- return new BaseMatcher<Instant>() {
- @Override
- public boolean matches(Object item) {
- ReadableInstant instant = (ReadableInstant) item;
- return instant.isBefore(laterInstant);
- }
-
- @Override
- public void describeTo(Description description) {
- description.appendText("earlier than ").appendValue(laterInstant);
- }
- };
- }
-
- private static Matcher<Instant> laterThan(final Instant shouldBeEarlier) {
- return new BaseMatcher<Instant>() {
- @Override
- public boolean matches(Object item) {
- ReadableInstant instant = (ReadableInstant) item;
- return instant.isAfter(shouldBeEarlier);
- }
- @Override
- public void describeTo(Description description) {
- description.appendText("later than ").appendValue(shouldBeEarlier);
- }
- };
- }
-
- @SafeVarargs
- private final <T> CommittedBundle<T> timestampedBundle(
- PCollection<T> pc, TimestampedValue<T>... values) {
- UncommittedBundle<T> bundle = bundleFactory.createRootBundle(pc);
- for (TimestampedValue<T> value : values) {
- bundle.add(
- WindowedValue.timestampedValueInGlobalWindow(value.getValue(), value.getTimestamp()));
- }
- return bundle.commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
- }
-
- @SafeVarargs
- private final <T> CommittedBundle<T> multiWindowedBundle(PCollection<T> pc, T... values) {
- UncommittedBundle<T> bundle = bundleFactory.createRootBundle(pc);
- Collection<BoundedWindow> windows =
- ImmutableList.of(
- GlobalWindow.INSTANCE,
- new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(0)));
- for (T value : values) {
- bundle.add(
- WindowedValue.of(value, BoundedWindow.TIMESTAMP_MIN_VALUE, windows, PaneInfo.NO_FIRING));
- }
- return bundle.commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
- }
-
- private final CommittedResult result(
- AppliedPTransform<?, ?, ?> transform,
- @Nullable CommittedBundle<?> unprocessedBundle,
- Iterable<? extends CommittedBundle<?>> bundles) {
- return CommittedResult.create(StepTransformResult.withoutHold(transform).build(),
- unprocessedBundle,
- bundles);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/4649eebe/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessBundleFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessBundleFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessBundleFactoryTest.java
deleted file mode 100644
index 3a3ac8c..0000000
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/InProcessBundleFactoryTest.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertThat;
-
-import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
-import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
-import org.apache.beam.sdk.coders.ByteArrayCoder;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.coders.VarIntCoder;
-import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
-import org.apache.beam.sdk.transforms.windowing.PaneInfo;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-
-import com.google.common.collect.ImmutableList;
-
-import org.hamcrest.Matcher;
-import org.hamcrest.Matchers;
-import org.joda.time.Instant;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-
-/**
- * Tests for {@link InProcessBundleFactory}.
- */
-@RunWith(JUnit4.class)
-public class InProcessBundleFactoryTest {
- @Rule public ExpectedException thrown = ExpectedException.none();
-
- private InProcessBundleFactory bundleFactory = InProcessBundleFactory.create();
-
- private PCollection<Integer> created;
- private PCollection<KV<String, Integer>> downstream;
-
- @Before
- public void setup() {
- TestPipeline p = TestPipeline.create();
- created = p.apply(Create.of(1, 2, 3));
- downstream = created.apply(WithKeys.<String, Integer>of("foo"));
- }
-
- @Test
- public void createRootBundleShouldCreateWithEmptyKey() {
- PCollection<Integer> pcollection = TestPipeline.create().apply(Create.of(1));
-
- UncommittedBundle<Integer> inFlightBundle = bundleFactory.createRootBundle(pcollection);
-
- CommittedBundle<Integer> bundle = inFlightBundle.commit(Instant.now());
-
- assertThat(bundle.getKey(),
- Matchers.<StructuralKey<?>>equalTo(StructuralKey.of(null, VoidCoder.of())));
- }
-
- private <T> void createKeyedBundle(Coder<T> coder, T key) throws Exception {
- PCollection<Integer> pcollection = TestPipeline.create().apply(Create.of(1));
- StructuralKey skey = StructuralKey.of(key, coder);
-
- UncommittedBundle<Integer> inFlightBundle =
- bundleFactory.createKeyedBundle(null, skey, pcollection);
-
- CommittedBundle<Integer> bundle = inFlightBundle.commit(Instant.now());
- assertThat(bundle.getKey(), equalTo(skey));
- }
-
- @Test
- public void keyedWithNullKeyShouldCreateKeyedBundle() throws Exception {
- createKeyedBundle(VoidCoder.of(), null);
- }
-
- @Test
- public void keyedWithKeyShouldCreateKeyedBundle() throws Exception {
- createKeyedBundle(StringUtf8Coder.of(), "foo");
- createKeyedBundle(VarIntCoder.of(), 1234);
- createKeyedBundle(ByteArrayCoder.of(), new byte[] {0, 2, 4, 99});
- }
-
- private <T> CommittedBundle<T>
- afterCommitGetElementsShouldHaveAddedElements(Iterable<WindowedValue<T>> elems) {
- PCollection<T> pcollection = TestPipeline.create().apply(Create.<T>of());
-
- UncommittedBundle<T> bundle = bundleFactory.createRootBundle(pcollection);
- Collection<Matcher<? super WindowedValue<T>>> expectations = new ArrayList<>();
- for (WindowedValue<T> elem : elems) {
- bundle.add(elem);
- expectations.add(equalTo(elem));
- }
- Matcher<Iterable<? extends WindowedValue<T>>> containsMatcher =
- Matchers.<WindowedValue<T>>containsInAnyOrder(expectations);
- CommittedBundle<T> committed = bundle.commit(Instant.now());
- assertThat(committed.getElements(), containsMatcher);
-
- return committed;
- }
-
- @Test
- public void getElementsBeforeAddShouldReturnEmptyIterable() {
- afterCommitGetElementsShouldHaveAddedElements(Collections.<WindowedValue<Integer>>emptyList());
- }
-
- @Test
- public void getElementsAfterAddShouldReturnAddedElements() {
- WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1);
- WindowedValue<Integer> secondValue =
- WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L));
-
- afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue));
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void withElementsShouldReturnIndependentBundle() {
- WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1);
- WindowedValue<Integer> secondValue =
- WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L));
-
- CommittedBundle<Integer> committed =
- afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue));
-
- WindowedValue<Integer> firstReplacement =
- WindowedValue.of(
- 9,
- new Instant(2048L),
- new IntervalWindow(new Instant(2044L), Instant.now()),
- PaneInfo.NO_FIRING);
- WindowedValue<Integer> secondReplacement =
- WindowedValue.timestampedValueInGlobalWindow(-1, Instant.now());
- CommittedBundle<Integer> withed =
- committed.withElements(ImmutableList.of(firstReplacement, secondReplacement));
-
- assertThat(withed.getElements(), containsInAnyOrder(firstReplacement, secondReplacement));
- assertThat(committed.getElements(), containsInAnyOrder(firstValue, secondValue));
- assertThat(withed.getKey(), Matchers.<StructuralKey<?>>equalTo(committed.getKey()));
- assertThat(withed.getPCollection(), equalTo(committed.getPCollection()));
- assertThat(
- withed.getSynchronizedProcessingOutputWatermark(),
- equalTo(committed.getSynchronizedProcessingOutputWatermark()));
- }
-
- @Test
- public void addAfterCommitShouldThrowException() {
- PCollection<Integer> pcollection = TestPipeline.create().apply(Create.<Integer>of());
-
- UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(pcollection);
- bundle.add(WindowedValue.valueInGlobalWindow(1));
- CommittedBundle<Integer> firstCommit = bundle.commit(Instant.now());
- assertThat(firstCommit.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(1)));
-
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage("3");
- thrown.expectMessage("committed");
-
- bundle.add(WindowedValue.valueInGlobalWindow(3));
- }
-
- @Test
- public void commitAfterCommitShouldThrowException() {
- PCollection<Integer> pcollection = TestPipeline.create().apply(Create.<Integer>of());
-
- UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(pcollection);
- bundle.add(WindowedValue.valueInGlobalWindow(1));
- CommittedBundle<Integer> firstCommit = bundle.commit(Instant.now());
- assertThat(firstCommit.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(1)));
-
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage("committed");
-
- bundle.commit(Instant.now());
- }
-
- @Test
- public void createBundleUnkeyedResultUnkeyed() {
- CommittedBundle<KV<String, Integer>> newBundle =
- bundleFactory
- .createBundle(bundleFactory.createRootBundle(created).commit(Instant.now()), downstream)
- .commit(Instant.now());
- }
-
- @Test
- public void createBundleKeyedResultPropagatesKey() {
- CommittedBundle<KV<String, Integer>> newBundle =
- bundleFactory.createBundle(
- bundleFactory.createKeyedBundle(
- null,
- StructuralKey.of("foo", StringUtf8Coder.of()),
- created).commit(Instant.now()),
- downstream).commit(Instant.now());
- assertThat(newBundle.getKey().getKey(), Matchers.<Object>equalTo("foo"));
- }
-
- @Test
- public void createKeyedBundleKeyed() {
- CommittedBundle<KV<String, Integer>> keyedBundle = bundleFactory.createKeyedBundle(
- bundleFactory.createRootBundle(created).commit(Instant.now()),
- StructuralKey.of("foo", StringUtf8Coder.of()),
- downstream).commit(Instant.now());
- assertThat(keyedBundle.getKey().getKey(), Matchers.<Object>equalTo("foo"));
- }
-}
[02/12] incubator-beam git commit: Rename InProcessPipelineRunner to
DirectRunner
Posted by ke...@apache.org.
Rename InProcessPipelineRunner to DirectRunner
Completes BEAM-243
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2bafda1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2bafda1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2bafda1d
Branch: refs/heads/master
Commit: 2bafda1de06c2f7a410b242c5a44787dd21a7396
Parents: 021fa1e
Author: Thomas Groh <tg...@google.com>
Authored: Wed Jun 15 10:45:15 2016 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Wed Jun 15 15:53:47 2016 -0700
----------------------------------------------------------------------
runners/direct-java/pom.xml | 2 +-
.../direct/AbstractModelEnforcement.java | 2 +-
.../direct/BoundedReadEvaluatorFactory.java | 4 +-
.../beam/runners/direct/BundleFactory.java | 4 +-
.../beam/runners/direct/CommittedResult.java | 2 +-
.../beam/runners/direct/CompletionCallback.java | 2 +-
.../beam/runners/direct/DirectOptions.java | 101 +++++
.../beam/runners/direct/DirectRegistrar.java | 55 +++
.../beam/runners/direct/DirectRunner.java | 371 +++++++++++++++++++
.../direct/EncodabilityEnforcementFactory.java | 2 +-
.../direct/ExecutorServiceParallelExecutor.java | 2 +-
.../runners/direct/FlattenEvaluatorFactory.java | 6 +-
.../ImmutabilityCheckingBundleFactory.java | 4 +-
.../direct/ImmutabilityEnforcementFactory.java | 2 +-
.../direct/InMemoryWatermarkManager.java | 2 +-
.../runners/direct/InProcessBundleFactory.java | 4 +-
.../direct/InProcessBundleOutputManager.java | 6 +-
.../direct/InProcessEvaluationContext.java | 20 +-
.../direct/InProcessExecutionContext.java | 4 +-
.../beam/runners/direct/InProcessExecutor.java | 2 +-
...rocessGroupAlsoByWindowEvaluatorFactory.java | 4 +-
...InProcessGroupByKeyOnlyEvaluatorFactory.java | 6 +-
.../direct/InProcessPipelineOptions.java | 101 -----
.../runners/direct/InProcessPipelineRunner.java | 370 ------------------
.../beam/runners/direct/InProcessRegistrar.java | 55 ---
.../direct/InProcessTransformResult.java | 2 +-
.../beam/runners/direct/ModelEnforcement.java | 2 +-
.../runners/direct/ModelEnforcementFactory.java | 2 +-
.../runners/direct/ParDoInProcessEvaluator.java | 4 +-
.../direct/ParDoMultiEvaluatorFactory.java | 4 +-
.../direct/ParDoSingleEvaluatorFactory.java | 4 +-
.../direct/PassthroughTransformEvaluator.java | 2 +-
.../runners/direct/StepTransformResult.java | 2 +-
.../beam/runners/direct/TransformEvaluator.java | 2 +-
.../direct/TransformEvaluatorFactory.java | 2 +-
.../direct/TransformEvaluatorRegistry.java | 2 +-
.../beam/runners/direct/TransformExecutor.java | 2 +-
.../direct/UnboundedReadEvaluatorFactory.java | 4 +-
.../runners/direct/ViewEvaluatorFactory.java | 6 +-
.../runners/direct/WindowEvaluatorFactory.java | 6 +-
.../direct/AvroIOShardedWriteFactoryTest.java | 2 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 4 +-
.../runners/direct/CommittedResultTest.java | 12 +-
.../runners/direct/DirectRegistrarTest.java | 74 ++++
.../beam/runners/direct/DirectRunnerTest.java | 339 +++++++++++++++++
.../EncodabilityEnforcementFactoryTest.java | 2 +-
.../direct/FlattenEvaluatorFactoryTest.java | 4 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 4 +-
.../ImmutabilityCheckingBundleFactoryTest.java | 4 +-
.../ImmutabilityEnforcementFactoryTest.java | 2 +-
.../direct/InMemoryWatermarkManagerTest.java | 4 +-
.../direct/InProcessBundleFactoryTest.java | 4 +-
.../direct/InProcessEvaluationContextTest.java | 10 +-
...ocessGroupByKeyOnlyEvaluatorFactoryTest.java | 4 +-
.../direct/InProcessPipelineRegistrarTest.java | 74 ----
.../direct/InProcessPipelineRunnerTest.java | 339 -----------------
.../direct/ParDoInProcessEvaluatorTest.java | 6 +-
.../direct/ParDoMultiEvaluatorFactoryTest.java | 4 +-
.../direct/ParDoSingleEvaluatorFactoryTest.java | 4 +-
.../direct/TextIOShardedWriteFactoryTest.java | 2 +-
.../runners/direct/TransformExecutorTest.java | 2 +-
.../UnboundedReadEvaluatorFactoryTest.java | 4 +-
.../direct/ViewEvaluatorFactoryTest.java | 4 +-
.../direct/WindowEvaluatorFactoryTest.java | 4 +-
.../translation/TransformTranslatorTest.java | 4 +-
.../beam/sdk/options/PipelineOptions.java | 2 +-
testing/travis/test_wordcount.sh | 4 +-
67 files changed, 1051 insertions(+), 1050 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index 8667aee..3010757 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -102,7 +102,7 @@
<systemPropertyVariables>
<beamTestPipelineOptions>
[
- "--runner=org.apache.beam.runners.direct.InProcessPipelineRunner"
+ "--runner=DirectRunner"
]
</beamTestPipelineOptions>
</systemPropertyVariables>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/AbstractModelEnforcement.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/AbstractModelEnforcement.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/AbstractModelEnforcement.java
index 948beb6..2ae0275 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/AbstractModelEnforcement.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/AbstractModelEnforcement.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.util.WindowedValue;
/**
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
index f15d446..63d248a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BoundedReadEvaluatorFactory.java
@@ -17,8 +17,8 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.BoundedSource.BoundedReader;
import org.apache.beam.sdk.io.Read.Bounded;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BundleFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BundleFactory.java
index a0511df..a546cfb 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BundleFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/BundleFactory.java
@@ -18,8 +18,8 @@
package org.apache.beam.runners.direct;
import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
index 4a42e34..b241493 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CommittedResult.java
@@ -18,7 +18,7 @@
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import com.google.auto.value.AutoValue;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
index 7c2c068..8ee4b44 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CompletionCallback.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
/**
* A callback for completing a bundle of input.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
new file mode 100644
index 0000000..3901c04
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectOptions.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.ApplicationNameOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.Hidden;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.Validation.Required;
+import org.apache.beam.sdk.transforms.PTransform;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Options that can be used to configure the {@link org.apache.beam.runners.direct.DirectRunner}.
+ */
+public interface DirectOptions extends PipelineOptions, ApplicationNameOptions {
+ /**
+ * Gets the {@link ExecutorServiceFactory} to use to create instances of {@link ExecutorService}
+ * to execute {@link PTransform PTransforms}.
+ *
+ * <p>Note that {@link ExecutorService ExecutorServices} returned by the factory must ensure that
+ * it cannot enter a state in which it will not schedule additional pending work unless currently
+ * scheduled work completes, as this may cause the {@link Pipeline} to cease processing.
+ *
+ * <p>Defaults to a {@link FixedThreadPoolExecutorServiceFactory}, which produces instances of
+ * {@link Executors#newCachedThreadPool()}.
+ */
+ @JsonIgnore
+ @Required
+ @Hidden
+ @Default.InstanceFactory(FixedThreadPoolExecutorServiceFactory.class)
+ ExecutorServiceFactory getExecutorServiceFactory();
+
+ void setExecutorServiceFactory(ExecutorServiceFactory executorService);
+
+ /**
+ * Gets the {@link Clock} used by this pipeline. The clock is used in place of accessing the
+ * system time when time values are required by the evaluator.
+ */
+ @Default.InstanceFactory(NanosOffsetClock.Factory.class)
+ @JsonIgnore
+ @Required
+ @Hidden
+ @Description(
+ "The processing time source used by the pipeline. When the current time is "
+ + "needed by the evaluator, the result of clock#now() is used.")
+ Clock getClock();
+
+ void setClock(Clock clock);
+
+ @Default.Boolean(false)
+ @Description(
+ "If the pipeline should shut down producers which have reached the maximum "
+ + "representable watermark. If this is set to true, a pipeline in which all PTransforms "
+ + "have reached the maximum watermark will be shut down, even if there are unbounded "
+ + "sources that could produce additional (late) data. By default, if the pipeline "
+ + "contains any unbounded PCollections, it will run until explicitly shut down.")
+ boolean isShutdownUnboundedProducersWithMaxWatermark();
+
+ void setShutdownUnboundedProducersWithMaxWatermark(boolean shutdown);
+
+ @Default.Boolean(true)
+ @Description(
+ "If the pipeline should block awaiting completion of the pipeline. If set to true, "
+ + "a call to Pipeline#run() will block until all PTransforms are complete. Otherwise, "
+ + "the Pipeline will execute asynchronously. If set to false, the completion of the "
+ + "pipeline can be awaited on by use of DirectPipelineResult#awaitCompletion().")
+ boolean isBlockOnRun();
+
+ void setBlockOnRun(boolean b);
+
+ @Default.Boolean(true)
+ @Description(
+ "Controls whether the runner should ensure that all of the elements of every "
+ + "PCollection are not mutated. PTransforms are not permitted to mutate input elements "
+ + "at any point, or output elements after they are output.")
+ boolean isTestImmutability();
+
+ void setTestImmutability(boolean test);
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
new file mode 100644
index 0000000..eb027fa
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRegistrar.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
+import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
+
+import com.google.auto.service.AutoService;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Contains the {@link PipelineRunnerRegistrar} and {@link PipelineOptionsRegistrar} for the
+ * {@link DirectRunner}.
+ */
+public class DirectRegistrar {
+ private DirectRegistrar() {}
+ /**
+ * Registers the {@link DirectRunner}.
+ */
+ @AutoService(PipelineRunnerRegistrar.class)
+ public static class InProcessRunner implements PipelineRunnerRegistrar {
+ @Override
+ public Iterable<Class<? extends PipelineRunner<?>>> getPipelineRunners() {
+ return ImmutableList.<Class<? extends PipelineRunner<?>>>of(DirectRunner.class);
+ }
+ }
+
+ /**
+ * Registers the {@link DirectOptions}.
+ */
+ @AutoService(PipelineOptionsRegistrar.class)
+ public static class InProcessOptions implements PipelineOptionsRegistrar {
+ @Override
+ public Iterable<Class<? extends PipelineOptions>> getPipelineOptions() {
+ return ImmutableList.<Class<? extends PipelineOptions>>of(DirectOptions.class);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
new file mode 100644
index 0000000..2f5a0bc
--- /dev/null
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.direct;
+
+import org.apache.beam.runners.direct.DirectRunner.DirectPipelineResult;
+import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
+import org.apache.beam.runners.direct.ViewEvaluatorFactory.InProcessViewOverrideFactory;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.io.AvroIO;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.runners.AggregatorPipelineExtractor;
+import org.apache.beam.sdk.runners.AggregatorRetrievalException;
+import org.apache.beam.sdk.runners.AggregatorValues;
+import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.transforms.Aggregator;
+import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
+import org.apache.beam.sdk.util.MapAggregatorValues;
+import org.apache.beam.sdk.util.TimerInternals.TimerData;
+import org.apache.beam.sdk.util.UserCodeException;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.util.common.Counter;
+import org.apache.beam.sdk.util.common.CounterSet;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.POutput;
+import org.apache.beam.sdk.values.PValue;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+import org.joda.time.Instant;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * An In-Memory implementation of the Dataflow Programming Model. Supports Unbounded
+ * {@link PCollection PCollections}.
+ */
+@Experimental
+public class DirectRunner
+ extends PipelineRunner<DirectPipelineResult> {
+ /**
+ * The default set of transform overrides to use in the {@link DirectRunner}.
+ *
+ * <p>A transform override must have a single-argument constructor that takes an instance of the
+ * type of transform it is overriding.
+ */
+ @SuppressWarnings("rawtypes")
+ private static Map<Class<? extends PTransform>, PTransformOverrideFactory>
+ defaultTransformOverrides =
+ ImmutableMap.<Class<? extends PTransform>, PTransformOverrideFactory>builder()
+ .put(GroupByKey.class, new InProcessGroupByKeyOverrideFactory())
+ .put(CreatePCollectionView.class, new InProcessViewOverrideFactory())
+ .put(AvroIO.Write.Bound.class, new AvroIOShardedWriteFactory())
+ .put(TextIO.Write.Bound.class, new TextIOShardedWriteFactory())
+ .build();
+
+ /**
+ * Part of a {@link PCollection}. Elements are output to a bundle, which will cause them to be
+ * executed by {@link PTransform PTransforms} that consume the {@link PCollection} this bundle is
+ * a part of at a later point. This is an uncommitted bundle and can have elements added to it.
+ *
+ * @param <T> the type of elements that can be added to this bundle
+ */
+ public static interface UncommittedBundle<T> {
+ /**
+ * Returns the PCollection that the elements of this {@link UncommittedBundle} belong to.
+ */
+ PCollection<T> getPCollection();
+
+ /**
+ * Outputs an element to this bundle.
+ *
+ * @param element the element to add to this bundle
+ * @return this bundle
+ */
+ UncommittedBundle<T> add(WindowedValue<T> element);
+
+ /**
+ * Commits this {@link UncommittedBundle}, returning an immutable {@link CommittedBundle}
+ * containing all of the elements that were added to it. The {@link #add(WindowedValue)} method
+ * will throw an {@link IllegalStateException} if called after a call to commit.
+ * @param synchronizedProcessingTime the synchronized processing time at which this bundle was
+ * committed
+ */
+ CommittedBundle<T> commit(Instant synchronizedProcessingTime);
+ }
+
+ /**
+ * Part of a {@link PCollection}. Elements are output to an {@link UncommittedBundle}, which will
+ * eventually committed. Committed elements are executed by the {@link PTransform PTransforms}
+ * that consume the {@link PCollection} this bundle is
+ * a part of at a later point.
+ * @param <T> the type of elements contained within this bundle
+ */
+ public static interface CommittedBundle<T> {
+ /**
+ * Returns the PCollection that the elements of this bundle belong to.
+ */
+ PCollection<T> getPCollection();
+
+ /**
+ * Returns the key that was output in the most recent {@link GroupByKey} in the
+ * execution of this bundle.
+ */
+ StructuralKey<?> getKey();
+
+ /**
+ * Returns an {@link Iterable} containing all of the elements that have been added to this
+ * {@link CommittedBundle}.
+ */
+ Iterable<WindowedValue<T>> getElements();
+
+ /**
+ * Returns the processing time output watermark at the time the producing {@link PTransform}
+ * committed this bundle. Downstream synchronized processing time watermarks cannot progress
+ * past this point before consuming this bundle.
+ *
+ * <p>This value is no greater than the earliest incomplete processing time or synchronized
+ * processing time {@link TimerData timer} at the time this bundle was committed, including any
+ * timers that fired to produce this bundle.
+ */
+ Instant getSynchronizedProcessingOutputWatermark();
+
+ /**
+ * Return a new {@link CommittedBundle} that is like this one, except calls to
+ * {@link #getElements()} will return the provided elements. This bundle is unchanged.
+ *
+ * <p>
+ * The value of the {@link #getSynchronizedProcessingOutputWatermark() synchronized processing
+ * output watermark} of the returned {@link CommittedBundle} is equal to the value returned from
+ * the current bundle. This is used to ensure a {@link PTransform} that could not complete
+ * processing on input elements properly holds the synchronized processing time to the
+ * appropriate value.
+ */
+ CommittedBundle<T> withElements(Iterable<WindowedValue<T>> elements);
+ }
+
+ /**
+ * A {@link PCollectionViewWriter} is responsible for writing contents of a {@link PCollection} to
+ * a storage mechanism that can be read from while constructing a {@link PCollectionView}.
+ * @param <ElemT> the type of elements the input {@link PCollection} contains.
+ * @param <ViewT> the type of the PCollectionView this writer writes to.
+ */
+ public static interface PCollectionViewWriter<ElemT, ViewT> {
+ void add(Iterable<WindowedValue<ElemT>> values);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////////////////////
+ private final DirectOptions options;
+
+ public static DirectRunner fromOptions(PipelineOptions options) {
+ return new DirectRunner(options.as(DirectOptions.class));
+ }
+
+ private DirectRunner(DirectOptions options) {
+ this.options = options;
+ }
+
+ /**
+ * Returns the {@link PipelineOptions} used to create this {@link DirectRunner}.
+ */
+ public DirectOptions getPipelineOptions() {
+ return options;
+ }
+
+ @Override
+ public <OutputT extends POutput, InputT extends PInput> OutputT apply(
+ PTransform<InputT, OutputT> transform, InputT input) {
+ PTransformOverrideFactory overrideFactory = defaultTransformOverrides.get(transform.getClass());
+ if (overrideFactory != null) {
+ PTransform<InputT, OutputT> customTransform = overrideFactory.override(transform);
+
+ return super.apply(customTransform, input);
+ }
+ // If there is no override, or we should not apply the override, apply the original transform
+ return super.apply(transform, input);
+ }
+
+ @Override
+ public DirectPipelineResult run(Pipeline pipeline) {
+ ConsumerTrackingPipelineVisitor consumerTrackingVisitor = new ConsumerTrackingPipelineVisitor();
+ pipeline.traverseTopologically(consumerTrackingVisitor);
+ for (PValue unfinalized : consumerTrackingVisitor.getUnfinalizedPValues()) {
+ unfinalized.finishSpecifying();
+ }
+ @SuppressWarnings("rawtypes")
+ KeyedPValueTrackingVisitor keyedPValueVisitor =
+ KeyedPValueTrackingVisitor.create(
+ ImmutableSet.<Class<? extends PTransform>>of(
+ GroupByKey.class, InProcessGroupByKeyOnly.class));
+ pipeline.traverseTopologically(keyedPValueVisitor);
+
+ DisplayDataValidator.validatePipeline(pipeline);
+
+ InProcessEvaluationContext context =
+ InProcessEvaluationContext.create(
+ getPipelineOptions(),
+ createBundleFactory(getPipelineOptions()),
+ consumerTrackingVisitor.getRootTransforms(),
+ consumerTrackingVisitor.getValueToConsumers(),
+ consumerTrackingVisitor.getStepNames(),
+ consumerTrackingVisitor.getViews());
+
+ // independent executor service for each run
+ ExecutorService executorService =
+ context.getPipelineOptions().getExecutorServiceFactory().create();
+ InProcessExecutor executor =
+ ExecutorServiceParallelExecutor.create(
+ executorService,
+ consumerTrackingVisitor.getValueToConsumers(),
+ keyedPValueVisitor.getKeyedPValues(),
+ TransformEvaluatorRegistry.defaultRegistry(),
+ defaultModelEnforcements(options),
+ context);
+ executor.start(consumerTrackingVisitor.getRootTransforms());
+
+ Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
+ new AggregatorPipelineExtractor(pipeline).getAggregatorSteps();
+ DirectPipelineResult result =
+ new DirectPipelineResult(executor, context, aggregatorSteps);
+ if (options.isBlockOnRun()) {
+ try {
+ result.awaitCompletion();
+ } catch (UserCodeException userException) {
+ throw new PipelineExecutionException(userException.getCause());
+ } catch (Throwable t) {
+ if (t instanceof RuntimeException) {
+ throw (RuntimeException) t;
+ }
+ throw new RuntimeException(t);
+ }
+ }
+ return result;
+ }
+
+ private Map<Class<? extends PTransform>, Collection<ModelEnforcementFactory>>
+ defaultModelEnforcements(DirectOptions options) {
+ ImmutableMap.Builder<Class<? extends PTransform>, Collection<ModelEnforcementFactory>>
+ enforcements = ImmutableMap.builder();
+ Collection<ModelEnforcementFactory> parDoEnforcements = createParDoEnforcements(options);
+ enforcements.put(ParDo.Bound.class, parDoEnforcements);
+ enforcements.put(ParDo.BoundMulti.class, parDoEnforcements);
+ return enforcements.build();
+ }
+
+ private Collection<ModelEnforcementFactory> createParDoEnforcements(
+ DirectOptions options) {
+ ImmutableList.Builder<ModelEnforcementFactory> enforcements = ImmutableList.builder();
+ if (options.isTestImmutability()) {
+ enforcements.add(ImmutabilityEnforcementFactory.create());
+ }
+ return enforcements.build();
+ }
+
+ private BundleFactory createBundleFactory(DirectOptions pipelineOptions) {
+ BundleFactory bundleFactory = InProcessBundleFactory.create();
+ if (pipelineOptions.isTestImmutability()) {
+ bundleFactory = ImmutabilityCheckingBundleFactory.create(bundleFactory);
+ }
+ return bundleFactory;
+ }
+
+ /**
+ * The result of running a {@link Pipeline} with the {@link DirectRunner}.
+ *
+ * Throws {@link UnsupportedOperationException} for all methods.
+ */
+ public static class DirectPipelineResult implements PipelineResult {
+ private final InProcessExecutor executor;
+ private final InProcessEvaluationContext evaluationContext;
+ private final Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps;
+ private State state;
+
+ private DirectPipelineResult(
+ InProcessExecutor executor,
+ InProcessEvaluationContext evaluationContext,
+ Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps) {
+ this.executor = executor;
+ this.evaluationContext = evaluationContext;
+ this.aggregatorSteps = aggregatorSteps;
+ // Only ever constructed after the executor has started.
+ this.state = State.RUNNING;
+ }
+
+ @Override
+ public State getState() {
+ return state;
+ }
+
+ @Override
+ public <T> AggregatorValues<T> getAggregatorValues(Aggregator<?, T> aggregator)
+ throws AggregatorRetrievalException {
+ CounterSet counters = evaluationContext.getCounters();
+ Collection<PTransform<?, ?>> steps = aggregatorSteps.get(aggregator);
+ Map<String, T> stepValues = new HashMap<>();
+ for (AppliedPTransform<?, ?, ?> transform : evaluationContext.getSteps()) {
+ if (steps.contains(transform.getTransform())) {
+ String stepName =
+ String.format(
+ "user-%s-%s", evaluationContext.getStepName(transform), aggregator.getName());
+ Counter<T> counter = (Counter<T>) counters.getExistingCounter(stepName);
+ if (counter != null) {
+ stepValues.put(transform.getFullName(), counter.getAggregate());
+ }
+ }
+ }
+ return new MapAggregatorValues<>(stepValues);
+ }
+
+ /**
+ * Blocks until the {@link Pipeline} execution represented by this
+ * {@link DirectPipelineResult} is complete, returning the terminal state.
+ *
+ * <p>If the pipeline terminates abnormally by throwing an exception, this will rethrow the
+ * exception. Future calls to {@link #getState()} will return
+ * {@link org.apache.beam.sdk.PipelineResult.State#FAILED}.
+ *
+ * <p>NOTE: if the {@link Pipeline} contains an {@link IsBounded#UNBOUNDED unbounded}
+ * {@link PCollection}, and the {@link PipelineRunner} was created with
+ * {@link DirectOptions#isShutdownUnboundedProducersWithMaxWatermark()} set to false,
+ * this method will never return.
+ *
+ * See also {@link InProcessExecutor#awaitCompletion()}.
+ */
+ public State awaitCompletion() throws Throwable {
+ if (!state.isTerminal()) {
+ try {
+ executor.awaitCompletion();
+ state = State.DONE;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw e;
+ } catch (Throwable t) {
+ state = State.FAILED;
+ throw t;
+ }
+ }
+ return state;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactory.java
index ccf4c2b..bed61ec 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EncodabilityEnforcementFactory.java
@@ -19,7 +19,7 @@ package org.apache.beam.runners.direct;
import static com.google.common.base.Preconditions.checkArgument;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.util.CoderUtils;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
index 980d764..14570a5 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
@@ -18,7 +18,7 @@
package org.apache.beam.runners.direct;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.FiredTimers;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
index 2efaad3..bbe8787 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FlattenEvaluatorFactory.java
@@ -17,8 +17,8 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.Flatten.FlattenPCollectionList;
@@ -28,7 +28,7 @@ import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionList;
/**
- * The {@link InProcessPipelineRunner} {@link TransformEvaluatorFactory} for the {@link Flatten}
+ * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the {@link Flatten}
* {@link PTransform}.
*/
class FlattenEvaluatorFactory implements TransformEvaluatorFactory {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java
index 2a965ed..dcbe3d1 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java
@@ -19,8 +19,8 @@ package org.apache.beam.runners.direct;
import static com.google.common.base.Preconditions.checkNotNull;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.transforms.DoFn;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
index bfecc9d..d121442 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactory.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
import org.apache.beam.sdk.transforms.AppliedPTransform;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
index 95095fa..fb8eb7c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InMemoryWatermarkManager.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleFactory.java
index 52bc575..0c7449c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleFactory.java
@@ -19,8 +19,8 @@ package org.apache.beam.runners.direct;
import static com.google.common.base.Preconditions.checkState;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleOutputManager.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleOutputManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleOutputManager.java
index f374f99..bd07040 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleOutputManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessBundleOutputManager.java
@@ -17,8 +17,8 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.util.DoFnRunners.OutputManager;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.TupleTag;
@@ -27,7 +27,7 @@ import java.util.Map;
/**
* An {@link OutputManager} that outputs to {@link CommittedBundle Bundles} used by the
- * {@link InProcessPipelineRunner}.
+ * {@link DirectRunner}.
*/
public class InProcessBundleOutputManager implements OutputManager {
private final Map<TupleTag<?>, UncommittedBundle<?>> bundles;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessEvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessEvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessEvaluationContext.java
index db8baa0..220ff83 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessEvaluationContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessEvaluationContext.java
@@ -22,9 +22,9 @@ import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.FiredTimers;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.TransformWatermarks;
import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.PCollectionViewWriter;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.PCollectionViewWriter;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
@@ -58,11 +58,11 @@ import javax.annotation.Nullable;
/**
* The evaluation context for a specific pipeline being executed by the
- * {@link InProcessPipelineRunner}. Contains state shared within the execution across all
+ * {@link DirectRunner}. Contains state shared within the execution across all
* transforms.
*
* <p>{@link InProcessEvaluationContext} contains shared state for an execution of the
- * {@link InProcessPipelineRunner} that can be used while evaluating a {@link PTransform}. This
+ * {@link DirectRunner} that can be used while evaluating a {@link PTransform}. This
* consists of views into underlying state and watermark implementations, access to read and write
* {@link PCollectionView PCollectionViews}, and constructing {@link CounterSet CounterSets} and
* {@link ExecutionContext ExecutionContexts}. This includes executing callbacks asynchronously when
@@ -79,7 +79,7 @@ class InProcessEvaluationContext {
private final Map<AppliedPTransform<?, ?, ?>, String> stepNames;
/** The options that were used to create this {@link Pipeline}. */
- private final InProcessPipelineOptions options;
+ private final DirectOptions options;
private final BundleFactory bundleFactory;
/** The current processing time and event time watermarks and timers. */
@@ -97,7 +97,7 @@ class InProcessEvaluationContext {
private final CounterSet mergedCounters;
public static InProcessEvaluationContext create(
- InProcessPipelineOptions options,
+ DirectOptions options,
BundleFactory bundleFactory,
Collection<AppliedPTransform<?, ?, ?>> rootTransforms,
Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> valueToConsumers,
@@ -108,7 +108,7 @@ class InProcessEvaluationContext {
}
private InProcessEvaluationContext(
- InProcessPipelineOptions options,
+ DirectOptions options,
BundleFactory bundleFactory,
Collection<AppliedPTransform<?, ?, ?>> rootTransforms,
Map<PValue, Collection<AppliedPTransform<?, ?, ?>>> valueToConsumers,
@@ -295,7 +295,7 @@ class InProcessEvaluationContext {
/**
* Get the options used by this {@link Pipeline}.
*/
- public InProcessPipelineOptions getPipelineOptions() {
+ public DirectOptions getPipelineOptions() {
return options;
}
@@ -389,7 +389,7 @@ class InProcessEvaluationContext {
*
* <p>If the provided transform produces any {@link IsBounded#UNBOUNDED}
* {@link PCollection PCollections}, returns the value of
- * {@link InProcessPipelineOptions#isShutdownUnboundedProducersWithMaxWatermark()}.
+ * {@link DirectOptions#isShutdownUnboundedProducersWithMaxWatermark()}.
*/
public boolean isDone(AppliedPTransform<?, ?, ?> transform) {
// if the PTransform's watermark isn't at the max value, it isn't done
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutionContext.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutionContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutionContext.java
index 4f10b3a..d2558ce 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutionContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutionContext.java
@@ -25,7 +25,7 @@ import org.apache.beam.sdk.util.TimerInternals;
import org.apache.beam.sdk.util.state.CopyOnAccessInMemoryStateInternals;
/**
- * Execution Context for the {@link InProcessPipelineRunner}.
+ * Execution Context for the {@link DirectRunner}.
*
* This implementation is not thread safe. A new {@link InProcessExecutionContext} must be created
* for each thread that requires it.
@@ -51,7 +51,7 @@ class InProcessExecutionContext
}
/**
- * Step Context for the {@link InProcessPipelineRunner}.
+ * Step Context for the {@link DirectRunner}.
*/
public class InProcessStepContext
extends org.apache.beam.sdk.util.BaseExecutionContext.StepContext {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutor.java
index d811e1b..1cfa544 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessExecutor.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupAlsoByWindowEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupAlsoByWindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupAlsoByWindowEvaluatorFactory.java
index 5ded8b6..53b93d0 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupAlsoByWindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupAlsoByWindowEvaluatorFactory.java
@@ -19,7 +19,7 @@ package org.apache.beam.runners.direct;
import org.apache.beam.runners.core.GroupAlsoByWindowViaWindowSetDoFn;
import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupAlsoByWindow;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.DoFn;
@@ -41,7 +41,7 @@ import com.google.common.collect.ImmutableMap;
import java.util.Collections;
/**
- * The {@link InProcessPipelineRunner} {@link TransformEvaluatorFactory} for the
+ * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
* {@link GroupByKeyOnly} {@link PTransform}.
*/
class InProcessGroupAlsoByWindowEvaluatorFactory implements TransformEvaluatorFactory {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactory.java
index a10d496..3604bca 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessGroupByKeyOnlyEvaluatorFactory.java
@@ -22,8 +22,8 @@ import static org.apache.beam.sdk.util.CoderUtils.encodeToByteArray;
import static com.google.common.base.Preconditions.checkState;
import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.runners.direct.StepTransformResult.Builder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderException;
@@ -45,7 +45,7 @@ import java.util.List;
import java.util.Map;
/**
- * The {@link InProcessPipelineRunner} {@link TransformEvaluatorFactory} for the
+ * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
* {@link GroupByKeyOnly} {@link PTransform}.
*/
class InProcessGroupByKeyOnlyEvaluatorFactory implements TransformEvaluatorFactory {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineOptions.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineOptions.java
deleted file mode 100644
index 0498521..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineOptions.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.options.ApplicationNameOptions;
-import org.apache.beam.sdk.options.Default;
-import org.apache.beam.sdk.options.Description;
-import org.apache.beam.sdk.options.Hidden;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.Validation.Required;
-import org.apache.beam.sdk.transforms.PTransform;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-/**
- * Options that can be used to configure the {@link InProcessPipelineRunner}.
- */
-public interface InProcessPipelineOptions extends PipelineOptions, ApplicationNameOptions {
- /**
- * Gets the {@link ExecutorServiceFactory} to use to create instances of {@link ExecutorService}
- * to execute {@link PTransform PTransforms}.
- *
- * <p>Note that {@link ExecutorService ExecutorServices} returned by the factory must ensure that
- * it cannot enter a state in which it will not schedule additional pending work unless currently
- * scheduled work completes, as this may cause the {@link Pipeline} to cease processing.
- *
- * <p>Defaults to a {@link FixedThreadPoolExecutorServiceFactory}, which produces instances of
- * {@link Executors#newCachedThreadPool()}.
- */
- @JsonIgnore
- @Required
- @Hidden
- @Default.InstanceFactory(FixedThreadPoolExecutorServiceFactory.class)
- ExecutorServiceFactory getExecutorServiceFactory();
-
- void setExecutorServiceFactory(ExecutorServiceFactory executorService);
-
- /**
- * Gets the {@link Clock} used by this pipeline. The clock is used in place of accessing the
- * system time when time values are required by the evaluator.
- */
- @Default.InstanceFactory(NanosOffsetClock.Factory.class)
- @JsonIgnore
- @Required
- @Hidden
- @Description(
- "The processing time source used by the pipeline. When the current time is "
- + "needed by the evaluator, the result of clock#now() is used.")
- Clock getClock();
-
- void setClock(Clock clock);
-
- @Default.Boolean(false)
- @Description(
- "If the pipeline should shut down producers which have reached the maximum "
- + "representable watermark. If this is set to true, a pipeline in which all PTransforms "
- + "have reached the maximum watermark will be shut down, even if there are unbounded "
- + "sources that could produce additional (late) data. By default, if the pipeline "
- + "contains any unbounded PCollections, it will run until explicitly shut down.")
- boolean isShutdownUnboundedProducersWithMaxWatermark();
-
- void setShutdownUnboundedProducersWithMaxWatermark(boolean shutdown);
-
- @Default.Boolean(true)
- @Description(
- "If the pipeline should block awaiting completion of the pipeline. If set to true, "
- + "a call to Pipeline#run() will block until all PTransforms are complete. Otherwise, "
- + "the Pipeline will execute asynchronously. If set to false, the completion of the "
- + "pipeline can be awaited on by use of InProcessPipelineResult#awaitCompletion().")
- boolean isBlockOnRun();
-
- void setBlockOnRun(boolean b);
-
- @Default.Boolean(true)
- @Description(
- "Controls whether the runner should ensure that all of the elements of every "
- + "PCollection are not mutated. PTransforms are not permitted to mutate input elements "
- + "at any point, or output elements after they are output.")
- boolean isTestImmutability();
-
- void setTestImmutability(boolean test);
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineRunner.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineRunner.java
deleted file mode 100644
index 8847c58..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineRunner.java
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.runners.direct.InProcessGroupByKey.InProcessGroupByKeyOnly;
-import org.apache.beam.runners.direct.ViewEvaluatorFactory.InProcessViewOverrideFactory;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
-import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.io.AvroIO;
-import org.apache.beam.sdk.io.TextIO;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.AggregatorPipelineExtractor;
-import org.apache.beam.sdk.runners.AggregatorRetrievalException;
-import org.apache.beam.sdk.runners.AggregatorValues;
-import org.apache.beam.sdk.runners.PipelineRunner;
-import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
-import org.apache.beam.sdk.util.MapAggregatorValues;
-import org.apache.beam.sdk.util.TimerInternals.TimerData;
-import org.apache.beam.sdk.util.UserCodeException;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.util.common.Counter;
-import org.apache.beam.sdk.util.common.CounterSet;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollection.IsBounded;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PInput;
-import org.apache.beam.sdk.values.POutput;
-import org.apache.beam.sdk.values.PValue;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
-import org.joda.time.Instant;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-
-/**
- * An In-Memory implementation of the Dataflow Programming Model. Supports Unbounded
- * {@link PCollection PCollections}.
- */
-@Experimental
-public class InProcessPipelineRunner
- extends PipelineRunner<InProcessPipelineRunner.InProcessPipelineResult> {
- /**
- * The default set of transform overrides to use in the {@link InProcessPipelineRunner}.
- *
- * <p>A transform override must have a single-argument constructor that takes an instance of the
- * type of transform it is overriding.
- */
- @SuppressWarnings("rawtypes")
- private static Map<Class<? extends PTransform>, PTransformOverrideFactory>
- defaultTransformOverrides =
- ImmutableMap.<Class<? extends PTransform>, PTransformOverrideFactory>builder()
- .put(GroupByKey.class, new InProcessGroupByKeyOverrideFactory())
- .put(CreatePCollectionView.class, new InProcessViewOverrideFactory())
- .put(AvroIO.Write.Bound.class, new AvroIOShardedWriteFactory())
- .put(TextIO.Write.Bound.class, new TextIOShardedWriteFactory())
- .build();
-
- /**
- * Part of a {@link PCollection}. Elements are output to a bundle, which will cause them to be
- * executed by {@link PTransform PTransforms} that consume the {@link PCollection} this bundle is
- * a part of at a later point. This is an uncommitted bundle and can have elements added to it.
- *
- * @param <T> the type of elements that can be added to this bundle
- */
- public static interface UncommittedBundle<T> {
- /**
- * Returns the PCollection that the elements of this {@link UncommittedBundle} belong to.
- */
- PCollection<T> getPCollection();
-
- /**
- * Outputs an element to this bundle.
- *
- * @param element the element to add to this bundle
- * @return this bundle
- */
- UncommittedBundle<T> add(WindowedValue<T> element);
-
- /**
- * Commits this {@link UncommittedBundle}, returning an immutable {@link CommittedBundle}
- * containing all of the elements that were added to it. The {@link #add(WindowedValue)} method
- * will throw an {@link IllegalStateException} if called after a call to commit.
- * @param synchronizedProcessingTime the synchronized processing time at which this bundle was
- * committed
- */
- CommittedBundle<T> commit(Instant synchronizedProcessingTime);
- }
-
- /**
- * Part of a {@link PCollection}. Elements are output to an {@link UncommittedBundle}, which will
- * eventually committed. Committed elements are executed by the {@link PTransform PTransforms}
- * that consume the {@link PCollection} this bundle is
- * a part of at a later point.
- * @param <T> the type of elements contained within this bundle
- */
- public static interface CommittedBundle<T> {
- /**
- * Returns the PCollection that the elements of this bundle belong to.
- */
- PCollection<T> getPCollection();
-
- /**
- * Returns the key that was output in the most recent {@link GroupByKey} in the
- * execution of this bundle.
- */
- StructuralKey<?> getKey();
-
- /**
- * Returns an {@link Iterable} containing all of the elements that have been added to this
- * {@link CommittedBundle}.
- */
- Iterable<WindowedValue<T>> getElements();
-
- /**
- * Returns the processing time output watermark at the time the producing {@link PTransform}
- * committed this bundle. Downstream synchronized processing time watermarks cannot progress
- * past this point before consuming this bundle.
- *
- * <p>This value is no greater than the earliest incomplete processing time or synchronized
- * processing time {@link TimerData timer} at the time this bundle was committed, including any
- * timers that fired to produce this bundle.
- */
- Instant getSynchronizedProcessingOutputWatermark();
-
- /**
- * Return a new {@link CommittedBundle} that is like this one, except calls to
- * {@link #getElements()} will return the provided elements. This bundle is unchanged.
- *
- * <p>
- * The value of the {@link #getSynchronizedProcessingOutputWatermark() synchronized processing
- * output watermark} of the returned {@link CommittedBundle} is equal to the value returned from
- * the current bundle. This is used to ensure a {@link PTransform} that could not complete
- * processing on input elements properly holds the synchronized processing time to the
- * appropriate value.
- */
- CommittedBundle<T> withElements(Iterable<WindowedValue<T>> elements);
- }
-
- /**
- * A {@link PCollectionViewWriter} is responsible for writing contents of a {@link PCollection} to
- * a storage mechanism that can be read from while constructing a {@link PCollectionView}.
- * @param <ElemT> the type of elements the input {@link PCollection} contains.
- * @param <ViewT> the type of the PCollectionView this writer writes to.
- */
- public static interface PCollectionViewWriter<ElemT, ViewT> {
- void add(Iterable<WindowedValue<ElemT>> values);
- }
-
- ////////////////////////////////////////////////////////////////////////////////////////////////
- private final InProcessPipelineOptions options;
-
- public static InProcessPipelineRunner fromOptions(PipelineOptions options) {
- return new InProcessPipelineRunner(options.as(InProcessPipelineOptions.class));
- }
-
- private InProcessPipelineRunner(InProcessPipelineOptions options) {
- this.options = options;
- }
-
- /**
- * Returns the {@link PipelineOptions} used to create this {@link InProcessPipelineRunner}.
- */
- public InProcessPipelineOptions getPipelineOptions() {
- return options;
- }
-
- @Override
- public <OutputT extends POutput, InputT extends PInput> OutputT apply(
- PTransform<InputT, OutputT> transform, InputT input) {
- PTransformOverrideFactory overrideFactory = defaultTransformOverrides.get(transform.getClass());
- if (overrideFactory != null) {
- PTransform<InputT, OutputT> customTransform = overrideFactory.override(transform);
-
- return super.apply(customTransform, input);
- }
- // If there is no override, or we should not apply the override, apply the original transform
- return super.apply(transform, input);
- }
-
- @Override
- public InProcessPipelineResult run(Pipeline pipeline) {
- ConsumerTrackingPipelineVisitor consumerTrackingVisitor = new ConsumerTrackingPipelineVisitor();
- pipeline.traverseTopologically(consumerTrackingVisitor);
- for (PValue unfinalized : consumerTrackingVisitor.getUnfinalizedPValues()) {
- unfinalized.finishSpecifying();
- }
- @SuppressWarnings("rawtypes")
- KeyedPValueTrackingVisitor keyedPValueVisitor =
- KeyedPValueTrackingVisitor.create(
- ImmutableSet.<Class<? extends PTransform>>of(
- GroupByKey.class, InProcessGroupByKeyOnly.class));
- pipeline.traverseTopologically(keyedPValueVisitor);
-
- DisplayDataValidator.validatePipeline(pipeline);
-
- InProcessEvaluationContext context =
- InProcessEvaluationContext.create(
- getPipelineOptions(),
- createBundleFactory(getPipelineOptions()),
- consumerTrackingVisitor.getRootTransforms(),
- consumerTrackingVisitor.getValueToConsumers(),
- consumerTrackingVisitor.getStepNames(),
- consumerTrackingVisitor.getViews());
-
- // independent executor service for each run
- ExecutorService executorService =
- context.getPipelineOptions().getExecutorServiceFactory().create();
- InProcessExecutor executor =
- ExecutorServiceParallelExecutor.create(
- executorService,
- consumerTrackingVisitor.getValueToConsumers(),
- keyedPValueVisitor.getKeyedPValues(),
- TransformEvaluatorRegistry.defaultRegistry(),
- defaultModelEnforcements(options),
- context);
- executor.start(consumerTrackingVisitor.getRootTransforms());
-
- Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps =
- new AggregatorPipelineExtractor(pipeline).getAggregatorSteps();
- InProcessPipelineResult result =
- new InProcessPipelineResult(executor, context, aggregatorSteps);
- if (options.isBlockOnRun()) {
- try {
- result.awaitCompletion();
- } catch (UserCodeException userException) {
- throw new PipelineExecutionException(userException.getCause());
- } catch (Throwable t) {
- if (t instanceof RuntimeException) {
- throw (RuntimeException) t;
- }
- throw new RuntimeException(t);
- }
- }
- return result;
- }
-
- private Map<Class<? extends PTransform>, Collection<ModelEnforcementFactory>>
- defaultModelEnforcements(InProcessPipelineOptions options) {
- ImmutableMap.Builder<Class<? extends PTransform>, Collection<ModelEnforcementFactory>>
- enforcements = ImmutableMap.builder();
- Collection<ModelEnforcementFactory> parDoEnforcements = createParDoEnforcements(options);
- enforcements.put(ParDo.Bound.class, parDoEnforcements);
- enforcements.put(ParDo.BoundMulti.class, parDoEnforcements);
- return enforcements.build();
- }
-
- private Collection<ModelEnforcementFactory> createParDoEnforcements(
- InProcessPipelineOptions options) {
- ImmutableList.Builder<ModelEnforcementFactory> enforcements = ImmutableList.builder();
- if (options.isTestImmutability()) {
- enforcements.add(ImmutabilityEnforcementFactory.create());
- }
- return enforcements.build();
- }
-
- private BundleFactory createBundleFactory(InProcessPipelineOptions pipelineOptions) {
- BundleFactory bundleFactory = InProcessBundleFactory.create();
- if (pipelineOptions.isTestImmutability()) {
- bundleFactory = ImmutabilityCheckingBundleFactory.create(bundleFactory);
- }
- return bundleFactory;
- }
-
- /**
- * The result of running a {@link Pipeline} with the {@link InProcessPipelineRunner}.
- *
- * Throws {@link UnsupportedOperationException} for all methods.
- */
- public static class InProcessPipelineResult implements PipelineResult {
- private final InProcessExecutor executor;
- private final InProcessEvaluationContext evaluationContext;
- private final Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps;
- private State state;
-
- private InProcessPipelineResult(
- InProcessExecutor executor,
- InProcessEvaluationContext evaluationContext,
- Map<Aggregator<?, ?>, Collection<PTransform<?, ?>>> aggregatorSteps) {
- this.executor = executor;
- this.evaluationContext = evaluationContext;
- this.aggregatorSteps = aggregatorSteps;
- // Only ever constructed after the executor has started.
- this.state = State.RUNNING;
- }
-
- @Override
- public State getState() {
- return state;
- }
-
- @Override
- public <T> AggregatorValues<T> getAggregatorValues(Aggregator<?, T> aggregator)
- throws AggregatorRetrievalException {
- CounterSet counters = evaluationContext.getCounters();
- Collection<PTransform<?, ?>> steps = aggregatorSteps.get(aggregator);
- Map<String, T> stepValues = new HashMap<>();
- for (AppliedPTransform<?, ?, ?> transform : evaluationContext.getSteps()) {
- if (steps.contains(transform.getTransform())) {
- String stepName =
- String.format(
- "user-%s-%s", evaluationContext.getStepName(transform), aggregator.getName());
- Counter<T> counter = (Counter<T>) counters.getExistingCounter(stepName);
- if (counter != null) {
- stepValues.put(transform.getFullName(), counter.getAggregate());
- }
- }
- }
- return new MapAggregatorValues<>(stepValues);
- }
-
- /**
- * Blocks until the {@link Pipeline} execution represented by this
- * {@link InProcessPipelineResult} is complete, returning the terminal state.
- *
- * <p>If the pipeline terminates abnormally by throwing an exception, this will rethrow the
- * exception. Future calls to {@link #getState()} will return
- * {@link org.apache.beam.sdk.PipelineResult.State#FAILED}.
- *
- * <p>NOTE: if the {@link Pipeline} contains an {@link IsBounded#UNBOUNDED unbounded}
- * {@link PCollection}, and the {@link PipelineRunner} was created with
- * {@link InProcessPipelineOptions#isShutdownUnboundedProducersWithMaxWatermark()} set to false,
- * this method will never return.
- *
- * See also {@link InProcessExecutor#awaitCompletion()}.
- */
- public State awaitCompletion() throws Throwable {
- if (!state.isTerminal()) {
- try {
- executor.awaitCompletion();
- state = State.DONE;
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw e;
- } catch (Throwable t) {
- state = State.FAILED;
- throw t;
- }
- }
- return state;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessRegistrar.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessRegistrar.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessRegistrar.java
deleted file mode 100644
index 4a09de7..0000000
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessRegistrar.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.direct;
-
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsRegistrar;
-import org.apache.beam.sdk.runners.PipelineRunner;
-import org.apache.beam.sdk.runners.PipelineRunnerRegistrar;
-
-import com.google.auto.service.AutoService;
-import com.google.common.collect.ImmutableList;
-
-/**
- * Contains the {@link PipelineRunnerRegistrar} and {@link PipelineOptionsRegistrar} for the
- * {@link InProcessPipelineRunner}.
- */
-public class InProcessRegistrar {
- private InProcessRegistrar() {}
- /**
- * Registers the {@link InProcessPipelineRunner}.
- */
- @AutoService(PipelineRunnerRegistrar.class)
- public static class InProcessRunner implements PipelineRunnerRegistrar {
- @Override
- public Iterable<Class<? extends PipelineRunner<?>>> getPipelineRunners() {
- return ImmutableList.<Class<? extends PipelineRunner<?>>>of(InProcessPipelineRunner.class);
- }
- }
-
- /**
- * Registers the {@link InProcessPipelineOptions}.
- */
- @AutoService(PipelineOptionsRegistrar.class)
- public static class InProcessOptions implements PipelineOptionsRegistrar {
- @Override
- public Iterable<Class<? extends PipelineOptions>> getPipelineOptions() {
- return ImmutableList.<Class<? extends PipelineOptions>>of(InProcessPipelineOptions.class);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTransformResult.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTransformResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTransformResult.java
index 0bc3ea1..92127b4 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTransformResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessTransformResult.java
@@ -18,7 +18,7 @@
package org.apache.beam.runners.direct;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
index cc9b6da..758ee24 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcement.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcementFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcementFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcementFactory.java
index 6162ba0..e0bbfcb 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcementFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ModelEnforcementFactory.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
/**
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoInProcessEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoInProcessEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoInProcessEvaluator.java
index a3e2f18..b9f4808 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoInProcessEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoInProcessEvaluator.java
@@ -18,8 +18,8 @@
package org.apache.beam.runners.direct;
import org.apache.beam.runners.direct.InProcessExecutionContext.InProcessStepContext;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.util.DoFnRunner;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactory.java
index 8945242..58d6f00 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoMultiEvaluatorFactory.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
@@ -32,7 +32,7 @@ import com.google.common.cache.LoadingCache;
import java.util.Map;
/**
- * The {@link InProcessPipelineRunner} {@link TransformEvaluatorFactory} for the
+ * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
* {@link BoundMulti} primitive {@link PTransform}.
*/
class ParDoMultiEvaluatorFactory implements TransformEvaluatorFactory {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactory.java
index 15704d7..afbb6ed 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoSingleEvaluatorFactory.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
@@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableMap;
import java.util.Collections;
/**
- * The {@link InProcessPipelineRunner} {@link TransformEvaluatorFactory} for the
+ * The {@link DirectRunner} {@link TransformEvaluatorFactory} for the
* {@link Bound ParDo.Bound} primitive {@link PTransform}.
*/
class ParDoSingleEvaluatorFactory implements TransformEvaluatorFactory {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
index aef62b2..ba792d3 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/PassthroughTransformEvaluator.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.runners.direct;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2bafda1d/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
index b2e3897..eacea91 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StepTransformResult.java
@@ -20,7 +20,7 @@ package org.apache.beam.runners.direct;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.beam.runners.direct.InMemoryWatermarkManager.TimerUpdate;
-import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
+import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;