You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/06/12 16:55:16 UTC
[01/50] [abbrv] beam git commit: [BEAM-1779] Port
UnboundedSourceWrapperTest to use Flink operator test harness
Repository: beam
Updated Branches:
refs/heads/gearpump-runner 99221e739 -> 559e3c341
[BEAM-1779] Port UnboundedSourceWrapperTest to use Flink operator test harness
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c1dc8f53
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c1dc8f53
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c1dc8f53
Branch: refs/heads/gearpump-runner
Commit: c1dc8f53c5438b575a7e84e9f680616ead49d61e
Parents: 62b942a
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Wed Mar 22 11:43:30 2017 +0100
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Wed Jun 7 19:43:11 2017 +0200
----------------------------------------------------------------------
.../flink/streaming/TestCountingSource.java | 48 +++--
.../streaming/UnboundedSourceWrapperTest.java | 198 +++++++------------
2 files changed, 110 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c1dc8f53/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TestCountingSource.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TestCountingSource.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TestCountingSource.java
index 3a08088..edf548a 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TestCountingSource.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/TestCountingSource.java
@@ -133,18 +133,8 @@ public class TestCountingSource
public Coder<CounterMark> getCheckpointMarkCoder() {
return DelegateCoder.of(
VarIntCoder.of(),
- new DelegateCoder.CodingFunction<CounterMark, Integer>() {
- @Override
- public Integer apply(CounterMark input) {
- return input.current;
- }
- },
- new DelegateCoder.CodingFunction<Integer, CounterMark>() {
- @Override
- public CounterMark apply(Integer input) {
- return new CounterMark(input);
- }
- });
+ new FromCounterMark(),
+ new ToCounterMark());
}
@Override
@@ -251,4 +241,38 @@ public class TestCountingSource
public Coder<KV<Integer, Integer>> getDefaultOutputCoder() {
return KvCoder.of(VarIntCoder.of(), VarIntCoder.of());
}
+
+ private class FromCounterMark implements DelegateCoder.CodingFunction<CounterMark, Integer> {
+ @Override
+ public Integer apply(CounterMark input) {
+ return input.current;
+ }
+
+ @Override
+ public int hashCode() {
+ return FromCounterMark.class.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof FromCounterMark;
+ }
+ }
+
+ private class ToCounterMark implements DelegateCoder.CodingFunction<Integer, CounterMark> {
+ @Override
+ public CounterMark apply(Integer input) {
+ return new CounterMark(input);
+ }
+
+ @Override
+ public int hashCode() {
+ return ToCounterMark.class.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof ToCounterMark;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/c1dc8f53/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java
index e3875bc..716e71d 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/streaming/UnboundedSourceWrapperTest.java
@@ -20,36 +20,20 @@ package org.apache.beam.runners.flink.streaming;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
import org.apache.beam.runners.flink.translation.wrappers.streaming.io.UnboundedSourceWrapper;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.ValueWithRecordId;
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.state.ListState;
-import org.apache.flink.api.common.state.ListStateDescriptor;
-import org.apache.flink.api.common.state.OperatorStateStore;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.execution.Environment;
-import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
-import org.apache.flink.runtime.state.StateInitializationContext;
-import org.apache.flink.runtime.state.StateSnapshotContextSynchronousImpl;
import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.Output;
import org.apache.flink.streaming.api.operators.StreamSource;
import org.apache.flink.streaming.api.watermark.Watermark;
@@ -57,15 +41,14 @@ import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.streamstatus.StreamStatus;
import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer;
-import org.apache.flink.streaming.runtime.tasks.StreamTask;
-import org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService;
+import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
+import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness;
import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.OutputTag;
import org.junit.Test;
import org.junit.experimental.runners.Enclosed;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.mockito.Matchers;
/**
* Tests for {@link UnboundedSourceWrapper}.
@@ -125,10 +108,18 @@ public class UnboundedSourceWrapperTest {
KV<Integer, Integer>,
TestCountingSource.CounterMark>> sourceOperator = new StreamSource<>(flinkWrapper);
- setupSourceOperator(sourceOperator, numTasks);
+ AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
+ testHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ sourceOperator,
+ numTasks /* max parallelism */,
+ numTasks /* parallelism */,
+ 0 /* subtask index */);
+
+ testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
try {
- sourceOperator.open();
+ testHarness.open();
sourceOperator.run(checkpointLock,
new TestStreamStatusMaintainer(),
new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
@@ -200,29 +191,22 @@ public class UnboundedSourceWrapperTest {
TestCountingSource.CounterMark>> sourceOperator = new StreamSource<>(flinkWrapper);
- OperatorStateStore backend = mock(OperatorStateStore.class);
-
- TestingListState<KV<UnboundedSource, TestCountingSource.CounterMark>>
- listState = new TestingListState<>();
-
- when(backend.getOperatorState(Matchers.any(ListStateDescriptor.class)))
- .thenReturn(listState);
-
- StateInitializationContext initializationContext = mock(StateInitializationContext.class);
-
- when(initializationContext.getOperatorStateStore()).thenReturn(backend);
- when(initializationContext.isRestored()).thenReturn(false, true);
+ AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
+ testHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ sourceOperator,
+ numTasks /* max parallelism */,
+ numTasks /* parallelism */,
+ 0 /* subtask index */);
- flinkWrapper.initializeState(initializationContext);
-
- setupSourceOperator(sourceOperator, numTasks);
+ testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
final Set<KV<Integer, Integer>> emittedElements = new HashSet<>();
boolean readFirstBatchOfElements = false;
try {
- sourceOperator.open();
+ testHarness.open();
sourceOperator.run(checkpointLock,
new TestStreamStatusMaintainer(),
new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
@@ -265,21 +249,12 @@ public class UnboundedSourceWrapperTest {
assertTrue("Did not successfully read first batch of elements.", readFirstBatchOfElements);
// draw a snapshot
- flinkWrapper.snapshotState(new StateSnapshotContextSynchronousImpl(0, 0));
-
- // test snapshot offsets
- assertEquals(flinkWrapper.getLocalSplitSources().size(),
- listState.getList().size());
- int totalEmit = 0;
- for (KV<UnboundedSource, TestCountingSource.CounterMark> kv : listState.get()) {
- totalEmit += kv.getValue().current + 1;
- }
- assertEquals(numElements / 2, totalEmit);
+ OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
// test that finalizeCheckpoint on CheckpointMark is called
final ArrayList<Integer> finalizeList = new ArrayList<>();
TestCountingSource.setFinalizeTracker(finalizeList);
- flinkWrapper.notifyCheckpointComplete(0);
+ testHarness.notifyOfCompletedCheckpoint(0);
assertEquals(flinkWrapper.getLocalSplitSources().size(), finalizeList.size());
// create a completely new source but restore from the snapshot
@@ -297,16 +272,25 @@ public class UnboundedSourceWrapperTest {
TestCountingSource.CounterMark>> restoredSourceOperator =
new StreamSource<>(restoredFlinkWrapper);
- setupSourceOperator(restoredSourceOperator, numTasks);
+ // set parallelism to 1 to ensure that our testing operator gets all checkpointed state
+ AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
+ restoredTestHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ restoredSourceOperator,
+ numTasks /* max parallelism */,
+ 1 /* parallelism */,
+ 0 /* subtask index */);
+
+ restoredTestHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
// restore snapshot
- restoredFlinkWrapper.initializeState(initializationContext);
+ restoredTestHarness.initializeState(snapshot);
boolean readSecondBatchOfElements = false;
// run again and verify that we see the other elements
try {
- restoredSourceOperator.open();
+ restoredTestHarness.open();
restoredSourceOperator.run(checkpointLock,
new TestStreamStatusMaintainer(),
new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
@@ -345,7 +329,9 @@ public class UnboundedSourceWrapperTest {
readSecondBatchOfElements = true;
}
- assertEquals(Math.max(1, numSplits / numTasks), flinkWrapper.getLocalSplitSources().size());
+ assertEquals(
+ Math.max(1, numSplits / numTasks),
+ restoredFlinkWrapper.getLocalSplitSources().size());
assertTrue("Did not successfully read second batch of elements.", readSecondBatchOfElements);
@@ -364,68 +350,57 @@ public class UnboundedSourceWrapperTest {
return null;
}
};
+
UnboundedSourceWrapper<KV<Integer, Integer>, TestCountingSource.CounterMark> flinkWrapper =
new UnboundedSourceWrapper<>("stepName", options, source, numSplits);
- OperatorStateStore backend = mock(OperatorStateStore.class);
-
- TestingListState<KV<UnboundedSource, TestCountingSource.CounterMark>>
- listState = new TestingListState<>();
-
- when(backend.getOperatorState(Matchers.any(ListStateDescriptor.class)))
- .thenReturn(listState);
-
- StateInitializationContext initializationContext = mock(StateInitializationContext.class);
-
- when(initializationContext.getOperatorStateStore()).thenReturn(backend);
- when(initializationContext.isRestored()).thenReturn(false, true);
+ StreamSource<
+ WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>,
+ UnboundedSourceWrapper<KV<Integer, Integer>, TestCountingSource.CounterMark>>
+ sourceOperator = new StreamSource<>(flinkWrapper);
- flinkWrapper.initializeState(initializationContext);
+ AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
+ testHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ sourceOperator,
+ numTasks /* max parallelism */,
+ numTasks /* parallelism */,
+ 0 /* subtask index */);
- StreamSource sourceOperator = new StreamSource<>(flinkWrapper);
- setupSourceOperator(sourceOperator, numTasks);
- sourceOperator.open();
+ testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
- flinkWrapper.snapshotState(new StateSnapshotContextSynchronousImpl(0, 0));
+ testHarness.open();
- assertEquals(0, listState.getList().size());
+ OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
UnboundedSourceWrapper<
KV<Integer, Integer>, TestCountingSource.CounterMark> restoredFlinkWrapper =
- new UnboundedSourceWrapper<>("stepName", options, new TestCountingSource(numElements),
- numSplits);
-
- StreamSource restoredSourceOperator = new StreamSource<>(flinkWrapper);
- setupSourceOperator(restoredSourceOperator, numTasks);
- sourceOperator.open();
-
- restoredFlinkWrapper.initializeState(initializationContext);
-
- assertEquals(Math.max(1, numSplits / numTasks), flinkWrapper.getLocalSplitSources().size());
-
- }
+ new UnboundedSourceWrapper<>(
+ "stepName", options, new TestCountingSource(numElements), numSplits);
- @SuppressWarnings("unchecked")
- private static <T> void setupSourceOperator(StreamSource<T, ?> operator, int numSubTasks) {
- ExecutionConfig executionConfig = new ExecutionConfig();
- StreamConfig cfg = new StreamConfig(new Configuration());
+ StreamSource<
+ WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>,
+ UnboundedSourceWrapper<KV<Integer, Integer>, TestCountingSource.CounterMark>>
+ restoredSourceOperator =
+ new StreamSource<>(restoredFlinkWrapper);
- cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
+ // set parallelism to 1 to ensure that our testing operator gets all checkpointed state
+ AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
+ restoredTestHarness =
+ new AbstractStreamOperatorTestHarness<>(
+ restoredSourceOperator,
+ numTasks /* max parallelism */,
+ 1 /* parallelism */,
+ 0 /* subtask index */);
- Environment env = new DummyEnvironment("MockTwoInputTask", numSubTasks, 0);
+ restoredTestHarness.setup();
+ restoredTestHarness.initializeState(snapshot);
+ restoredTestHarness.open();
- StreamTask<?, ?> mockTask = mock(StreamTask.class);
- when(mockTask.getName()).thenReturn("Mock Task");
- when(mockTask.getCheckpointLock()).thenReturn(new Object());
- when(mockTask.getConfiguration()).thenReturn(cfg);
- when(mockTask.getEnvironment()).thenReturn(env);
- when(mockTask.getExecutionConfig()).thenReturn(executionConfig);
- when(mockTask.getAccumulatorMap())
- .thenReturn(Collections.<String, Accumulator<?, ?>>emptyMap());
- TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
- when(mockTask.getProcessingTimeService()).thenReturn(testProcessingTimeService);
+ // when the source checkpointed a null we don't re-initialize the splits, that is we
+ // will have no splits.
+ assertEquals(0, restoredFlinkWrapper.getLocalSplitSources().size());
- operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}
/**
@@ -458,31 +433,6 @@ public class UnboundedSourceWrapperTest {
}
- private static final class TestingListState<T> implements ListState<T> {
-
- private final List<T> list = new ArrayList<>();
-
- @Override
- public void clear() {
- list.clear();
- }
-
- @Override
- public Iterable<T> get() throws Exception {
- return list;
- }
-
- @Override
- public void add(T value) throws Exception {
- list.add(value);
- }
-
- public List<T> getList() {
- return list;
- }
-
- }
-
private static final class TestStreamStatusMaintainer implements StreamStatusMaintainer {
StreamStatus currentStreamStatus = StreamStatus.ACTIVE;
[35/50] [abbrv] beam git commit: [BEAM-2410] This closes #3328
Posted by ke...@apache.org.
[BEAM-2410] This closes #3328
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1c6861f2
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1c6861f2
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1c6861f2
Branch: refs/heads/gearpump-runner
Commit: 1c6861f22249dadacb34994564d33b14f1d57d3d
Parents: e980ae9 7caea7a
Author: Jean-Baptiste Onofré <jb...@apache.org>
Authored: Fri Jun 9 07:31:27 2017 +0200
Committer: Jean-Baptiste Onofré <jb...@apache.org>
Committed: Fri Jun 9 07:31:27 2017 +0200
----------------------------------------------------------------------
.../sdk/io/common/IOTestPipelineOptions.java | 6 +-
.../sdk/io/elasticsearch/ElasticsearchIO.java | 4 +-
.../elasticsearch/ElasticSearchIOTestUtils.java | 81 +++++++++++---------
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 14 ++--
.../io/elasticsearch/ElasticsearchIOTest.java | 36 +++++----
.../elasticsearch/ElasticsearchTestDataSet.java | 37 ++++-----
6 files changed, 87 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
[45/50] [abbrv] beam git commit: Port DirectGroupByKey to
SDK-agnostic APIs
Posted by ke...@apache.org.
Port DirectGroupByKey to SDK-agnostic APIs
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/02dbaefd
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/02dbaefd
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/02dbaefd
Branch: refs/heads/gearpump-runner
Commit: 02dbaefd2bbad0f0ff0b87469d184137b220fae7
Parents: 8c5b57e
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri May 26 14:27:23 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Jun 9 19:56:52 2017 -0700
----------------------------------------------------------------------
.../apache/beam/runners/direct/DirectGroupByKey.java | 13 +++++++------
.../direct/DirectGroupByKeyOverrideFactory.java | 14 +++++++++++---
2 files changed, 18 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/02dbaefd/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
index 2fc0dd4..06b8e29 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKey.java
@@ -36,13 +36,17 @@ import org.apache.beam.sdk.values.WindowingStrategy;
class DirectGroupByKey<K, V>
extends ForwardingPTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> {
- private final GroupByKey<K, V> original;
+ private final PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> original;
static final String DIRECT_GBKO_URN = "urn:beam:directrunner:transforms:gbko:v1";
static final String DIRECT_GABW_URN = "urn:beam:directrunner:transforms:gabw:v1";
+ private final WindowingStrategy<?, ?> outputWindowingStrategy;
- DirectGroupByKey(GroupByKey<K, V> from) {
- this.original = from;
+ DirectGroupByKey(
+ PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> original,
+ WindowingStrategy<?, ?> outputWindowingStrategy) {
+ this.original = original;
+ this.outputWindowingStrategy = outputWindowingStrategy;
}
@Override
@@ -57,9 +61,6 @@ class DirectGroupByKey<K, V>
// key/value input elements and the window merge operation of the
// window function associated with the input PCollection.
WindowingStrategy<?, ?> inputWindowingStrategy = input.getWindowingStrategy();
- // Update the windowing strategy as appropriate.
- WindowingStrategy<?, ?> outputWindowingStrategy =
- original.updateWindowingStrategy(inputWindowingStrategy);
// By default, implement GroupByKey via a series of lower-level operations.
return input
http://git-wip-us.apache.org/repos/asf/beam/blob/02dbaefd/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
index c2eb5e7..9c2de3d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGroupByKeyOverrideFactory.java
@@ -17,26 +17,34 @@
*/
package org.apache.beam.runners.direct;
+import com.google.common.collect.Iterables;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
/** A {@link PTransformOverrideFactory} for {@link GroupByKey} PTransforms. */
final class DirectGroupByKeyOverrideFactory<K, V>
extends SingleInputOutputOverrideFactory<
- PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>, GroupByKey<K, V>> {
+ PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>,
+ PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>>> {
@Override
public PTransformReplacement<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>>
getReplacementTransform(
AppliedPTransform<
- PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>, GroupByKey<K, V>>
+ PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>,
+ PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>>>
transform) {
+
+ PCollection<KV<K, Iterable<V>>> output =
+ (PCollection<KV<K, Iterable<V>>>) Iterables.getOnlyElement(transform.getOutputs().values());
+
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new DirectGroupByKey<>(transform.getTransform()));
+ new DirectGroupByKey<>(transform.getTransform(), output.getWindowingStrategy()));
}
}
[02/50] [abbrv] beam git commit: This closes #2286
Posted by ke...@apache.org.
This closes #2286
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/609016d7
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/609016d7
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/609016d7
Branch: refs/heads/gearpump-runner
Commit: 609016d700c84800cf942482fb7cd2ddaa420b00
Parents: b2de3db 8f4fa43
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Wed Jun 7 19:43:19 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Wed Jun 7 19:43:19 2017 +0200
----------------------------------------------------------------------
.../translation/types/CoderTypeSerializer.java | 41 ++-
.../streaming/io/UnboundedSourceWrapper.java | 2 +
.../flink/streaming/TestCountingSource.java | 48 ++-
.../streaming/UnboundedSourceWrapperTest.java | 309 +++++++++++--------
4 files changed, 254 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
[42/50] [abbrv] beam git commit: Roll-forward Include Additional
PTransform inputs in Transform Nodes
Posted by ke...@apache.org.
Roll-forward Include Additional PTransform inputs in Transform Nodes
Update DirectGraph to have All and Non-Additional Inputs
This reverts commit 247f9bc1581984d026764b3d433cb594e700bc21
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/696f8b28
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/696f8b28
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/696f8b28
Branch: refs/heads/gearpump-runner
Commit: 696f8b28a3a17e7de81e2d46bb9774d57d6e265e
Parents: 33883ed
Author: Thomas Groh <tg...@google.com>
Authored: Tue Jun 6 17:00:09 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Fri Jun 9 15:00:33 2017 -0700
----------------------------------------------------------------------
.../apex/translation/TranslationContext.java | 4 +-
.../core/construction/TransformInputs.java | 50 ++++++
.../core/construction/TransformInputsTest.java | 166 +++++++++++++++++++
.../apache/beam/runners/direct/DirectGraph.java | 34 +++-
.../beam/runners/direct/DirectGraphVisitor.java | 26 ++-
.../direct/ExecutorServiceParallelExecutor.java | 2 +-
.../runners/direct/ParDoEvaluatorFactory.java | 9 +-
...littableProcessElementsEvaluatorFactory.java | 2 +
.../direct/StatefulParDoEvaluatorFactory.java | 1 +
.../beam/runners/direct/WatermarkManager.java | 14 +-
.../runners/direct/DirectGraphVisitorTest.java | 10 +-
.../runners/direct/EvaluationContextTest.java | 2 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 6 +-
.../flink/FlinkBatchTranslationContext.java | 3 +-
.../flink/FlinkStreamingTranslationContext.java | 3 +-
.../dataflow/DataflowPipelineTranslator.java | 5 +-
.../spark/translation/EvaluationContext.java | 4 +-
.../beam/sdk/runners/TransformHierarchy.java | 28 +++-
18 files changed, 323 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
index aff3863..94d13e1 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/TranslationContext.java
@@ -34,6 +34,7 @@ import org.apache.beam.runners.apex.translation.utils.ApexStateInternals;
import org.apache.beam.runners.apex.translation.utils.ApexStateInternals.ApexStateBackend;
import org.apache.beam.runners.apex.translation.utils.ApexStreamTuple;
import org.apache.beam.runners.apex.translation.utils.CoderAdapterStreamCodec;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
@@ -93,7 +94,8 @@ class TranslationContext {
}
public <InputT extends PValue> InputT getInput() {
- return (InputT) Iterables.getOnlyElement(getCurrentTransform().getInputs().values());
+ return (InputT)
+ Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
}
public Map<TupleTag<?>, PValue> getOutputs() {
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
new file mode 100644
index 0000000..2baf93a
--- /dev/null
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformInputs.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.ImmutableList;
+import java.util.Collection;
+import java.util.Map;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Utilities for extracting subsets of inputs from an {@link AppliedPTransform}. */
+public class TransformInputs {
+ /**
+ * Gets all inputs of the {@link AppliedPTransform} that are not returned by {@link
+ * PTransform#getAdditionalInputs()}.
+ */
+ public static Collection<PValue> nonAdditionalInputs(AppliedPTransform<?, ?, ?> application) {
+ ImmutableList.Builder<PValue> mainInputs = ImmutableList.builder();
+ PTransform<?, ?> transform = application.getTransform();
+ for (Map.Entry<TupleTag<?>, PValue> input : application.getInputs().entrySet()) {
+ if (!transform.getAdditionalInputs().containsKey(input.getKey())) {
+ mainInputs.add(input.getValue());
+ }
+ }
+ checkArgument(
+ !mainInputs.build().isEmpty() || application.getInputs().isEmpty(),
+ "Expected at least one main input if any inputs exist");
+ return mainInputs.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
new file mode 100644
index 0000000..f5b2c11
--- /dev/null
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/TransformInputsTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.core.construction;
+
+import static org.junit.Assert.assertThat;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.PInput;
+import org.apache.beam.sdk.values.POutput;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.hamcrest.Matchers;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link TransformInputs}. */
+@RunWith(JUnit4.class)
+public class TransformInputsTest {
+ @Rule public TestPipeline pipeline = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void nonAdditionalInputsWithNoInputSucceeds() {
+ AppliedPTransform<PInput, POutput, TestTransform> transform =
+ AppliedPTransform.of(
+ "input-free",
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ new TestTransform(),
+ pipeline);
+
+ assertThat(TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>empty());
+ }
+
+ @Test
+ public void nonAdditionalInputsWithOneMainInputSucceeds() {
+ PCollection<Long> input = pipeline.apply(GenerateSequence.from(1L));
+ AppliedPTransform<PInput, POutput, TestTransform> transform =
+ AppliedPTransform.of(
+ "input-single",
+ Collections.<TupleTag<?>, PValue>singletonMap(new TupleTag<Long>() {}, input),
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ new TestTransform(),
+ pipeline);
+
+ assertThat(
+ TransformInputs.nonAdditionalInputs(transform), Matchers.<PValue>containsInAnyOrder(input));
+ }
+
+ @Test
+ public void nonAdditionalInputsWithMultipleNonAdditionalInputsSucceeds() {
+ Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
+ PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
+ allInputs.put(new TupleTag<Integer>() {}, mainInts);
+ PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
+ allInputs.put(new TupleTag<Void>() {}, voids);
+ AppliedPTransform<PInput, POutput, TestTransform> transform =
+ AppliedPTransform.of(
+ "additional-free",
+ allInputs,
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ new TestTransform(),
+ pipeline);
+
+ assertThat(
+ TransformInputs.nonAdditionalInputs(transform),
+ Matchers.<PValue>containsInAnyOrder(voids, mainInts));
+ }
+
+ @Test
+ public void nonAdditionalInputsWithAdditionalInputsSucceeds() {
+ Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
+ additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
+ additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
+
+ Map<TupleTag<?>, PValue> allInputs = new HashMap<>();
+ PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3));
+ allInputs.put(new TupleTag<Integer>() {}, mainInts);
+ PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of()));
+ allInputs.put(
+ new TupleTag<Void>() {}, voids);
+ allInputs.putAll(additionalInputs);
+
+ AppliedPTransform<PInput, POutput, TestTransform> transform =
+ AppliedPTransform.of(
+ "additional",
+ allInputs,
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ new TestTransform(additionalInputs),
+ pipeline);
+
+ assertThat(
+ TransformInputs.nonAdditionalInputs(transform),
+ Matchers.<PValue>containsInAnyOrder(mainInts, voids));
+ }
+
+ @Test
+ public void nonAdditionalInputsWithOnlyAdditionalInputsThrows() {
+ Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>();
+ additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3")));
+ additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L)));
+
+ AppliedPTransform<PInput, POutput, TestTransform> transform =
+ AppliedPTransform.of(
+ "additional-only",
+ additionalInputs,
+ Collections.<TupleTag<?>, PValue>emptyMap(),
+ new TestTransform(additionalInputs),
+ pipeline);
+
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("at least one");
+ TransformInputs.nonAdditionalInputs(transform);
+ }
+
+ private static class TestTransform extends PTransform<PInput, POutput> {
+ private final Map<TupleTag<?>, PValue> additionalInputs;
+
+ private TestTransform() {
+ this(Collections.<TupleTag<?>, PValue>emptyMap());
+ }
+
+ private TestTransform(Map<TupleTag<?>, PValue> additionalInputs) {
+ this.additionalInputs = additionalInputs;
+ }
+
+ @Override
+ public POutput expand(PInput input) {
+ return PDone.in(input.getPipeline());
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return additionalInputs;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
index 9ca745d..ad17b2b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
@@ -17,6 +17,8 @@
*/
package org.apache.beam.runners.direct;
+import static com.google.common.base.Preconditions.checkArgument;
+
import com.google.common.collect.ListMultimap;
import java.util.Collection;
import java.util.List;
@@ -36,7 +38,8 @@ import org.apache.beam.sdk.values.PValue;
class DirectGraph {
private final Map<PCollection<?>, AppliedPTransform<?, ?, ?>> producers;
private final Map<PCollectionView<?>, AppliedPTransform<?, ?, ?>> viewWriters;
- private final ListMultimap<PInput, AppliedPTransform<?, ?, ?>> primitiveConsumers;
+ private final ListMultimap<PInput, AppliedPTransform<?, ?, ?>> perElementConsumers;
+ private final ListMultimap<PValue, AppliedPTransform<?, ?, ?>> allConsumers;
private final Set<AppliedPTransform<?, ?, ?>> rootTransforms;
private final Map<AppliedPTransform<?, ?, ?>, String> stepNames;
@@ -44,23 +47,36 @@ class DirectGraph {
public static DirectGraph create(
Map<PCollection<?>, AppliedPTransform<?, ?, ?>> producers,
Map<PCollectionView<?>, AppliedPTransform<?, ?, ?>> viewWriters,
- ListMultimap<PInput, AppliedPTransform<?, ?, ?>> primitiveConsumers,
+ ListMultimap<PInput, AppliedPTransform<?, ?, ?>> perElementConsumers,
+ ListMultimap<PValue, AppliedPTransform<?, ?, ?>> allConsumers,
Set<AppliedPTransform<?, ?, ?>> rootTransforms,
Map<AppliedPTransform<?, ?, ?>, String> stepNames) {
- return new DirectGraph(producers, viewWriters, primitiveConsumers, rootTransforms, stepNames);
+ return new DirectGraph(
+ producers, viewWriters, perElementConsumers, allConsumers, rootTransforms, stepNames);
}
private DirectGraph(
Map<PCollection<?>, AppliedPTransform<?, ?, ?>> producers,
Map<PCollectionView<?>, AppliedPTransform<?, ?, ?>> viewWriters,
- ListMultimap<PInput, AppliedPTransform<?, ?, ?>> primitiveConsumers,
+ ListMultimap<PInput, AppliedPTransform<?, ?, ?>> perElementConsumers,
+ ListMultimap<PValue, AppliedPTransform<?, ?, ?>> allConsumers,
Set<AppliedPTransform<?, ?, ?>> rootTransforms,
Map<AppliedPTransform<?, ?, ?>, String> stepNames) {
this.producers = producers;
this.viewWriters = viewWriters;
- this.primitiveConsumers = primitiveConsumers;
+ this.perElementConsumers = perElementConsumers;
+ this.allConsumers = allConsumers;
this.rootTransforms = rootTransforms;
this.stepNames = stepNames;
+ for (AppliedPTransform<?, ?, ?> step : stepNames.keySet()) {
+ for (PValue input : step.getInputs().values()) {
+ checkArgument(
+ allConsumers.get(input).contains(step),
+ "Step %s lists value %s as input, but it is not in the graph of consumers",
+ step.getFullName(),
+ input);
+ }
+ }
}
AppliedPTransform<?, ?, ?> getProducer(PCollection<?> produced) {
@@ -71,8 +87,12 @@ class DirectGraph {
return viewWriters.get(view);
}
- List<AppliedPTransform<?, ?, ?>> getPrimitiveConsumers(PValue consumed) {
- return primitiveConsumers.get(consumed);
+ List<AppliedPTransform<?, ?, ?>> getPerElementConsumers(PValue consumed) {
+ return perElementConsumers.get(consumed);
+ }
+
+ List<AppliedPTransform<?, ?, ?>> getAllConsumers(PValue consumed) {
+ return allConsumers.get(consumed);
}
Set<AppliedPTransform<?, ?, ?>> getRootTransforms() {
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
index 07bcf06..675de2c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
@@ -22,10 +22,12 @@ import static com.google.common.base.Preconditions.checkState;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Sets;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.direct.ViewOverrideFactory.WriteView;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
@@ -37,6 +39,8 @@ import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.PValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Tracks the {@link AppliedPTransform AppliedPTransforms} that consume each {@link PValue} in the
@@ -44,12 +48,15 @@ import org.apache.beam.sdk.values.PValue;
* input after the upstream transform has produced and committed output.
*/
class DirectGraphVisitor extends PipelineVisitor.Defaults {
+ private static final Logger LOG = LoggerFactory.getLogger(DirectGraphVisitor.class);
private Map<PCollection<?>, AppliedPTransform<?, ?, ?>> producers = new HashMap<>();
private Map<PCollectionView<?>, AppliedPTransform<?, ?, ?>> viewWriters = new HashMap<>();
private Set<PCollectionView<?>> consumedViews = new HashSet<>();
- private ListMultimap<PInput, AppliedPTransform<?, ?, ?>> primitiveConsumers =
+ private ListMultimap<PInput, AppliedPTransform<?, ?, ?>> perElementConsumers =
+ ArrayListMultimap.create();
+ private ListMultimap<PValue, AppliedPTransform<?, ?, ?>> allConsumers =
ArrayListMultimap.create();
private Set<AppliedPTransform<?, ?, ?>> rootTransforms = new HashSet<>();
@@ -94,8 +101,19 @@ class DirectGraphVisitor extends PipelineVisitor.Defaults {
if (node.getInputs().isEmpty()) {
rootTransforms.add(appliedTransform);
} else {
+ Collection<PValue> mainInputs =
+ TransformInputs.nonAdditionalInputs(node.toAppliedPTransform(getPipeline()));
+ if (!mainInputs.containsAll(node.getInputs().values())) {
+ LOG.debug(
+ "Inputs reduced to {} from {} by removing additional inputs",
+ mainInputs,
+ node.getInputs().values());
+ }
+ for (PValue value : mainInputs) {
+ perElementConsumers.put(value, appliedTransform);
+ }
for (PValue value : node.getInputs().values()) {
- primitiveConsumers.put(value, appliedTransform);
+ allConsumers.put(value, appliedTransform);
}
}
if (node.getTransform() instanceof ParDo.MultiOutput) {
@@ -106,7 +124,7 @@ class DirectGraphVisitor extends PipelineVisitor.Defaults {
}
}
- @Override
+ @Override
public void visitValue(PValue value, TransformHierarchy.Node producer) {
AppliedPTransform<?, ?, ?> appliedTransform = getAppliedTransform(producer);
if (value instanceof PCollection && !producers.containsKey(value)) {
@@ -131,6 +149,6 @@ class DirectGraphVisitor extends PipelineVisitor.Defaults {
public DirectGraph getGraph() {
checkState(finalized, "Can't get a graph before the Pipeline has been completely traversed");
return DirectGraph.create(
- producers, viewWriters, primitiveConsumers, rootTransforms, stepNames);
+ producers, viewWriters, perElementConsumers, allConsumers, rootTransforms, stepNames);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
index 71ab4cc..6fe8ebd 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
@@ -355,7 +355,7 @@ final class ExecutorServiceParallelExecutor implements PipelineExecutor {
for (CommittedBundle<?> outputBundle : committedResult.getOutputs()) {
allUpdates.offer(
ExecutorUpdate.fromBundle(
- outputBundle, graph.getPrimitiveConsumers(outputBundle.getPCollection())));
+ outputBundle, graph.getPerElementConsumers(outputBundle.getPCollection())));
}
CommittedBundle<?> unprocessedInputs = committedResult.getUnprocessedInputs();
if (unprocessedInputs != null && !Iterables.isEmpty(unprocessedInputs.getElements())) {
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
index 8aa75cf..516f798 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ParDoEvaluatorFactory.java
@@ -20,7 +20,6 @@ package org.apache.beam.runners.direct;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Iterables;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -79,6 +78,7 @@ final class ParDoEvaluatorFactory<InputT, OutputT> implements TransformEvaluator
(TransformEvaluator<T>)
createEvaluator(
(AppliedPTransform) application,
+ (PCollection<InputT>) inputBundle.getPCollection(),
inputBundle.getKey(),
doFn,
transform.getSideInputs(),
@@ -102,6 +102,7 @@ final class ParDoEvaluatorFactory<InputT, OutputT> implements TransformEvaluator
@SuppressWarnings({"unchecked", "rawtypes"})
DoFnLifecycleManagerRemovingTransformEvaluator<InputT> createEvaluator(
AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
+ PCollection<InputT> mainInput,
StructuralKey<?> inputBundleKey,
DoFn<InputT, OutputT> doFn,
List<PCollectionView<?>> sideInputs,
@@ -120,6 +121,7 @@ final class ParDoEvaluatorFactory<InputT, OutputT> implements TransformEvaluator
createParDoEvaluator(
application,
inputBundleKey,
+ mainInput,
sideInputs,
mainOutputTag,
additionalOutputTags,
@@ -132,6 +134,7 @@ final class ParDoEvaluatorFactory<InputT, OutputT> implements TransformEvaluator
ParDoEvaluator<InputT> createParDoEvaluator(
AppliedPTransform<PCollection<InputT>, PCollectionTuple, ?> application,
StructuralKey<?> key,
+ PCollection<InputT> mainInput,
List<PCollectionView<?>> sideInputs,
TupleTag<OutputT> mainOutputTag,
List<TupleTag<?>> additionalOutputTags,
@@ -144,8 +147,7 @@ final class ParDoEvaluatorFactory<InputT, OutputT> implements TransformEvaluator
evaluationContext,
stepContext,
application,
- ((PCollection<InputT>) Iterables.getOnlyElement(application.getInputs().values()))
- .getWindowingStrategy(),
+ mainInput.getWindowingStrategy(),
fn,
key,
sideInputs,
@@ -173,5 +175,4 @@ final class ParDoEvaluatorFactory<InputT, OutputT> implements TransformEvaluator
}
return pcs;
}
-
}
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
index b85f481c..eccc83a 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/SplittableProcessElementsEvaluatorFactory.java
@@ -116,6 +116,8 @@ class SplittableProcessElementsEvaluatorFactory<
delegateFactory.createParDoEvaluator(
application,
inputBundle.getKey(),
+ (PCollection<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>)
+ inputBundle.getPCollection(),
transform.getSideInputs(),
transform.getMainOutputTag(),
transform.getAdditionalOutputTags().getAll(),
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
index 506c84c..3619d05 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/StatefulParDoEvaluatorFactory.java
@@ -117,6 +117,7 @@ final class StatefulParDoEvaluatorFactory<K, InputT, OutputT> implements Transfo
DoFnLifecycleManagerRemovingTransformEvaluator<KV<K, InputT>> delegateEvaluator =
delegateFactory.createEvaluator(
(AppliedPTransform) application,
+ (PCollection) inputBundle.getPCollection(),
inputBundle.getKey(),
doFn,
application.getTransform().getUnderlyingParDo().getSideInputs(),
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index 40ce163..80a3504 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -54,6 +54,7 @@ import javax.annotation.concurrent.GuardedBy;
import org.apache.beam.runners.core.StateNamespace;
import org.apache.beam.runners.core.TimerInternals;
import org.apache.beam.runners.core.TimerInternals.TimerData;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.state.TimeDomain;
@@ -62,7 +63,6 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TupleTag;
import org.joda.time.Instant;
/**
@@ -831,11 +831,11 @@ class WatermarkManager {
private Collection<Watermark> getInputProcessingWatermarks(AppliedPTransform<?, ?, ?> transform) {
ImmutableList.Builder<Watermark> inputWmsBuilder = ImmutableList.builder();
- Map<TupleTag<?>, PValue> inputs = transform.getInputs();
+ Collection<PValue> inputs = TransformInputs.nonAdditionalInputs(transform);
if (inputs.isEmpty()) {
inputWmsBuilder.add(THE_END_OF_TIME);
}
- for (PValue pvalue : inputs.values()) {
+ for (PValue pvalue : inputs) {
Watermark producerOutputWatermark =
getValueWatermark(pvalue).synchronizedProcessingOutputWatermark;
inputWmsBuilder.add(producerOutputWatermark);
@@ -845,11 +845,11 @@ class WatermarkManager {
private List<Watermark> getInputWatermarks(AppliedPTransform<?, ?, ?> transform) {
ImmutableList.Builder<Watermark> inputWatermarksBuilder = ImmutableList.builder();
- Map<TupleTag<?>, PValue> inputs = transform.getInputs();
+ Collection< PValue> inputs = TransformInputs.nonAdditionalInputs(transform);
if (inputs.isEmpty()) {
inputWatermarksBuilder.add(THE_END_OF_TIME);
}
- for (PValue pvalue : inputs.values()) {
+ for (PValue pvalue : inputs) {
Watermark producerOutputWatermark = getValueWatermark(pvalue).outputWatermark;
inputWatermarksBuilder.add(producerOutputWatermark);
}
@@ -987,7 +987,7 @@ class WatermarkManager {
// refresh.
for (CommittedBundle<?> bundle : result.getOutputs()) {
for (AppliedPTransform<?, ?, ?> consumer :
- graph.getPrimitiveConsumers(bundle.getPCollection())) {
+ graph.getPerElementConsumers(bundle.getPCollection())) {
TransformWatermarks watermarks = transformToWatermarks.get(consumer);
watermarks.addPending(bundle);
}
@@ -1035,7 +1035,7 @@ class WatermarkManager {
if (updateResult.isAdvanced()) {
Set<AppliedPTransform<?, ?, ?>> additionalRefreshes = new HashSet<>();
for (PValue outputPValue : toRefresh.getOutputs().values()) {
- additionalRefreshes.addAll(graph.getPrimitiveConsumers(outputPValue));
+ additionalRefreshes.addAll(graph.getPerElementConsumers(outputPValue));
}
return additionalRefreshes;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
index 576edf3..bf3e83e 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
@@ -151,13 +151,13 @@ public class DirectGraphVisitorTest implements Serializable {
graph.getProducer(flattened);
assertThat(
- graph.getPrimitiveConsumers(created),
+ graph.getPerElementConsumers(created),
Matchers.<AppliedPTransform<?, ?, ?>>containsInAnyOrder(
transformedProducer, flattenedProducer));
assertThat(
- graph.getPrimitiveConsumers(transformed),
+ graph.getPerElementConsumers(transformed),
Matchers.<AppliedPTransform<?, ?, ?>>containsInAnyOrder(flattenedProducer));
- assertThat(graph.getPrimitiveConsumers(flattened), emptyIterable());
+ assertThat(graph.getPerElementConsumers(flattened), emptyIterable());
}
@Test
@@ -173,10 +173,10 @@ public class DirectGraphVisitorTest implements Serializable {
AppliedPTransform<?, ?, ?> flattenedProducer = graph.getProducer(flattened);
assertThat(
- graph.getPrimitiveConsumers(created),
+ graph.getPerElementConsumers(created),
Matchers.<AppliedPTransform<?, ?, ?>>containsInAnyOrder(flattenedProducer,
flattenedProducer));
- assertThat(graph.getPrimitiveConsumers(flattened), emptyIterable());
+ assertThat(graph.getPerElementConsumers(flattened), emptyIterable());
}
@Test
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index f3edf55..699a318 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -414,7 +414,7 @@ public class EvaluationContextTest {
StepTransformResult.withoutHold(unboundedProducer).build());
assertThat(context.isDone(), is(false));
- for (AppliedPTransform<?, ?, ?> consumers : graph.getPrimitiveConsumers(created)) {
+ for (AppliedPTransform<?, ?, ?> consumers : graph.getPerElementConsumers(created)) {
context.handleResult(
committedBundle,
ImmutableList.<TimerData>of(),
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index df84cbf..7912538 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -98,7 +98,7 @@ public class ParDoEvaluatorTest {
when(evaluationContext.createBundle(output)).thenReturn(outputBundle);
ParDoEvaluator<Integer> evaluator =
- createEvaluator(singletonView, fn, output);
+ createEvaluator(singletonView, fn, inputPc, output);
IntervalWindow nonGlobalWindow = new IntervalWindow(new Instant(0), new Instant(10_000L));
WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(3);
@@ -132,6 +132,7 @@ public class ParDoEvaluatorTest {
private ParDoEvaluator<Integer> createEvaluator(
PCollectionView<Integer> singletonView,
RecorderFn fn,
+ PCollection<Integer> input,
PCollection<Integer> output) {
when(
evaluationContext.createSideInputReader(
@@ -157,8 +158,7 @@ public class ParDoEvaluatorTest {
evaluationContext,
stepContext,
transform,
- ((PCollection<?>) Iterables.getOnlyElement(transform.getInputs().values()))
- .getWindowingStrategy(),
+ input.getWindowingStrategy(),
fn,
null /* key */,
ImmutableList.<PCollectionView<?>>of(singletonView),
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
index 0439119..6e70198 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
@@ -20,6 +20,7 @@ package org.apache.beam.runners.flink;
import com.google.common.collect.Iterables;
import java.util.HashMap;
import java.util.Map;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -143,7 +144,7 @@ class FlinkBatchTranslationContext {
@SuppressWarnings("unchecked")
<T extends PValue> T getInput(PTransform<T, ?> transform) {
- return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
+ return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
}
Map<TupleTag<?>, PValue> getOutputs(PTransform<?, ?> transform) {
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
index ea5f6b3..74a5fb9 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTranslationContext.java
@@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.collect.Iterables;
import java.util.HashMap;
import java.util.Map;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -113,7 +114,7 @@ class FlinkStreamingTranslationContext {
@SuppressWarnings("unchecked")
public <T extends PValue> T getInput(PTransform<T, ?> transform) {
- return (T) Iterables.getOnlyElement(currentTransform.getInputs().values());
+ return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform));
}
public <T extends PInput> Map<TupleTag<?>, PValue> getInputs(PTransform<T, ?> transform) {
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index a3a7ab6..afc34e6 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -56,6 +56,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.core.construction.WindowingStrategyTranslation;
import org.apache.beam.runners.dataflow.BatchViewOverrides.GroupByKeyAndSortValuesOnly;
import org.apache.beam.runners.dataflow.DataflowRunner.CombineGroupedValues;
@@ -395,7 +396,9 @@ public class DataflowPipelineTranslator {
@Override
public <InputT extends PValue> InputT getInput(PTransform<InputT, ?> transform) {
- return (InputT) Iterables.getOnlyElement(getInputs(transform).values());
+ return (InputT)
+ Iterables.getOnlyElement(
+ TransformInputs.nonAdditionalInputs(getCurrentTransform(transform)));
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 8102926..0c6c4d1 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -26,6 +26,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.beam.runners.core.construction.TransformInputs;
import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.coders.CoderHelpers;
import org.apache.beam.sdk.Pipeline;
@@ -103,7 +104,8 @@ public class EvaluationContext {
public <T extends PValue> T getInput(PTransform<T, ?> transform) {
@SuppressWarnings("unchecked")
- T input = (T) Iterables.getOnlyElement(getInputs(transform).values());
+ T input =
+ (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
return input;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/696f8b28/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index 5e048eb..9c5f148 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -34,7 +34,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import javax.annotation.Nullable;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.Pipeline.PipelineVisitor.CompositeBehavior;
@@ -71,7 +70,7 @@ public class TransformHierarchy {
producers = new HashMap<>();
producerInput = new HashMap<>();
unexpandedInputs = new HashMap<>();
- root = new Node(null, null, "", null);
+ root = new Node();
current = root;
}
@@ -297,25 +296,36 @@ public class TransformHierarchy {
boolean finishedSpecifying = false;
/**
+ * Creates the root-level node. The root level node has a null enclosing node, a null transform,
+ * an empty map of inputs, and a name equal to the empty string.
+ */
+ private Node() {
+ this.enclosingNode = null;
+ this.transform = null;
+ this.fullName = "";
+ this.inputs = Collections.emptyMap();
+ }
+
+ /**
* Creates a new Node with the given parent and transform.
*
- * <p>EnclosingNode and transform may both be null for a root-level node, which holds all other
- * nodes.
- *
* @param enclosingNode the composite node containing this node
* @param transform the PTransform tracked by this node
* @param fullName the fully qualified name of the transform
* @param input the unexpanded input to the transform
*/
private Node(
- @Nullable Node enclosingNode,
- @Nullable PTransform<?, ?> transform,
+ Node enclosingNode,
+ PTransform<?, ?> transform,
String fullName,
- @Nullable PInput input) {
+ PInput input) {
this.enclosingNode = enclosingNode;
this.transform = transform;
this.fullName = fullName;
- this.inputs = input == null ? Collections.<TupleTag<?>, PValue>emptyMap() : input.expand();
+ ImmutableMap.Builder<TupleTag<?>, PValue> inputs = ImmutableMap.builder();
+ inputs.putAll(input.expand());
+ inputs.putAll(transform.getAdditionalInputs());
+ this.inputs = inputs.build();
}
/**
[24/50] [abbrv] beam git commit: Add Runner API oriented
PTransformMatchers for DirectRunner overrides
Posted by ke...@apache.org.
Add Runner API oriented PTransformMatchers for DirectRunner overrides
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ca7b9c28
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ca7b9c28
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ca7b9c28
Branch: refs/heads/gearpump-runner
Commit: ca7b9c288151d318898ab000b91d26fcf62046ca
Parents: b53e6f0
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu May 25 06:29:09 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu Jun 8 11:36:28 2017 -0700
----------------------------------------------------------------------
.../core/construction/PTransformMatchers.java | 94 +++++++++++++++++++-
.../construction/PTransformTranslation.java | 7 +-
.../construction/PTransformMatchersTest.java | 32 +++++++
3 files changed, 128 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/ca7b9c28/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
index bfe24a0..c339891 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
@@ -18,6 +18,7 @@
package org.apache.beam.runners.core.construction;
import com.google.common.base.MoreObjects;
+import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.apache.beam.sdk.annotations.Experimental;
@@ -50,6 +51,34 @@ public class PTransformMatchers {
private PTransformMatchers() {}
/**
+ * Returns a {@link PTransformMatcher} that matches a {@link PTransform} if the URN of the
+ * {@link PTransform} is equal to the URN provided ot this matcher.
+ */
+ public static PTransformMatcher urnEqualTo(String urn) {
+ return new EqualUrnPTransformMatcher(urn);
+ }
+
+ private static class EqualUrnPTransformMatcher implements PTransformMatcher {
+ private final String urn;
+
+ private EqualUrnPTransformMatcher(String urn) {
+ this.urn = urn;
+ }
+
+ @Override
+ public boolean matches(AppliedPTransform<?, ?, ?> application) {
+ return urn.equals(PTransformTranslation.urnForTransformOrNull(application.getTransform()));
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("urn", urn)
+ .toString();
+ }
+ }
+
+ /**
* Returns a {@link PTransformMatcher} that matches a {@link PTransform} if the class of the
* {@link PTransform} is equal to the {@link Class} provided ot this matcher.
*/
@@ -151,6 +180,68 @@ public class PTransformMatchers {
}
/**
+ * A {@link PTransformMatcher} that matches a {@link ParDo} by URN if it has a splittable {@link
+ * DoFn}.
+ */
+ public static PTransformMatcher splittableParDo() {
+ return new PTransformMatcher() {
+ @Override
+ public boolean matches(AppliedPTransform<?, ?, ?> application) {
+ if (PTransformTranslation.PAR_DO_TRANSFORM_URN.equals(
+ PTransformTranslation.urnForTransformOrNull(application.getTransform()))) {
+
+ try {
+ return ParDoTranslation.isSplittable(application);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format(
+ "Transform with URN %s could not be translated",
+ PTransformTranslation.PAR_DO_TRANSFORM_URN),
+ e);
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper("SplittableParDoMultiMatcher").toString();
+ }
+ };
+ }
+
+ /**
+ * A {@link PTransformMatcher} that matches a {@link ParDo} transform by URN
+ * and whether it contains state or timers as specified by {@link ParDoTranslation}.
+ */
+ public static PTransformMatcher stateOrTimerParDo() {
+ return new PTransformMatcher() {
+ @Override
+ public boolean matches(AppliedPTransform<?, ?, ?> application) {
+ if (PTransformTranslation.PAR_DO_TRANSFORM_URN.equals(
+ PTransformTranslation.urnForTransformOrNull(application.getTransform()))) {
+
+ try {
+ return ParDoTranslation.usesStateOrTimers(application);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format(
+ "Transform with URN %s could not be translated",
+ PTransformTranslation.PAR_DO_TRANSFORM_URN),
+ e);
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper("StateOrTimerParDoMatcher").toString();
+ }
+ };
+ }
+
+ /**
* A {@link PTransformMatcher} that matches a {@link ParDo.MultiOutput} containing a {@link DoFn}
* that uses state or timers, as specified by {@link DoFnSignature#usesState()} and
* {@link DoFnSignature#usesTimers()}.
@@ -268,7 +359,8 @@ public class PTransformMatchers {
return new PTransformMatcher() {
@Override
public boolean matches(AppliedPTransform<?, ?, ?> application) {
- if (application.getTransform() instanceof WriteFiles) {
+ if (PTransformTranslation.WRITE_FILES_TRANSFORM_URN.equals(
+ PTransformTranslation.urnForTransformOrNull(application.getTransform()))) {
WriteFiles write = (WriteFiles) application.getTransform();
return write.getSharding() == null && write.getNumShards() == null;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ca7b9c28/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
index 32ecf43..bae7b05 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
@@ -179,13 +179,12 @@ public class PTransformTranslation {
* Returns the URN for the transform if it is known, otherwise throws.
*/
public static String urnForTransform(PTransform<?, ?> transform) {
- TransformPayloadTranslator translator = KNOWN_PAYLOAD_TRANSLATORS.get(transform.getClass());
- if (translator == null) {
+ String urn = urnForTransformOrNull(transform);
+ if (urn == null) {
throw new IllegalStateException(
String.format("No translator known for %s", transform.getClass().getName()));
}
-
- return translator.getUrn(transform);
+ return urn;
}
/**
http://git-wip-us.apache.org/repos/asf/beam/blob/ca7b9c28/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
index 2497598..6459849 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PTransformMatchersTest.java
@@ -27,6 +27,8 @@ import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableMap;
import java.io.Serializable;
import java.util.Collections;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.io.DefaultFilenamePolicy;
@@ -95,9 +97,14 @@ public class PTransformMatchersTest implements Serializable {
PCollection<KV<String, Integer>> input =
PCollection.createPrimitiveOutputInternal(
p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ input.setName("dummy input");
+ input.setCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
+
PCollection<Integer> output =
PCollection.createPrimitiveOutputInternal(
p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED);
+ output.setName("dummy output");
+ output.setCoder(VarIntCoder.of());
return AppliedPTransform.of("pardo", input.expand(), output.expand(), pardo, p);
}
@@ -272,6 +279,18 @@ public class PTransformMatchersTest implements Serializable {
}
@Test
+ public void parDoSplittable() {
+ AppliedPTransform<?, ?, ?> parDoApplication =
+ getAppliedTransform(
+ ParDo.of(splittableDoFn).withOutputTags(new TupleTag<Integer>(), TupleTagList.empty()));
+ assertThat(PTransformMatchers.splittableParDo().matches(parDoApplication), is(true));
+
+ assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false));
+ assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false));
+ assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false));
+ }
+
+ @Test
public void parDoMultiWithState() {
AppliedPTransform<?, ?, ?> parDoApplication =
getAppliedTransform(
@@ -284,6 +303,19 @@ public class PTransformMatchersTest implements Serializable {
}
@Test
+ public void parDoWithState() {
+ AppliedPTransform<?, ?, ?> statefulApplication =
+ getAppliedTransform(
+ ParDo.of(doFnWithState).withOutputTags(new TupleTag<Integer>(), TupleTagList.empty()));
+ assertThat(PTransformMatchers.stateOrTimerParDo().matches(statefulApplication), is(true));
+
+ AppliedPTransform<?, ?, ?> splittableApplication =
+ getAppliedTransform(
+ ParDo.of(splittableDoFn).withOutputTags(new TupleTag<Integer>(), TupleTagList.empty()));
+ assertThat(PTransformMatchers.stateOrTimerParDo().matches(splittableApplication), is(false));
+ }
+
+ @Test
public void parDoMultiWithTimers() {
AppliedPTransform<?, ?, ?> parDoApplication =
getAppliedTransform(
[28/50] [abbrv] beam git commit: Implements HCatalogIO
Posted by ke...@apache.org.
Implements HCatalogIO
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/349898c4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/349898c4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/349898c4
Branch: refs/heads/gearpump-runner
Commit: 349898c4702fc3e52d8c0cd1c5a04f14cd40fd27
Parents: 7c5a70e
Author: Seshadri Chakkravarthy <se...@gmail.com>
Authored: Thu May 18 12:07:01 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Jun 8 14:05:38 2017 -0700
----------------------------------------------------------------------
sdks/java/io/hcatalog/pom.xml | 163 ++++++
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 511 +++++++++++++++++++
.../beam/sdk/io/hcatalog/package-info.java | 22 +
.../io/hcatalog/EmbeddedMetastoreService.java | 88 ++++
.../beam/sdk/io/hcatalog/HCatalogIOTest.java | 271 ++++++++++
.../sdk/io/hcatalog/HCatalogIOTestUtils.java | 106 ++++
.../hcatalog/src/test/resources/hive-site.xml | 301 +++++++++++
sdks/java/io/pom.xml | 1 +
8 files changed, 1463 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/349898c4/sdks/java/io/hcatalog/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/hcatalog/pom.xml b/sdks/java/io/hcatalog/pom.xml
new file mode 100644
index 0000000..19b62a5
--- /dev/null
+++ b/sdks/java/io/hcatalog/pom.xml
@@ -0,0 +1,163 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-parent</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-java-io-hcatalog</artifactId>
+ <name>Apache Beam :: SDKs :: Java :: IO :: HCatalog</name>
+ <description>IO to read and write for HCatalog source.</description>
+
+ <properties>
+ <hive.version>2.1.0</hive.version>
+ <apache.commons.version>2.5</apache.commons.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-io-hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${apache.commons.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-core</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-core</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ <version>${hive.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${hive.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-cli</artifactId>
+ <version>${hive.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/349898c4/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
new file mode 100644
index 0000000..07b56e3
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hcatalog;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.hadoop.WritableCoder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.stats.StatsUtils;
+import org.apache.hive.hcatalog.common.HCatConstants;
+import org.apache.hive.hcatalog.common.HCatException;
+import org.apache.hive.hcatalog.common.HCatUtil;
+import org.apache.hive.hcatalog.data.DefaultHCatRecord;
+import org.apache.hive.hcatalog.data.HCatRecord;
+import org.apache.hive.hcatalog.data.transfer.DataTransferFactory;
+import org.apache.hive.hcatalog.data.transfer.HCatReader;
+import org.apache.hive.hcatalog.data.transfer.HCatWriter;
+import org.apache.hive.hcatalog.data.transfer.ReadEntity;
+import org.apache.hive.hcatalog.data.transfer.ReaderContext;
+import org.apache.hive.hcatalog.data.transfer.WriteEntity;
+import org.apache.hive.hcatalog.data.transfer.WriterContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * IO to read and write data using HCatalog.
+ *
+ * <h3>Reading using HCatalog</h3>
+ *
+ * <p>HCatalog source supports reading of HCatRecord from a HCatalog managed source, for eg. Hive.
+ *
+ * <p>To configure a HCatalog source, you must specify a metastore URI and a table name. Other
+ * optional parameters are database & filter For instance:
+ *
+ * <pre>{@code
+ * Map<String, String> configProperties = new HashMap<String, String>();
+ * configProperties.put("hive.metastore.uris","thrift://metastore-host:port");
+ *
+ * pipeline
+ * .apply(HCatalogIO.read()
+ * .withConfigProperties(configProperties) //mandatory
+ * .withTable("employee") //mandatory
+ * .withDatabase("default") //optional, assumes default if none specified
+ * .withFilter(filterString) //optional,
+ * should be specified if the table is partitioned
+ * }</pre>
+ *
+ * <h3>Writing using HCatalog</h3>
+ *
+ * <p>HCatalog sink supports writing of HCatRecord to a HCatalog managed source, for eg. Hive.
+ *
+ * <p>To configure a HCatalog sink, you must specify a metastore URI and a table name. Other
+ * optional parameters are database, partition & batchsize The destination table should exist
+ * beforehand, the transform does not create a new table if it does not exist For instance:
+ *
+ * <pre>{@code
+ * Map<String, String> configProperties = new HashMap<String, String>();
+ * configProperties.put("hive.metastore.uris","thrift://metastore-host:port");
+ *
+ * pipeline
+ * .apply(...)
+ * .apply(HiveIO.write()
+ * .withConfigProperties(configProperties) //mandatory
+ * .withTable("employee") //mandatory
+ * .withDatabase("default") //optional, assumes default if none specified
+ * .withFilter(partitionValues) //optional,
+ * should be specified if the table is partitioned
+ * .withBatchSize(1024L)) //optional,
+ * assumes a default batch size of 1024 if none specified
+ * }</pre>
+ */
+@Experimental
+public class HCatalogIO {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HCatalogIO.class);
+
+ /** Write data to Hive. */
+ public static Write write() {
+ return new AutoValue_HCatalogIO_Write.Builder().setBatchSize(1024L).build();
+ }
+
+ /** Read data from Hive. */
+ public static Read read() {
+ return new AutoValue_HCatalogIO_Read.Builder().setDatabase("default").build();
+ }
+
+ private HCatalogIO() {}
+
+ /** A {@link PTransform} to read data using HCatalog. */
+ @VisibleForTesting
+ @AutoValue
+ public abstract static class Read extends PTransform<PBegin, PCollection<HCatRecord>> {
+ @Nullable
+ abstract Map<String, String> getConfigProperties();
+
+ @Nullable
+ abstract String getDatabase();
+
+ @Nullable
+ abstract String getTable();
+
+ @Nullable
+ abstract String getFilter();
+
+ @Nullable
+ abstract ReaderContext getContext();
+
+ @Nullable
+ abstract Integer getSplitId();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setConfigProperties(Map<String, String> configProperties);
+
+ abstract Builder setDatabase(String database);
+
+ abstract Builder setTable(String table);
+
+ abstract Builder setFilter(String filter);
+
+ abstract Builder setSplitId(Integer splitId);
+
+ abstract Builder setContext(ReaderContext context);
+
+ abstract Read build();
+ }
+
+ /** Sets the configuration properties like metastore URI. This is mandatory */
+ public Read withConfigProperties(Map<String, String> configProperties) {
+ return toBuilder().setConfigProperties(new HashMap<>(configProperties)).build();
+ }
+
+ /** Sets the database name. This is optional, assumes 'default' database if none specified */
+ public Read withDatabase(String database) {
+ return toBuilder().setDatabase(database).build();
+ }
+
+ /** Sets the table name to read from. This is mandatory */
+ public Read withTable(String table) {
+ return toBuilder().setTable(table).build();
+ }
+
+ /** Sets the filter (partition) details. This is optional, assumes none if not specified */
+ public Read withFilter(String filter) {
+ return toBuilder().setFilter(filter).build();
+ }
+
+ Read withSplitId(int splitId) {
+ checkArgument(splitId >= 0, "Invalid split id-" + splitId);
+ return toBuilder().setSplitId(splitId).build();
+ }
+
+ Read withContext(ReaderContext context) {
+ return toBuilder().setContext(context).build();
+ }
+
+ @Override
+ public PCollection<HCatRecord> expand(PBegin input) {
+ return input.apply(org.apache.beam.sdk.io.Read.from(new BoundedHCatalogSource(this)));
+ }
+
+ @Override
+ public void validate(PipelineOptions options) {
+ checkNotNull(getTable(), "table");
+ checkNotNull(getConfigProperties(), "configProperties");
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.add(DisplayData.item("configProperties", getConfigProperties().toString()));
+ builder.add(DisplayData.item("table", getTable()));
+ builder.addIfNotNull(DisplayData.item("database", getDatabase()));
+ builder.addIfNotNull(DisplayData.item("filter", getFilter()));
+ }
+ }
+
+ /** A HCatalog {@link BoundedSource} reading {@link HCatRecord} from a given instance. */
+ @VisibleForTesting
+ static class BoundedHCatalogSource extends BoundedSource<HCatRecord> {
+ private Read spec;
+
+ BoundedHCatalogSource(Read spec) {
+ this.spec = spec;
+ }
+
+ @Override
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public Coder<HCatRecord> getDefaultOutputCoder() {
+ return (Coder) WritableCoder.of(DefaultHCatRecord.class);
+ }
+
+ @Override
+ public void validate() {
+ spec.validate(null);
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ spec.populateDisplayData(builder);
+ }
+
+ @Override
+ public BoundedReader<HCatRecord> createReader(PipelineOptions options) {
+ return new BoundedHCatalogReader(this);
+ }
+
+ /**
+ * Returns the size of the table in bytes, does not take into consideration filter/partition
+ * details passed, if any.
+ */
+ @Override
+ public long getEstimatedSizeBytes(PipelineOptions pipelineOptions) throws Exception {
+ Configuration conf = new Configuration();
+ for (Entry<String, String> entry : spec.getConfigProperties().entrySet()) {
+ conf.set(entry.getKey(), entry.getValue());
+ }
+ IMetaStoreClient client = null;
+ try {
+ HiveConf hiveConf = HCatUtil.getHiveConf(conf);
+ client = HCatUtil.getHiveMetastoreClient(hiveConf);
+ Table table = HCatUtil.getTable(client, spec.getDatabase(), spec.getTable());
+ return StatsUtils.getFileSizeForTable(hiveConf, table);
+ } finally {
+ // IMetaStoreClient is not AutoCloseable, closing it manually
+ if (client != null) {
+ client.close();
+ }
+ }
+ }
+
+ /**
+ * Calculates the 'desired' number of splits based on desiredBundleSizeBytes which is passed as
+ * a hint to native API. Retrieves the actual splits generated by native API, which could be
+ * different from the 'desired' split count calculated using desiredBundleSizeBytes
+ */
+ @Override
+ public List<BoundedSource<HCatRecord>> split(
+ long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
+ int desiredSplitCount = 1;
+ long estimatedSizeBytes = getEstimatedSizeBytes(options);
+ if (desiredBundleSizeBytes > 0 && estimatedSizeBytes > 0) {
+ desiredSplitCount = (int) Math.ceil((double) estimatedSizeBytes / desiredBundleSizeBytes);
+ }
+ ReaderContext readerContext = getReaderContext(desiredSplitCount);
+ //process the splits returned by native API
+ //this could be different from 'desiredSplitCount' calculated above
+ LOG.info(
+ "Splitting into bundles of {} bytes: "
+ + "estimated size {}, desired split count {}, actual split count {}",
+ desiredBundleSizeBytes,
+ estimatedSizeBytes,
+ desiredSplitCount,
+ readerContext.numSplits());
+
+ List<BoundedSource<HCatRecord>> res = new ArrayList<>();
+ for (int split = 0; split < readerContext.numSplits(); split++) {
+ res.add(new BoundedHCatalogSource(spec.withContext(readerContext).withSplitId(split)));
+ }
+ return res;
+ }
+
+ private ReaderContext getReaderContext(long desiredSplitCount) throws HCatException {
+ ReadEntity entity =
+ new ReadEntity.Builder()
+ .withDatabase(spec.getDatabase())
+ .withTable(spec.getTable())
+ .withFilter(spec.getFilter())
+ .build();
+ // pass the 'desired' split count as an hint to the API
+ Map<String, String> configProps = new HashMap<>(spec.getConfigProperties());
+ configProps.put(
+ HCatConstants.HCAT_DESIRED_PARTITION_NUM_SPLITS, String.valueOf(desiredSplitCount));
+ return DataTransferFactory.getHCatReader(entity, configProps).prepareRead();
+ }
+
+ static class BoundedHCatalogReader extends BoundedSource.BoundedReader<HCatRecord> {
+ private final BoundedHCatalogSource source;
+ private HCatRecord current;
+ private Iterator<HCatRecord> hcatIterator;
+
+ public BoundedHCatalogReader(BoundedHCatalogSource source) {
+ this.source = source;
+ }
+
+ @Override
+ public boolean start() throws HCatException {
+ HCatReader reader =
+ DataTransferFactory.getHCatReader(source.spec.getContext(), source.spec.getSplitId());
+ hcatIterator = reader.read();
+ return advance();
+ }
+
+ @Override
+ public boolean advance() {
+ if (hcatIterator.hasNext()) {
+ current = hcatIterator.next();
+ return true;
+ } else {
+ current = null;
+ return false;
+ }
+ }
+
+ @Override
+ public BoundedHCatalogSource getCurrentSource() {
+ return source;
+ }
+
+ @Override
+ public HCatRecord getCurrent() {
+ if (current == null) {
+ throw new NoSuchElementException("Current element is null");
+ }
+ return current;
+ }
+
+ @Override
+ public void close() {
+ // nothing to close/release
+ }
+ }
+ }
+
+ /** A {@link PTransform} to write to a HCatalog managed source. */
+ @AutoValue
+ public abstract static class Write extends PTransform<PCollection<HCatRecord>, PDone> {
+ @Nullable
+ abstract Map<String, String> getConfigProperties();
+
+ @Nullable
+ abstract String getDatabase();
+
+ @Nullable
+ abstract String getTable();
+
+ @Nullable
+ abstract Map getFilter();
+
+ abstract long getBatchSize();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setConfigProperties(Map<String, String> configProperties);
+
+ abstract Builder setDatabase(String database);
+
+ abstract Builder setTable(String table);
+
+ abstract Builder setFilter(Map partition);
+
+ abstract Builder setBatchSize(long batchSize);
+
+ abstract Write build();
+ }
+
+ /** Sets the configuration properties like metastore URI. This is mandatory */
+ public Write withConfigProperties(Map<String, String> configProperties) {
+ return toBuilder().setConfigProperties(new HashMap<>(configProperties)).build();
+ }
+
+ /** Sets the database name. This is optional, assumes 'default' database if none specified */
+ public Write withDatabase(String database) {
+ return toBuilder().setDatabase(database).build();
+ }
+
+ /** Sets the table name to write to, the table should exist beforehand. This is mandatory */
+ public Write withTable(String table) {
+ return toBuilder().setTable(table).build();
+ }
+
+ /** Sets the filter (partition) details. This is required if the table is partitioned */
+ public Write withFilter(Map filter) {
+ return toBuilder().setFilter(filter).build();
+ }
+
+ /**
+ * Sets batch size for the write operation. This is optional, assumes a default batch size of
+ * 1024 if not set
+ */
+ public Write withBatchSize(long batchSize) {
+ return toBuilder().setBatchSize(batchSize).build();
+ }
+
+ @Override
+ public PDone expand(PCollection<HCatRecord> input) {
+ input.apply(ParDo.of(new WriteFn(this)));
+ return PDone.in(input.getPipeline());
+ }
+
+ @Override
+ public void validate(PipelineOptions options) {
+ checkNotNull(getConfigProperties(), "configProperties");
+ checkNotNull(getTable(), "table");
+ }
+
+ private static class WriteFn extends DoFn<HCatRecord, Void> {
+ private final Write spec;
+ private WriterContext writerContext;
+ private HCatWriter slaveWriter;
+ private HCatWriter masterWriter;
+ private List<HCatRecord> hCatRecordsBatch;
+
+ public WriteFn(Write spec) {
+ this.spec = spec;
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.addIfNotNull(DisplayData.item("database", spec.getDatabase()));
+ builder.add(DisplayData.item("table", spec.getTable()));
+ builder.addIfNotNull(DisplayData.item("filter", String.valueOf(spec.getFilter())));
+ builder.add(DisplayData.item("configProperties", spec.getConfigProperties().toString()));
+ builder.add(DisplayData.item("batchSize", spec.getBatchSize()));
+ }
+
+ @Setup
+ public void initiateWrite() throws HCatException {
+ WriteEntity entity =
+ new WriteEntity.Builder()
+ .withDatabase(spec.getDatabase())
+ .withTable(spec.getTable())
+ .withPartition(spec.getFilter())
+ .build();
+ masterWriter = DataTransferFactory.getHCatWriter(entity, spec.getConfigProperties());
+ writerContext = masterWriter.prepareWrite();
+ slaveWriter = DataTransferFactory.getHCatWriter(writerContext);
+ }
+
+ @StartBundle
+ public void startBundle() {
+ hCatRecordsBatch = new ArrayList<>();
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext ctx) throws HCatException {
+ hCatRecordsBatch.add(ctx.element());
+ if (hCatRecordsBatch.size() >= spec.getBatchSize()) {
+ flush();
+ }
+ }
+
+ @FinishBundle
+ public void finishBundle() throws HCatException {
+ flush();
+ }
+
+ private void flush() throws HCatException {
+ if (hCatRecordsBatch.isEmpty()) {
+ return;
+ }
+ try {
+ slaveWriter.write(hCatRecordsBatch.iterator());
+ masterWriter.commit(writerContext);
+ } catch (HCatException e) {
+ LOG.error("Exception in flush - write/commit data to Hive", e);
+ //abort on exception
+ masterWriter.abort(writerContext);
+ throw e;
+ } finally {
+ hCatRecordsBatch.clear();
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/349898c4/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/package-info.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/package-info.java b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/package-info.java
new file mode 100644
index 0000000..dff5bd1
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Transforms for reading and writing using HCatalog.
+ */
+package org.apache.beam.sdk.io.hcatalog;
http://git-wip-us.apache.org/repos/asf/beam/blob/349898c4/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/EmbeddedMetastoreService.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/EmbeddedMetastoreService.java b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/EmbeddedMetastoreService.java
new file mode 100644
index 0000000..5792bf6
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/EmbeddedMetastoreService.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hcatalog;
+
+import static org.apache.hive.hcatalog.common.HCatUtil.makePathASafeFileName;
+
+import java.io.File;
+import java.io.IOException;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ * Implementation of a light-weight embedded metastore. This class is a trimmed-down version of <a
+ * href="https://github.com/apache/hive/blob/master
+ * /hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java">
+ * https://github.com/apache/hive/blob/master/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce
+ * /HCatBaseTest.java </a>
+ */
+public final class EmbeddedMetastoreService implements AutoCloseable {
+ private final Driver driver;
+ private final HiveConf hiveConf;
+ private final SessionState sessionState;
+
+ EmbeddedMetastoreService(String baseDirPath) throws IOException {
+ FileUtils.forceDeleteOnExit(new File(baseDirPath));
+
+ String hiveDirPath = makePathASafeFileName(baseDirPath + "/hive");
+ String testDataDirPath =
+ makePathASafeFileName(
+ hiveDirPath
+ + "/data/"
+ + EmbeddedMetastoreService.class.getCanonicalName()
+ + System.currentTimeMillis());
+ String testWarehouseDirPath = makePathASafeFileName(testDataDirPath + "/warehouse");
+
+ hiveConf = new HiveConf(getClass());
+ hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
+ hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, "");
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, testWarehouseDirPath);
+ hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, true);
+ hiveConf.setVar(
+ HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
+ "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd."
+ + "SQLStdHiveAuthorizerFactory");
+ hiveConf.set("test.tmp.dir", hiveDirPath);
+
+ System.setProperty("derby.stream.error.file", "/dev/null");
+ driver = new Driver(hiveConf);
+ sessionState = SessionState.start(new CliSessionState(hiveConf));
+ }
+
+ /** Executes the passed query on the embedded metastore service. */
+ void executeQuery(String query) throws CommandNeedRetryException {
+ driver.run(query);
+ }
+
+ /** Returns the HiveConf object for the embedded metastore. */
+ HiveConf getHiveConf() {
+ return hiveConf;
+ }
+
+ @Override
+ public void close() throws Exception {
+ driver.close();
+ sessionState.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/349898c4/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
new file mode 100644
index 0000000..49c538f
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hcatalog;
+
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.TEST_RECORDS_COUNT;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.TEST_TABLE_NAME;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.getConfigPropertiesAsMap;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.getExpectedRecords;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.getHCatRecords;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.getReaderContext;
+import static org.apache.beam.sdk.io.hcatalog.HCatalogIOTestUtils.insertTestData;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.isA;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.hcatalog.HCatalogIO.BoundedHCatalogSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.util.UserCodeException;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hive.hcatalog.data.HCatRecord;
+import org.apache.hive.hcatalog.data.transfer.ReaderContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+/** Test for HCatalogIO. */
+public class HCatalogIOTest implements Serializable {
+ public static final PipelineOptions OPTIONS = PipelineOptionsFactory.create();
+
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+
+ @Rule public final transient TestPipeline defaultPipeline = TestPipeline.create();
+
+ @Rule public final transient TestPipeline readAfterWritePipeline = TestPipeline.create();
+
+ @Rule public transient ExpectedException thrown = ExpectedException.none();
+
+ @Rule
+ public final transient TestRule testDataSetupRule =
+ new TestWatcher() {
+ public Statement apply(final Statement base, final Description description) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ if (description.getAnnotation(NeedsTestData.class) != null) {
+ prepareTestData();
+ } else if (description.getAnnotation(NeedsEmptyTestTables.class) != null) {
+ reCreateTestTable();
+ }
+ base.evaluate();
+ }
+ };
+ }
+ };
+
+ private static EmbeddedMetastoreService service;
+
+ /** Use this annotation to setup complete test data(table populated with records). */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ElementType.METHOD})
+ @interface NeedsTestData {}
+
+ /** Use this annotation to setup test tables alone(empty tables, no records are populated). */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ElementType.METHOD})
+ @interface NeedsEmptyTestTables {}
+
+ @BeforeClass
+ public static void setupEmbeddedMetastoreService () throws IOException {
+ service = new EmbeddedMetastoreService(TMP_FOLDER.getRoot().getAbsolutePath());
+ }
+
+ @AfterClass
+ public static void shutdownEmbeddedMetastoreService () throws Exception {
+ service.executeQuery("drop table " + TEST_TABLE_NAME);
+ service.close();
+ }
+
+ /** Perform end-to-end test of Write-then-Read operation. */
+ @Test
+ @NeedsEmptyTestTables
+ public void testWriteThenReadSuccess() throws Exception {
+ defaultPipeline
+ .apply(Create.of(getHCatRecords(TEST_RECORDS_COUNT)))
+ .apply(
+ HCatalogIO.write()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withTable(TEST_TABLE_NAME));
+ defaultPipeline.run();
+
+ PCollection<String> output =
+ readAfterWritePipeline
+ .apply(
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withTable(HCatalogIOTestUtils.TEST_TABLE_NAME))
+ .apply(
+ ParDo.of(
+ new DoFn<HCatRecord, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element().get(0).toString());
+ }
+ }));
+ PAssert.that(output).containsInAnyOrder(getExpectedRecords(TEST_RECORDS_COUNT));
+ readAfterWritePipeline.run();
+ }
+
+ /** Test of Write to a non-existent table. */
+ @Test
+ public void testWriteFailureTableDoesNotExist() throws Exception {
+ thrown.expectCause(isA(UserCodeException.class));
+ thrown.expectMessage(containsString("org.apache.hive.hcatalog.common.HCatException"));
+ thrown.expectMessage(containsString("NoSuchObjectException"));
+ defaultPipeline
+ .apply(Create.of(getHCatRecords(TEST_RECORDS_COUNT)))
+ .apply(
+ HCatalogIO.write()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withTable("myowntable"));
+ defaultPipeline.run();
+ }
+
+ /** Test of Write without specifying a table. */
+ @Test
+ public void testWriteFailureValidationTable() throws Exception {
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage(containsString("table"));
+ HCatalogIO.write()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .validate(null);
+ }
+
+ /** Test of Write without specifying configuration properties. */
+ @Test
+ public void testWriteFailureValidationConfigProp() throws Exception {
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage(containsString("configProperties"));
+ HCatalogIO.write().withTable("myowntable").validate(null);
+ }
+
+ /** Test of Read from a non-existent table. */
+ @Test
+ public void testReadFailureTableDoesNotExist() throws Exception {
+ defaultPipeline.apply(
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withTable("myowntable"));
+ thrown.expectCause(isA(NoSuchObjectException.class));
+ defaultPipeline.run();
+ }
+
+ /** Test of Read without specifying configuration properties. */
+ @Test
+ public void testReadFailureValidationConfig() throws Exception {
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage(containsString("configProperties"));
+ HCatalogIO.read().withTable("myowntable").validate(null);
+ }
+
+ /** Test of Read without specifying a table. */
+ @Test
+ public void testReadFailureValidationTable() throws Exception {
+ thrown.expect(NullPointerException.class);
+ thrown.expectMessage(containsString("table"));
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .validate(null);
+ }
+
+ /** Test of Read using SourceTestUtils.readFromSource(..). */
+ @Test
+ @NeedsTestData
+ public void testReadFromSource() throws Exception {
+ ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
+ HCatalogIO.Read spec =
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withContext(context)
+ .withTable(TEST_TABLE_NAME);
+
+ List<String> records = new ArrayList<>();
+ for (int i = 0; i < context.numSplits(); i++) {
+ BoundedHCatalogSource source = new BoundedHCatalogSource(spec.withSplitId(i));
+ for (HCatRecord record : SourceTestUtils.readFromSource(source, OPTIONS)) {
+ records.add(record.get(0).toString());
+ }
+ }
+ assertThat(records, containsInAnyOrder(getExpectedRecords(TEST_RECORDS_COUNT).toArray()));
+ }
+
+ /** Test of Read using SourceTestUtils.assertSourcesEqualReferenceSource(..). */
+ @Test
+ @NeedsTestData
+ public void testSourceEqualsSplits() throws Exception {
+ final int numRows = 1500;
+ final int numSamples = 10;
+ final long bytesPerRow = 15;
+ ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
+ HCatalogIO.Read spec =
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withContext(context)
+ .withTable(TEST_TABLE_NAME);
+
+ BoundedHCatalogSource source = new BoundedHCatalogSource(spec);
+ List<BoundedSource<HCatRecord>> unSplitSource = source.split(-1, OPTIONS);
+ assertEquals(1, unSplitSource.size());
+
+ List<BoundedSource<HCatRecord>> splits =
+ source.split(numRows * bytesPerRow / numSamples, OPTIONS);
+ assertTrue(splits.size() >= 1);
+
+ SourceTestUtils.assertSourcesEqualReferenceSource(unSplitSource.get(0), splits, OPTIONS);
+ }
+
+ private void reCreateTestTable() throws CommandNeedRetryException {
+ service.executeQuery("drop table " + TEST_TABLE_NAME);
+ service.executeQuery("create table " + TEST_TABLE_NAME + "(mycol1 string, mycol2 int)");
+ }
+
+ private void prepareTestData() throws Exception {
+ reCreateTestTable();
+ insertTestData(getConfigPropertiesAsMap(service.getHiveConf()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/349898c4/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTestUtils.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTestUtils.java b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTestUtils.java
new file mode 100644
index 0000000..f66e0bc
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTestUtils.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.hcatalog;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hive.hcatalog.common.HCatException;
+import org.apache.hive.hcatalog.data.DefaultHCatRecord;
+import org.apache.hive.hcatalog.data.HCatRecord;
+import org.apache.hive.hcatalog.data.transfer.DataTransferFactory;
+import org.apache.hive.hcatalog.data.transfer.ReadEntity;
+import org.apache.hive.hcatalog.data.transfer.ReaderContext;
+import org.apache.hive.hcatalog.data.transfer.WriteEntity;
+import org.apache.hive.hcatalog.data.transfer.WriterContext;
+
+/** Utility class for HCatalogIOTest. */
+public class HCatalogIOTestUtils {
+ public static final String TEST_TABLE_NAME = "mytable";
+
+ public static final int TEST_RECORDS_COUNT = 1000;
+
+ private static final ReadEntity READ_ENTITY =
+ new ReadEntity.Builder().withTable(TEST_TABLE_NAME).build();
+ private static final WriteEntity WRITE_ENTITY =
+ new WriteEntity.Builder().withTable(TEST_TABLE_NAME).build();
+
+ /** Returns a ReaderContext instance for the passed datastore config params. */
+ static ReaderContext getReaderContext(Map<String, String> config) throws HCatException {
+ return DataTransferFactory.getHCatReader(READ_ENTITY, config).prepareRead();
+ }
+
+ /** Returns a WriterContext instance for the passed datastore config params. */
+ static WriterContext getWriterContext(Map<String, String> config) throws HCatException {
+ return DataTransferFactory.getHCatWriter(WRITE_ENTITY, config).prepareWrite();
+ }
+
+ /** Writes records to the table using the passed WriterContext. */
+ static void writeRecords(WriterContext context) throws HCatException {
+ DataTransferFactory.getHCatWriter(context).write(getHCatRecords(TEST_RECORDS_COUNT).iterator());
+ }
+
+ /** Commits the pending writes to the database. */
+ static void commitRecords(Map<String, String> config, WriterContext context) throws IOException {
+ DataTransferFactory.getHCatWriter(WRITE_ENTITY, config).commit(context);
+ }
+
+ /** Returns a list of strings containing 'expected' test data for verification. */
+ static List<String> getExpectedRecords(int count) {
+ List<String> expected = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ expected.add("record " + i);
+ }
+ return expected;
+ }
+
+ /** Returns a list of HCatRecords of passed size. */
+ static List<HCatRecord> getHCatRecords(int size) {
+ List<HCatRecord> expected = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ expected.add(toHCatRecord(i));
+ }
+ return expected;
+ }
+
+ /** Inserts data into test datastore. */
+ static void insertTestData(Map<String, String> configMap) throws Exception {
+ WriterContext cntxt = getWriterContext(configMap);
+ writeRecords(cntxt);
+ commitRecords(configMap, cntxt);
+ }
+
+ /** Returns config params for the test datastore as a Map. */
+ static Map<String, String> getConfigPropertiesAsMap(HiveConf hiveConf) {
+ Map<String, String> map = new HashMap<>();
+ for (Entry<String, String> kv : hiveConf) {
+ map.put(kv.getKey(), kv.getValue());
+ }
+ return map;
+ }
+
+ /** returns a DefaultHCatRecord instance for passed value. */
+ static DefaultHCatRecord toHCatRecord(int value) {
+ return new DefaultHCatRecord(Arrays.<Object>asList("record " + value, value));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/349898c4/sdks/java/io/hcatalog/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/hcatalog/src/test/resources/hive-site.xml b/sdks/java/io/hcatalog/src/test/resources/hive-site.xml
new file mode 100644
index 0000000..5bb1496
--- /dev/null
+++ b/sdks/java/io/hcatalog/src/test/resources/hive-site.xml
@@ -0,0 +1,301 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- This file is a copy of https://github.com/apache/hive/blob/master/data/conf/hive-site.xml used to support embedded Hive metastore-->
+<configuration>
+
+<property>
+ <name>hive.in.test</name>
+ <value>true</value>
+ <description>Internal marker for test. Used for masking env-dependent values</description>
+</property>
+
+<!-- Hive Configuration can either be stored in this file or in the hadoop configuration files -->
+<!-- that are implied by Hadoop setup variables. -->
+<!-- Aside from Hadoop setup variables - this file is provided as a convenience so that Hive -->
+<!-- users do not have to edit hadoop configuration files (that may be managed as a centralized -->
+<!-- resource). -->
+
+<!-- Hive Execution Parameters -->
+<property>
+ <name>hadoop.tmp.dir</name>
+ <value>${test.tmp.dir}/hadoop-tmp</value>
+ <description>A base for other temporary directories.</description>
+</property>
+
+<!--
+<property>
+ <name>hive.exec.reducers.max</name>
+ <value>1</value>
+ <description>maximum number of reducers</description>
+</property>
+-->
+
+<property>
+ <name>hive.exec.scratchdir</name>
+ <value>${test.tmp.dir}/scratchdir</value>
+ <description>Scratch space for Hive jobs</description>
+</property>
+
+<property>
+ <name>hive.exec.local.scratchdir</name>
+ <value>${test.tmp.dir}/localscratchdir/</value>
+ <description>Local scratch space for Hive jobs</description>
+</property>
+
+<property>
+ <name>datanucleus.schema.autoCreateAll</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>APP</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>mine</value>
+</property>
+
+<property>
+ <!-- this should eventually be deprecated since the metastore should supply this -->
+ <name>hive.metastore.warehouse.dir</name>
+ <value>${test.warehouse.dir}</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.metastore.metadb.dir</name>
+ <value>file://${test.tmp.dir}/metadb/</value>
+ <description>
+ Required by metastore server or if the uris argument below is not supplied
+ </description>
+</property>
+
+<property>
+ <name>test.log.dir</name>
+ <value>${test.tmp.dir}/log/</value>
+ <description></description>
+</property>
+
+<property>
+ <name>test.data.files</name>
+ <value>${hive.root}/data/files</value>
+ <description></description>
+</property>
+
+<property>
+ <name>test.data.scripts</name>
+ <value>${hive.root}/data/scripts</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.jar.path</name>
+ <value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.metastore.rawstore.impl</name>
+ <value>org.apache.hadoop.hive.metastore.ObjectStore</value>
+ <description>Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. This class is used to store and retrieval of raw metadata objects such as table, database</description>
+</property>
+
+<property>
+ <name>hive.querylog.location</name>
+ <value>${test.tmp.dir}/tmp</value>
+ <description>Location of the structured hive logs</description>
+</property>
+
+<property>
+ <name>hive.exec.pre.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables</value>
+ <description>Pre Execute Hook for Tests</description>
+</property>
+
+<property>
+ <name>hive.exec.post.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value>
+ <description>Post Execute Hook for Tests</description>
+</property>
+
+<property>
+ <name>hive.support.concurrency</name>
+ <value>true</value>
+ <description>Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks.</description>
+</property>
+
+<property>
+ <key>hive.unlock.numretries</key>
+ <value>2</value>
+ <description>The number of times you want to retry to do one unlock</description>
+</property>
+
+<property>
+ <key>hive.lock.sleep.between.retries</key>
+ <value>2</value>
+ <description>The sleep time (in seconds) between various retries</description>
+</property>
+
+
+<property>
+ <name>fs.pfile.impl</name>
+ <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
+ <description>A proxy for local file system used for cross file system testing</description>
+</property>
+
+<property>
+ <name>hive.exec.mode.local.auto</name>
+ <value>false</value>
+ <description>
+ Let hive determine whether to run in local mode automatically
+ Disabling this for tests so that minimr is not affected
+ </description>
+</property>
+
+<property>
+ <name>hive.auto.convert.join</name>
+ <value>false</value>
+ <description>Whether Hive enable the optimization about converting common join into mapjoin based on the input file size</description>
+</property>
+
+<property>
+ <name>hive.ignore.mapjoin.hint</name>
+ <value>false</value>
+ <description>Whether Hive ignores the mapjoin hint</description>
+</property>
+
+<property>
+ <name>hive.input.format</name>
+ <value>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</value>
+ <description>The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. </description>
+</property>
+
+<property>
+ <name>hive.default.rcfile.serde</name>
+ <value>org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe</value>
+ <description>The default SerDe hive will use for the rcfile format</description>
+</property>
+
+<property>
+ <name>hive.stats.key.prefix.reserve.length</name>
+ <value>0</value>
+</property>
+
+<property>
+ <name>hive.conf.restricted.list</name>
+ <value>dummy.config.value</value>
+ <description>Using dummy config value above because you cannot override config with empty value</description>
+</property>
+
+<property>
+ <name>hive.exec.submit.local.task.via.child</name>
+ <value>false</value>
+</property>
+
+
+<property>
+ <name>hive.dummyparam.test.server.specific.config.override</name>
+ <value>from.hive-site.xml</value>
+ <description>Using dummy param to test server specific configuration</description>
+</property>
+
+<property>
+ <name>hive.dummyparam.test.server.specific.config.hivesite</name>
+ <value>from.hive-site.xml</value>
+ <description>Using dummy param to test server specific configuration</description>
+</property>
+
+<property>
+ <name>test.var.hiveconf.property</name>
+ <value>${hive.exec.default.partition.name}</value>
+ <description>Test hiveconf property substitution</description>
+</property>
+
+<property>
+ <name>test.property1</name>
+ <value>value1</value>
+ <description>Test property defined in hive-site.xml only</description>
+</property>
+
+<property>
+ <name>hive.test.dummystats.aggregator</name>
+ <value>value2</value>
+</property>
+
+<property>
+ <name>hive.fetch.task.conversion</name>
+ <value>minimal</value>
+</property>
+
+<property>
+ <name>hive.users.in.admin.role</name>
+ <value>hive_admin_user</value>
+</property>
+
+<property>
+ <name>hive.llap.io.cache.orc.size</name>
+ <value>8388608</value>
+</property>
+
+<property>
+ <name>hive.llap.io.cache.orc.arena.size</name>
+ <value>8388608</value>
+</property>
+
+<property>
+ <name>hive.llap.io.cache.orc.alloc.max</name>
+ <value>2097152</value>
+</property>
+
+
+<property>
+ <name>hive.llap.io.cache.orc.alloc.min</name>
+ <value>32768</value>
+</property>
+
+<property>
+ <name>hive.llap.cache.allow.synthetic.fileid</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>hive.llap.io.use.lrfu</name>
+ <value>true</value>
+</property>
+
+
+<property>
+ <name>hive.llap.io.allocator.direct</name>
+ <value>false</value>
+</property>
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/beam/blob/349898c4/sdks/java/io/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/pom.xml b/sdks/java/io/pom.xml
index 44f3baa..13cd418 100644
--- a/sdks/java/io/pom.xml
+++ b/sdks/java/io/pom.xml
@@ -72,6 +72,7 @@
<module>hadoop-file-system</module>
<module>hadoop</module>
<module>hbase</module>
+ <module>hcatalog</module>
<module>jdbc</module>
<module>jms</module>
<module>kafka</module>
[36/50] [abbrv] beam git commit: Fix compile error occurs in some JDKs
Posted by ke...@apache.org.
Fix compile error occurs in some JDKs
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b7ae7ecf
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b7ae7ecf
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b7ae7ecf
Branch: refs/heads/gearpump-runner
Commit: b7ae7ecffcd08b6a0ccc8296210d36b90306c171
Parents: 1c6861f
Author: Mark Liu <ma...@google.com>
Authored: Wed Jun 7 16:27:34 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Fri Jun 9 10:54:12 2017 -0700
----------------------------------------------------------------------
.../runners/flink/FlinkStreamingTransformTranslators.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/b7ae7ecf/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
index ef46b63..fef32de 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
@@ -363,8 +363,13 @@ class FlinkStreamingTransformTranslators {
Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags = Maps.newHashMap();
for (Map.Entry<TupleTag<?>, PValue> entry : outputs.entrySet()) {
if (!tagsToOutputTags.containsKey(entry.getKey())) {
- tagsToOutputTags.put(entry.getKey(), new OutputTag<>(entry.getKey().getId(),
- (TypeInformation) context.getTypeInfo((PCollection<?>) entry.getValue())));
+ tagsToOutputTags.put(
+ entry.getKey(),
+ new OutputTag<WindowedValue<?>>(
+ entry.getKey().getId(),
+ (TypeInformation) context.getTypeInfo((PCollection<?>) entry.getValue())
+ )
+ );
}
}
[46/50] [abbrv] beam git commit: Port DirectRunner WriteFiles
override to SDK-agnostic APIs
Posted by ke...@apache.org.
Port DirectRunner WriteFiles override to SDK-agnostic APIs
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ed6bd18b
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ed6bd18b
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ed6bd18b
Branch: refs/heads/gearpump-runner
Commit: ed6bd18bffe8a51d5fc2a59ff9aaa731b196d58a
Parents: 02dbaef
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri May 26 16:07:45 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Jun 9 19:56:52 2017 -0700
----------------------------------------------------------------------
.../core/construction/PTransformMatchers.java | 17 ++++++++---
.../direct/WriteWithShardingFactory.java | 30 ++++++++++++++------
.../direct/WriteWithShardingFactoryTest.java | 26 +++++++++++------
3 files changed, 52 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/ed6bd18b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
index c339891..0d27241 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformMatchers.java
@@ -17,13 +17,14 @@
*/
package org.apache.beam.runners.core.construction;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.WRITE_FILES_TRANSFORM_URN;
+
import com.google.common.base.MoreObjects;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.io.WriteFiles;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformMatcher;
import org.apache.beam.sdk.transforms.DoFn;
@@ -359,10 +360,18 @@ public class PTransformMatchers {
return new PTransformMatcher() {
@Override
public boolean matches(AppliedPTransform<?, ?, ?> application) {
- if (PTransformTranslation.WRITE_FILES_TRANSFORM_URN.equals(
+ if (WRITE_FILES_TRANSFORM_URN.equals(
PTransformTranslation.urnForTransformOrNull(application.getTransform()))) {
- WriteFiles write = (WriteFiles) application.getTransform();
- return write.getSharding() == null && write.getNumShards() == null;
+ try {
+ return WriteFilesTranslation.isRunnerDeterminedSharding(
+ (AppliedPTransform) application);
+ } catch (IOException exc) {
+ throw new RuntimeException(
+ String.format(
+ "Transform with URN %s failed to parse: %s",
+ WRITE_FILES_TRANSFORM_URN, application.getTransform()),
+ exc);
+ }
}
return false;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ed6bd18b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
index 65a5a19..d8734a1 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WriteWithShardingFactory.java
@@ -21,11 +21,13 @@ package org.apache.beam.runners.direct;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
+import java.io.IOException;
import java.io.Serializable;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.beam.runners.core.construction.PTransformReplacements;
+import org.apache.beam.runners.core.construction.WriteFilesTranslation;
import org.apache.beam.sdk.io.WriteFiles;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
@@ -43,23 +45,33 @@ import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.TupleTag;
/**
- * A {@link PTransformOverrideFactory} that overrides {@link WriteFiles}
- * {@link PTransform PTransforms} with an unspecified number of shards with a write with a
- * specified number of shards. The number of shards is the log base 10 of the number of input
- * records, with up to 2 additional shards.
+ * A {@link PTransformOverrideFactory} that overrides {@link WriteFiles} {@link PTransform
+ * PTransforms} with an unspecified number of shards with a write with a specified number of shards.
+ * The number of shards is the log base 10 of the number of input records, with up to 2 additional
+ * shards.
*/
class WriteWithShardingFactory<InputT>
- implements PTransformOverrideFactory<PCollection<InputT>, PDone, WriteFiles<InputT>> {
+ implements PTransformOverrideFactory<
+ PCollection<InputT>, PDone, PTransform<PCollection<InputT>, PDone>> {
static final int MAX_RANDOM_EXTRA_SHARDS = 3;
@VisibleForTesting static final int MIN_SHARDS_FOR_LOG = 3;
@Override
public PTransformReplacement<PCollection<InputT>, PDone> getReplacementTransform(
- AppliedPTransform<PCollection<InputT>, PDone, WriteFiles<InputT>> transform) {
+ AppliedPTransform<PCollection<InputT>, PDone, PTransform<PCollection<InputT>, PDone>>
+ transform) {
- return PTransformReplacement.of(
- PTransformReplacements.getSingletonMainInput(transform),
- transform.getTransform().withSharding(new LogElementShardsWithDrift<InputT>()));
+ try {
+ WriteFiles<InputT> replacement = WriteFiles.to(WriteFilesTranslation.getSink(transform));
+ if (WriteFilesTranslation.isWindowedWrites(transform)) {
+ replacement = replacement.withWindowedWrites();
+ }
+ return PTransformReplacement.of(
+ PTransformReplacements.getSingletonMainInput(transform),
+ replacement.withSharding(new LogElementShardsWithDrift<InputT>()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/ed6bd18b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
index a88d95e..41d671f 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WriteWithShardingFactoryTest.java
@@ -30,6 +30,7 @@ import static org.junit.Assert.assertThat;
import java.io.File;
import java.io.FileReader;
import java.io.Reader;
+import java.io.Serializable;
import java.nio.CharBuffer;
import java.util.ArrayList;
import java.util.Collections;
@@ -53,6 +54,7 @@ import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.DoFnTester;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
@@ -71,11 +73,17 @@ import org.junit.runners.JUnit4;
* Tests for {@link WriteWithShardingFactory}.
*/
@RunWith(JUnit4.class)
-public class WriteWithShardingFactoryTest {
+public class WriteWithShardingFactoryTest implements Serializable {
+
private static final int INPUT_SIZE = 10000;
- @Rule public TemporaryFolder tmp = new TemporaryFolder();
- private WriteWithShardingFactory<Object> factory = new WriteWithShardingFactory<>();
- @Rule public final TestPipeline p = TestPipeline.create().enableAbandonedNodeEnforcement(false);
+
+ @Rule public transient TemporaryFolder tmp = new TemporaryFolder();
+
+ private transient WriteWithShardingFactory<Object> factory = new WriteWithShardingFactory<>();
+
+ @Rule
+ public final transient TestPipeline p =
+ TestPipeline.create().enableAbandonedNodeEnforcement(false);
@Test
public void dynamicallyReshardedWrite() throws Exception {
@@ -135,7 +143,8 @@ public class WriteWithShardingFactoryTest {
DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE,
"",
false);
- WriteFiles<Object> original =
+
+ PTransform<PCollection<Object>, PDone> original =
WriteFiles.to(
new FileBasedSink<Object>(StaticValueProvider.of(outputDirectory), policy) {
@Override
@@ -146,9 +155,10 @@ public class WriteWithShardingFactoryTest {
@SuppressWarnings("unchecked")
PCollection<Object> objs = (PCollection) p.apply(Create.empty(VoidCoder.of()));
- AppliedPTransform<PCollection<Object>, PDone, WriteFiles<Object>> originalApplication =
- AppliedPTransform.of(
- "write", objs.expand(), Collections.<TupleTag<?>, PValue>emptyMap(), original, p);
+ AppliedPTransform<PCollection<Object>, PDone, PTransform<PCollection<Object>, PDone>>
+ originalApplication =
+ AppliedPTransform.of(
+ "write", objs.expand(), Collections.<TupleTag<?>, PValue>emptyMap(), original, p);
assertThat(
factory.getReplacementTransform(originalApplication).getTransform(),
[09/50] [abbrv] beam git commit: This closes #3321
Posted by ke...@apache.org.
This closes #3321
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/66460cb2
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/66460cb2
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/66460cb2
Branch: refs/heads/gearpump-runner
Commit: 66460cb2d3db56261483c39acdea4786fec156fd
Parents: 32f22b7 f2e3088
Author: Ahmet Altay <al...@google.com>
Authored: Wed Jun 7 18:45:42 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Wed Jun 7 18:45:42 2017 -0700
----------------------------------------------------------------------
.../runners/direct/transform_evaluator.py | 62 +-------------------
1 file changed, 1 insertion(+), 61 deletions(-)
----------------------------------------------------------------------
[22/50] [abbrv] beam git commit: This closes #3302
Posted by ke...@apache.org.
This closes #3302
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b53e6f0d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b53e6f0d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b53e6f0d
Branch: refs/heads/gearpump-runner
Commit: b53e6f0dc91948040364d1e742b023d5a2be8433
Parents: fdfc70e 156f326
Author: Ahmet Altay <al...@google.com>
Authored: Thu Jun 8 10:57:11 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Thu Jun 8 10:57:11 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 2 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 16 +++++++++-------
2 files changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[21/50] [abbrv] beam git commit: Raise entity limit per RPC to 9MB.
Posted by ke...@apache.org.
Raise entity limit per RPC to 9MB.
This is closer to the API limit, while still leaving room for overhead. Brings
the Java SDK into line with the Python SDK.
Switch the unit test to use the size of each entity, which is what the
connector is actually using, rather than the property size (which is slightly
smaller and would cause the test to fail for some values).
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/156f326a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/156f326a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/156f326a
Branch: refs/heads/gearpump-runner
Commit: 156f326a16e15b4e22a189a2a263d11d7b273656
Parents: fdfc70e
Author: Colin Phipps <fi...@google.com>
Authored: Mon Jun 5 12:12:49 2017 +0000
Committer: Ahmet Altay <al...@google.com>
Committed: Thu Jun 8 10:57:09 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 2 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 16 +++++++++-------
2 files changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/156f326a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index b198a6f..06b9c8a 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -213,7 +213,7 @@ public class DatastoreV1 {
* the mutations themselves and not the CommitRequest wrapper around them.
*/
@VisibleForTesting
- static final int DATASTORE_BATCH_UPDATE_BYTES_LIMIT = 5_000_000;
+ static final int DATASTORE_BATCH_UPDATE_BYTES_LIMIT = 9_000_000;
/**
* Returns an empty {@link DatastoreV1.Read} builder. Configure the source {@code projectId},
http://git-wip-us.apache.org/repos/asf/beam/blob/156f326a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
index 460049e..229b1fb 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
@@ -651,12 +651,14 @@ public class DatastoreV1Test {
@Test
public void testDatatoreWriterFnWithLargeEntities() throws Exception {
List<Mutation> mutations = new ArrayList<>();
- int propertySize = 900_000;
+ int entitySize = 0;
for (int i = 0; i < 12; ++i) {
- Entity.Builder entity = Entity.newBuilder().setKey(makeKey("key" + i, i + 1));
- entity.putProperties("long", makeValue(new String(new char[propertySize])
- ).setExcludeFromIndexes(true).build());
- mutations.add(makeUpsert(entity.build()).build());
+ Entity entity = Entity.newBuilder().setKey(makeKey("key" + i, i + 1))
+ .putProperties("long", makeValue(new String(new char[900_000])
+ ).setExcludeFromIndexes(true).build())
+ .build();
+ entitySize = entity.getSerializedSize(); // Take the size of any one entity.
+ mutations.add(makeUpsert(entity).build());
}
DatastoreWriterFn datastoreWriter = new DatastoreWriterFn(StaticValueProvider.of(PROJECT_ID),
@@ -667,10 +669,10 @@ public class DatastoreV1Test {
// This test is over-specific currently; it requires that we split the 12 entity writes into 3
// requests, but we only need each CommitRequest to be less than 10MB in size.
- int propertiesPerRpc = DATASTORE_BATCH_UPDATE_BYTES_LIMIT / propertySize;
+ int entitiesPerRpc = DATASTORE_BATCH_UPDATE_BYTES_LIMIT / entitySize;
int start = 0;
while (start < mutations.size()) {
- int end = Math.min(mutations.size(), start + propertiesPerRpc);
+ int end = Math.min(mutations.size(), start + entitiesPerRpc);
CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
commitRequest.addAllMutations(mutations.subList(start, end));
[50/50] [abbrv] beam git commit: This closes #3292: Merge master into
gearpump-runner
Posted by ke...@apache.org.
This closes #3292: Merge master into gearpump-runner
Adjustments to gearpump-runner:
Fix side input handling in DoFnFunction
Respect WindowFn#getOutputTime in gearpump-runner
Activate Gearpump local-validates-runner-tests in precommit
Update against master changes
Merged from master:
[BEAM-972] Add more unit test to Gearpump runner
[BEAM-972] Add unit tests to Gearpump runner
[BEAM-79] Fix gearpump-runner merge conflicts and test failure
enable ParDoTest
[BEAM-79] Add SideInput support for GearpumpRunner
[BEAM-79] Support merging windows in GearpumpRunner
[BEAM-79] Fix PostCommit test confs for Gearpump runner
note thread is interrupted on InterruptedException
Remove cache for Gearpump on travis
reduce timeout to wait for result
fix ParDo.BoundMulti translation
return encoded key for GroupByKey translation
support OutputTimeFn
update to latest gearpump dsl function interface
fix group by window
activate ROS on Gearpump by default
update ROS configurations
[BEAM-1180] Implement GearpumpPipelineResult
[BEAM-79] Upgrade to beam-0.5.0-incubating-SNAPSHOT
[BEAM-79] Update to latest Gearpump API
Fix NoOpAggregatorFactory
Remove print to stdout
Skip window assignment when windows don't change
Add Window.Bound translator
Upgrade Gearpump version
[BEAM-79] fix gearpump runner build failure
[BEAM-79] update GearpumpPipelineResult
[BEAM-79] Port Gearpump runner from OldDoFn to new DoFn
upgrade gearpump-runner to 0.4.0-incubating-SNAPSHOT
remove "pipeline" in runner name
post-merge fix
[BEAM-79] fix integration-test failure
fix import order
!fixup Minor javadoc clean-up
Added even more javadoc to TextIO#withHeader and TextIO#withFooter (2).
Added even more javadoc to TextIO#withHeader and TextIO#withFooter.
Added javadoc to TextIO#withHeader and TextIO#withFooter.
Reverted header and footer to be of type String.
Revised according to comments following a code review.
Add header/footer support to TextIO.Write
[BEAM-242] Enable and fix checkstyle in Flink runner examples
Remove timeout in JAXBCoderTest
Be more accepting in UnboundedReadDeduplicatorTest
BigQuery: limit max job polling time to 1 minute
[BEAM-242] Enable checkstyle and fix checkstyle errors in Flink runner
[BEAM-456] Add MongoDbIO
FluentBackoff: a replacement for a variety of custom backoff implementations
Remove the DataflowRunner instructions from examples
Put classes in runners-core package into runners.core namespace
Delegate populateDipslayData to wrapped combineFn's
Fixed Combine display data
Cloud Datastore naming clean-up
DatastoreIO SplitQueryFn integration test
Add Latest CombineFn and PTransforms
Remove empty unused method in TestStreamEvaluatorFactory
Test that multiple instances of TestStream are supported
Correct some accidental renames
Fix condition in FlinkStreamingPipelineTranslator
Address comments of Flink Side-Input PR
[BEAM-569] Define maxNumRecords default value to Long.MAX_VALUE in JmsIO
Add LeaderBoardTest
take advantage of setup/teardown for KafkaWriter
Returned KafkaIO getWatermark log line in debug mode
[BEAM-572] Remove Spark Reference in WordCount
Update Dataflow Container Version
[BEAM-313] Provide a context for SparkRunner
DataflowRunner: get PBegin from PInput
[BEAM-592] Fix SparkRunner Dependency Problem in WordCount
Fix javadoc in Kinesis
Organize imports in Kinesis
kinesis: a connector for Amazon Kinesis
[BEAM-589] Fixing IO.Read transformation
Query latest timestamp
travis.yml: disable updating snapshots
Added support for reporting aggregator values to Spark sinks
[BEAM-294] Rename dataflow references to beam
Modified BigtableIO to use DoFn setup/tearDown methods instead of startBundle/finishBundle
checkstyle: prohibit API client repackaged Guava
Make WriteTest more resilient to Randomness
Update DoFn javadocs to remove references to OldDoFn and Dataflow
[BEAM-545] Promote JobName to PipelineOptions
Move the samples data to gs://apache-beam-samples/
Cleanup some javadoc that referring Dataflow
BigQueryIO.Write: raise size limit to 11 TiB
Optimize imports
Update checkstyle.xml to put all imports in one group
Fix Exception Unwrapping in TestFlinkRunner
Make ParDoLifecycleTest Serializable to Fix Test with TupleTag
Use AllPanes as the PaneExtractor in IterableAssert
Fix combine tests with Accumulation Mode
Enable Flink Streaming Runner RunnableOnService tests
Fix Flink Runner Pom for Batch RunnableOnService tests
Don't Suppress Throwable in PAssert in Streaming Mode
...
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/559e3c34
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/559e3c34
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/559e3c34
Branch: refs/heads/gearpump-runner
Commit: 559e3c341f879cc06f823363ed0fa4c1eab8a6a5
Parents: 1597f3c 7653e7e
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Jun 12 09:48:29 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Jun 12 09:48:29 2017 -0700
----------------------------------------------------------------------
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 2 +-
examples/java/pom.xml | 12 +
pom.xml | 6 +
runners/gearpump/README.md | 61 +++
runners/gearpump/pom.xml | 281 ++++++++++
.../gearpump/GearpumpPipelineOptions.java | 67 +++
.../gearpump/GearpumpPipelineResult.java | 109 ++++
.../gearpump/GearpumpPipelineTranslator.java | 524 +++++++++++++++++++
.../beam/runners/gearpump/GearpumpRunner.java | 116 ++++
.../gearpump/GearpumpRunnerRegistrar.java | 64 +++
.../runners/gearpump/TestGearpumpRunner.java | 63 +++
.../beam/runners/gearpump/package-info.java | 22 +
...CreateGearpumpPCollectionViewTranslator.java | 46 ++
.../FlattenPCollectionsTranslator.java | 83 +++
.../translators/GroupByKeyTranslator.java | 253 +++++++++
.../translators/ParDoMultiOutputTranslator.java | 103 ++++
.../translators/ReadBoundedTranslator.java | 46 ++
.../translators/ReadUnboundedTranslator.java | 48 ++
.../translators/TransformTranslator.java | 30 ++
.../translators/TranslationContext.java | 104 ++++
.../translators/WindowAssignTranslator.java | 103 ++++
.../translators/functions/DoFnFunction.java | 194 +++++++
.../translators/functions/package-info.java | 22 +
.../translators/io/BoundedSourceWrapper.java | 45 ++
.../gearpump/translators/io/GearpumpSource.java | 121 +++++
.../translators/io/UnboundedSourceWrapper.java | 46 ++
.../gearpump/translators/io/ValuesSource.java | 173 ++++++
.../gearpump/translators/io/package-info.java | 22 +
.../gearpump/translators/package-info.java | 22 +
.../translators/utils/DoFnRunnerFactory.java | 81 +++
.../translators/utils/NoOpStepContext.java | 41 ++
.../translators/utils/TranslatorUtils.java | 198 +++++++
.../translators/utils/package-info.java | 22 +
.../gearpump/GearpumpRunnerRegistrarTest.java | 55 ++
.../runners/gearpump/PipelineOptionsTest.java | 73 +++
...teGearpumpPCollectionViewTranslatorTest.java | 57 ++
.../FlattenPCollectionsTranslatorTest.java | 149 ++++++
.../translators/GroupByKeyTranslatorTest.java | 152 ++++++
.../translators/ReadBoundedTranslatorTest.java | 70 +++
.../ReadUnboundedTranslatorTest.java | 70 +++
.../translators/WindowAssignTranslatorTest.java | 110 ++++
.../translators/io/GearpumpSourceTest.java | 90 ++++
.../gearpump/translators/io/ValueSoureTest.java | 79 +++
.../translators/utils/TranslatorUtilsTest.java | 74 +++
runners/pom.xml | 9 +
sdks/java/pom.xml | 2 +-
46 files changed, 4118 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[30/50] [abbrv] beam git commit: Slight debuggability improvements in
BigtableIO
Posted by ke...@apache.org.
Slight debuggability improvements in BigtableIO
- ByteKeyRangeTracker.splitAtPosition logs the "insane" case first.
- BigtableIO logs the split position at INFO
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/82a6cb61
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/82a6cb61
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/82a6cb61
Branch: refs/heads/gearpump-runner
Commit: 82a6cb6104c0bb179832dabf4433c183743ea983
Parents: 911bfbd
Author: Eugene Kirpichov <ki...@google.com>
Authored: Thu Jun 8 14:51:15 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Jun 8 14:51:15 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 22 ++++++++++++--------
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 2 +-
2 files changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/82a6cb61/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
index 99717a4..b889ec7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRangeTracker.java
@@ -71,6 +71,10 @@ public final class ByteKeyRangeTracker implements RangeTracker<ByteKey> {
"Trying to return record which is before the last-returned record");
if (position == null) {
+ LOG.info(
+ "Adjusting range start from {} to {} as position of first returned record",
+ range.getStartKey(),
+ recordStart);
range = range.withStartKey(recordStart);
}
position = recordStart;
@@ -87,6 +91,15 @@ public final class ByteKeyRangeTracker implements RangeTracker<ByteKey> {
@Override
public synchronized boolean trySplitAtPosition(ByteKey splitPosition) {
+ // Sanity check.
+ if (!range.containsKey(splitPosition)) {
+ LOG.warn(
+ "{}: Rejecting split request at {} because it is not within the range.",
+ this,
+ splitPosition);
+ return false;
+ }
+
// Unstarted.
if (position == null) {
LOG.warn(
@@ -106,15 +119,6 @@ public final class ByteKeyRangeTracker implements RangeTracker<ByteKey> {
return false;
}
- // Sanity check.
- if (!range.containsKey(splitPosition)) {
- LOG.warn(
- "{}: Rejecting split request at {} because it is not within the range.",
- this,
- splitPosition);
- return false;
- }
-
range = range.withEndKey(splitPosition);
return true;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/82a6cb61/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index 1692cda..62679bb 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -1027,7 +1027,7 @@ public class BigtableIO {
"{}: Failed to interpolate key for fraction {}.", rangeTracker.getRange(), fraction, e);
return null;
}
- LOG.debug(
+ LOG.info(
"Proposing to split {} at fraction {} (key {})", rangeTracker, fraction, splitKey);
BigtableSource primary;
BigtableSource residual;
[05/50] [abbrv] beam git commit: [BEAM-2421] Swap to use an Impulse
primitive + DoFn for Create when executing with the Fn API.
Posted by ke...@apache.org.
[BEAM-2421] Swap to use an Impulse primitive + DoFn for Create when executing with the Fn API.
This closes #3312
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/caecac3b
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/caecac3b
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/caecac3b
Branch: refs/heads/gearpump-runner
Commit: caecac3b4acb5bfa6e36143d3868b2d80ab119da
Parents: 609016d 1cdb80c
Author: Luke Cwik <lc...@google.com>
Authored: Wed Jun 7 13:43:38 2017 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Wed Jun 7 13:43:38 2017 -0700
----------------------------------------------------------------------
.../beam/runners/dataflow/DataflowRunner.java | 154 ++++++++++++++++++-
1 file changed, 146 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[06/50] [abbrv] beam git commit: Shutdown Flink Streaming Pipeline
when reaching +Inf watermark
Posted by ke...@apache.org.
Shutdown Flink Streaming Pipeline when reaching +Inf watermark
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9c83ffe0
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9c83ffe0
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9c83ffe0
Branch: refs/heads/gearpump-runner
Commit: 9c83ffe0cdc6636d2187bf9439a73a3b45756d50
Parents: caecac3
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Mon Jun 5 12:19:00 2017 +0200
Committer: Ismaël MejÃa <ie...@apache.org>
Committed: Wed Jun 7 23:13:52 2017 +0200
----------------------------------------------------------------------
.../wrappers/streaming/io/UnboundedSourceWrapper.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/9c83ffe0/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
index 6055a43..e75072a 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java
@@ -31,6 +31,7 @@ import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.SerializableCoder;
import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
@@ -436,6 +437,10 @@ public class UnboundedSourceWrapper<
}
}
context.emitWatermark(new Watermark(watermarkMillis));
+
+ if (watermarkMillis >= BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()) {
+ this.isRunning = false;
+ }
}
setNextWatermarkTimer(this.runtimeContext);
}
[23/50] [abbrv] beam git commit: DirectRunner override matchers using
Runner API
Posted by ke...@apache.org.
DirectRunner override matchers using Runner API
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d8d90878
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d8d90878
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d8d90878
Branch: refs/heads/gearpump-runner
Commit: d8d9087877c01f1786271726a541fb3eeda7f939
Parents: ca7b9c2
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu May 25 06:31:16 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu Jun 8 11:36:28 2017 -0700
----------------------------------------------------------------------
.../beam/runners/direct/DirectRunner.java | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/d8d90878/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index dbd1ec4..136ccf3 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.beam.runners.core.SplittableParDoViaKeyedWorkItems;
import org.apache.beam.runners.core.construction.PTransformMatchers;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
import org.apache.beam.runners.core.construction.SplittableParDo;
import org.apache.beam.runners.direct.DirectRunner.DirectPipelineResult;
import org.apache.beam.runners.direct.TestStreamEvaluatorFactory.DirectTestStreamFactory;
@@ -42,12 +43,9 @@ import org.apache.beam.sdk.metrics.MetricResults;
import org.apache.beam.sdk.metrics.MetricsEnvironment;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.runners.PTransformOverride;
-import org.apache.beam.sdk.testing.TestStream;
-import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
-import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.values.PCollection;
import org.joda.time.Duration;
@@ -230,33 +228,33 @@ public class DirectRunner extends PipelineRunner<DirectPipelineResult> {
new WriteWithShardingFactory())) /* Uses a view internally. */
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(CreatePCollectionView.class),
+ PTransformMatchers.urnEqualTo(PTransformTranslation.CREATE_VIEW_TRANSFORM_URN),
new ViewOverrideFactory())) /* Uses pardos and GBKs */
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(TestStream.class),
+ PTransformMatchers.urnEqualTo(PTransformTranslation.TEST_STREAM_TRANSFORM_URN),
new DirectTestStreamFactory(this))) /* primitive */
// SplittableParMultiDo is implemented in terms of nonsplittable simple ParDos and extra
// primitives
.add(
PTransformOverride.of(
- PTransformMatchers.splittableParDoMulti(), new ParDoMultiOverrideFactory()))
+ PTransformMatchers.splittableParDo(), new ParDoMultiOverrideFactory()))
// state and timer pardos are implemented in terms of simple ParDos and extra primitives
.add(
PTransformOverride.of(
- PTransformMatchers.stateOrTimerParDoMulti(), new ParDoMultiOverrideFactory()))
+ PTransformMatchers.stateOrTimerParDo(), new ParDoMultiOverrideFactory()))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(SplittableParDo.ProcessKeyedElements.class),
+ PTransformMatchers.urnEqualTo(
+ SplittableParDo.SPLITTABLE_PROCESS_KEYED_ELEMENTS_URN),
new SplittableParDoViaKeyedWorkItems.OverrideFactory()))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(
- SplittableParDoViaKeyedWorkItems.GBKIntoKeyedWorkItems.class),
+ PTransformMatchers.urnEqualTo(SplittableParDo.SPLITTABLE_GBKIKWI_URN),
new DirectGBKIntoKeyedWorkItemsOverrideFactory())) /* Returns a GBKO */
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(GroupByKey.class),
+ PTransformMatchers.urnEqualTo(PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN),
new DirectGroupByKeyOverrideFactory())) /* returns two chained primitives. */
.build();
}
[03/50] [abbrv] beam git commit: [BEAM-2421] Swap to use an Impulse
primitive + DoFn for Create when executing with the Fn API.
Posted by ke...@apache.org.
[BEAM-2421] Swap to use an Impulse primitive + DoFn for Create when executing with the Fn API.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1cdb80cb
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1cdb80cb
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1cdb80cb
Branch: refs/heads/gearpump-runner
Commit: 1cdb80cb6319c04fa94961c14c038a5e15736d68
Parents: 5f7e73b
Author: Luke Cwik <lc...@google.com>
Authored: Wed Jun 7 08:53:14 2017 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Wed Jun 7 13:41:20 2017 -0700
----------------------------------------------------------------------
.../beam/runners/dataflow/DataflowRunner.java | 158 +++++++++++++++++--
1 file changed, 145 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/1cdb80cb/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index ed29330..3e7c8ce 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -49,6 +49,7 @@ import java.net.URLClassLoader;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -57,6 +58,7 @@ import java.util.Random;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import org.apache.beam.runners.core.construction.CoderTranslation;
import org.apache.beam.runners.core.construction.DeduplicatedFlattenFactory;
import org.apache.beam.runners.core.construction.EmptyFlattenAsCreateFactory;
import org.apache.beam.runners.core.construction.PTransformMatchers;
@@ -79,10 +81,12 @@ import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.PipelineResult.State;
import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.Coder.NonDeterministicException;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.common.runner.v1.RunnerApi;
import org.apache.beam.sdk.extensions.gcp.storage.PathValidator;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.FileSystems;
@@ -103,6 +107,7 @@ import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.runners.TransformHierarchy.Node;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.Combine.GroupedValues;
+import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
@@ -113,6 +118,7 @@ import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.WithKeys;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.util.InstanceBuilder;
import org.apache.beam.sdk.util.MimeTypes;
import org.apache.beam.sdk.util.NameUtils;
@@ -312,6 +318,12 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
PTransformMatchers.classEqualTo(PubsubUnboundedSink.class),
new StreamingPubsubIOWriteOverrideFactory(this)));
}
+ if (hasExperiment(options, "beam_fn_api")) {
+ overridesBuilder.add(
+ PTransformOverride.of(
+ PTransformMatchers.classEqualTo(Create.Values.class),
+ new StreamingFnApiCreateOverrideFactory()));
+ }
overridesBuilder
.add(
// Streaming Bounded Read is implemented in terms of Streaming Unbounded Read, and
@@ -428,15 +440,12 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
AppliedPTransform<PBegin, PCollection<T>, PTransform<PInput, PCollection<T>>> transform) {
PTransform<PInput, PCollection<T>> original = transform.getTransform();
- PCollection<T> output =
- (PCollection) Iterables.getOnlyElement(transform.getOutputs().values());
return PTransformReplacement.of(
transform.getPipeline().begin(),
InstanceBuilder.ofType(replacement)
.withArg(DataflowRunner.class, runner)
.withArg(
(Class<? super PTransform<PInput, PCollection<T>>>) original.getClass(), original)
- .withArg((Class<? super PCollection<T>>) output.getClass(), output)
.build());
}
@@ -814,10 +823,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
/** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public StreamingPubsubIORead(
- DataflowRunner runner,
- PubsubUnboundedSource transform,
- PCollection<PubsubMessage> originalOutput) {
+ public StreamingPubsubIORead(DataflowRunner runner, PubsubUnboundedSource transform) {
this.transform = transform;
}
@@ -986,6 +992,136 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
// ================================================================================
/**
+ * A PTranform override factory which maps Create.Values PTransforms for streaming pipelines
+ * into a Dataflow specific variant.
+ */
+ private static class StreamingFnApiCreateOverrideFactory<T>
+ implements PTransformOverrideFactory<PBegin, PCollection<T>, Create.Values<T>> {
+
+ @Override
+ public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
+ AppliedPTransform<PBegin, PCollection<T>, Create.Values<T>> transform) {
+ Create.Values<T> original = transform.getTransform();
+ PCollection<T> output =
+ (PCollection) Iterables.getOnlyElement(transform.getOutputs().values());
+ return PTransformReplacement.of(
+ transform.getPipeline().begin(),
+ new StreamingFnApiCreate<>(original, output));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollection<T> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
+ }
+ }
+
+ /**
+ * Specialized implementation for
+ * {@link org.apache.beam.sdk.transforms.Create.Values Create.Values} for the Dataflow runner in
+ * streaming mode over the Fn API.
+ */
+ private static class StreamingFnApiCreate<T> extends PTransform<PBegin, PCollection<T>> {
+ private final Create.Values<T> transform;
+ private final PCollection<T> originalOutput;
+
+ private StreamingFnApiCreate(
+ Create.Values<T> transform,
+ PCollection<T> originalOutput) {
+ this.transform = transform;
+ this.originalOutput = originalOutput;
+ }
+
+ @Override
+ public final PCollection<T> expand(PBegin input) {
+ try {
+ PCollection<T> pc = Pipeline
+ .applyTransform(input, new Impulse(IsBounded.BOUNDED))
+ .apply(ParDo.of(DecodeAndEmitDoFn
+ .fromIterable(transform.getElements(), originalOutput.getCoder())));
+ pc.setCoder(originalOutput.getCoder());
+ return pc;
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to encode elements.", e);
+ }
+ }
+
+ /**
+ * A DoFn which stores encoded versions of elements and a representation of a Coder
+ * capable of decoding those elements.
+ *
+ * <p>TODO: BEAM-2422 - Make this a SplittableDoFn.
+ */
+ private static class DecodeAndEmitDoFn<T> extends DoFn<byte[], T> {
+ public static <T> DecodeAndEmitDoFn<T> fromIterable(Iterable<T> elements, Coder<T> elemCoder)
+ throws IOException {
+ ImmutableList.Builder<byte[]> allElementsBytes = ImmutableList.builder();
+ for (T element : elements) {
+ byte[] bytes = CoderUtils.encodeToByteArray(elemCoder, element);
+ allElementsBytes.add(bytes);
+ }
+ return new DecodeAndEmitDoFn<>(allElementsBytes.build(), elemCoder);
+ }
+
+ private final Collection<byte[]> elements;
+ private final RunnerApi.MessageWithComponents coderSpec;
+
+ private DecodeAndEmitDoFn(Collection<byte[]> elements, Coder<T> coder) throws IOException {
+ this.elements = elements;
+ this.coderSpec = CoderTranslation.toProto(coder);
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext context) throws IOException {
+ Coder<T> coder =
+ (Coder) CoderTranslation.fromProto(coderSpec.getCoder(), coderSpec.getComponents());
+ for (byte[] element : elements) {
+ context.output(CoderUtils.decodeFromByteArray(coder, element));
+ }
+ }
+ }
+ }
+
+ /** The Dataflow specific override for the impulse primitive. */
+ private static class Impulse extends PTransform<PBegin, PCollection<byte[]>> {
+ private final IsBounded isBounded;
+
+ private Impulse(IsBounded isBounded) {
+ this.isBounded = isBounded;
+ }
+
+ @Override
+ public PCollection<byte[]> expand(PBegin input) {
+ return PCollection.createPrimitiveOutputInternal(
+ input.getPipeline(), WindowingStrategy.globalDefault(), isBounded);
+ }
+
+ @Override
+ protected Coder<?> getDefaultOutputCoder() {
+ return ByteArrayCoder.of();
+ }
+
+ private static class Translator implements TransformTranslator<Impulse> {
+ @Override
+ public void translate(Impulse transform, TranslationContext context) {
+ if (context.getPipelineOptions().isStreaming()) {
+ StepTranslationContext stepContext = context.addStep(transform, "ParallelRead");
+ stepContext.addInput(PropertyNames.FORMAT, "pubsub");
+ stepContext.addInput(PropertyNames.PUBSUB_SUBSCRIPTION, "_starting_signal/");
+ stepContext.addOutput(context.getOutput(transform));
+ } else {
+ throw new UnsupportedOperationException(
+ "Impulse source for batch pipelines has not been defined.");
+ }
+ }
+ }
+
+ static {
+ DataflowPipelineTranslator.registerTransformTranslator(Impulse.class, new Translator());
+ }
+ }
+
+ /**
* Specialized implementation for
* {@link org.apache.beam.sdk.io.Read.Unbounded Read.Unbounded} for the
* Dataflow runner in streaming mode.
@@ -998,9 +1134,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
/** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public StreamingUnboundedRead(DataflowRunner runner,
- Read.Unbounded<T> transform,
- PCollection<T> originalOutput) {
+ public StreamingUnboundedRead(DataflowRunner runner, Read.Unbounded<T> transform) {
this.source = transform.getSource();
}
@@ -1115,9 +1249,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
/** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public StreamingBoundedRead(DataflowRunner runner,
- Read.Bounded<T> transform,
- PCollection<T> originalOutput) {
+ public StreamingBoundedRead(DataflowRunner runner, Read.Bounded<T> transform) {
this.source = transform.getSource();
}
[26/50] [abbrv] beam git commit: Use inner module for non-public
coders.
Posted by ke...@apache.org.
Use inner module for non-public coders.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/36aea2d2
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/36aea2d2
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/36aea2d2
Branch: refs/heads/gearpump-runner
Commit: 36aea2d26d7c8ea3299d7a25d617a6ba99794e18
Parents: 4aef938
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Wed Jun 7 23:35:11 2017 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Thu Jun 8 13:58:31 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/runners/worker/operation_specs.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/36aea2d2/sdks/python/apache_beam/runners/worker/operation_specs.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/operation_specs.py b/sdks/python/apache_beam/runners/worker/operation_specs.py
index b8d19a1..bdafbea 100644
--- a/sdks/python/apache_beam/runners/worker/operation_specs.py
+++ b/sdks/python/apache_beam/runners/worker/operation_specs.py
@@ -326,11 +326,12 @@ def get_coder_from_spec(coder_spec):
assert len(coder_spec['component_encodings']) == 2
value_coder, window_coder = [
get_coder_from_spec(c) for c in coder_spec['component_encodings']]
- return coders.WindowedValueCoder(value_coder, window_coder=window_coder)
+ return coders.coders.WindowedValueCoder(
+ value_coder, window_coder=window_coder)
elif coder_spec['@type'] == 'kind:interval_window':
assert ('component_encodings' not in coder_spec
or not coder_spec['component_encodings'])
- return coders.IntervalWindowCoder()
+ return coders.coders.IntervalWindowCoder()
elif coder_spec['@type'] == 'kind:global_window':
assert ('component_encodings' not in coder_spec
or not coder_spec['component_encodings'])
[48/50] [abbrv] beam git commit: This closes #3338: [BEAM-2371] Port
some DirectRunner overrides to SDK-agnostic APIs
Posted by ke...@apache.org.
This closes #3338: [BEAM-2371] Port some DirectRunner overrides to SDK-agnostic APIs
Port DirectRunner TestStream override to SDK-agnostic APIs
Port DirectRunner WriteFiles override to SDK-agnostic APIs
Port DirectGroupByKey to SDK-agnostic APIs
Port ViewOverrideFactory to SDK-agnostic APIs
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1597f3ca
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1597f3ca
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1597f3ca
Branch: refs/heads/gearpump-runner
Commit: 1597f3ca64558f0099237aeb618b144e132ddcc6
Parents: b4c7716 eaaf45f
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Jun 9 19:57:17 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Jun 9 19:57:17 2017 -0700
----------------------------------------------------------------------
.../CreatePCollectionViewTranslation.java | 4 +-
.../core/construction/PTransformMatchers.java | 17 +++++--
.../construction/TestStreamTranslation.java | 49 +++++++++++++++++++-
.../beam/runners/direct/DirectGroupByKey.java | 13 +++---
.../direct/DirectGroupByKeyOverrideFactory.java | 14 ++++--
.../direct/TestStreamEvaluatorFactory.java | 20 ++++++--
.../runners/direct/ViewOverrideFactory.java | 48 +++++++++++--------
.../direct/WriteWithShardingFactory.java | 30 ++++++++----
.../direct/ViewEvaluatorFactoryTest.java | 3 +-
.../runners/direct/ViewOverrideFactoryTest.java | 23 +++++++--
.../direct/WriteWithShardingFactoryTest.java | 26 +++++++----
.../org/apache/beam/sdk/testing/TestStream.java | 12 +++++
.../beam/sdk/values/PCollectionViews.java | 10 ++++
13 files changed, 207 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
[08/50] [abbrv] beam git commit: Remove support for NativeSinks from
the Python DirectRunner
Posted by ke...@apache.org.
Remove support for NativeSinks from the Python DirectRunner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f2e30886
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f2e30886
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f2e30886
Branch: refs/heads/gearpump-runner
Commit: f2e3088633fef10f19bfd11ff9b508930916a740
Parents: 32f22b7
Author: Charles Chen <cc...@google.com>
Authored: Wed Jun 7 17:00:57 2017 -0700
Committer: Charles Chen <cc...@google.com>
Committed: Wed Jun 7 17:01:33 2017 -0700
----------------------------------------------------------------------
.../runners/direct/transform_evaluator.py | 62 +-------------------
1 file changed, 1 insertion(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/f2e30886/sdks/python/apache_beam/runners/direct/transform_evaluator.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py
index b1cb626..0fec8b8 100644
--- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py
+++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py
@@ -29,7 +29,6 @@ from apache_beam.runners.common import DoFnRunner
from apache_beam.runners.common import DoFnState
from apache_beam.runners.direct.watermark_manager import WatermarkManager
from apache_beam.runners.direct.transform_result import TransformResult
-from apache_beam.runners.dataflow.native_io.iobase import _NativeWrite # pylint: disable=protected-access
from apache_beam.transforms import core
from apache_beam.transforms.window import GlobalWindows
from apache_beam.transforms.window import WindowedValue
@@ -54,7 +53,6 @@ class TransformEvaluatorRegistry(object):
core.Flatten: _FlattenEvaluator,
core.ParDo: _ParDoEvaluator,
core._GroupByKeyOnly: _GroupByKeyOnlyEvaluator,
- _NativeWrite: _NativeWriteEvaluator,
}
def for_application(
@@ -98,8 +96,7 @@ class TransformEvaluatorRegistry(object):
Returns:
True if executor should execute applied_ptransform serially.
"""
- return isinstance(applied_ptransform.transform,
- (core._GroupByKeyOnly, _NativeWrite))
+ return isinstance(applied_ptransform.transform, core._GroupByKeyOnly)
class _TransformEvaluator(object):
@@ -403,60 +400,3 @@ class _GroupByKeyOnlyEvaluator(_TransformEvaluator):
return TransformResult(
self._applied_ptransform, bundles, state, None, None, hold)
-
-
-class _NativeWriteEvaluator(_TransformEvaluator):
- """TransformEvaluator for _NativeWrite transform."""
-
- def __init__(self, evaluation_context, applied_ptransform,
- input_committed_bundle, side_inputs, scoped_metrics_container):
- assert not side_inputs
- super(_NativeWriteEvaluator, self).__init__(
- evaluation_context, applied_ptransform, input_committed_bundle,
- side_inputs, scoped_metrics_container)
-
- assert applied_ptransform.transform.sink
- self._sink = applied_ptransform.transform.sink
-
- @property
- def _is_final_bundle(self):
- return (self._execution_context.watermarks.input_watermark
- == WatermarkManager.WATERMARK_POS_INF)
-
- @property
- def _has_already_produced_output(self):
- return (self._execution_context.watermarks.output_watermark
- == WatermarkManager.WATERMARK_POS_INF)
-
- def start_bundle(self):
- # state: [values]
- self.state = (self._execution_context.existing_state
- if self._execution_context.existing_state else [])
-
- def process_element(self, element):
- self.state.append(element)
-
- def finish_bundle(self):
- # finish_bundle will append incoming bundles in memory until all the bundles
- # carrying data is processed. This is done to produce only a single output
- # shard (some tests depends on this behavior). It is possible to have
- # incoming empty bundles after the output is produced, these bundles will be
- # ignored and would not generate additional output files.
- # TODO(altay): Do not wait until the last bundle to write in a single shard.
- if self._is_final_bundle:
- if self._has_already_produced_output:
- # Ignore empty bundles that arrive after the output is produced.
- assert self.state == []
- else:
- self._sink.pipeline_options = self._evaluation_context.pipeline_options
- with self._sink.writer() as writer:
- for v in self.state:
- writer.Write(v.value)
- state = None
- hold = WatermarkManager.WATERMARK_POS_INF
- else:
- state = self.state
- hold = WatermarkManager.WATERMARK_NEG_INF
-
- return TransformResult(
- self._applied_ptransform, [], state, None, None, hold)
[37/50] [abbrv] beam git commit: This closes #3320
Posted by ke...@apache.org.
This closes #3320
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/97957ea0
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/97957ea0
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/97957ea0
Branch: refs/heads/gearpump-runner
Commit: 97957ea0ec1625b0e937a4f3c9ea2bee316d24d4
Parents: 1c6861f b7ae7ec
Author: Thomas Groh <tg...@google.com>
Authored: Fri Jun 9 10:54:13 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Fri Jun 9 10:54:13 2017 -0700
----------------------------------------------------------------------
.../runners/flink/FlinkStreamingTransformTranslators.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[39/50] [abbrv] beam git commit: Closes #3332
Posted by ke...@apache.org.
Closes #3332
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d2c4093e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d2c4093e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d2c4093e
Branch: refs/heads/gearpump-runner
Commit: d2c4093ed2fd47e28b31898a17e28d0575aa9ff9
Parents: 97957ea fb61c54
Author: Robert Bradshaw <ro...@google.com>
Authored: Fri Jun 9 13:00:55 2017 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Fri Jun 9 13:00:55 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/io/gcp/pubsub.py | 16 +++-------------
sdks/python/apache_beam/io/gcp/pubsub_test.py | 10 ----------
2 files changed, 3 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
[18/50] [abbrv] beam git commit: Avoid flakiness in data channel for
empty streams.
Posted by ke...@apache.org.
Avoid flakiness in data channel for empty streams.
This closes #3324
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d81ed217
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d81ed217
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d81ed217
Branch: refs/heads/gearpump-runner
Commit: d81ed21726cb50962bc4fcfb57d50e11eacdcf3b
Parents: e066a9d 4ebebfd
Author: Luke Cwik <lc...@google.com>
Authored: Thu Jun 8 10:09:12 2017 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Thu Jun 8 10:09:12 2017 -0700
----------------------------------------------------------------------
.../python/apache_beam/runners/worker/data_plane.py | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[49/50] [abbrv] beam git commit: Fix side input handling in
DoFnFunction
Posted by ke...@apache.org.
Fix side input handling in DoFnFunction
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7653e7ed
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7653e7ed
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7653e7ed
Branch: refs/heads/gearpump-runner
Commit: 7653e7ed6de3d9db822dcd390d2bf70819954fa5
Parents: 98854d4
Author: manuzhang <ow...@gmail.com>
Authored: Wed Jun 7 14:08:04 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Jun 12 11:45:37 2017 +0800
----------------------------------------------------------------------
.../translators/TranslationContext.java | 2 ++
.../translators/functions/DoFnFunction.java | 23 ++++----------------
.../gearpump/translators/io/GearpumpSource.java | 1 -
.../translators/utils/TranslatorUtils.java | 5 ++---
4 files changed, 8 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/7653e7ed/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
index 4090354..64a1e0d 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/TranslationContext.java
@@ -68,6 +68,8 @@ public class TranslationContext {
public <OutputT> void setOutputStream(PValue output, JavaStream<OutputT> outputStream) {
if (!streams.containsKey(output)) {
streams.put(output, outputStream);
+ } else {
+ throw new RuntimeException("set stream for duplicated output " + output);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/7653e7ed/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
index f521d7b..6e4fbeb 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/functions/DoFnFunction.java
@@ -24,7 +24,6 @@ import com.google.common.collect.Lists;
import java.io.Serializable;
import java.util.Collection;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
@@ -66,7 +65,6 @@ public class DoFnFunction<InputT, OutputT> extends
private transient PushbackSideInputDoFnRunner<InputT, OutputT> doFnRunner;
private transient SideInputHandler sideInputReader;
private transient List<WindowedValue<InputT>> pushedBackValues;
- private transient Map<PCollectionView<?>, List<WindowedValue<Iterable<?>>>> sideInputValues;
private final Collection<PCollectionView<?>> sideInputs;
private final Map<String, PCollectionView<?>> tagsToSideInputs;
private final TupleTag<OutputT> mainOutput;
@@ -109,7 +107,6 @@ public class DoFnFunction<InputT, OutputT> extends
doFnRunner = doFnRunnerFactory.createRunner(sideInputReader);
pushedBackValues = new LinkedList<>();
- sideInputValues = new HashMap<>();
outputManager.setup(mainOutput, sideOutputs);
}
@@ -132,25 +129,14 @@ public class DoFnFunction<InputT, OutputT> extends
} else {
// side input
PCollectionView<?> sideInput = tagsToSideInputs.get(unionValue.getUnionTag());
- WindowedValue<?> sideInputValue =
- (WindowedValue<?>) unionValue.getValue();
- Object value = sideInputValue.getValue();
- if (!(value instanceof Iterable)) {
- sideInputValue = sideInputValue.withValue(Lists.newArrayList(value));
- }
- if (!sideInputValues.containsKey(sideInput)) {
- sideInputValues.put(sideInput, new LinkedList<WindowedValue<Iterable<?>>>());
- }
- sideInputValues.get(sideInput).add((WindowedValue<Iterable<?>>) sideInputValue);
+ WindowedValue<Iterable<?>> sideInputValue =
+ (WindowedValue<Iterable<?>>) unionValue.getValue();
+ sideInputReader.addSideInputValue(sideInput, sideInputValue);
}
}
+
for (PCollectionView<?> sideInput: sideInputs) {
- if (sideInputValues.containsKey(sideInput)) {
- for (WindowedValue<Iterable<?>> value: sideInputValues.get(sideInput)) {
- sideInputReader.addSideInputValue(sideInput, value);
- }
- }
for (WindowedValue<InputT> value : pushedBackValues) {
for (BoundedWindow win: value.getWindows()) {
BoundedWindow sideInputWindow =
@@ -171,7 +157,6 @@ public class DoFnFunction<InputT, OutputT> extends
}
pushedBackValues.clear();
Iterables.addAll(pushedBackValues, nextPushedBackValues);
- sideInputValues.clear();
doFnRunner.finishBundle();
http://git-wip-us.apache.org/repos/asf/beam/blob/7653e7ed/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
index 5e79151..60f319d 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/io/GearpumpSource.java
@@ -28,7 +28,6 @@ import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
import org.apache.beam.sdk.io.Source;
import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
-// import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
http://git-wip-us.apache.org/repos/asf/beam/blob/7653e7ed/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
index 999afae..282f261 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/utils/TranslatorUtils.java
@@ -70,11 +70,10 @@ public class TranslatorUtils {
JavaStream<WindowedValue<InputT>> inputStream,
Map<String, PCollectionView<?>> tagsToSideInputs) {
JavaStream<RawUnionValue> mainStream =
- inputStream.map(new ToRawUnionValue<InputT>("0"), "map_to_RawUnionValue");
+ inputStream.map(new ToRawUnionValue<>("0"), "map_to_RawUnionValue");
for (Map.Entry<String, PCollectionView<?>> tagToSideInput: tagsToSideInputs.entrySet()) {
- // actually JavaStream<WindowedValue<List<?>>>
- JavaStream<WindowedValue<Object>> sideInputStream = context.getInputStream(
+ JavaStream<WindowedValue<List<?>>> sideInputStream = context.getInputStream(
tagToSideInput.getValue());
mainStream = mainStream.merge(sideInputStream.map(new ToRawUnionValue<>(
tagToSideInput.getKey()), "map_to_RawUnionValue"), "merge_to_MainStream");
[10/50] [abbrv] beam git commit: Make BytesCoder to be a known type
Posted by ke...@apache.org.
Make BytesCoder to be a known type
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d94ac58e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d94ac58e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d94ac58e
Branch: refs/heads/gearpump-runner
Commit: d94ac58ea2d12f55743e8ad27a02bdb83c194da7
Parents: 66460cb
Author: Vikas Kedigehalli <vi...@google.com>
Authored: Wed Jun 7 16:26:21 2017 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Wed Jun 7 20:05:40 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/coders/coders.py | 5 +++++
sdks/python/apache_beam/runners/worker/operation_specs.py | 4 ++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/d94ac58e/sdks/python/apache_beam/coders/coders.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py
index f40045d..f3e0b43 100644
--- a/sdks/python/apache_beam/coders/coders.py
+++ b/sdks/python/apache_beam/coders/coders.py
@@ -286,6 +286,11 @@ class BytesCoder(FastCoder):
def is_deterministic(self):
return True
+ def as_cloud_object(self):
+ return {
+ '@type': 'kind:bytes',
+ }
+
def __eq__(self, other):
return type(self) == type(other)
http://git-wip-us.apache.org/repos/asf/beam/blob/d94ac58e/sdks/python/apache_beam/runners/worker/operation_specs.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/operation_specs.py b/sdks/python/apache_beam/runners/worker/operation_specs.py
index db5eb76..b8d19a1 100644
--- a/sdks/python/apache_beam/runners/worker/operation_specs.py
+++ b/sdks/python/apache_beam/runners/worker/operation_specs.py
@@ -339,6 +339,10 @@ def get_coder_from_spec(coder_spec):
assert len(coder_spec['component_encodings']) == 1
return coders.coders.LengthPrefixCoder(
get_coder_from_spec(coder_spec['component_encodings'][0]))
+ elif coder_spec['@type'] == 'kind:bytes':
+ assert ('component_encodings' not in coder_spec
+ or len(coder_spec['component_encodings'] == 0))
+ return coders.BytesCoder()
# We pass coders in the form "<coder_name>$<pickled_data>" to make the job
# description JSON more readable.
[25/50] [abbrv] beam git commit: This closes #3239: [BEAM-2371] Port
DirectRunner override matchers to language-agnostic construction APIs
Posted by ke...@apache.org.
This closes #3239: [BEAM-2371] Port DirectRunner override matchers to language-agnostic construction APIs
DirectRunner override matchers using Runner API
Add Runner API oriented PTransformMatchers for DirectRunner overrides
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4aef9385
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4aef9385
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4aef9385
Branch: refs/heads/gearpump-runner
Commit: 4aef9385477f0d13bdaa345cb5a629b2fd00b867
Parents: b53e6f0 d8d9087
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Jun 8 11:37:15 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Thu Jun 8 11:37:15 2017 -0700
----------------------------------------------------------------------
.../core/construction/PTransformMatchers.java | 94 +++++++++++++++++++-
.../construction/PTransformTranslation.java | 7 +-
.../construction/PTransformMatchersTest.java | 32 +++++++
.../beam/runners/direct/DirectRunner.java | 20 ++---
4 files changed, 137 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
[07/50] [abbrv] beam git commit: This closes #3294
Posted by ke...@apache.org.
This closes #3294
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/32f22b7d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/32f22b7d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/32f22b7d
Branch: refs/heads/gearpump-runner
Commit: 32f22b7d9cfd5dcd22555272c5a7365fd1323e5f
Parents: caecac3 9c83ffe
Author: Ismaël MejÃa <ie...@apache.org>
Authored: Wed Jun 7 23:14:02 2017 +0200
Committer: Ismaël MejÃa <ie...@apache.org>
Committed: Wed Jun 7 23:14:02 2017 +0200
----------------------------------------------------------------------
.../wrappers/streaming/io/UnboundedSourceWrapper.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
[11/50] [abbrv] beam git commit: [BEAM-1226] Make BytesCoder to be a
known type
Posted by ke...@apache.org.
[BEAM-1226] Make BytesCoder to be a known type
This closes #3316
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0a0a1bc7
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0a0a1bc7
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0a0a1bc7
Branch: refs/heads/gearpump-runner
Commit: 0a0a1bc7493ad71f1a70b1c9921ba09ecfe4c2e8
Parents: 66460cb d94ac58
Author: Luke Cwik <lc...@google.com>
Authored: Wed Jun 7 20:08:49 2017 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Wed Jun 7 20:08:49 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/coders/coders.py | 5 +++++
sdks/python/apache_beam/runners/worker/operation_specs.py | 4 ++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
[15/50] [abbrv] beam git commit: Expand all PValues to component
PCollections always
Posted by ke...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java
index 6c385d7..1853248 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/StreamingViewOverrides.java
@@ -42,12 +42,12 @@ import org.apache.beam.sdk.values.PCollectionView;
class StreamingViewOverrides {
static class StreamingCreatePCollectionViewFactory<ElemT, ViewT>
extends SingleInputOutputOverrideFactory<
- PCollection<ElemT>, PCollectionView<ViewT>, CreatePCollectionView<ElemT, ViewT>> {
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>> {
@Override
- public PTransformReplacement<PCollection<ElemT>, PCollectionView<ViewT>>
+ public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>>
getReplacementTransform(
AppliedPTransform<
- PCollection<ElemT>, PCollectionView<ViewT>, CreatePCollectionView<ElemT, ViewT>>
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>>
transform) {
StreamingCreatePCollectionView<ElemT, ViewT> streamingView =
new StreamingCreatePCollectionView<>(transform.getTransform().getView());
@@ -56,7 +56,7 @@ class StreamingViewOverrides {
}
private static class StreamingCreatePCollectionView<ElemT, ViewT>
- extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
private final PCollectionView<ViewT> view;
private StreamingCreatePCollectionView(PCollectionView<ViewT> view) {
@@ -64,7 +64,7 @@ class StreamingViewOverrides {
}
@Override
- public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
return input
.apply(Combine.globally(new Concatenate<ElemT>()).withoutDefaults())
.apply(ParDo.of(StreamingPCollectionViewWriterFn.create(view, input.getCoder())))
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
index 89dc2d5..53215f6 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslatorTest.java
@@ -920,15 +920,15 @@ public class DataflowPipelineTranslatorTest implements Serializable {
assertAllStepOutputsHaveUniqueIds(job);
List<Step> steps = job.getSteps();
- assertEquals(5, steps.size());
+ assertEquals(9, steps.size());
@SuppressWarnings("unchecked")
List<Map<String, Object>> toIsmRecordOutputs =
- (List<Map<String, Object>>) steps.get(3).getProperties().get(PropertyNames.OUTPUT_INFO);
+ (List<Map<String, Object>>) steps.get(7).getProperties().get(PropertyNames.OUTPUT_INFO);
assertTrue(
Structs.getBoolean(Iterables.getOnlyElement(toIsmRecordOutputs), "use_indexed_format"));
- Step collectionToSingletonStep = steps.get(4);
+ Step collectionToSingletonStep = steps.get(8);
assertEquals("CollectionToSingleton", collectionToSingletonStep.getKind());
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index 64aa35a..ac5e0cd 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -508,50 +508,6 @@ public final class TransformTranslator {
};
}
- private static <T> TransformEvaluator<View.AsSingleton<T>> viewAsSingleton() {
- return new TransformEvaluator<View.AsSingleton<T>>() {
- @Override
- public void evaluate(View.AsSingleton<T> transform, EvaluationContext context) {
- Iterable<? extends WindowedValue<?>> iter =
- context.getWindowedValues(context.getInput(transform));
- PCollectionView<T> output = context.getOutput(transform);
- Coder<Iterable<WindowedValue<?>>> coderInternal = output.getCoderInternal();
-
- @SuppressWarnings("unchecked")
- Iterable<WindowedValue<?>> iterCast = (Iterable<WindowedValue<?>>) iter;
-
- context.putPView(output, iterCast, coderInternal);
- }
-
- @Override
- public String toNativeString() {
- return "collect()";
- }
- };
- }
-
- private static <T> TransformEvaluator<View.AsIterable<T>> viewAsIter() {
- return new TransformEvaluator<View.AsIterable<T>>() {
- @Override
- public void evaluate(View.AsIterable<T> transform, EvaluationContext context) {
- Iterable<? extends WindowedValue<?>> iter =
- context.getWindowedValues(context.getInput(transform));
- PCollectionView<Iterable<T>> output = context.getOutput(transform);
- Coder<Iterable<WindowedValue<?>>> coderInternal = output.getCoderInternal();
-
- @SuppressWarnings("unchecked")
- Iterable<WindowedValue<?>> iterCast = (Iterable<WindowedValue<?>>) iter;
-
- context.putPView(output, iterCast, coderInternal);
- }
-
- @Override
- public String toNativeString() {
- return "collect()";
- }
- };
- }
-
private static <ReadT, WriteT> TransformEvaluator<View.CreatePCollectionView<ReadT, WriteT>>
createPCollView() {
return new TransformEvaluator<View.CreatePCollectionView<ReadT, WriteT>>() {
@@ -560,7 +516,7 @@ public final class TransformTranslator {
EvaluationContext context) {
Iterable<? extends WindowedValue<?>> iter =
context.getWindowedValues(context.getInput(transform));
- PCollectionView<WriteT> output = context.getOutput(transform);
+ PCollectionView<WriteT> output = transform.getView();
Coder<Iterable<WindowedValue<?>>> coderInternal = output.getCoderInternal();
@SuppressWarnings("unchecked")
@@ -645,8 +601,8 @@ public final class TransformTranslator {
EVALUATORS.put(Combine.PerKey.class, combinePerKey());
EVALUATORS.put(Flatten.PCollections.class, flattenPColl());
EVALUATORS.put(Create.Values.class, create());
- EVALUATORS.put(View.AsSingleton.class, viewAsSingleton());
- EVALUATORS.put(View.AsIterable.class, viewAsIter());
+// EVALUATORS.put(View.AsSingleton.class, viewAsSingleton());
+// EVALUATORS.put(View.AsIterable.class, viewAsIter());
EVALUATORS.put(View.CreatePCollectionView.class, createPCollView());
EVALUATORS.put(Window.Assign.class, window());
EVALUATORS.put(Reshuffle.class, reshuffle());
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index 2f0e8ef..ee1ce7b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -24,10 +24,12 @@ import static com.google.common.base.Preconditions.checkState;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -39,6 +41,7 @@ import org.apache.beam.sdk.Pipeline.PipelineVisitor.CompositeBehavior;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.runners.PTransformOverrideFactory.ReplacementOutput;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.POutput;
import org.apache.beam.sdk.values.PValue;
@@ -165,7 +168,7 @@ public class TransformHierarchy {
* nodes.
*/
public void setOutput(POutput output) {
- for (PValue value : output.expand().values()) {
+ for (PCollection<?> value : fullyExpand(output).values()) {
if (!producers.containsKey(value)) {
producers.put(value, current);
value.finishSpecifyingOutput(
@@ -226,6 +229,47 @@ public class TransformHierarchy {
return current;
}
+ private Map<TupleTag<?>, PCollection<?>> fullyExpand(POutput output) {
+ Map<TupleTag<?>, PCollection<?>> result = new LinkedHashMap<>();
+ for (Map.Entry<TupleTag<?>, PValue> value : output.expand().entrySet()) {
+ if (value.getValue() instanceof PCollection) {
+ PCollection<?> previous = result.put(value.getKey(), (PCollection<?>) value.getValue());
+ checkArgument(
+ previous == null,
+ "Found conflicting %ss in flattened expansion of %s: %s maps to %s and %s",
+ output,
+ TupleTag.class.getSimpleName(),
+ value.getKey(),
+ previous,
+ value.getValue());
+ } else {
+ if (value.getValue().expand().size() == 1
+ && Iterables.getOnlyElement(value.getValue().expand().values())
+ .equals(value.getValue())) {
+ throw new IllegalStateException(
+ String.format(
+ "Non %s %s that expands into itself %s",
+ PCollection.class.getSimpleName(),
+ PValue.class.getSimpleName(),
+ value.getValue()));
+ }
+ for (Map.Entry<TupleTag<?>, PCollection<?>> valueComponent :
+ fullyExpand(value.getValue()).entrySet()) {
+ PCollection<?> previous = result.put(valueComponent.getKey(), valueComponent.getValue());
+ checkArgument(
+ previous == null,
+ "Found conflicting %ss in flattened expansion of %s: %s maps to %s and %s",
+ output,
+ TupleTag.class.getSimpleName(),
+ valueComponent.getKey(),
+ previous,
+ valueComponent.getValue());
+ }
+ }
+ }
+ return result;
+ }
+
/**
* Provides internal tracking of transform relationships with helper methods
* for initialization and ordered visitation.
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 9e1cc71..6a90bcf 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -1277,14 +1277,15 @@ public class Combine {
public PCollectionView<OutputT> expand(PCollection<InputT> input) {
PCollection<OutputT> combined =
input.apply(Combine.<InputT, OutputT>globally(fn).withoutDefaults().withFanout(fanout));
- return combined.apply(
- CreatePCollectionView.<OutputT, OutputT>of(
- PCollectionViews.singletonView(
- combined,
- input.getWindowingStrategy(),
- insertDefault,
- insertDefault ? fn.defaultValue() : null,
- combined.getCoder())));
+ PCollectionView<OutputT> view =
+ PCollectionViews.singletonView(
+ combined,
+ input.getWindowingStrategy(),
+ insertDefault,
+ insertDefault ? fn.defaultValue() : null,
+ combined.getCoder());
+ combined.apply(CreatePCollectionView.<OutputT, OutputT>of(view));
+ return view;
}
public int getFanout() {
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
index 073c750..331b143 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
@@ -257,8 +257,10 @@ public class View {
throw new IllegalStateException("Unable to create a side-input view from input", e);
}
- return input.apply(CreatePCollectionView.<T, List<T>>of(PCollectionViews.listView(
- input, input.getWindowingStrategy(), input.getCoder())));
+ PCollectionView<List<T>> view =
+ PCollectionViews.listView(input, input.getWindowingStrategy(), input.getCoder());
+ input.apply(CreatePCollectionView.<T, List<T>>of(view));
+ return view;
}
}
@@ -282,8 +284,10 @@ public class View {
throw new IllegalStateException("Unable to create a side-input view from input", e);
}
- return input.apply(CreatePCollectionView.<T, Iterable<T>>of(PCollectionViews.iterableView(
- input, input.getWindowingStrategy(), input.getCoder())));
+ PCollectionView<Iterable<T>> view =
+ PCollectionViews.iterableView(input, input.getWindowingStrategy(), input.getCoder());
+ input.apply(CreatePCollectionView.<T, Iterable<T>>of(view));
+ return view;
}
}
@@ -423,11 +427,10 @@ public class View {
throw new IllegalStateException("Unable to create a side-input view from input", e);
}
- return input.apply(CreatePCollectionView.<KV<K, V>, Map<K, Iterable<V>>>of(
- PCollectionViews.multimapView(
- input,
- input.getWindowingStrategy(),
- input.getCoder())));
+ PCollectionView<Map<K, Iterable<V>>> view =
+ PCollectionViews.multimapView(input, input.getWindowingStrategy(), input.getCoder());
+ input.apply(CreatePCollectionView.<KV<K, V>, Map<K, Iterable<V>>>of(view));
+ return view;
}
}
@@ -459,11 +462,10 @@ public class View {
throw new IllegalStateException("Unable to create a side-input view from input", e);
}
- return input.apply(CreatePCollectionView.<KV<K, V>, Map<K, V>>of(
- PCollectionViews.mapView(
- input,
- input.getWindowingStrategy(),
- input.getCoder())));
+ PCollectionView<Map<K, V>> view =
+ PCollectionViews.mapView(input, input.getWindowingStrategy(), input.getCoder());
+ input.apply(CreatePCollectionView.<KV<K, V>, Map<K, V>>of(view));
+ return view;
}
}
@@ -480,7 +482,7 @@ public class View {
*/
@Internal
public static class CreatePCollectionView<ElemT, ViewT>
- extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
private PCollectionView<ViewT> view;
private CreatePCollectionView(PCollectionView<ViewT> view) {
@@ -506,8 +508,10 @@ public class View {
}
@Override
- public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
- return view;
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
+ return PCollection.<ElemT>createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded())
+ .setCoder(input.getCoder());
}
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
index f210fd8..4063d11 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollection.java
@@ -20,6 +20,8 @@ package org.apache.beam.sdk.values;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
+import java.util.Collections;
+import java.util.Map;
import javax.annotation.Nullable;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.annotations.Internal;
@@ -226,6 +228,11 @@ public class PCollection<T> extends PValueBase implements PValue {
return super.getName();
}
+ @Override
+ public final Map<TupleTag<?>, PValue> expand() {
+ return Collections.<TupleTag<?>, PValue>singletonMap(tag, this);
+ }
+
/**
* Sets the name of this {@link PCollection}. Returns {@code this}.
*
@@ -314,6 +321,11 @@ public class PCollection<T> extends PValueBase implements PValue {
private IsBounded isBounded;
+ /**
+ * A local {@link TupleTag} used in the expansion of this {@link PValueBase}.
+ */
+ private final TupleTag<?> tag = new TupleTag<>();
+
private PCollection(Pipeline p) {
super(p);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
index 74887c7..5e2e2c3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
@@ -170,6 +170,15 @@ public class PCollectionViews {
}
/**
+ * Returns if a default value was specified.
+ */
+ @Deprecated
+ @Internal
+ public boolean hasDefault() {
+ return hasDefault;
+ }
+
+ /**
* Returns the default value that was specified.
*
* <p>For internal use only.
@@ -491,5 +500,10 @@ public class PCollectionViews {
public String toString() {
return MoreObjects.toStringHelper(this).add("tag", tag).toString();
}
+
+ @Override
+ public Map<TupleTag<?>, PValue> expand() {
+ return Collections.<TupleTag<?>, PValue>singletonMap(tag, pCollection);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
index 6f638d7..f312eac 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValueBase.java
@@ -19,8 +19,6 @@ package org.apache.beam.sdk.values;
import static com.google.common.base.Preconditions.checkState;
-import java.util.Collections;
-import java.util.Map;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.transforms.PTransform;
@@ -87,11 +85,6 @@ public abstract class PValueBase implements PValue {
private String name;
/**
- * A local {@link TupleTag} used in the expansion of this {@link PValueBase}.
- */
- private TupleTag<?> tag = new TupleTag<>();
-
- /**
* Whether this {@link PValueBase} has been finalized, and its core
* properties, e.g., name, can no longer be changed.
*/
@@ -108,11 +101,6 @@ public abstract class PValueBase implements PValue {
}
@Override
- public final Map<TupleTag<?>, PValue> expand() {
- return Collections.<TupleTag<?>, PValue>singletonMap(tag, this);
- }
-
- @Override
public void finishSpecifying(PInput input, PTransform<?, ?> transform) {
finishedSpecifying = true;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
index adf27f8..aaf8b91 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/PCollectionViewTesting.java
@@ -22,7 +22,9 @@ import com.google.common.base.Function;
import com.google.common.base.MoreObjects;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.IterableCoder;
@@ -37,6 +39,7 @@ import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.PValueBase;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
@@ -349,5 +352,10 @@ public final class PCollectionViewTesting {
.add("viewFn", viewFn)
.toString();
}
+
+ @Override
+ public Map<TupleTag<?>, PValue> expand() {
+ return Collections.<TupleTag<?>, PValue>singletonMap(tag, pCollection);
+ }
}
}
[20/50] [abbrv] beam git commit: This closes #3322
Posted by ke...@apache.org.
This closes #3322
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fdfc70ee
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fdfc70ee
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fdfc70ee
Branch: refs/heads/gearpump-runner
Commit: fdfc70eecb8a3833c155e8bb7b6e128d956a61de
Parents: d81ed21 3e04902
Author: Ahmet Altay <al...@google.com>
Authored: Thu Jun 8 10:55:47 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Thu Jun 8 10:55:47 2017 -0700
----------------------------------------------------------------------
.../runners/direct/watermark_manager.py | 20 +++++++++++++++++---
sdks/python/apache_beam/utils/timestamp.py | 5 +++++
2 files changed, 22 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[16/50] [abbrv] beam git commit: Expand all PValues to component
PCollections always
Posted by ke...@apache.org.
Expand all PValues to component PCollections always
Update the implementation of WriteView
The PCollectionView is constructed within the composite override, but
WriteView just produces a primitive PCollection which has no consumers.
Track the ViewWriter within the Direct Runner, and utilize that
transform rather than the producer to update PCollection Watermarks.
Remove most Flink View overrides. All of the overrides are materially
identical within the flink runner, so use a single override to replace
all of them.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ccf73448
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ccf73448
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ccf73448
Branch: refs/heads/gearpump-runner
Commit: ccf7344820d6c69ca922aa3176dc141718382629
Parents: 86e1fab
Author: Thomas Groh <tg...@google.com>
Authored: Thu Jun 1 18:39:58 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Thu Jun 8 09:13:57 2017 -0700
----------------------------------------------------------------------
.../apache/beam/runners/apex/ApexRunner.java | 59 +--
.../translation/ApexPipelineTranslator.java | 16 +-
.../construction/RunnerPCollectionView.java | 8 +
.../apache/beam/runners/direct/DirectGraph.java | 4 +
.../beam/runners/direct/DirectGraphVisitor.java | 22 +-
.../runners/direct/ViewEvaluatorFactory.java | 8 +-
.../runners/direct/ViewOverrideFactory.java | 29 +-
.../beam/runners/direct/DirectGraphs.java | 7 +
.../runners/direct/EvaluationContextTest.java | 5 +-
.../ImmutabilityEnforcementFactoryTest.java | 4 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 1 +
.../runners/direct/TransformExecutorTest.java | 1 +
.../direct/ViewEvaluatorFactoryTest.java | 5 +-
.../runners/direct/ViewOverrideFactoryTest.java | 16 +-
.../direct/WatermarkCallbackExecutorTest.java | 1 +
.../runners/direct/WatermarkManagerTest.java | 1 +
.../runners/flink/CreateStreamingFlinkView.java | 154 ++++++++
.../flink/FlinkStreamingPipelineTranslator.java | 36 +-
.../FlinkStreamingTransformTranslators.java | 8 +-
.../flink/FlinkStreamingViewOverrides.java | 372 -------------------
.../runners/dataflow/BatchViewOverrides.java | 182 +++------
.../runners/dataflow/CreateDataflowView.java | 8 +-
.../dataflow/DataflowPipelineTranslator.java | 11 +-
.../beam/runners/dataflow/DataflowRunner.java | 17 +-
.../dataflow/StreamingViewOverrides.java | 10 +-
.../DataflowPipelineTranslatorTest.java | 6 +-
.../spark/translation/TransformTranslator.java | 50 +--
.../beam/sdk/runners/TransformHierarchy.java | 46 ++-
.../org/apache/beam/sdk/transforms/Combine.java | 17 +-
.../org/apache/beam/sdk/transforms/View.java | 38 +-
.../org/apache/beam/sdk/values/PCollection.java | 12 +
.../beam/sdk/values/PCollectionViews.java | 14 +
.../org/apache/beam/sdk/values/PValueBase.java | 12 -
.../sdk/testing/PCollectionViewTesting.java | 8 +
34 files changed, 458 insertions(+), 730 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index c595b3f..95b354a 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -62,8 +62,6 @@ import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.transforms.View.AsIterable;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
@@ -214,7 +212,7 @@ public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
* @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
*/
public static class CreateApexPCollectionView<ElemT, ViewT>
- extends PTransform<PCollection<List<ElemT>>, PCollectionView<ViewT>> {
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
private static final long serialVersionUID = 1L;
private PCollectionView<ViewT> view;
@@ -228,7 +226,13 @@ public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
}
@Override
- public PCollectionView<ViewT> expand(PCollection<List<ElemT>> input) {
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
+ return PCollection.<ElemT>createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded())
+ .setCoder(input.getCoder());
+ }
+
+ public PCollectionView<ViewT> getView() {
return view;
}
}
@@ -241,7 +245,7 @@ public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
}
private static class StreamingWrapSingletonInList<T>
- extends PTransform<PCollection<T>, PCollectionView<T>> {
+ extends PTransform<PCollection<T>, PCollection<T>> {
private static final long serialVersionUID = 1L;
CreatePCollectionView<T, T> transform;
@@ -254,10 +258,11 @@ public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
}
@Override
- public PCollectionView<T> expand(PCollection<T> input) {
- return input
+ public PCollection<T> expand(PCollection<T> input) {
+ input
.apply(ParDo.of(new WrapAsList<T>()))
- .apply(CreateApexPCollectionView.<T, T>of(transform.getView()));
+ .apply(CreateApexPCollectionView.<List<T>, T>of(transform.getView()));
+ return input;
}
@Override
@@ -267,15 +272,12 @@ public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
static class Factory<T>
extends SingleInputOutputOverrideFactory<
- PCollection<T>, PCollectionView<T>,
+ PCollection<T>, PCollection<T>,
CreatePCollectionView<T, T>> {
@Override
- public PTransformReplacement<PCollection<T>, PCollectionView<T>>
- getReplacementTransform(
- AppliedPTransform<
- PCollection<T>, PCollectionView<T>,
- CreatePCollectionView<T, T>>
- transform) {
+ public PTransformReplacement<PCollection<T>, PCollection<T>> getReplacementTransform(
+ AppliedPTransform<PCollection<T>, PCollection<T>, CreatePCollectionView<T, T>>
+ transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
new StreamingWrapSingletonInList<>(transform.getTransform()));
@@ -284,18 +286,19 @@ public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
}
private static class StreamingViewAsIterable<T>
- extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
+ extends PTransform<PCollection<T>, PCollection<T>> {
private static final long serialVersionUID = 1L;
+ private final PCollectionView<Iterable<T>> view;
- private StreamingViewAsIterable() {}
+ private StreamingViewAsIterable(PCollectionView<Iterable<T>> view) {
+ this.view = view;
+ }
@Override
- public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
- PCollectionView<Iterable<T>> view =
- PCollectionViews.iterableView(input, input.getWindowingStrategy(), input.getCoder());
-
- return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
- .apply(CreateApexPCollectionView.<T, Iterable<T>> of(view));
+ public PCollection<T> expand(PCollection<T> input) {
+ return ((PCollection<T>)
+ input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults()))
+ .apply(CreateApexPCollectionView.<T, Iterable<T>>of(view));
}
@Override
@@ -305,15 +308,17 @@ public class ApexRunner extends PipelineRunner<ApexRunnerResult> {
static class Factory<T>
extends SingleInputOutputOverrideFactory<
- PCollection<T>, PCollectionView<Iterable<T>>, View.AsIterable<T>> {
+ PCollection<T>, PCollection<T>, CreatePCollectionView<T, Iterable<T>>> {
@Override
- public PTransformReplacement<PCollection<T>, PCollectionView<Iterable<T>>>
+ public PTransformReplacement<PCollection<T>, PCollection<T>>
getReplacementTransform(
- AppliedPTransform<PCollection<T>, PCollectionView<Iterable<T>>, AsIterable<T>>
+ AppliedPTransform<
+ PCollection<T>, PCollection<T>,
+ CreatePCollectionView<T, Iterable<T>>>
transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new StreamingViewAsIterable<T>());
+ new StreamingViewAsIterable<T>(transform.getTransform().getView()));
}
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
index bda074b..02f53ec 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/translation/ApexPipelineTranslator.java
@@ -39,7 +39,6 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -154,7 +153,6 @@ public class ApexPipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
unboundedSource, true, context.getPipelineOptions());
context.addOperator(operator, operator.output);
}
-
}
private static class CreateApexPCollectionViewTranslator<ElemT, ViewT>
@@ -162,11 +160,10 @@ public class ApexPipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
private static final long serialVersionUID = 1L;
@Override
- public void translate(CreateApexPCollectionView<ElemT, ViewT> transform,
- TranslationContext context) {
- PCollectionView<ViewT> view = (PCollectionView<ViewT>) context.getOutput();
- context.addView(view);
- LOG.debug("view {}", view.getName());
+ public void translate(
+ CreateApexPCollectionView<ElemT, ViewT> transform, TranslationContext context) {
+ context.addView(transform.getView());
+ LOG.debug("view {}", transform.getView().getName());
}
}
@@ -177,9 +174,8 @@ public class ApexPipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
@Override
public void translate(
CreatePCollectionView<ElemT, ViewT> transform, TranslationContext context) {
- PCollectionView<ViewT> view = (PCollectionView<ViewT>) context.getOutput();
- context.addView(view);
- LOG.debug("view {}", view.getName());
+ context.addView(transform.getView());
+ LOG.debug("view {}", transform.getView().getName());
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
index 89e8784..c359cec 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/RunnerPCollectionView.java
@@ -18,6 +18,7 @@
package org.apache.beam.runners.core.construction;
+import java.util.Map;
import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.common.runner.v1.RunnerApi.SideInput;
@@ -26,6 +27,7 @@ import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
import org.apache.beam.sdk.values.PValueBase;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
@@ -85,4 +87,10 @@ class RunnerPCollectionView<T> extends PValueBase implements PCollectionView<T>
public Coder<Iterable<WindowedValue<?>>> getCoderInternal() {
return coder;
}
+
+ @Override
+ public Map<TupleTag<?>, PValue> expand() {
+ throw new UnsupportedOperationException(String.format(
+ "A %s cannot be expanded", RunnerPCollectionView.class.getSimpleName()));
+ }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
index c2c0afa..9ca745d 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraph.java
@@ -79,6 +79,10 @@ class DirectGraph {
return rootTransforms;
}
+ Set<PCollection<?>> getPCollections() {
+ return producers.keySet();
+ }
+
Set<PCollectionView<?>> getViews() {
return viewWriters.keySet();
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
index d54de5d..07bcf06 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectGraphVisitor.java
@@ -21,15 +21,18 @@ import static com.google.common.base.Preconditions.checkState;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Sets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.beam.runners.direct.ViewOverrideFactory.WriteView;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PInput;
@@ -44,6 +47,7 @@ class DirectGraphVisitor extends PipelineVisitor.Defaults {
private Map<PCollection<?>, AppliedPTransform<?, ?, ?>> producers = new HashMap<>();
private Map<PCollectionView<?>, AppliedPTransform<?, ?, ?>> viewWriters = new HashMap<>();
+ private Set<PCollectionView<?>> consumedViews = new HashSet<>();
private ListMultimap<PInput, AppliedPTransform<?, ?, ?>> primitiveConsumers =
ArrayListMultimap.create();
@@ -73,6 +77,13 @@ class DirectGraphVisitor extends PipelineVisitor.Defaults {
getClass().getSimpleName());
if (node.isRootNode()) {
finalized = true;
+ checkState(
+ viewWriters.keySet().containsAll(consumedViews),
+ "All %ss that are consumed must be written by some %s %s: Missing %s",
+ PCollectionView.class.getSimpleName(),
+ WriteView.class.getSimpleName(),
+ PTransform.class.getSimpleName(),
+ Sets.difference(consumedViews, viewWriters.keySet()));
}
}
@@ -86,11 +97,12 @@ class DirectGraphVisitor extends PipelineVisitor.Defaults {
for (PValue value : node.getInputs().values()) {
primitiveConsumers.put(value, appliedTransform);
}
- if (node.getTransform() instanceof ViewOverrideFactory.WriteView) {
- viewWriters.put(
- ((ViewOverrideFactory.WriteView<?, ?>) node.getTransform()).getView(),
- node.toAppliedPTransform(getPipeline()));
- }
+ }
+ if (node.getTransform() instanceof ParDo.MultiOutput) {
+ consumedViews.addAll(((ParDo.MultiOutput<?, ?>) node.getTransform()).getSideInputs());
+ } else if (node.getTransform() instanceof ViewOverrideFactory.WriteView) {
+ viewWriters.put(
+ ((WriteView) node.getTransform()).getView(), node.toAppliedPTransform(getPipeline()));
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
index 057f4a1..8a281a7 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewEvaluatorFactory.java
@@ -28,7 +28,6 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
/**
* The {@link DirectRunner} {@link TransformEvaluatorFactory} for the {@link CreatePCollectionView}
@@ -60,12 +59,13 @@ class ViewEvaluatorFactory implements TransformEvaluatorFactory {
public void cleanup() throws Exception {}
private <InT, OuT> TransformEvaluator<Iterable<InT>> createEvaluator(
- final AppliedPTransform<PCollection<Iterable<InT>>, PCollectionView<OuT>, WriteView<InT, OuT>>
+ final AppliedPTransform<
+ PCollection<Iterable<InT>>, PCollection<Iterable<InT>>, WriteView<InT, OuT>>
application) {
PCollection<Iterable<InT>> input =
(PCollection<Iterable<InT>>) Iterables.getOnlyElement(application.getInputs().values());
- final PCollectionViewWriter<InT, OuT> writer = context.createPCollectionViewWriter(input,
- (PCollectionView<OuT>) Iterables.getOnlyElement(application.getOutputs().values()));
+ final PCollectionViewWriter<InT, OuT> writer =
+ context.createPCollectionViewWriter(input, application.getTransform().getView());
return new TransformEvaluator<Iterable<InT>>() {
private final List<WindowedValue<InT>> elements = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
index fdff63d..06a7388 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
@@ -18,11 +18,11 @@
package org.apache.beam.runners.direct;
-import java.util.Collections;
import java.util.Map;
import org.apache.beam.runners.core.construction.ForwardingPTransform;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.PTransformTranslation.RawPTransform;
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.runners.AppliedPTransform;
@@ -43,12 +43,12 @@ import org.apache.beam.sdk.values.TupleTag;
*/
class ViewOverrideFactory<ElemT, ViewT>
implements PTransformOverrideFactory<
- PCollection<ElemT>, PCollectionView<ViewT>, CreatePCollectionView<ElemT, ViewT>> {
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>> {
@Override
- public PTransformReplacement<PCollection<ElemT>, PCollectionView<ViewT>> getReplacementTransform(
+ public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>> getReplacementTransform(
AppliedPTransform<
- PCollection<ElemT>, PCollectionView<ViewT>, CreatePCollectionView<ElemT, ViewT>>
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>>
transform) {
return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
@@ -57,13 +57,13 @@ class ViewOverrideFactory<ElemT, ViewT>
@Override
public Map<PValue, ReplacementOutput> mapOutputs(
- Map<TupleTag<?>, PValue> outputs, PCollectionView<ViewT> newOutput) {
- return Collections.emptyMap();
+ Map<TupleTag<?>, PValue> outputs, PCollection<ElemT> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
}
/** The {@link DirectRunner} composite override for {@link CreatePCollectionView}. */
static class GroupAndWriteView<ElemT, ViewT>
- extends ForwardingPTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
+ extends ForwardingPTransform<PCollection<ElemT>, PCollection<ElemT>> {
private final CreatePCollectionView<ElemT, ViewT> og;
private GroupAndWriteView(CreatePCollectionView<ElemT, ViewT> og) {
@@ -71,17 +71,18 @@ class ViewOverrideFactory<ElemT, ViewT>
}
@Override
- public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
- return input
+ public PCollection<ElemT> expand(final PCollection<ElemT> input) {
+ input
.apply(WithKeys.<Void, ElemT>of((Void) null))
.setCoder(KvCoder.of(VoidCoder.of(), input.getCoder()))
.apply(GroupByKey.<Void, ElemT>create())
.apply(Values.<Iterable<ElemT>>create())
.apply(new WriteView<ElemT, ViewT>(og));
+ return input;
}
@Override
- protected PTransform<PCollection<ElemT>, PCollectionView<ViewT>> delegate() {
+ protected PTransform<PCollection<ElemT>, PCollection<ElemT>> delegate() {
return og;
}
}
@@ -94,7 +95,7 @@ class ViewOverrideFactory<ElemT, ViewT>
* to {@link ViewT}.
*/
static final class WriteView<ElemT, ViewT>
- extends RawPTransform<PCollection<Iterable<ElemT>>, PCollectionView<ViewT>> {
+ extends RawPTransform<PCollection<Iterable<ElemT>>, PCollection<Iterable<ElemT>>> {
private final CreatePCollectionView<ElemT, ViewT> og;
WriteView(CreatePCollectionView<ElemT, ViewT> og) {
@@ -103,8 +104,10 @@ class ViewOverrideFactory<ElemT, ViewT>
@Override
@SuppressWarnings("deprecation")
- public PCollectionView<ViewT> expand(PCollection<Iterable<ElemT>> input) {
- return og.getView();
+ public PCollection<Iterable<ElemT>> expand(PCollection<Iterable<ElemT>> input) {
+ return PCollection.<Iterable<ElemT>>createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded())
+ .setCoder(input.getCoder());
}
@SuppressWarnings("deprecation")
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java
index 43de091..7707f7f 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphs.java
@@ -18,6 +18,7 @@
package org.apache.beam.runners.direct;
import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
@@ -25,6 +26,12 @@ import org.apache.beam.sdk.values.PValue;
/** Test utilities for the {@link DirectRunner}. */
final class DirectGraphs {
+ public static void performDirectOverrides(Pipeline p) {
+ p.replaceAll(
+ DirectRunner.fromOptions(PipelineOptionsFactory.create().as(DirectOptions.class))
+ .defaultTransformOverrides());
+ }
+
public static DirectGraph getGraph(Pipeline p) {
DirectGraphVisitor visitor = new DirectGraphVisitor();
p.traverseTopologically(visitor);
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index c0e43d6..f3edf55 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -101,14 +101,13 @@ public class EvaluationContextTest {
view = created.apply(View.<Integer>asIterable());
unbounded = p.apply(GenerateSequence.from(0));
- p.replaceAll(
- DirectRunner.fromOptions(TestPipeline.testingPipelineOptions())
- .defaultTransformOverrides());
+ p.replaceAll(runner.defaultTransformOverrides());
KeyedPValueTrackingVisitor keyedPValueTrackingVisitor = KeyedPValueTrackingVisitor.create();
p.traverseTopologically(keyedPValueTrackingVisitor);
BundleFactory bundleFactory = ImmutableListBundleFactory.create();
+ DirectGraphs.performDirectOverrides(p);
graph = DirectGraphs.getGraph(p);
context =
EvaluationContext.create(
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
index c0919b9..365b6c4 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutabilityEnforcementFactoryTest.java
@@ -64,7 +64,9 @@ public class ImmutabilityEnforcementFactoryTest implements Serializable {
c.element()[0] = 'b';
}
}));
- consumer = DirectGraphs.getProducer(pcollection.apply(Count.<byte[]>globally()));
+ PCollection<Long> consumer = pcollection.apply(Count.<byte[]>globally());
+ DirectGraphs.performDirectOverrides(p);
+ this.consumer = DirectGraphs.getProducer(consumer);
}
@Test
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
index 09a21ac..df84cbf 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ParDoEvaluatorTest.java
@@ -149,6 +149,7 @@ public class ParDoEvaluatorTest {
Mockito.any(AppliedPTransform.class), Mockito.any(StructuralKey.class)))
.thenReturn(executionContext);
+ DirectGraphs.performDirectOverrides(p);
@SuppressWarnings("unchecked")
AppliedPTransform<PCollection<Integer>, ?, ?> transform =
(AppliedPTransform<PCollection<Integer>, ?, ?>) DirectGraphs.getProducer(output);
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
index 86412a0..3dd4028 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/TransformExecutorTest.java
@@ -90,6 +90,7 @@ public class TransformExecutorTest {
created = p.apply(Create.of("foo", "spam", "third"));
PCollection<KV<Integer, String>> downstream = created.apply(WithKeys.<Integer, String>of(3));
+ DirectGraphs.performDirectOverrides(p);
DirectGraph graph = DirectGraphs.getGraph(p);
createdProducer = graph.getProducer(created);
downstreamProducer = graph.getProducer(downstream);
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
index 419698e..ad1aecc 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
@@ -36,7 +36,6 @@ import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.transforms.WithKeys;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PCollectionViews;
import org.joda.time.Instant;
import org.junit.Rule;
@@ -66,12 +65,12 @@ public class ViewEvaluatorFactoryTest {
.setCoder(KvCoder.of(VoidCoder.of(), StringUtf8Coder.of()))
.apply(GroupByKey.<Void, String>create())
.apply(Values.<Iterable<String>>create());
- PCollectionView<Iterable<String>> view =
+ PCollection<Iterable<String>> view =
concat.apply(new ViewOverrideFactory.WriteView<>(createView));
EvaluationContext context = mock(EvaluationContext.class);
TestViewWriter<String, Iterable<String>> viewWriter = new TestViewWriter<>();
- when(context.createPCollectionViewWriter(concat, view)).thenReturn(viewWriter);
+ when(context.createPCollectionViewWriter(concat, createView.getView())).thenReturn(viewWriter);
CommittedBundle<String> inputBundle = bundleFactory.createBundle(input).commit(Instant.now());
AppliedPTransform<?, ?, ?> producer = DirectGraphs.getProducer(view);
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
index 024e15c..94728c7 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
@@ -18,7 +18,6 @@
package org.apache.beam.runners.direct;
-import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
@@ -63,11 +62,11 @@ public class ViewOverrideFactoryTest implements Serializable {
PCollection<Integer> ints = p.apply("CreateContents", Create.of(1, 2, 3));
final PCollectionView<List<Integer>> view =
PCollectionViews.listView(ints, WindowingStrategy.globalDefault(), ints.getCoder());
- PTransformReplacement<PCollection<Integer>, PCollectionView<List<Integer>>>
+ PTransformReplacement<PCollection<Integer>, PCollection<Integer>>
replacementTransform =
factory.getReplacementTransform(
AppliedPTransform
- .<PCollection<Integer>, PCollectionView<List<Integer>>,
+ .<PCollection<Integer>, PCollection<Integer>,
CreatePCollectionView<Integer, List<Integer>>>
of(
"foo",
@@ -75,12 +74,7 @@ public class ViewOverrideFactoryTest implements Serializable {
view.expand(),
CreatePCollectionView.<Integer, List<Integer>>of(view),
p));
- PCollectionView<List<Integer>> afterReplacement =
- ints.apply(replacementTransform.getTransform());
- assertThat(
- "The CreatePCollectionView replacement should return the same View",
- afterReplacement,
- equalTo(view));
+ ints.apply(replacementTransform.getTransform());
PCollection<Set<Integer>> outputViewContents =
p.apply("CreateSingleton", Create.of(0))
@@ -104,10 +98,10 @@ public class ViewOverrideFactoryTest implements Serializable {
final PCollection<Integer> ints = p.apply("CreateContents", Create.of(1, 2, 3));
final PCollectionView<List<Integer>> view =
PCollectionViews.listView(ints, WindowingStrategy.globalDefault(), ints.getCoder());
- PTransformReplacement<PCollection<Integer>, PCollectionView<List<Integer>>> replacement =
+ PTransformReplacement<PCollection<Integer>, PCollection<Integer>> replacement =
factory.getReplacementTransform(
AppliedPTransform
- .<PCollection<Integer>, PCollectionView<List<Integer>>,
+ .<PCollection<Integer>, PCollection<Integer>,
CreatePCollectionView<Integer, List<Integer>>>
of(
"foo",
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
index b667346..1d8aac1 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkCallbackExecutorTest.java
@@ -59,6 +59,7 @@ public class WatermarkCallbackExecutorTest {
public void setup() {
PCollection<Integer> created = p.apply(Create.of(1, 2, 3));
PCollection<Integer> summed = created.apply(Sum.integersGlobally());
+ DirectGraphs.performDirectOverrides(p);
DirectGraph graph = DirectGraphs.getGraph(p);
create = graph.getProducer(created);
sum = graph.getProducer(summed);
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
index 9528ac9..e0b5251 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
@@ -121,6 +121,7 @@ public class WatermarkManagerTest implements Serializable {
flattened = preFlatten.apply("flattened", Flatten.<Integer>pCollections());
clock = MockClock.fromInstant(new Instant(1000));
+ DirectGraphs.performDirectOverrides(p);
graph = DirectGraphs.getGraph(p);
manager = WatermarkManager.create(clock, graph);
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java
new file mode 100644
index 0000000..0cc3aec
--- /dev/null
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.flink;
+
+import com.google.common.collect.Iterables;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderRegistry;
+import org.apache.beam.sdk.coders.ListCoder;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.runners.PTransformOverrideFactory;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Flink streaming overrides for various view (side input) transforms. */
+class CreateStreamingFlinkView<ElemT, ViewT>
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
+ private final PCollectionView<ViewT> view;
+
+ public CreateStreamingFlinkView(PCollectionView<ViewT> view) {
+ this.view = view;
+ }
+
+ @Override
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
+ input
+ .apply(Combine.globally(new Concatenate<ElemT>()).withoutDefaults())
+ .apply(CreateFlinkPCollectionView.<ElemT, ViewT>of(view));
+ return input;
+ }
+
+ /**
+ * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
+ *
+ * <p>For internal use by {@link CreateStreamingFlinkView}. This combiner requires that the input
+ * {@link PCollection} fits in memory. For a large {@link PCollection} this is expected to crash!
+ *
+ * @param <T> the type of elements to concatenate.
+ */
+ private static class Concatenate<T> extends Combine.CombineFn<T, List<T>, List<T>> {
+ @Override
+ public List<T> createAccumulator() {
+ return new ArrayList<T>();
+ }
+
+ @Override
+ public List<T> addInput(List<T> accumulator, T input) {
+ accumulator.add(input);
+ return accumulator;
+ }
+
+ @Override
+ public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
+ List<T> result = createAccumulator();
+ for (List<T> accumulator : accumulators) {
+ result.addAll(accumulator);
+ }
+ return result;
+ }
+
+ @Override
+ public List<T> extractOutput(List<T> accumulator) {
+ return accumulator;
+ }
+
+ @Override
+ public Coder<List<T>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+
+ @Override
+ public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) {
+ return ListCoder.of(inputCoder);
+ }
+ }
+
+ /**
+ * Creates a primitive {@link PCollectionView}.
+ *
+ * <p>For internal use only by runner implementors.
+ *
+ * @param <ElemT> The type of the elements of the input PCollection
+ * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
+ */
+ public static class CreateFlinkPCollectionView<ElemT, ViewT>
+ extends PTransform<PCollection<List<ElemT>>, PCollection<List<ElemT>>> {
+ private PCollectionView<ViewT> view;
+
+ private CreateFlinkPCollectionView(PCollectionView<ViewT> view) {
+ this.view = view;
+ }
+
+ public static <ElemT, ViewT> CreateFlinkPCollectionView<ElemT, ViewT> of(
+ PCollectionView<ViewT> view) {
+ return new CreateFlinkPCollectionView<>(view);
+ }
+
+ @Override
+ public PCollection<List<ElemT>> expand(PCollection<List<ElemT>> input) {
+ return PCollection.<List<ElemT>>createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded())
+ .setCoder(input.getCoder());
+ }
+
+ public PCollectionView<ViewT> getView() {
+ return view;
+ }
+ }
+
+ public static class Factory<ElemT, ViewT>
+ implements PTransformOverrideFactory<
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>> {
+ public Factory() {}
+
+ @Override
+ public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>> getReplacementTransform(
+ AppliedPTransform<
+ PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>>
+ transform) {
+ return PTransformReplacement.of(
+ (PCollection<ElemT>) Iterables.getOnlyElement(transform.getInputs().values()),
+ new CreateStreamingFlinkView<ElemT, ViewT>(transform.getTransform().getView()));
+ }
+
+ @Override
+ public Map<PValue, ReplacementOutput> mapOutputs(
+ Map<TupleTag<?>, PValue> outputs, PCollection<ElemT> newOutput) {
+ return ReplacementOutputs.singleton(outputs, newOutput);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
index 8da68c5..a88ff07 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPipelineTranslator.java
@@ -33,10 +33,9 @@ import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverride;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo.MultiOutput;
-import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.util.InstanceBuilder;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
@@ -85,37 +84,8 @@ class FlinkStreamingPipelineTranslator extends FlinkPipelineTranslator {
new SplittableParDoViaKeyedWorkItems.OverrideFactory()))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsIterable.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingViewAsIterable.class, flinkRunner)))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsList.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingViewAsList.class, flinkRunner)))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsMap.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingViewAsMap.class, flinkRunner)))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsMultimap.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingViewAsMultimap.class, flinkRunner)))
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsSingleton.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingViewAsSingleton.class, flinkRunner)))
- // this has to be last since the ViewAsSingleton override
- // can expand to a Combine.GloballyAsSingletonView
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
- new ReflectiveOneToOneOverrideFactory(
- FlinkStreamingViewOverrides.StreamingCombineGloballyAsSingletonView.class,
- flinkRunner)))
+ PTransformMatchers.classEqualTo(CreatePCollectionView.class),
+ new CreateStreamingFlinkView.Factory()))
.build();
// Ensure all outputs of all reads are consumed.
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
index 2a7c5d6..ef46b63 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java
@@ -124,7 +124,7 @@ class FlinkStreamingTransformTranslators {
TRANSLATORS.put(Window.Assign.class, new WindowAssignTranslator());
TRANSLATORS.put(Flatten.PCollections.class, new FlattenPCollectionTranslator());
TRANSLATORS.put(
- FlinkStreamingViewOverrides.CreateFlinkPCollectionView.class,
+ CreateStreamingFlinkView.CreateFlinkPCollectionView.class,
new CreateViewStreamingTranslator());
TRANSLATORS.put(Reshuffle.class, new ReshuffleTranslatorStreaming());
@@ -584,17 +584,17 @@ class FlinkStreamingTransformTranslators {
private static class CreateViewStreamingTranslator<ElemT, ViewT>
extends FlinkStreamingPipelineTranslator.StreamTransformTranslator<
- FlinkStreamingViewOverrides.CreateFlinkPCollectionView<ElemT, ViewT>> {
+ CreateStreamingFlinkView.CreateFlinkPCollectionView<ElemT, ViewT>> {
@Override
public void translateNode(
- FlinkStreamingViewOverrides.CreateFlinkPCollectionView<ElemT, ViewT> transform,
+ CreateStreamingFlinkView.CreateFlinkPCollectionView<ElemT, ViewT> transform,
FlinkStreamingTranslationContext context) {
// just forward
DataStream<WindowedValue<List<ElemT>>> inputDataSet =
context.getInputDataStream(context.getInput(transform));
- PCollectionView<ViewT> view = context.getOutput(transform);
+ PCollectionView<ViewT> view = transform.getView();
context.setOutputDataStream(view, inputDataSet);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java
----------------------------------------------------------------------
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java
deleted file mode 100644
index ce1c895..0000000
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkStreamingViewOverrides.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.flink;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.CoderRegistry;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.ListCoder;
-import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PCollectionViews;
-
-/**
- * Flink streaming overrides for various view (side input) transforms.
- */
-class FlinkStreamingViewOverrides {
-
- /**
- * Specialized implementation for
- * {@link org.apache.beam.sdk.transforms.View.AsMap View.AsMap}
- * for the Flink runner in streaming mode.
- */
- static class StreamingViewAsMap<K, V>
- extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, V>>> {
-
- private final transient FlinkRunner runner;
-
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingViewAsMap(FlinkRunner runner, View.AsMap<K, V> transform) {
- this.runner = runner;
- }
-
- @Override
- public PCollectionView<Map<K, V>> expand(PCollection<KV<K, V>> input) {
- PCollectionView<Map<K, V>> view =
- PCollectionViews.mapView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
- try {
- inputCoder.getKeyCoder().verifyDeterministic();
- } catch (Coder.NonDeterministicException e) {
- runner.recordViewUsesNonDeterministicKeyCoder(this);
- }
-
- return input
- .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
- .apply(CreateFlinkPCollectionView.<KV<K, V>, Map<K, V>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsMap";
- }
- }
-
- /**
- * Specialized expansion for {@link
- * View.AsMultimap View.AsMultimap} for the
- * Flink runner in streaming mode.
- */
- static class StreamingViewAsMultimap<K, V>
- extends PTransform<PCollection<KV<K, V>>, PCollectionView<Map<K, Iterable<V>>>> {
-
- private final transient FlinkRunner runner;
-
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingViewAsMultimap(FlinkRunner runner, View.AsMultimap<K, V> transform) {
- this.runner = runner;
- }
-
- @Override
- public PCollectionView<Map<K, Iterable<V>>> expand(PCollection<KV<K, V>> input) {
- PCollectionView<Map<K, Iterable<V>>> view =
- PCollectionViews.multimapView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
- try {
- inputCoder.getKeyCoder().verifyDeterministic();
- } catch (Coder.NonDeterministicException e) {
- runner.recordViewUsesNonDeterministicKeyCoder(this);
- }
-
- return input
- .apply(Combine.globally(new Concatenate<KV<K, V>>()).withoutDefaults())
- .apply(CreateFlinkPCollectionView.<KV<K, V>, Map<K, Iterable<V>>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsMultimap";
- }
- }
-
- /**
- * Specialized implementation for
- * {@link View.AsList View.AsList} for the
- * Flink runner in streaming mode.
- */
- static class StreamingViewAsList<T>
- extends PTransform<PCollection<T>, PCollectionView<List<T>>> {
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingViewAsList(FlinkRunner runner, View.AsList<T> transform) {}
-
- @Override
- public PCollectionView<List<T>> expand(PCollection<T> input) {
- PCollectionView<List<T>> view =
- PCollectionViews.listView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
- .apply(CreateFlinkPCollectionView.<T, List<T>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsList";
- }
- }
-
- /**
- * Specialized implementation for
- * {@link View.AsIterable View.AsIterable} for the
- * Flink runner in streaming mode.
- */
- static class StreamingViewAsIterable<T>
- extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingViewAsIterable(FlinkRunner runner, View.AsIterable<T> transform) { }
-
- @Override
- public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
- PCollectionView<Iterable<T>> view =
- PCollectionViews.iterableView(
- input,
- input.getWindowingStrategy(),
- input.getCoder());
-
- return input.apply(Combine.globally(new Concatenate<T>()).withoutDefaults())
- .apply(CreateFlinkPCollectionView.<T, Iterable<T>>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsIterable";
- }
- }
-
- /**
- * Specialized expansion for
- * {@link View.AsSingleton View.AsSingleton} for the
- * Flink runner in streaming mode.
- */
- static class StreamingViewAsSingleton<T>
- extends PTransform<PCollection<T>, PCollectionView<T>> {
- private View.AsSingleton<T> transform;
-
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingViewAsSingleton(FlinkRunner runner, View.AsSingleton<T> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollectionView<T> expand(PCollection<T> input) {
- Combine.Globally<T, T> combine = Combine.globally(
- new SingletonCombine<>(transform.hasDefaultValue(), transform.defaultValue()));
- if (!transform.hasDefaultValue()) {
- combine = combine.withoutDefaults();
- }
- return input.apply(combine.asSingletonView());
- }
-
- @Override
- protected String getKindString() {
- return "StreamingViewAsSingleton";
- }
-
- private static class SingletonCombine<T> extends Combine.BinaryCombineFn<T> {
- private boolean hasDefaultValue;
- private T defaultValue;
-
- SingletonCombine(boolean hasDefaultValue, T defaultValue) {
- this.hasDefaultValue = hasDefaultValue;
- this.defaultValue = defaultValue;
- }
-
- @Override
- public T apply(T left, T right) {
- throw new IllegalArgumentException("PCollection with more than one element "
- + "accessed as a singleton view. Consider using Combine.globally().asSingleton() to "
- + "combine the PCollection into a single value");
- }
-
- @Override
- public T identity() {
- if (hasDefaultValue) {
- return defaultValue;
- } else {
- throw new IllegalArgumentException(
- "Empty PCollection accessed as a singleton view. "
- + "Consider setting withDefault to provide a default value");
- }
- }
- }
- }
-
- static class StreamingCombineGloballyAsSingletonView<InputT, OutputT>
- extends PTransform<PCollection<InputT>, PCollectionView<OutputT>> {
- Combine.GloballyAsSingletonView<InputT, OutputT> transform;
-
- /**
- * Builds an instance of this class from the overridden transform.
- */
- @SuppressWarnings("unused") // used via reflection in FlinkRunner#apply()
- public StreamingCombineGloballyAsSingletonView(
- FlinkRunner runner,
- Combine.GloballyAsSingletonView<InputT, OutputT> transform) {
- this.transform = transform;
- }
-
- @Override
- public PCollectionView<OutputT> expand(PCollection<InputT> input) {
- PCollection<OutputT> combined =
- input.apply(Combine.globally(transform.getCombineFn())
- .withoutDefaults()
- .withFanout(transform.getFanout()));
-
- PCollectionView<OutputT> view = PCollectionViews.singletonView(
- combined,
- combined.getWindowingStrategy(),
- transform.getInsertDefault(),
- transform.getInsertDefault()
- ? transform.getCombineFn().defaultValue() : null,
- combined.getCoder());
- return combined
- .apply(ParDo.of(new WrapAsList<OutputT>()))
- .apply(CreateFlinkPCollectionView.<OutputT, OutputT>of(view));
- }
-
- @Override
- protected String getKindString() {
- return "StreamingCombineGloballyAsSingletonView";
- }
- }
-
- private static class WrapAsList<T> extends DoFn<T, List<T>> {
- @ProcessElement
- public void processElement(ProcessContext c) {
- c.output(Collections.singletonList(c.element()));
- }
- }
-
- /**
- * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
- *
- * <p>For internal use by {@link StreamingViewAsMap}, {@link StreamingViewAsMultimap},
- * {@link StreamingViewAsList}, {@link StreamingViewAsIterable}.
- * They require the input {@link PCollection} fits in memory.
- * For a large {@link PCollection} this is expected to crash!
- *
- * @param <T> the type of elements to concatenate.
- */
- private static class Concatenate<T> extends Combine.CombineFn<T, List<T>, List<T>> {
- @Override
- public List<T> createAccumulator() {
- return new ArrayList<T>();
- }
-
- @Override
- public List<T> addInput(List<T> accumulator, T input) {
- accumulator.add(input);
- return accumulator;
- }
-
- @Override
- public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
- List<T> result = createAccumulator();
- for (List<T> accumulator : accumulators) {
- result.addAll(accumulator);
- }
- return result;
- }
-
- @Override
- public List<T> extractOutput(List<T> accumulator) {
- return accumulator;
- }
-
- @Override
- public Coder<List<T>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) {
- return ListCoder.of(inputCoder);
- }
-
- @Override
- public Coder<List<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) {
- return ListCoder.of(inputCoder);
- }
- }
-
- /**
- * Creates a primitive {@link PCollectionView}.
- *
- * <p>For internal use only by runner implementors.
- *
- * @param <ElemT> The type of the elements of the input PCollection
- * @param <ViewT> The type associated with the {@link PCollectionView} used as a side input
- */
- public static class CreateFlinkPCollectionView<ElemT, ViewT>
- extends PTransform<PCollection<List<ElemT>>, PCollectionView<ViewT>> {
- private PCollectionView<ViewT> view;
-
- private CreateFlinkPCollectionView(PCollectionView<ViewT> view) {
- this.view = view;
- }
-
- public static <ElemT, ViewT> CreateFlinkPCollectionView<ElemT, ViewT> of(
- PCollectionView<ViewT> view) {
- return new CreateFlinkPCollectionView<>(view);
- }
-
- @Override
- public PCollectionView<ViewT> expand(PCollection<List<ElemT>> input) {
- return view;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
index b4a6e64..ad3faed 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BatchViewOverrides.java
@@ -39,8 +39,6 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.beam.runners.core.construction.PTransformReplacements;
-import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.runners.dataflow.internal.IsmFormat;
import org.apache.beam.runners.dataflow.internal.IsmFormat.IsmRecord;
import org.apache.beam.runners.dataflow.internal.IsmFormat.IsmRecordCoder;
@@ -57,17 +55,11 @@ import org.apache.beam.sdk.coders.SerializableCoder;
import org.apache.beam.sdk.coders.StructuredCoder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VarLongCoder;
-import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.Combine;
-import org.apache.beam.sdk.transforms.Combine.GloballyAsSingletonView;
-import org.apache.beam.sdk.transforms.CombineFnBase.GlobalCombineFn;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.transforms.GroupByKey;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.View;
-import org.apache.beam.sdk.transforms.View.AsSingleton;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -83,7 +75,6 @@ import org.apache.beam.sdk.values.PCollection.IsBounded;
import org.apache.beam.sdk.values.PCollectionList;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PCollectionViews;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
import org.apache.beam.sdk.values.WindowingStrategy;
@@ -192,12 +183,13 @@ class BatchViewOverrides {
}
private final DataflowRunner runner;
- /**
- * Builds an instance of this class from the overridden transform.
- */
+ private final PCollectionView<Map<K, V>> view;
+ /** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public BatchViewAsMap(DataflowRunner runner, View.AsMap<K, V> transform) {
+ public BatchViewAsMap(
+ DataflowRunner runner, CreatePCollectionView<KV<K, V>, Map<K, V>> transform) {
this.runner = runner;
+ this.view = transform.getView();
}
@Override
@@ -207,12 +199,7 @@ class BatchViewOverrides {
private <W extends BoundedWindow> PCollectionView<Map<K, V>>
applyInternal(PCollection<KV<K, V>> input) {
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
try {
- PCollectionView<Map<K, V>> view = PCollectionViews.mapView(
- input, input.getWindowingStrategy(), inputCoder);
return BatchViewAsMultimap.applyForMapLike(runner, input, view, true /* unique keys */);
} catch (NonDeterministicException e) {
runner.recordViewUsesNonDeterministicKeyCoder(this);
@@ -249,19 +236,14 @@ class BatchViewOverrides {
inputCoder.getKeyCoder(),
FullWindowedValueCoder.of(inputCoder.getValueCoder(), windowCoder)));
- TransformedMap<K, WindowedValue<V>, V> defaultValue = new TransformedMap<>(
- WindowedValueToValue.<V>of(),
- ImmutableMap.<K, WindowedValue<V>>of());
-
return BatchViewAsSingleton.<KV<K, V>, TransformedMap<K, WindowedValue<V>, V>,
Map<K, V>,
W> applyForSingleton(
runner,
input,
new ToMapDoFn<K, V, W>(windowCoder),
- true,
- defaultValue,
- finalValueCoder);
+ finalValueCoder,
+ view);
}
}
@@ -680,12 +662,13 @@ class BatchViewOverrides {
}
private final DataflowRunner runner;
- /**
- * Builds an instance of this class from the overridden transform.
- */
+ private final PCollectionView<Map<K, Iterable<V>>> view;
+ /** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public BatchViewAsMultimap(DataflowRunner runner, View.AsMultimap<K, V> transform) {
+ public BatchViewAsMultimap(
+ DataflowRunner runner, CreatePCollectionView<KV<K, V>, Map<K, Iterable<V>>> transform) {
this.runner = runner;
+ this.view = transform.getView();
}
@Override
@@ -695,12 +678,7 @@ class BatchViewOverrides {
private <W extends BoundedWindow> PCollectionView<Map<K, Iterable<V>>>
applyInternal(PCollection<KV<K, V>> input) {
- @SuppressWarnings({"rawtypes", "unchecked"})
- KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
try {
- PCollectionView<Map<K, Iterable<V>>> view = PCollectionViews.multimapView(
- input, input.getWindowingStrategy(), inputCoder);
-
return applyForMapLike(runner, input, view, false /* unique keys not expected */);
} catch (NonDeterministicException e) {
runner.recordViewUsesNonDeterministicKeyCoder(this);
@@ -738,16 +716,15 @@ class BatchViewOverrides {
IterableWithWindowedValuesToIterable.<V>of(),
ImmutableMap.<K, Iterable<WindowedValue<V>>>of());
- return BatchViewAsSingleton.<KV<K, V>,
- TransformedMap<K, Iterable<WindowedValue<V>>, Iterable<V>>,
- Map<K, Iterable<V>>,
- W> applyForSingleton(
- runner,
- input,
- new ToMultimapDoFn<K, V, W>(windowCoder),
- true,
- defaultValue,
- finalValueCoder);
+ return BatchViewAsSingleton
+ .<KV<K, V>, TransformedMap<K, Iterable<WindowedValue<V>>, Iterable<V>>,
+ Map<K, Iterable<V>>, W>
+ applyForSingleton(
+ runner,
+ input,
+ new ToMultimapDoFn<K, V, W>(windowCoder),
+ finalValueCoder,
+ view);
}
private static <K, V, W extends BoundedWindow, ViewT> PCollectionView<ViewT> applyForMapLike(
@@ -827,10 +804,9 @@ class BatchViewOverrides {
PCollectionList.of(ImmutableList.of(
perHashWithReifiedWindows, windowMapSizeMetadata, windowMapKeysMetadata));
- return Pipeline.applyTransform(outputs,
- Flatten.<IsmRecord<WindowedValue<V>>>pCollections())
- .apply(CreateDataflowView.<IsmRecord<WindowedValue<V>>,
- ViewT>of(view));
+ Pipeline.applyTransform(outputs, Flatten.<IsmRecord<WindowedValue<V>>>pCollections())
+ .apply(CreateDataflowView.<IsmRecord<WindowedValue<V>>, ViewT>of(view));
+ return view;
}
@Override
@@ -915,14 +891,12 @@ class BatchViewOverrides {
}
private final DataflowRunner runner;
- private final View.AsSingleton<T> transform;
- /**
- * Builds an instance of this class from the overridden transform.
- */
+ private final PCollectionView<T> view;
+ /** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public BatchViewAsSingleton(DataflowRunner runner, View.AsSingleton<T> transform) {
+ public BatchViewAsSingleton(DataflowRunner runner, CreatePCollectionView<T, T> transform) {
this.runner = runner;
- this.transform = transform;
+ this.view = transform.getView();
}
@Override
@@ -935,9 +909,8 @@ class BatchViewOverrides {
runner,
input,
new IsmRecordForSingularValuePerWindowDoFn<T, BoundedWindow>(windowCoder),
- transform.hasDefaultValue(),
- transform.defaultValue(),
- input.getCoder());
+ input.getCoder(),
+ view);
}
static <T, FinalT, ViewT, W extends BoundedWindow> PCollectionView<ViewT>
@@ -946,23 +919,13 @@ class BatchViewOverrides {
PCollection<T> input,
DoFn<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>,
IsmRecord<WindowedValue<FinalT>>> doFn,
- boolean hasDefault,
- FinalT defaultValue,
- Coder<FinalT> defaultValueCoder) {
+ Coder<FinalT> defaultValueCoder,
+ PCollectionView<ViewT> view) {
@SuppressWarnings("unchecked")
Coder<W> windowCoder = (Coder<W>)
input.getWindowingStrategy().getWindowFn().windowCoder();
- @SuppressWarnings({"rawtypes", "unchecked"})
- PCollectionView<ViewT> view =
- (PCollectionView<ViewT>) PCollectionViews.<FinalT, W>singletonView(
- (PCollection) input,
- (WindowingStrategy) input.getWindowingStrategy(),
- hasDefault,
- defaultValue,
- defaultValueCoder);
-
IsmRecordCoder<WindowedValue<FinalT>> ismCoder =
coderForSingleton(windowCoder, defaultValueCoder);
@@ -972,8 +935,9 @@ class BatchViewOverrides {
reifiedPerWindowAndSorted.setCoder(ismCoder);
runner.addPCollectionRequiringIndexedFormat(reifiedPerWindowAndSorted);
- return reifiedPerWindowAndSorted.apply(
+ reifiedPerWindowAndSorted.apply(
CreateDataflowView.<IsmRecord<WindowedValue<FinalT>>, ViewT>of(view));
+ return view;
}
@Override
@@ -1079,18 +1043,18 @@ class BatchViewOverrides {
}
private final DataflowRunner runner;
+ private final PCollectionView<List<T>> view;
/**
* Builds an instance of this class from the overridden transform.
*/
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public BatchViewAsList(DataflowRunner runner, View.AsList<T> transform) {
+ public BatchViewAsList(DataflowRunner runner, CreatePCollectionView<T, List<T>> transform) {
this.runner = runner;
+ this.view = transform.getView();
}
@Override
public PCollectionView<List<T>> expand(PCollection<T> input) {
- PCollectionView<List<T>> view = PCollectionViews.listView(
- input, input.getWindowingStrategy(), input.getCoder());
return applyForIterableLike(runner, input, view);
}
@@ -1116,8 +1080,9 @@ class BatchViewOverrides {
reifiedPerWindowAndSorted.setCoder(ismCoder);
runner.addPCollectionRequiringIndexedFormat(reifiedPerWindowAndSorted);
- return reifiedPerWindowAndSorted.apply(
+ reifiedPerWindowAndSorted.apply(
CreateDataflowView.<IsmRecord<WindowedValue<T>>, ViewT>of(view));
+ return view;
}
PCollection<IsmRecord<WindowedValue<T>>> reifiedPerWindowAndSorted = input
@@ -1126,8 +1091,9 @@ class BatchViewOverrides {
reifiedPerWindowAndSorted.setCoder(ismCoder);
runner.addPCollectionRequiringIndexedFormat(reifiedPerWindowAndSorted);
- return reifiedPerWindowAndSorted.apply(
+ reifiedPerWindowAndSorted.apply(
CreateDataflowView.<IsmRecord<WindowedValue<T>>, ViewT>of(view));
+ return view;
}
@Override
@@ -1164,18 +1130,17 @@ class BatchViewOverrides {
extends PTransform<PCollection<T>, PCollectionView<Iterable<T>>> {
private final DataflowRunner runner;
- /**
- * Builds an instance of this class from the overridden transform.
- */
+ private final PCollectionView<Iterable<T>> view;
+ /** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
- public BatchViewAsIterable(DataflowRunner runner, View.AsIterable<T> transform) {
+ public BatchViewAsIterable(
+ DataflowRunner runner, CreatePCollectionView<T, Iterable<T>> transform) {
this.runner = runner;
+ this.view = transform.getView();
}
@Override
public PCollectionView<Iterable<T>> expand(PCollection<T> input) {
- PCollectionView<Iterable<T>> view = PCollectionViews.iterableView(
- input, input.getWindowingStrategy(), input.getCoder());
return BatchViewAsList.applyForIterableLike(runner, input, view);
}
}
@@ -1377,59 +1342,4 @@ class BatchViewOverrides {
verifyDeterministic(this, "Expected map coder to be deterministic.", originalMapCoder);
}
}
-
- static class BatchCombineGloballyAsSingletonViewFactory<ElemT, ViewT>
- extends SingleInputOutputOverrideFactory<
- PCollection<ElemT>, PCollectionView<ViewT>,
- Combine.GloballyAsSingletonView<ElemT, ViewT>> {
- private final DataflowRunner runner;
-
- BatchCombineGloballyAsSingletonViewFactory(DataflowRunner runner) {
- this.runner = runner;
- }
-
- @Override
- public PTransformReplacement<PCollection<ElemT>, PCollectionView<ViewT>>
- getReplacementTransform(
- AppliedPTransform<
- PCollection<ElemT>, PCollectionView<ViewT>,
- GloballyAsSingletonView<ElemT, ViewT>>
- transform) {
- GloballyAsSingletonView<ElemT, ViewT> combine = transform.getTransform();
- return PTransformReplacement.of(
- PTransformReplacements.getSingletonMainInput(transform),
- new BatchCombineGloballyAsSingletonView<>(
- runner, combine.getCombineFn(), combine.getFanout(), combine.getInsertDefault()));
- }
-
- private static class BatchCombineGloballyAsSingletonView<ElemT, ViewT>
- extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
- private final DataflowRunner runner;
- private final GlobalCombineFn<? super ElemT, ?, ViewT> combineFn;
- private final int fanout;
- private final boolean insertDefault;
-
- BatchCombineGloballyAsSingletonView(
- DataflowRunner runner,
- GlobalCombineFn<? super ElemT, ?, ViewT> combineFn,
- int fanout,
- boolean insertDefault) {
- this.runner = runner;
- this.combineFn = combineFn;
- this.fanout = fanout;
- this.insertDefault = insertDefault;
- }
-
- @Override
- public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
- PCollection<ViewT> combined =
- input.apply(Combine.globally(combineFn).withoutDefaults().withFanout(fanout));
- AsSingleton<ViewT> viewAsSingleton = View.asSingleton();
- if (insertDefault) {
- viewAsSingleton.withDefaultValue(combineFn.defaultValue());
- }
- return combined.apply(new BatchViewAsSingleton<>(runner, viewAsSingleton));
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/CreateDataflowView.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/CreateDataflowView.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/CreateDataflowView.java
index e7542cb..caad7f8 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/CreateDataflowView.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/CreateDataflowView.java
@@ -24,7 +24,7 @@ import org.apache.beam.sdk.values.PCollectionView;
/** A {@link DataflowRunner} marker class for creating a {@link PCollectionView}. */
public class CreateDataflowView<ElemT, ViewT>
- extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
public static <ElemT, ViewT> CreateDataflowView<ElemT, ViewT> of(PCollectionView<ViewT> view) {
return new CreateDataflowView<>(view);
}
@@ -36,8 +36,10 @@ public class CreateDataflowView<ElemT, ViewT>
}
@Override
- public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
- return view;
+ public PCollection<ElemT> expand(PCollection<ElemT> input) {
+ return PCollection.<ElemT>createPrimitiveOutputInternal(
+ input.getPipeline(), input.getWindowingStrategy(), input.isBounded())
+ .setCoder(input.getCoder());
}
public PCollectionView<ViewT> getView() {
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
index 8eaf61b..a3a7ab6 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java
@@ -440,6 +440,14 @@ public class DataflowPipelineTranslator {
public void visitValue(PValue value, TransformHierarchy.Node producer) {
LOG.debug("Checking translation of {}", value);
// Primitive transforms are the only ones assigned step names.
+ if (producer.getTransform() instanceof CreateDataflowView) {
+ // CreateDataflowView produces a dummy output (as it must be a primitive transform) but
+ // in the Dataflow Job graph produces only the view and not the output PCollection.
+ asOutputReference(
+ ((CreateDataflowView) producer.getTransform()).getView(),
+ producer.toAppliedPTransform(getPipeline()));
+ return;
+ }
asOutputReference(value, producer.toAppliedPTransform(getPipeline()));
}
@@ -465,6 +473,7 @@ public class DataflowPipelineTranslator {
StepTranslator stepContext = new StepTranslator(this, step);
stepContext.addInput(PropertyNames.USER_NAME, getFullName(transform));
stepContext.addDisplayData(step, stepName, transform);
+ LOG.info("Adding {} as step {}", getCurrentTransform(transform).getFullName(), stepName);
return stepContext;
}
@@ -677,7 +686,7 @@ public class DataflowPipelineTranslator {
context.addStep(transform, "CollectionToSingleton");
PCollection<ElemT> input = context.getInput(transform);
stepContext.addInput(PropertyNames.PARALLEL_INPUT, input);
- stepContext.addCollectionToSingletonOutput(input, context.getOutput(transform));
+ stepContext.addCollectionToSingletonOutput(input, transform.getView());
}
});
http://git-wip-us.apache.org/repos/asf/beam/blob/ccf73448/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 3e7c8ce..ea9db24 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -67,7 +67,6 @@ import org.apache.beam.runners.core.construction.ReplacementOutputs;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource;
import org.apache.beam.runners.core.construction.UnconsumedReads;
-import org.apache.beam.runners.dataflow.BatchViewOverrides.BatchCombineGloballyAsSingletonViewFactory;
import org.apache.beam.runners.dataflow.DataflowPipelineTranslator.JobSpecification;
import org.apache.beam.runners.dataflow.StreamingViewOverrides.StreamingCreatePCollectionViewFactory;
import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
@@ -129,6 +128,7 @@ import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollection.IsBounded;
import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PCollectionViews;
import org.apache.beam.sdk.values.PDone;
import org.apache.beam.sdk.values.PInput;
import org.apache.beam.sdk.values.PValue;
@@ -350,34 +350,29 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
PTransformOverride.of(
PTransformMatchers.stateOrTimerParDoSingle(),
BatchStatefulParDoOverrides.singleOutputOverrideFactory()))
-
- .add(
- PTransformOverride.of(
- PTransformMatchers.classEqualTo(Combine.GloballyAsSingletonView.class),
- new BatchCombineGloballyAsSingletonViewFactory(this)))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsMap.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.MapViewFn.class),
new ReflectiveOneToOneOverrideFactory(
BatchViewOverrides.BatchViewAsMap.class, this)))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsMultimap.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.MultimapViewFn.class),
new ReflectiveOneToOneOverrideFactory(
BatchViewOverrides.BatchViewAsMultimap.class, this)))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsSingleton.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.SingletonViewFn.class),
new ReflectiveOneToOneOverrideFactory(
BatchViewOverrides.BatchViewAsSingleton.class, this)))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsList.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.ListViewFn.class),
new ReflectiveOneToOneOverrideFactory(
BatchViewOverrides.BatchViewAsList.class, this)))
.add(
PTransformOverride.of(
- PTransformMatchers.classEqualTo(View.AsIterable.class),
+ PTransformMatchers.createViewWithViewFn(PCollectionViews.IterableViewFn.class),
new ReflectiveOneToOneOverrideFactory(
BatchViewOverrides.BatchViewAsIterable.class, this)));
}
[32/50] [abbrv] beam git commit: Move Runner API protos to
portability/runners/api
Posted by ke...@apache.org.
Move Runner API protos to portability/runners/api
This fixes a circular import issue between transforms/ and runners/
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7689e43a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7689e43a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7689e43a
Branch: refs/heads/gearpump-runner
Commit: 7689e43ac4b88c85962cea14d65d788ad27dbe93
Parents: bc2f97c
Author: Charles Chen <cc...@google.com>
Authored: Wed Jun 7 16:08:43 2017 -0700
Committer: chamikara@google.com <ch...@google.com>
Committed: Thu Jun 8 15:29:39 2017 -0700
----------------------------------------------------------------------
.gitignore | 2 +-
sdks/python/apache_beam/coders/coders.py | 2 +-
sdks/python/apache_beam/pipeline.py | 4 ++--
sdks/python/apache_beam/portability/__init__.py | 18 +++++++++++++++++
.../apache_beam/portability/runners/__init__.py | 18 +++++++++++++++++
.../portability/runners/api/__init__.py | 21 ++++++++++++++++++++
sdks/python/apache_beam/pvalue.py | 2 +-
sdks/python/apache_beam/runners/api/__init__.py | 21 --------------------
.../runners/dataflow/dataflow_runner.py | 4 ++--
.../apache_beam/runners/pipeline_context.py | 2 +-
.../runners/portability/fn_api_runner.py | 2 +-
.../apache_beam/runners/worker/data_plane.py | 2 +-
.../runners/worker/data_plane_test.py | 2 +-
.../apache_beam/runners/worker/log_handler.py | 2 +-
.../runners/worker/log_handler_test.py | 2 +-
.../apache_beam/runners/worker/sdk_worker.py | 2 +-
.../runners/worker/sdk_worker_main.py | 2 +-
.../runners/worker/sdk_worker_test.py | 2 +-
sdks/python/apache_beam/transforms/core.py | 2 +-
.../python/apache_beam/transforms/ptransform.py | 2 +-
sdks/python/apache_beam/transforms/trigger.py | 2 +-
sdks/python/apache_beam/transforms/window.py | 4 ++--
sdks/python/apache_beam/utils/urns.py | 2 +-
sdks/python/gen_protos.py | 2 +-
sdks/python/run_pylint.sh | 2 +-
25 files changed, 81 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index bd419a7..631d7f3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,7 +25,7 @@ sdks/python/**/*.egg
sdks/python/LICENSE
sdks/python/NOTICE
sdks/python/README.md
-sdks/python/apache_beam/runners/api/*pb2*.*
+sdks/python/apache_beam/portability/runners/api/*pb2*.*
# Ignore IntelliJ files.
.idea/
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/coders/coders.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py
index f3e0b43..1be1f3c 100644
--- a/sdks/python/apache_beam/coders/coders.py
+++ b/sdks/python/apache_beam/coders/coders.py
@@ -25,6 +25,7 @@ import cPickle as pickle
import google.protobuf
from apache_beam.coders import coder_impl
+from apache_beam.portability.runners.api import beam_runner_api_pb2
from apache_beam.utils import urns
from apache_beam.utils import proto_utils
@@ -205,7 +206,6 @@ class Coder(object):
"""For internal use only; no backwards-compatibility guarantees.
"""
# TODO(BEAM-115): Use specialized URNs and components.
- from apache_beam.runners.api import beam_runner_api_pb2
return beam_runner_api_pb2.Coder(
spec=beam_runner_api_pb2.SdkFunctionSpec(
spec=beam_runner_api_pb2.FunctionSpec(
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/pipeline.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py
index 9093abf..cea7215 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -339,7 +339,7 @@ class Pipeline(object):
def to_runner_api(self):
"""For internal use only; no backwards-compatibility guarantees."""
from apache_beam.runners import pipeline_context
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.runners.api import beam_runner_api_pb2
context = pipeline_context.PipelineContext()
# Mutates context; placing inline would force dependence on
# argument evaluation order.
@@ -525,7 +525,7 @@ class AppliedPTransform(object):
if isinstance(output, pvalue.PCollection)}
def to_runner_api(self, context):
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.runners.api import beam_runner_api_pb2
def transform_to_runner_api(transform, context):
if transform is None:
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/portability/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/portability/__init__.py b/sdks/python/apache_beam/portability/__init__.py
new file mode 100644
index 0000000..0bce5d6
--- /dev/null
+++ b/sdks/python/apache_beam/portability/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""For internal use only; no backwards-compatibility guarantees."""
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/portability/runners/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/portability/runners/__init__.py b/sdks/python/apache_beam/portability/runners/__init__.py
new file mode 100644
index 0000000..0bce5d6
--- /dev/null
+++ b/sdks/python/apache_beam/portability/runners/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""For internal use only; no backwards-compatibility guarantees."""
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/portability/runners/api/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/portability/runners/api/__init__.py b/sdks/python/apache_beam/portability/runners/api/__init__.py
new file mode 100644
index 0000000..2750859
--- /dev/null
+++ b/sdks/python/apache_beam/portability/runners/api/__init__.py
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""For internal use only; no backwards-compatibility guarantees.
+
+Automatically generated when running setup.py sdist or build[_py].
+"""
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/pvalue.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py
index 7385e82..8a774c4 100644
--- a/sdks/python/apache_beam/pvalue.py
+++ b/sdks/python/apache_beam/pvalue.py
@@ -128,7 +128,7 @@ class PCollection(PValue):
return _InvalidUnpickledPCollection, ()
def to_runner_api(self, context):
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.runners.api import beam_runner_api_pb2
from apache_beam.internal import pickler
return beam_runner_api_pb2.PCollection(
unique_name='%d%s.%s' % (
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/api/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/api/__init__.py b/sdks/python/apache_beam/runners/api/__init__.py
deleted file mode 100644
index 2750859..0000000
--- a/sdks/python/apache_beam/runners/api/__init__.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""For internal use only; no backwards-compatibility guarantees.
-
-Automatically generated when running setup.py sdist or build[_py].
-"""
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index d9aa1bf..a6cc25d 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -732,7 +732,7 @@ class DataflowRunner(PipelineRunner):
@classmethod
def serialize_windowing_strategy(cls, windowing):
from apache_beam.runners import pipeline_context
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.runners.api import beam_runner_api_pb2
context = pipeline_context.PipelineContext()
windowing_proto = windowing.to_runner_api(context)
return cls.byte_array_to_json_string(
@@ -745,7 +745,7 @@ class DataflowRunner(PipelineRunner):
# Imported here to avoid circular dependencies.
# pylint: disable=wrong-import-order, wrong-import-position
from apache_beam.runners import pipeline_context
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.runners.api import beam_runner_api_pb2
from apache_beam.transforms.core import Windowing
proto = beam_runner_api_pb2.MessageWithComponents()
proto.ParseFromString(cls.json_string_to_byte_array(serialized_data))
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/pipeline_context.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py
index 1c89d06..1330c39 100644
--- a/sdks/python/apache_beam/runners/pipeline_context.py
+++ b/sdks/python/apache_beam/runners/pipeline_context.py
@@ -24,7 +24,7 @@ For internal use only; no backwards-compatibility guarantees.
from apache_beam import pipeline
from apache_beam import pvalue
from apache_beam import coders
-from apache_beam.runners.api import beam_runner_api_pb2
+from apache_beam.portability.runners.api import beam_runner_api_pb2
from apache_beam.transforms import core
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/portability/fn_api_runner.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner.py
index db34ef9..a83eae4 100644
--- a/sdks/python/apache_beam/runners/portability/fn_api_runner.py
+++ b/sdks/python/apache_beam/runners/portability/fn_api_runner.py
@@ -33,7 +33,7 @@ from apache_beam.coders.coder_impl import create_OutputStream
from apache_beam.internal import pickler
from apache_beam.io import iobase
from apache_beam.transforms.window import GlobalWindows
-from apache_beam.runners.api import beam_fn_api_pb2
+from apache_beam.portability.runners.api import beam_fn_api_pb2
from apache_beam.runners.portability import maptask_executor_runner
from apache_beam.runners.worker import data_plane
from apache_beam.runners.worker import operation_specs
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/worker/data_plane.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/data_plane.py b/sdks/python/apache_beam/runners/worker/data_plane.py
index 7365db6..734ee9c 100644
--- a/sdks/python/apache_beam/runners/worker/data_plane.py
+++ b/sdks/python/apache_beam/runners/worker/data_plane.py
@@ -28,7 +28,7 @@ import Queue as queue
import threading
from apache_beam.coders import coder_impl
-from apache_beam.runners.api import beam_fn_api_pb2
+from apache_beam.portability.runners.api import beam_fn_api_pb2
import grpc
# This module is experimental. No backwards-compatibility guarantees.
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/worker/data_plane_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/data_plane_test.py b/sdks/python/apache_beam/runners/worker/data_plane_test.py
index e3e01ac..a2b31e8 100644
--- a/sdks/python/apache_beam/runners/worker/data_plane_test.py
+++ b/sdks/python/apache_beam/runners/worker/data_plane_test.py
@@ -29,7 +29,7 @@ import unittest
from concurrent import futures
import grpc
-from apache_beam.runners.api import beam_fn_api_pb2
+from apache_beam.portability.runners.api import beam_fn_api_pb2
from apache_beam.runners.worker import data_plane
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/worker/log_handler.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/log_handler.py b/sdks/python/apache_beam/runners/worker/log_handler.py
index 59ffbf4..dca0e4b 100644
--- a/sdks/python/apache_beam/runners/worker/log_handler.py
+++ b/sdks/python/apache_beam/runners/worker/log_handler.py
@@ -21,7 +21,7 @@ import math
import Queue as queue
import threading
-from apache_beam.runners.api import beam_fn_api_pb2
+from apache_beam.portability.runners.api import beam_fn_api_pb2
import grpc
# This module is experimental. No backwards-compatibility guarantees.
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/worker/log_handler_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/log_handler_test.py b/sdks/python/apache_beam/runners/worker/log_handler_test.py
index 8720ca8..6dd018f 100644
--- a/sdks/python/apache_beam/runners/worker/log_handler_test.py
+++ b/sdks/python/apache_beam/runners/worker/log_handler_test.py
@@ -22,7 +22,7 @@ import unittest
from concurrent import futures
import grpc
-from apache_beam.runners.api import beam_fn_api_pb2
+from apache_beam.portability.runners.api import beam_fn_api_pb2
from apache_beam.runners.worker import log_handler
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/worker/sdk_worker.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py
index 33c50ad..33f2b61 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py
@@ -38,7 +38,7 @@ from apache_beam.internal import pickler
from apache_beam.io import iobase
from apache_beam.runners.dataflow.native_io import iobase as native_iobase
from apache_beam.utils import counters
-from apache_beam.runners.api import beam_fn_api_pb2
+from apache_beam.portability.runners.api import beam_fn_api_pb2
from apache_beam.runners.worker import operation_specs
from apache_beam.runners.worker import operations
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
index b891779..9c11068 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
@@ -24,7 +24,7 @@ import sys
import grpc
from google.protobuf import text_format
-from apache_beam.runners.api import beam_fn_api_pb2
+from apache_beam.portability.runners.api import beam_fn_api_pb2
from apache_beam.runners.worker.log_handler import FnApiLogRecordHandler
from apache_beam.runners.worker.sdk_worker import SdkHarness
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py
index 0d0811b..93f60d3 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py
@@ -29,7 +29,7 @@ import grpc
from apache_beam.io.concat_source_test import RangeSource
from apache_beam.io.iobase import SourceBundle
-from apache_beam.runners.api import beam_fn_api_pb2
+from apache_beam.portability.runners.api import beam_fn_api_pb2
from apache_beam.runners.worker import data_plane
from apache_beam.runners.worker import sdk_worker
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/transforms/core.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index 0e497f9..d7fa770 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -27,7 +27,7 @@ from apache_beam import pvalue
from apache_beam import typehints
from apache_beam.coders import typecoders
from apache_beam.internal import util
-from apache_beam.runners.api import beam_runner_api_pb2
+from apache_beam.portability.runners.api import beam_runner_api_pb2
from apache_beam.transforms import ptransform
from apache_beam.transforms.display import DisplayDataItem
from apache_beam.transforms.display import HasDisplayData
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/transforms/ptransform.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py
index bd2a120..79fe3ad 100644
--- a/sdks/python/apache_beam/transforms/ptransform.py
+++ b/sdks/python/apache_beam/transforms/ptransform.py
@@ -430,7 +430,7 @@ class PTransform(WithTypeHints, HasDisplayData):
cls._known_urns[urn] = parameter_type, constructor
def to_runner_api(self, context):
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.runners.api import beam_runner_api_pb2
urn, typed_param = self.to_runner_api_parameter(context)
return beam_runner_api_pb2.FunctionSpec(
urn=urn,
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/transforms/trigger.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py
index 4200995..4151607 100644
--- a/sdks/python/apache_beam/transforms/trigger.py
+++ b/sdks/python/apache_beam/transforms/trigger.py
@@ -33,7 +33,7 @@ from apache_beam.transforms.window import GlobalWindow
from apache_beam.transforms.window import TimestampCombiner
from apache_beam.transforms.window import WindowedValue
from apache_beam.transforms.window import WindowFn
-from apache_beam.runners.api import beam_runner_api_pb2
+from apache_beam.portability.runners.api import beam_runner_api_pb2
from apache_beam.utils.timestamp import MAX_TIMESTAMP
from apache_beam.utils.timestamp import MIN_TIMESTAMP
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/transforms/window.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py
index e87a007..08c7a2d 100644
--- a/sdks/python/apache_beam/transforms/window.py
+++ b/sdks/python/apache_beam/transforms/window.py
@@ -55,8 +55,8 @@ from google.protobuf import duration_pb2
from google.protobuf import timestamp_pb2
from apache_beam.coders import coders
-from apache_beam.runners.api import beam_runner_api_pb2
-from apache_beam.runners.api import standard_window_fns_pb2
+from apache_beam.portability.runners.api import beam_runner_api_pb2
+from apache_beam.portability.runners.api import standard_window_fns_pb2
from apache_beam.transforms import timeutil
from apache_beam.utils import proto_utils
from apache_beam.utils import urns
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/apache_beam/utils/urns.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/urns.py b/sdks/python/apache_beam/utils/urns.py
index 849b8e3..b925bcc 100644
--- a/sdks/python/apache_beam/utils/urns.py
+++ b/sdks/python/apache_beam/utils/urns.py
@@ -102,7 +102,7 @@ class RunnerApiFn(object):
Prefer overriding self.to_runner_api_parameter.
"""
- from apache_beam.runners.api import beam_runner_api_pb2
+ from apache_beam.portability.runners.api import beam_runner_api_pb2
urn, typed_param = self.to_runner_api_parameter(context)
return beam_runner_api_pb2.SdkFunctionSpec(
spec=beam_runner_api_pb2.FunctionSpec(
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/gen_protos.py
----------------------------------------------------------------------
diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py
index 92b8414..a33c74b 100644
--- a/sdks/python/gen_protos.py
+++ b/sdks/python/gen_protos.py
@@ -35,7 +35,7 @@ BEAM_PROTO_PATHS = [
os.path.join('..', 'common', 'fn-api', 'src', 'main', 'proto')
]
-PYTHON_OUTPUT_PATH = os.path.join('apache_beam', 'runners', 'api')
+PYTHON_OUTPUT_PATH = os.path.join('apache_beam', 'portability', 'runners', 'api')
def generate_proto_files():
http://git-wip-us.apache.org/repos/asf/beam/blob/7689e43a/sdks/python/run_pylint.sh
----------------------------------------------------------------------
diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh
index 7808136..7434516 100755
--- a/sdks/python/run_pylint.sh
+++ b/sdks/python/run_pylint.sh
@@ -46,7 +46,7 @@ EXCLUDED_GENERATED_FILES=(
"apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py"
"apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py"
"apache_beam/coders/proto2_coder_test_messages_pb2.py"
-apache_beam/runners/api/*pb2*.py
+apache_beam/portability/runners/api/*pb2*.py
)
FILES_TO_IGNORE=""
[31/50] [abbrv] beam git commit: This closes #3331: Slight
debuggability improvements in BigtableIO
Posted by ke...@apache.org.
This closes #3331: Slight debuggability improvements in BigtableIO
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/bc2f97c7
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/bc2f97c7
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/bc2f97c7
Branch: refs/heads/gearpump-runner
Commit: bc2f97c7769bb9b81bc13a181c6e68e5fe219726
Parents: 911bfbd 82a6cb6
Author: Eugene Kirpichov <ki...@google.com>
Authored: Thu Jun 8 14:57:55 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Jun 8 14:57:55 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 22 ++++++++++++--------
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 2 +-
2 files changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
[47/50] [abbrv] beam git commit: Port DirectRunner TestStream
override to SDK-agnostic APIs
Posted by ke...@apache.org.
Port DirectRunner TestStream override to SDK-agnostic APIs
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/eaaf45fa
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/eaaf45fa
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/eaaf45fa
Branch: refs/heads/gearpump-runner
Commit: eaaf45fa33d500a9f0fd0c2861aac4889ee5086c
Parents: ed6bd18
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Jun 8 13:39:32 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Jun 9 19:56:52 2017 -0700
----------------------------------------------------------------------
.../construction/TestStreamTranslation.java | 49 +++++++++++++++++++-
.../direct/TestStreamEvaluatorFactory.java | 20 ++++++--
.../org/apache/beam/sdk/testing/TestStream.java | 12 +++++
3 files changed, 75 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/eaaf45fa/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
index 90e6304..515de57 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TestStreamTranslation.java
@@ -18,6 +18,9 @@
package org.apache.beam.runners.core.construction;
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.beam.runners.core.construction.PTransformTranslation.TEST_STREAM_TRANSFORM_URN;
+
import com.google.auto.service.AutoService;
import com.google.protobuf.Any;
import com.google.protobuf.ByteString;
@@ -33,6 +36,8 @@ import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.testing.TestStream;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TimestampedValue;
import org.joda.time.Duration;
import org.joda.time.Instant;
@@ -57,6 +62,48 @@ public class TestStreamTranslation {
return builder.build();
}
+ private static TestStream<?> fromProto(
+ RunnerApi.TestStreamPayload testStreamPayload, RunnerApi.Components components)
+ throws IOException {
+
+ Coder<Object> coder =
+ (Coder<Object>)
+ CoderTranslation.fromProto(
+ components.getCodersOrThrow(testStreamPayload.getCoderId()), components);
+
+ List<TestStream.Event<Object>> events = new ArrayList<>();
+
+ for (RunnerApi.TestStreamPayload.Event event : testStreamPayload.getEventsList()) {
+ events.add(fromProto(event, coder));
+ }
+ return TestStream.fromRawEvents(coder, events);
+ }
+
+ /**
+ * Converts an {@link AppliedPTransform}, which may be a rehydrated transform or an original
+ * {@link TestStream}, to a {@link TestStream}.
+ */
+ public static <T> TestStream<T> getTestStream(
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> application)
+ throws IOException {
+ // For robustness, we don't take this shortcut:
+ // if (application.getTransform() instanceof TestStream) {
+ // return application.getTransform()
+ // }
+
+ SdkComponents sdkComponents = SdkComponents.create();
+ RunnerApi.PTransform transformProto = PTransformTranslation.toProto(application, sdkComponents);
+ checkArgument(
+ TEST_STREAM_TRANSFORM_URN.equals(transformProto.getSpec().getUrn()),
+ "Attempt to get %s from a transform with wrong URN %s",
+ TestStream.class.getSimpleName(),
+ transformProto.getSpec().getUrn());
+ RunnerApi.TestStreamPayload testStreamPayload =
+ transformProto.getSpec().getParameter().unpack(RunnerApi.TestStreamPayload.class);
+
+ return (TestStream<T>) fromProto(testStreamPayload, sdkComponents.toComponents());
+ }
+
static <T> RunnerApi.TestStreamPayload.Event toProto(TestStream.Event<T> event, Coder<T> coder)
throws IOException {
switch (event.getType()) {
@@ -130,7 +177,7 @@ public class TestStreamTranslation {
static class TestStreamTranslator implements TransformPayloadTranslator<TestStream<?>> {
@Override
public String getUrn(TestStream<?> transform) {
- return PTransformTranslation.TEST_STREAM_TRANSFORM_URN;
+ return TEST_STREAM_TRANSFORM_URN;
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/eaaf45fa/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
index 2da7a71..16c8589 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TestStreamEvaluatorFactory.java
@@ -22,6 +22,7 @@ import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Supplier;
import com.google.common.collect.Iterables;
+import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -30,6 +31,7 @@ import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
import org.apache.beam.runners.core.construction.PTransformTranslation;
import org.apache.beam.runners.core.construction.ReplacementOutputs;
+import org.apache.beam.runners.core.construction.TestStreamTranslation;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.runners.PTransformOverrideFactory;
import org.apache.beam.sdk.testing.TestStream;
@@ -160,7 +162,8 @@ class TestStreamEvaluatorFactory implements TransformEvaluatorFactory {
}
static class DirectTestStreamFactory<T>
- implements PTransformOverrideFactory<PBegin, PCollection<T>, TestStream<T>> {
+ implements PTransformOverrideFactory<
+ PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> {
private final DirectRunner runner;
DirectTestStreamFactory(DirectRunner runner) {
@@ -169,10 +172,17 @@ class TestStreamEvaluatorFactory implements TransformEvaluatorFactory {
@Override
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
- AppliedPTransform<PBegin, PCollection<T>, TestStream<T>> transform) {
- return PTransformReplacement.of(
- transform.getPipeline().begin(),
- new DirectTestStream<T>(runner, transform.getTransform()));
+ AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform) {
+ try {
+ return PTransformReplacement.of(
+ transform.getPipeline().begin(),
+ new DirectTestStream<T>(runner, TestStreamTranslation.getTestStream(transform)));
+ } catch (IOException exc) {
+ throw new RuntimeException(
+ String.format(
+ "Transform could not be converted to %s", TestStream.class.getSimpleName()),
+ exc);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/eaaf45fa/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
index 9ad8fd8..d13fcf1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestStream.java
@@ -271,6 +271,18 @@ public final class TestStream<T> extends PTransform<PBegin, PCollection<T>> {
return events;
}
+ /**
+ * <b>For internal use only. No backwards-compatibility guarantees.</b>
+ *
+ * <p>Builder a test stream directly from events. No validation is performed on
+ * watermark monotonicity, etc. This is assumed to be a previously-serialized
+ * {@link TestStream} transform that is correct by construction.
+ */
+ @Internal
+ public static <T> TestStream<T> fromRawEvents(Coder<T> coder, List<Event<T>> events) {
+ return new TestStream<>(coder, events);
+ }
+
@Override
public boolean equals(Object other) {
if (!(other instanceof TestStream)) {
[41/50] [abbrv] beam git commit: Visit a Transform Hierarchy in
Topological Order
Posted by ke...@apache.org.
Visit a Transform Hierarchy in Topological Order
This reverts commit 6ad6433ec0c02aec8656e9e3b27f6e0f974f8ece.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/261e7df2
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/261e7df2
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/261e7df2
Branch: refs/heads/gearpump-runner
Commit: 261e7df2b860fe82d9f401e2621b020fe2020fea
Parents: d2c4093
Author: Thomas Groh <tg...@google.com>
Authored: Tue Jun 6 16:15:19 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Fri Jun 9 14:44:42 2017 -0700
----------------------------------------------------------------------
.../spark/translation/StorageLevelTest.java | 4 +-
.../beam/sdk/runners/TransformHierarchy.java | 79 +++++++-
.../sdk/runners/TransformHierarchyTest.java | 197 +++++++++++++++++++
3 files changed, 274 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/261e7df2/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
index 8f2e681..8bd6dae 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/StorageLevelTest.java
@@ -58,12 +58,12 @@ public class StorageLevelTest {
@Test
public void test() throws Exception {
- PCollection<String> pCollection = pipeline.apply(Create.of("foo"));
+ PCollection<String> pCollection = pipeline.apply("CreateFoo", Create.of("foo"));
// by default, the Spark runner doesn't cache the RDD if it accessed only one time.
// So, to "force" the caching of the RDD, we have to call the RDD at least two time.
// That's why we are using Count fn on the PCollection.
- pCollection.apply(Count.<String>globally());
+ pCollection.apply("CountAll", Count.<String>globally());
PCollection<String> output = pCollection.apply(new StorageLevelPTransform());
http://git-wip-us.apache.org/repos/asf/beam/blob/261e7df2/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
index ee1ce7b..5e048eb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/TransformHierarchy.java
@@ -208,7 +208,7 @@ public class TransformHierarchy {
public Set<PValue> visit(PipelineVisitor visitor) {
finishSpecifying();
Set<PValue> visitedValues = new HashSet<>();
- root.visit(visitor, visitedValues);
+ root.visit(visitor, visitedValues, new HashSet<Node>(), new HashSet<Node>());
return visitedValues;
}
@@ -503,10 +503,60 @@ public class TransformHierarchy {
/**
* Visit the transform node.
*
+ * <p>The visit proceeds in the following order:
+ *
+ * <ul>
+ * <li>Visit all input {@link PValue PValues} returned by the flattened expansion of {@link
+ * Node#getInputs()}.
+ * <li>If the node is a composite:
+ * <ul>
+ * <li>Enter the node via {@link PipelineVisitor#enterCompositeTransform(Node)}.
+ * <li>If the result of {@link PipelineVisitor#enterCompositeTransform(Node)} was {@link
+ * CompositeBehavior#ENTER_TRANSFORM}, visit each child node of this {@link Node}.
+ * <li>Leave the node via {@link PipelineVisitor#leaveCompositeTransform(Node)}.
+ * </ul>
+ * <li>If the node is a primitive, visit it via {@link
+ * PipelineVisitor#visitPrimitiveTransform(Node)}.
+ * <li>Visit each {@link PValue} that was output by this node.
+ * </ul>
+ *
+ * <p>Additionally, the following ordering restrictions are observed:
+ *
+ * <ul>
+ * <li>A {@link Node} will be visited after its enclosing node has been entered and before its
+ * enclosing node has been left
+ * <li>A {@link Node} will not be visited if any enclosing {@link Node} has returned {@link
+ * CompositeBehavior#DO_NOT_ENTER_TRANSFORM} from the call to {@link
+ * PipelineVisitor#enterCompositeTransform(Node)}.
+ * <li>A {@link PValue} will only be visited after the {@link Node} that originally produced
+ * it has been visited.
+ * </ul>
+ *
* <p>Provides an ordered visit of the input values, the primitive transform (or child nodes for
* composite transforms), then the output values.
*/
- private void visit(PipelineVisitor visitor, Set<PValue> visitedValues) {
+ private void visit(
+ PipelineVisitor visitor,
+ Set<PValue> visitedValues,
+ Set<Node> visitedNodes,
+ Set<Node> skippedComposites) {
+ if (getEnclosingNode() != null && !visitedNodes.contains(getEnclosingNode())) {
+ // Recursively enter all enclosing nodes, as appropriate.
+ getEnclosingNode().visit(visitor, visitedValues, visitedNodes, skippedComposites);
+ }
+ // These checks occur after visiting the enclosing node to ensure that if this node has been
+ // visited while visiting the enclosing node the node is not revisited, or, if an enclosing
+ // Node is skipped, this node is also skipped.
+ if (!visitedNodes.add(this)) {
+ LOG.debug("Not revisiting previously visited node {}", this);
+ return;
+ } else if (childNodeOf(skippedComposites)) {
+ // This node is a child of a node that has been passed over via CompositeBehavior, and
+ // should also be skipped. All child nodes of a skipped composite should always be skipped.
+ LOG.debug("Not revisiting Node {} which is a child of a previously passed composite", this);
+ return;
+ }
+
if (!finishedSpecifying) {
finishSpecifying();
}
@@ -514,22 +564,31 @@ public class TransformHierarchy {
if (!isRootNode()) {
// Visit inputs.
for (PValue inputValue : inputs.values()) {
+ Node valueProducer = getProducer(inputValue);
+ if (!visitedNodes.contains(valueProducer)) {
+ valueProducer.visit(visitor, visitedValues, visitedNodes, skippedComposites);
+ }
if (visitedValues.add(inputValue)) {
- visitor.visitValue(inputValue, getProducer(inputValue));
+ LOG.debug("Visiting input value {}", inputValue);
+ visitor.visitValue(inputValue, valueProducer);
}
}
}
if (isCompositeNode()) {
+ LOG.debug("Visiting composite node {}", this);
PipelineVisitor.CompositeBehavior recurse = visitor.enterCompositeTransform(this);
if (recurse.equals(CompositeBehavior.ENTER_TRANSFORM)) {
for (Node child : parts) {
- child.visit(visitor, visitedValues);
+ child.visit(visitor, visitedValues, visitedNodes, skippedComposites);
}
+ } else {
+ skippedComposites.add(this);
}
visitor.leaveCompositeTransform(this);
} else {
+ LOG.debug("Visiting primitive node {}", this);
visitor.visitPrimitiveTransform(this);
}
@@ -538,12 +597,24 @@ public class TransformHierarchy {
// Visit outputs.
for (PValue pValue : outputs.values()) {
if (visitedValues.add(pValue)) {
+ LOG.debug("Visiting output value {}", pValue);
visitor.visitValue(pValue, this);
}
}
}
}
+ private boolean childNodeOf(Set<Node> nodes) {
+ if (isRootNode()) {
+ return false;
+ }
+ Node parent = this.getEnclosingNode();
+ while (!parent.isRootNode() && !nodes.contains(parent)) {
+ parent = parent.getEnclosingNode();
+ }
+ return nodes.contains(parent);
+ }
+
/**
* Finish specifying a transform.
*
http://git-wip-us.apache.org/repos/asf/beam/blob/261e7df2/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
index 1197d1b..93650dd 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/TransformHierarchyTest.java
@@ -19,6 +19,7 @@ package org.apache.beam.sdk.runners;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertThat;
@@ -32,6 +33,8 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.beam.sdk.Pipeline.PipelineVisitor;
import org.apache.beam.sdk.Pipeline.PipelineVisitor.Defaults;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.io.CountingSource;
import org.apache.beam.sdk.io.GenerateSequence;
import org.apache.beam.sdk.io.Read;
@@ -492,4 +495,198 @@ public class TransformHierarchyTest implements Serializable {
assertThat(visitedPrimitiveNodes, containsInAnyOrder(upstreamNode, replacementParNode));
assertThat(visitedValues, Matchers.<PValue>containsInAnyOrder(upstream, output));
}
+
+ @Test
+ public void visitIsTopologicallyOrdered() {
+ PCollection<String> one =
+ PCollection.<String>createPrimitiveOutputInternal(
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)
+ .setCoder(StringUtf8Coder.of());
+ final PCollection<Integer> two =
+ PCollection.<Integer>createPrimitiveOutputInternal(
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED)
+ .setCoder(VarIntCoder.of());
+ final PDone done = PDone.in(pipeline);
+ final TupleTag<String> oneTag = new TupleTag<String>() {};
+ final TupleTag<Integer> twoTag = new TupleTag<Integer>() {};
+ final PCollectionTuple oneAndTwo = PCollectionTuple.of(oneTag, one).and(twoTag, two);
+
+ PTransform<PCollection<String>, PDone> multiConsumer =
+ new PTransform<PCollection<String>, PDone>() {
+ @Override
+ public PDone expand(PCollection<String> input) {
+ return done;
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return Collections.<TupleTag<?>, PValue>singletonMap(twoTag, two);
+ }
+ };
+ hierarchy.pushNode("consumes_both", one, multiConsumer);
+ hierarchy.setOutput(done);
+ hierarchy.popNode();
+
+ final PTransform<PBegin, PCollectionTuple> producer =
+ new PTransform<PBegin, PCollectionTuple>() {
+ @Override
+ public PCollectionTuple expand(PBegin input) {
+ return oneAndTwo;
+ }
+ };
+ hierarchy.pushNode(
+ "encloses_producer",
+ PBegin.in(pipeline),
+ new PTransform<PBegin, PCollectionTuple>() {
+ @Override
+ public PCollectionTuple expand(PBegin input) {
+ return input.apply(producer);
+ }
+ });
+ hierarchy.pushNode(
+ "creates_one_and_two",
+ PBegin.in(pipeline), producer);
+ hierarchy.setOutput(oneAndTwo);
+ hierarchy.popNode();
+ hierarchy.setOutput(oneAndTwo);
+ hierarchy.popNode();
+
+ hierarchy.pushNode("second_copy_of_consumes_both", one, multiConsumer);
+ hierarchy.setOutput(done);
+ hierarchy.popNode();
+
+ final Set<Node> visitedNodes = new HashSet<>();
+ final Set<Node> exitedNodes = new HashSet<>();
+ final Set<PValue> visitedValues = new HashSet<>();
+ hierarchy.visit(
+ new PipelineVisitor.Defaults() {
+
+ @Override
+ public CompositeBehavior enterCompositeTransform(Node node) {
+ for (PValue input : node.getInputs().values()) {
+ assertThat(visitedValues, hasItem(input));
+ }
+ assertThat(
+ "Nodes should not be visited more than once", visitedNodes, not(hasItem(node)));
+ if (!node.isRootNode()) {
+ assertThat(
+ "Nodes should always be visited after their enclosing nodes",
+ visitedNodes,
+ hasItem(node.getEnclosingNode()));
+ }
+ visitedNodes.add(node);
+ return CompositeBehavior.ENTER_TRANSFORM;
+ }
+
+ @Override
+ public void leaveCompositeTransform(Node node) {
+ assertThat(visitedNodes, hasItem(node));
+ if (!node.isRootNode()) {
+ assertThat(
+ "Nodes should always be left before their enclosing nodes are left",
+ exitedNodes,
+ not(hasItem(node.getEnclosingNode())));
+ }
+ assertThat(exitedNodes, not(hasItem(node)));
+ exitedNodes.add(node);
+ }
+
+ @Override
+ public void visitPrimitiveTransform(Node node) {
+ assertThat(visitedNodes, hasItem(node.getEnclosingNode()));
+ assertThat(exitedNodes, not(hasItem(node.getEnclosingNode())));
+ assertThat(
+ "Nodes should not be visited more than once", visitedNodes, not(hasItem(node)));
+ for (PValue input : node.getInputs().values()) {
+ assertThat(visitedValues, hasItem(input));
+ }
+ visitedNodes.add(node);
+ }
+
+ @Override
+ public void visitValue(PValue value, Node producer) {
+ assertThat(visitedNodes, hasItem(producer));
+ assertThat(visitedValues, not(hasItem(value)));
+ visitedValues.add(value);
+ }
+ });
+ assertThat("Should have visited all the nodes", visitedNodes.size(), equalTo(5));
+ assertThat("Should have left all of the visited composites", exitedNodes.size(), equalTo(2));
+ }
+
+ @Test
+ public void visitDoesNotVisitSkippedNodes() {
+ PCollection<String> one =
+ PCollection.<String>createPrimitiveOutputInternal(
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED)
+ .setCoder(StringUtf8Coder.of());
+ final PCollection<Integer> two =
+ PCollection.<Integer>createPrimitiveOutputInternal(
+ pipeline, WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED)
+ .setCoder(VarIntCoder.of());
+ final PDone done = PDone.in(pipeline);
+ final TupleTag<String> oneTag = new TupleTag<String>() {};
+ final TupleTag<Integer> twoTag = new TupleTag<Integer>() {};
+ final PCollectionTuple oneAndTwo = PCollectionTuple.of(oneTag, one).and(twoTag, two);
+
+ hierarchy.pushNode(
+ "consumes_both",
+ one,
+ new PTransform<PCollection<String>, PDone>() {
+ @Override
+ public PDone expand(PCollection<String> input) {
+ return done;
+ }
+
+ @Override
+ public Map<TupleTag<?>, PValue> getAdditionalInputs() {
+ return Collections.<TupleTag<?>, PValue>singletonMap(twoTag, two);
+ }
+ });
+ hierarchy.setOutput(done);
+ hierarchy.popNode();
+
+ final PTransform<PBegin, PCollectionTuple> producer =
+ new PTransform<PBegin, PCollectionTuple>() {
+ @Override
+ public PCollectionTuple expand(PBegin input) {
+ return oneAndTwo;
+ }
+ };
+ final Node enclosing =
+ hierarchy.pushNode(
+ "encloses_producer",
+ PBegin.in(pipeline),
+ new PTransform<PBegin, PCollectionTuple>() {
+ @Override
+ public PCollectionTuple expand(PBegin input) {
+ return input.apply(producer);
+ }
+ });
+ Node enclosed = hierarchy.pushNode("creates_one_and_two", PBegin.in(pipeline), producer);
+ hierarchy.setOutput(oneAndTwo);
+ hierarchy.popNode();
+ hierarchy.setOutput(oneAndTwo);
+ hierarchy.popNode();
+
+ final Set<Node> visitedNodes = new HashSet<>();
+ hierarchy.visit(
+ new PipelineVisitor.Defaults() {
+ @Override
+ public CompositeBehavior enterCompositeTransform(Node node) {
+ visitedNodes.add(node);
+ return node.equals(enclosing)
+ ? CompositeBehavior.DO_NOT_ENTER_TRANSFORM
+ : CompositeBehavior.ENTER_TRANSFORM;
+ }
+
+ @Override
+ public void visitPrimitiveTransform(Node node) {
+ visitedNodes.add(node);
+ }
+ });
+
+ assertThat(visitedNodes, hasItem(enclosing));
+ assertThat(visitedNodes, not(hasItem(enclosed)));
+ }
}
[27/50] [abbrv] beam git commit: Closes #3325
Posted by ke...@apache.org.
Closes #3325
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7c5a70ed
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7c5a70ed
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7c5a70ed
Branch: refs/heads/gearpump-runner
Commit: 7c5a70edb2d19f22b0a6b81ae516762703358986
Parents: 4aef938 36aea2d
Author: Robert Bradshaw <ro...@google.com>
Authored: Thu Jun 8 13:58:31 2017 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Thu Jun 8 13:58:31 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/runners/worker/operation_specs.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[34/50] [abbrv] beam git commit: [BEAM-2410] Remove TransportClient
from ElasticSearchIO to decouple IO and ES server versions
Posted by ke...@apache.org.
[BEAM-2410] Remove TransportClient from ElasticSearchIO to decouple IO and ES server versions
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7caea7a8
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7caea7a8
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7caea7a8
Branch: refs/heads/gearpump-runner
Commit: 7caea7a845eff072a647baf69b9b004db4523652
Parents: e980ae9
Author: Etienne Chauchot <ec...@gmail.com>
Authored: Mon Jun 5 16:21:58 2017 +0200
Committer: Jean-Baptiste Onofré <jb...@apache.org>
Committed: Fri Jun 9 07:31:06 2017 +0200
----------------------------------------------------------------------
.../sdk/io/common/IOTestPipelineOptions.java | 6 +-
.../sdk/io/elasticsearch/ElasticsearchIO.java | 4 +-
.../elasticsearch/ElasticSearchIOTestUtils.java | 81 +++++++++++---------
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 14 ++--
.../io/elasticsearch/ElasticsearchIOTest.java | 36 +++++----
.../elasticsearch/ElasticsearchTestDataSet.java | 37 ++++-----
6 files changed, 87 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/7caea7a8/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/IOTestPipelineOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/IOTestPipelineOptions.java b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/IOTestPipelineOptions.java
index 387fd22..25ab929 100644
--- a/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/IOTestPipelineOptions.java
+++ b/sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/IOTestPipelineOptions.java
@@ -71,11 +71,7 @@ public interface IOTestPipelineOptions extends TestPipelineOptions {
Integer getElasticsearchHttpPort();
void setElasticsearchHttpPort(Integer value);
- @Description("Tcp port for elasticsearch server")
- @Default.Integer(9300)
- Integer getElasticsearchTcpPort();
- void setElasticsearchTcpPort(Integer value);
-
+ /* Cassandra */
@Description("Host for Cassandra server (host name/ip address)")
@Default.String("cassandra-host")
String getCassandraHost();
http://git-wip-us.apache.org/repos/asf/beam/blob/7caea7a8/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java b/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
index f6ceef2..e3965dc 100644
--- a/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
+++ b/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
@@ -139,7 +139,7 @@ public class ElasticsearchIO {
private static final ObjectMapper mapper = new ObjectMapper();
- private static JsonNode parseResponse(Response response) throws IOException {
+ static JsonNode parseResponse(Response response) throws IOException {
return mapper.readValue(response.getEntity().getContent(), JsonNode.class);
}
@@ -264,7 +264,7 @@ public class ElasticsearchIO {
builder.addIfNotNull(DisplayData.item("username", getUsername()));
}
- private RestClient createClient() throws MalformedURLException {
+ RestClient createClient() throws MalformedURLException {
HttpHost[] hosts = new HttpHost[getAddresses().size()];
int i = 0;
for (String address : getAddresses()) {
http://git-wip-us.apache.org/repos/asf/beam/blob/7caea7a8/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java
index b0d161f..203963d 100644
--- a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java
+++ b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticSearchIOTestUtils.java
@@ -17,19 +17,17 @@
*/
package org.apache.beam.sdk.io.elasticsearch;
+import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.IndicesAdminClient;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.index.IndexNotFoundException;
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
/** Test utilities to use with {@link ElasticsearchIO}. */
class ElasticSearchIOTestUtils {
@@ -41,57 +39,68 @@ class ElasticSearchIOTestUtils {
}
/** Deletes the given index synchronously. */
- static void deleteIndex(String index, Client client) throws Exception {
- IndicesAdminClient indices = client.admin().indices();
- IndicesExistsResponse indicesExistsResponse =
- indices.exists(new IndicesExistsRequest(index)).get();
- if (indicesExistsResponse.isExists()) {
- indices.prepareClose(index).get();
- indices.delete(Requests.deleteIndexRequest(index)).get();
+ static void deleteIndex(String index, RestClient restClient) throws IOException {
+ try {
+ restClient.performRequest("DELETE", String.format("/%s", index), new BasicHeader("", ""));
+ } catch (IOException e) {
+ // it is fine to ignore this expression as deleteIndex occurs in @before,
+ // so when the first tests is run, the index does not exist yet
+ if (!e.getMessage().contains("index_not_found_exception")){
+ throw e;
+ }
}
}
/** Inserts the given number of test documents into Elasticsearch. */
- static void insertTestDocuments(String index, String type, long numDocs, Client client)
- throws Exception {
- final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk().setRefresh(true);
+ static void insertTestDocuments(String index, String type, long numDocs, RestClient restClient)
+ throws IOException {
List<String> data =
ElasticSearchIOTestUtils.createDocuments(
numDocs, ElasticSearchIOTestUtils.InjectionMode.DO_NOT_INJECT_INVALID_DOCS);
+ StringBuilder bulkRequest = new StringBuilder();
for (String document : data) {
- bulkRequestBuilder.add(client.prepareIndex(index, type, null).setSource(document));
+ bulkRequest.append(String.format("{ \"index\" : {} }%n%s%n", document));
}
- final BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
- if (bulkResponse.hasFailures()) {
+ String endPoint = String.format("/%s/%s/_bulk", index, type);
+ HttpEntity requestBody =
+ new NStringEntity(bulkRequest.toString(), ContentType.APPLICATION_JSON);
+ Response response = restClient.performRequest("POST", endPoint,
+ Collections.singletonMap("refresh", "true"), requestBody,
+ new BasicHeader("", ""));
+ JsonNode searchResult = ElasticsearchIO.parseResponse(response);
+ boolean errors = searchResult.path("errors").asBoolean();
+ if (errors){
throw new IOException(
- String.format(
- "Cannot insert test documents in index %s : %s",
- index, bulkResponse.buildFailureMessage()));
+ String.format("Failed to insert test documents in index %s", index));
}
}
/**
- * Forces an upgrade of the given index to make recently inserted documents available for search.
+ * Forces a refresh of the given index to make recently inserted documents available for search.
*
* @return The number of docs in the index
*/
- static long upgradeIndexAndGetCurrentNumDocs(String index, String type, Client client) {
+ static long refreshIndexAndGetCurrentNumDocs(String index, String type, RestClient restClient)
+ throws IOException {
+ long result = 0;
try {
- client.admin().indices().upgrade(new UpgradeRequest(index)).actionGet();
- SearchResponse response =
- client.prepareSearch(index).setTypes(type).execute().actionGet(5000);
- return response.getHits().getTotalHits();
+ String endPoint = String.format("/%s/_refresh", index);
+ restClient.performRequest("POST", endPoint, new BasicHeader("", ""));
+
+ endPoint = String.format("/%s/%s/_search", index, type);
+ Response response = restClient.performRequest("GET", endPoint, new BasicHeader("", ""));
+ JsonNode searchResult = ElasticsearchIO.parseResponse(response);
+ result = searchResult.path("hits").path("total").asLong();
+ } catch (IOException e) {
// it is fine to ignore bellow exceptions because in testWriteWithBatchSize* sometimes,
// we call upgrade before any doc have been written
// (when there are fewer docs processed than batchSize).
// In that cases index/type has not been created (created upon first doc insertion)
- } catch (IndexNotFoundException e) {
- } catch (java.lang.IllegalArgumentException e) {
- if (!e.getMessage().contains("No search type")) {
+ if (!e.getMessage().contains("index_not_found_exception")){
throw e;
}
}
- return 0;
+ return result;
}
/**
http://git-wip-us.apache.org/repos/asf/beam/blob/7caea7a8/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
index 2d6393a..7c37e87 100644
--- a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
+++ b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
@@ -32,7 +32,7 @@ import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.values.PCollection;
-import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.client.RestClient;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -57,7 +57,7 @@ import org.slf4j.LoggerFactory;
*/
public class ElasticsearchIOIT {
private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchIOIT.class);
- private static TransportClient client;
+ private static RestClient restClient;
private static IOTestPipelineOptions options;
private static ElasticsearchIO.ConnectionConfiguration readConnectionConfiguration;
@Rule public TestPipeline pipeline = TestPipeline.create();
@@ -66,16 +66,16 @@ public class ElasticsearchIOIT {
public static void beforeClass() throws Exception {
PipelineOptionsFactory.register(IOTestPipelineOptions.class);
options = TestPipeline.testingPipelineOptions().as(IOTestPipelineOptions.class);
- client = ElasticsearchTestDataSet.getClient(options);
readConnectionConfiguration =
ElasticsearchTestDataSet.getConnectionConfiguration(
options, ElasticsearchTestDataSet.ReadOrWrite.READ);
+ restClient = readConnectionConfiguration.createClient();
}
@AfterClass
public static void afterClass() throws Exception {
- ElasticsearchTestDataSet.deleteIndex(client, ElasticsearchTestDataSet.ReadOrWrite.WRITE);
- client.close();
+ ElasticsearchTestDataSet.deleteIndex(restClient, ElasticsearchTestDataSet.ReadOrWrite.WRITE);
+ restClient.close();
}
@Test
@@ -128,8 +128,8 @@ public class ElasticsearchIOIT {
pipeline.run();
long currentNumDocs =
- ElasticSearchIOTestUtils.upgradeIndexAndGetCurrentNumDocs(
- ElasticsearchTestDataSet.ES_INDEX, ElasticsearchTestDataSet.ES_TYPE, client);
+ ElasticSearchIOTestUtils.refreshIndexAndGetCurrentNumDocs(
+ ElasticsearchTestDataSet.ES_INDEX, ElasticsearchTestDataSet.ES_TYPE, restClient);
assertEquals(ElasticsearchTestDataSet.NUM_DOCS, currentNumDocs);
}
http://git-wip-us.apache.org/repos/asf/beam/blob/7caea7a8/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
index 260af79..b349a29 100644
--- a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
+++ b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java
@@ -39,11 +39,11 @@ import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFnTester;
import org.apache.beam.sdk.values.PCollection;
import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
import org.hamcrest.CustomMatcher;
import org.junit.AfterClass;
import org.junit.Before;
@@ -74,9 +74,10 @@ public class ElasticsearchIOTest implements Serializable {
private static final long BATCH_SIZE_BYTES = 2048L;
private static Node node;
+ private static RestClient restClient;
private static ElasticsearchIO.ConnectionConfiguration connectionConfiguration;
- @ClassRule public static TemporaryFolder folder = new TemporaryFolder();
+ @ClassRule public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder();
@Rule
public TestPipeline pipeline = TestPipeline.create();
@@ -91,8 +92,8 @@ public class ElasticsearchIOTest implements Serializable {
.put("cluster.name", "beam")
.put("http.enabled", "true")
.put("node.data", "true")
- .put("path.data", folder.getRoot().getPath())
- .put("path.home", folder.getRoot().getPath())
+ .put("path.data", TEMPORARY_FOLDER.getRoot().getPath())
+ .put("path.home", TEMPORARY_FOLDER.getRoot().getPath())
.put("node.name", "beam")
.put("network.host", ES_IP)
.put("http.port", esHttpPort)
@@ -100,27 +101,29 @@ public class ElasticsearchIOTest implements Serializable {
// had problems with some jdk, embedded ES was too slow for bulk insertion,
// and queue of 50 was full. No pb with real ES instance (cf testWrite integration test)
.put("threadpool.bulk.queue_size", 100);
- node = NodeBuilder.nodeBuilder().settings(settingsBuilder).build();
+ node = new Node(settingsBuilder.build());
LOG.info("Elasticsearch node created");
node.start();
connectionConfiguration =
ElasticsearchIO.ConnectionConfiguration.create(
new String[] {"http://" + ES_IP + ":" + esHttpPort}, ES_INDEX, ES_TYPE);
+ restClient = connectionConfiguration.createClient();
}
@AfterClass
- public static void afterClass() {
+ public static void afterClass() throws IOException{
+ restClient.close();
node.close();
}
@Before
public void before() throws Exception {
- ElasticSearchIOTestUtils.deleteIndex(ES_INDEX, node.client());
+ ElasticSearchIOTestUtils.deleteIndex(ES_INDEX, restClient);
}
@Test
public void testSizes() throws Exception {
- ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, node.client());
+ ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, restClient);
PipelineOptions options = PipelineOptionsFactory.create();
ElasticsearchIO.Read read =
ElasticsearchIO.read().withConnectionConfiguration(connectionConfiguration);
@@ -134,7 +137,7 @@ public class ElasticsearchIOTest implements Serializable {
@Test
public void testRead() throws Exception {
- ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, node.client());
+ ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, restClient);
PCollection<String> output =
pipeline.apply(
@@ -150,7 +153,7 @@ public class ElasticsearchIOTest implements Serializable {
@Test
public void testReadWithQuery() throws Exception {
- ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, node.client());
+ ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, restClient);
String query =
"{\n"
@@ -185,7 +188,7 @@ public class ElasticsearchIOTest implements Serializable {
pipeline.run();
long currentNumDocs =
- ElasticSearchIOTestUtils.upgradeIndexAndGetCurrentNumDocs(ES_INDEX, ES_TYPE, node.client());
+ ElasticSearchIOTestUtils.refreshIndexAndGetCurrentNumDocs(ES_INDEX, ES_TYPE, restClient);
assertEquals(NUM_DOCS, currentNumDocs);
QueryBuilder queryBuilder = QueryBuilders.queryStringQuery("Einstein").field("scientist");
@@ -258,9 +261,8 @@ public class ElasticsearchIOTest implements Serializable {
if ((numDocsProcessed % 100) == 0) {
// force the index to upgrade after inserting for the inserted docs
// to be searchable immediately
- long currentNumDocs =
- ElasticSearchIOTestUtils.upgradeIndexAndGetCurrentNumDocs(
- ES_INDEX, ES_TYPE, node.client());
+ long currentNumDocs = ElasticSearchIOTestUtils
+ .refreshIndexAndGetCurrentNumDocs(ES_INDEX, ES_TYPE, restClient);
if ((numDocsProcessed % BATCH_SIZE) == 0) {
/* bundle end */
assertEquals(
@@ -304,8 +306,8 @@ public class ElasticsearchIOTest implements Serializable {
// force the index to upgrade after inserting for the inserted docs
// to be searchable immediately
long currentNumDocs =
- ElasticSearchIOTestUtils.upgradeIndexAndGetCurrentNumDocs(
- ES_INDEX, ES_TYPE, node.client());
+ ElasticSearchIOTestUtils.refreshIndexAndGetCurrentNumDocs(
+ ES_INDEX, ES_TYPE, restClient);
if (sizeProcessed / BATCH_SIZE_BYTES > batchInserted) {
/* bundle end */
assertThat(
@@ -327,7 +329,7 @@ public class ElasticsearchIOTest implements Serializable {
@Test
public void testSplit() throws Exception {
- ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, node.client());
+ ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, restClient);
PipelineOptions options = PipelineOptionsFactory.create();
ElasticsearchIO.Read read =
ElasticsearchIO.read().withConnectionConfiguration(connectionConfiguration);
http://git-wip-us.apache.org/repos/asf/beam/blob/7caea7a8/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchTestDataSet.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchTestDataSet.java b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchTestDataSet.java
index 3a9aae6..2a2dbe9 100644
--- a/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchTestDataSet.java
+++ b/sdks/java/io/elasticsearch/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchTestDataSet.java
@@ -17,13 +17,11 @@
*/
package org.apache.beam.sdk.io.elasticsearch;
-import static java.net.InetAddress.getByName;
import java.io.IOException;
import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.client.RestClient;
/**
* Manipulates test data used by the {@link ElasticsearchIO}
@@ -51,7 +49,6 @@ public class ElasticsearchTestDataSet {
* -Dexec.mainClass=org.apache.beam.sdk.io.elasticsearch.ElasticsearchTestDataSet \
* -Dexec.args="--elasticsearchServer=1.2.3.4 \
* --elasticsearchHttpPort=9200 \
- * --elasticsearchTcpPort=9300" \
* -Dexec.classpathScope=test
* </pre>
*
@@ -62,29 +59,20 @@ public class ElasticsearchTestDataSet {
PipelineOptionsFactory.register(IOTestPipelineOptions.class);
IOTestPipelineOptions options =
PipelineOptionsFactory.fromArgs(args).as(IOTestPipelineOptions.class);
-
- createAndPopulateIndex(getClient(options), ReadOrWrite.READ);
+ createAndPopulateReadIndex(options);
}
- private static void createAndPopulateIndex(TransportClient client, ReadOrWrite rOw)
- throws Exception {
+ private static void createAndPopulateReadIndex(IOTestPipelineOptions options) throws Exception {
+ RestClient restClient = getConnectionConfiguration(options, ReadOrWrite.READ).createClient();
// automatically creates the index and insert docs
- ElasticSearchIOTestUtils.insertTestDocuments(
- (rOw == ReadOrWrite.READ) ? ES_INDEX : writeIndex, ES_TYPE, NUM_DOCS, client);
- }
-
- public static TransportClient getClient(IOTestPipelineOptions options) throws Exception {
- TransportClient client =
- TransportClient.builder()
- .build()
- .addTransportAddress(
- new InetSocketTransportAddress(
- getByName(options.getElasticsearchServer()),
- options.getElasticsearchTcpPort()));
- return client;
+ try {
+ ElasticSearchIOTestUtils.insertTestDocuments(ES_INDEX, ES_TYPE, NUM_DOCS, restClient);
+ } finally {
+ restClient.close();
+ }
}
- public static ElasticsearchIO.ConnectionConfiguration getConnectionConfiguration(
+ static ElasticsearchIO.ConnectionConfiguration getConnectionConfiguration(
IOTestPipelineOptions options, ReadOrWrite rOw) throws IOException {
ElasticsearchIO.ConnectionConfiguration connectionConfiguration =
ElasticsearchIO.ConnectionConfiguration.create(
@@ -99,8 +87,9 @@ public class ElasticsearchTestDataSet {
return connectionConfiguration;
}
- public static void deleteIndex(TransportClient client, ReadOrWrite rOw) throws Exception {
- ElasticSearchIOTestUtils.deleteIndex((rOw == ReadOrWrite.READ) ? ES_INDEX : writeIndex, client);
+ static void deleteIndex(RestClient restClient, ReadOrWrite rOw) throws Exception {
+ ElasticSearchIOTestUtils
+ .deleteIndex((rOw == ReadOrWrite.READ) ? ES_INDEX : writeIndex, restClient);
}
/** Enum that tells whether we use the index for reading or for writing. */
[13/50] [abbrv] beam git commit: Add coder info to pubsub io
Posted by ke...@apache.org.
Add coder info to pubsub io
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b5852d21
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b5852d21
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b5852d21
Branch: refs/heads/gearpump-runner
Commit: b5852d212cab060321c43a5800f8585aa3649aec
Parents: 0a0a1bc
Author: Vikas Kedigehalli <vi...@google.com>
Authored: Wed Jun 7 16:28:18 2017 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Wed Jun 7 22:55:00 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/io/gcp/pubsub.py | 32 +++++++++++++++-----
sdks/python/apache_beam/io/gcp/pubsub_test.py | 28 +++++++++++++++--
.../runners/dataflow/dataflow_runner.py | 23 ++++++++++----
3 files changed, 67 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/b5852d21/sdks/python/apache_beam/io/gcp/pubsub.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py
index 1ba8ac0..40326e1 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub.py
@@ -40,13 +40,15 @@ __all__ = ['ReadStringsFromPubSub', 'WriteStringsToPubSub',
class ReadStringsFromPubSub(PTransform):
"""A ``PTransform`` for reading utf-8 string payloads from Cloud Pub/Sub."""
- def __init__(self, topic, subscription=None, id_label=None):
+ def __init__(self, topic=None, subscription=None, id_label=None):
"""Initializes ``ReadStringsFromPubSub``.
Attributes:
- topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>".
- subscription: Optional existing Cloud Pub/Sub subscription to use in the
- form "projects/<project>/subscriptions/<subscription>".
+ topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>". If
+ provided then subscription must be None.
+ subscription: Existing Cloud Pub/Sub subscription to use in the
+ form "projects/<project>/subscriptions/<subscription>". If provided then
+ topic must be None.
id_label: The attribute on incoming Pub/Sub messages to use as a unique
record identifier. When specified, the value of this attribute (which
can be any string that uniquely identifies the record) will be used for
@@ -55,6 +57,12 @@ class ReadStringsFromPubSub(PTransform):
case, deduplication of the stream will be strictly best effort.
"""
super(ReadStringsFromPubSub, self).__init__()
+ if topic and subscription:
+ raise ValueError("Only one of topic or subscription should be provided.")
+
+ if not (topic or subscription):
+ raise ValueError("Either a topic or subscription must be provided.")
+
self._source = _PubSubPayloadSource(
topic,
subscription=subscription,
@@ -90,9 +98,11 @@ class _PubSubPayloadSource(dataflow_io.NativeSource):
"""Source for the payload of a message as bytes from a Cloud Pub/Sub topic.
Attributes:
- topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>".
- subscription: Optional existing Cloud Pub/Sub subscription to use in the
- form "projects/<project>/subscriptions/<subscription>".
+ topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>". If
+ provided then topic must be None.
+ subscription: Existing Cloud Pub/Sub subscription to use in the
+ form "projects/<project>/subscriptions/<subscription>". If provided then
+ subscription must be None.
id_label: The attribute on incoming Pub/Sub messages to use as a unique
record identifier. When specified, the value of this attribute (which can
be any string that uniquely identifies the record) will be used for
@@ -101,7 +111,10 @@ class _PubSubPayloadSource(dataflow_io.NativeSource):
case, deduplication of the stream will be strictly best effort.
"""
- def __init__(self, topic, subscription=None, id_label=None):
+ def __init__(self, topic=None, subscription=None, id_label=None):
+ # we are using this coder explicitly for portability reasons of PubsubIO
+ # across implementations in languages.
+ self.coder = coders.BytesCoder()
self.topic = topic
self.subscription = subscription
self.id_label = id_label
@@ -131,6 +144,9 @@ class _PubSubPayloadSink(dataflow_io.NativeSink):
"""Sink for the payload of a message as bytes to a Cloud Pub/Sub topic."""
def __init__(self, topic):
+ # we are using this coder explicitly for portability reasons of PubsubIO
+ # across implementations in languages.
+ self.coder = coders.BytesCoder()
self.topic = topic
@property
http://git-wip-us.apache.org/repos/asf/beam/blob/b5852d21/sdks/python/apache_beam/io/gcp/pubsub_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py b/sdks/python/apache_beam/io/gcp/pubsub_test.py
index 322d08a..cf14e8c 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub_test.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py
@@ -34,9 +34,9 @@ from apache_beam.transforms.display_test import DisplayDataItemMatcher
class TestReadStringsFromPubSub(unittest.TestCase):
- def test_expand(self):
+ def test_expand_with_topic(self):
p = TestPipeline()
- pcoll = p | ReadStringsFromPubSub('a_topic', 'a_subscription', 'a_label')
+ pcoll = p | ReadStringsFromPubSub('a_topic', None, 'a_label')
# Ensure that the output type is str
self.assertEqual(unicode, pcoll.element_type)
@@ -47,9 +47,33 @@ class TestReadStringsFromPubSub(unittest.TestCase):
# Ensure that the properties passed through correctly
source = read_pcoll.producer.transform.source
self.assertEqual('a_topic', source.topic)
+ self.assertEqual('a_label', source.id_label)
+
+ def test_expand_with_subscription(self):
+ p = TestPipeline()
+ pcoll = p | ReadStringsFromPubSub(None, 'a_subscription', 'a_label')
+ # Ensure that the output type is str
+ self.assertEqual(unicode, pcoll.element_type)
+
+ # Ensure that the type on the intermediate read output PCollection is bytes
+ read_pcoll = pcoll.producer.inputs[0]
+ self.assertEqual(bytes, read_pcoll.element_type)
+
+ # Ensure that the properties passed through correctly
+ source = read_pcoll.producer.transform.source
self.assertEqual('a_subscription', source.subscription)
self.assertEqual('a_label', source.id_label)
+ def test_expand_with_both_topic_and_subscription(self):
+ with self.assertRaisesRegexp(
+ ValueError, "Only one of topic or subscription should be provided."):
+ ReadStringsFromPubSub('a_topic', 'a_subscription', 'a_label')
+
+ def test_expand_with_no_topic_or_subscription(self):
+ with self.assertRaisesRegexp(
+ ValueError, "Either a topic or subscription must be provided."):
+ ReadStringsFromPubSub(None, None, 'a_label')
+
class TestWriteStringsToPubSub(unittest.TestCase):
def test_expand(self):
http://git-wip-us.apache.org/repos/asf/beam/blob/b5852d21/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index 3fc8983..d9aa1bf 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -618,10 +618,12 @@ class DataflowRunner(PipelineRunner):
if not standard_options.streaming:
raise ValueError('PubSubPayloadSource is currently available for use '
'only in streaming pipelines.')
- step.add_property(PropertyNames.PUBSUB_TOPIC, transform.source.topic)
- if transform.source.subscription:
+ # Only one of topic or subscription should be set.
+ if transform.source.topic:
+ step.add_property(PropertyNames.PUBSUB_TOPIC, transform.source.topic)
+ elif transform.source.subscription:
step.add_property(PropertyNames.PUBSUB_SUBSCRIPTION,
- transform.source.topic)
+ transform.source.subscription)
if transform.source.id_label:
step.add_property(PropertyNames.PUBSUB_ID_LABEL,
transform.source.id_label)
@@ -639,7 +641,12 @@ class DataflowRunner(PipelineRunner):
# step should be the type of value outputted by each step. Read steps
# automatically wrap output values in a WindowedValue wrapper, if necessary.
# This is also necessary for proper encoding for size estimation.
- coder = coders.WindowedValueCoder(transform._infer_output_coder()) # pylint: disable=protected-access
+ # Using a GlobalWindowCoder as a place holder instead of the default
+ # PickleCoder because GlobalWindowCoder is known coder.
+ # TODO(robertwb): Query the collection for the windowfn to extract the
+ # correct coder.
+ coder = coders.WindowedValueCoder(transform._infer_output_coder(),
+ coders.coders.GlobalWindowCoder()) # pylint: disable=protected-access
step.encoding = self._get_cloud_encoding(coder)
step.add_property(
@@ -708,8 +715,12 @@ class DataflowRunner(PipelineRunner):
step.add_property(PropertyNames.FORMAT, transform.sink.format)
# Wrap coder in WindowedValueCoder: this is necessary for proper encoding
- # for size estimation.
- coder = coders.WindowedValueCoder(transform.sink.coder)
+ # for size estimation. Using a GlobalWindowCoder as a place holder instead
+ # of the default PickleCoder because GlobalWindowCoder is known coder.
+ # TODO(robertwb): Query the collection for the windowfn to extract the
+ # correct coder.
+ coder = coders.WindowedValueCoder(transform.sink.coder,
+ coders.coders.GlobalWindowCoder())
step.encoding = self._get_cloud_encoding(coder)
step.add_property(PropertyNames.ENCODING, step.encoding)
step.add_property(
[04/50] [abbrv] beam git commit: This closes #2286
Posted by ke...@apache.org.
This closes #2286
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/5f7e73bb
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/5f7e73bb
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/5f7e73bb
Branch: refs/heads/gearpump-runner
Commit: 5f7e73bbacf7096eed44002a54910a560b195801
Parents: b2de3db 8f4fa43
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Wed Jun 7 19:43:19 2017 +0200
Committer: Luke Cwik <lc...@google.com>
Committed: Wed Jun 7 13:41:20 2017 -0700
----------------------------------------------------------------------
.../translation/types/CoderTypeSerializer.java | 41 ++-
.../streaming/io/UnboundedSourceWrapper.java | 2 +
.../flink/streaming/TestCountingSource.java | 48 ++-
.../streaming/UnboundedSourceWrapperTest.java | 309 +++++++++++--------
.../beam/runners/dataflow/DataflowRunner.java | 24 +-
5 files changed, 269 insertions(+), 155 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/5f7e73bb/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
----------------------------------------------------------------------
diff --cc runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index cce6ce7,cce6ce7..ed29330
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@@ -428,12 -428,12 +428,15 @@@ public class DataflowRunner extends Pip
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
AppliedPTransform<PBegin, PCollection<T>, PTransform<PInput, PCollection<T>>> transform) {
PTransform<PInput, PCollection<T>> original = transform.getTransform();
++ PCollection<T> output =
++ (PCollection) Iterables.getOnlyElement(transform.getOutputs().values());
return PTransformReplacement.of(
transform.getPipeline().begin(),
InstanceBuilder.ofType(replacement)
.withArg(DataflowRunner.class, runner)
.withArg(
(Class<? super PTransform<PInput, PCollection<T>>>) original.getClass(), original)
++ .withArg((Class<? super PCollection<T>>) output.getClass(), output)
.build());
}
@@@ -809,11 -809,11 +812,12 @@@
extends PTransform<PBegin, PCollection<PubsubMessage>> {
private final PubsubUnboundedSource transform;
-- /**
-- * Builds an instance of this class from the overridden transform.
-- */
++ /** Builds an instance of this class from the overridden transform. */
++ @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
public StreamingPubsubIORead(
-- DataflowRunner runner, PubsubUnboundedSource transform) {
++ DataflowRunner runner,
++ PubsubUnboundedSource transform,
++ PCollection<PubsubMessage> originalOutput) {
this.transform = transform;
}
@@@ -992,11 -992,11 +996,11 @@@
private static class StreamingUnboundedRead<T> extends PTransform<PBegin, PCollection<T>> {
private final UnboundedSource<T, ?> source;
-- /**
-- * Builds an instance of this class from the overridden transform.
-- */
++ /** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
-- public StreamingUnboundedRead(DataflowRunner runner, Read.Unbounded<T> transform) {
++ public StreamingUnboundedRead(DataflowRunner runner,
++ Read.Unbounded<T> transform,
++ PCollection<T> originalOutput) {
this.source = transform.getSource();
}
@@@ -1111,7 -1111,7 +1115,9 @@@
/** Builds an instance of this class from the overridden transform. */
@SuppressWarnings("unused") // used via reflection in DataflowRunner#apply()
-- public StreamingBoundedRead(DataflowRunner runner, Read.Bounded<T> transform) {
++ public StreamingBoundedRead(DataflowRunner runner,
++ Read.Bounded<T> transform,
++ PCollection<T> originalOutput) {
this.source = transform.getSource();
}
[43/50] [abbrv] beam git commit: This closes #3337: Include
Additional PTransform Inputs in Transform Nodes
Posted by ke...@apache.org.
This closes #3337: Include Additional PTransform Inputs in Transform Nodes
Roll-forward Include Additional PTransform inputs in Transform Nodes
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b4c77167
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b4c77167
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b4c77167
Branch: refs/heads/gearpump-runner
Commit: b4c77167f20b8346d22ccff440489592e0de4b52
Parents: 33883ed 696f8b2
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri Jun 9 15:04:02 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Jun 9 15:04:02 2017 -0700
----------------------------------------------------------------------
.../apex/translation/TranslationContext.java | 4 +-
.../core/construction/TransformInputs.java | 50 ++++++
.../core/construction/TransformInputsTest.java | 166 +++++++++++++++++++
.../apache/beam/runners/direct/DirectGraph.java | 34 +++-
.../beam/runners/direct/DirectGraphVisitor.java | 26 ++-
.../direct/ExecutorServiceParallelExecutor.java | 2 +-
.../runners/direct/ParDoEvaluatorFactory.java | 9 +-
...littableProcessElementsEvaluatorFactory.java | 2 +
.../direct/StatefulParDoEvaluatorFactory.java | 1 +
.../beam/runners/direct/WatermarkManager.java | 14 +-
.../runners/direct/DirectGraphVisitorTest.java | 10 +-
.../runners/direct/EvaluationContextTest.java | 2 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 6 +-
.../flink/FlinkBatchTranslationContext.java | 3 +-
.../flink/FlinkStreamingTranslationContext.java | 3 +-
.../dataflow/DataflowPipelineTranslator.java | 5 +-
.../spark/translation/EvaluationContext.java | 4 +-
.../beam/sdk/runners/TransformHierarchy.java | 28 +++-
18 files changed, 323 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
[19/50] [abbrv] beam git commit: Refine Python DirectRunner watermark
advancement behavior
Posted by ke...@apache.org.
Refine Python DirectRunner watermark advancement behavior
This change helps prepare for streaming pipeline execution.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3e049020
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3e049020
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3e049020
Branch: refs/heads/gearpump-runner
Commit: 3e04902008b410269b23179dc2146623ff1fbd0a
Parents: d81ed21
Author: Charles Chen <cc...@google.com>
Authored: Wed Jun 7 17:46:36 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Thu Jun 8 10:55:44 2017 -0700
----------------------------------------------------------------------
.../runners/direct/watermark_manager.py | 20 +++++++++++++++++---
sdks/python/apache_beam/utils/timestamp.py | 5 +++++
2 files changed, 22 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/3e049020/sdks/python/apache_beam/runners/direct/watermark_manager.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/direct/watermark_manager.py b/sdks/python/apache_beam/runners/direct/watermark_manager.py
index 3a13539..0d7cd4f 100644
--- a/sdks/python/apache_beam/runners/direct/watermark_manager.py
+++ b/sdks/python/apache_beam/runners/direct/watermark_manager.py
@@ -25,6 +25,7 @@ from apache_beam import pipeline
from apache_beam import pvalue
from apache_beam.utils.timestamp import MAX_TIMESTAMP
from apache_beam.utils.timestamp import MIN_TIMESTAMP
+from apache_beam.utils.timestamp import TIME_GRANULARITY
class WatermarkManager(object):
@@ -193,9 +194,22 @@ class _TransformWatermarks(object):
def refresh(self):
with self._lock:
- pending_holder = (WatermarkManager.WATERMARK_NEG_INF
- if self._pending else
- WatermarkManager.WATERMARK_POS_INF)
+ min_pending_timestamp = WatermarkManager.WATERMARK_POS_INF
+ has_pending_elements = False
+ for input_bundle in self._pending:
+ # TODO(ccy): we can have the Bundle class keep track of the minimum
+ # timestamp so we don't have to do an iteration here.
+ for wv in input_bundle.get_elements_iterable():
+ has_pending_elements = True
+ if wv.timestamp < min_pending_timestamp:
+ min_pending_timestamp = wv.timestamp
+
+ # If there is a pending element with a certain timestamp, we can at most
+ # advance our watermark to the maximum timestamp less than that
+ # timestamp.
+ pending_holder = WatermarkManager.WATERMARK_POS_INF
+ if has_pending_elements:
+ pending_holder = min_pending_timestamp - TIME_GRANULARITY
input_watermarks = [
tw.output_watermark for tw in self._input_transform_watermarks]
http://git-wip-us.apache.org/repos/asf/beam/blob/3e049020/sdks/python/apache_beam/utils/timestamp.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/timestamp.py b/sdks/python/apache_beam/utils/timestamp.py
index 5d1b48c..b3e840e 100644
--- a/sdks/python/apache_beam/utils/timestamp.py
+++ b/sdks/python/apache_beam/utils/timestamp.py
@@ -208,3 +208,8 @@ class Duration(object):
def __mod__(self, other):
other = Duration.of(other)
return Duration(micros=self.micros % other.micros)
+
+
+# The minimum granularity / interval expressible in a Timestamp / Duration
+# object.
+TIME_GRANULARITY = Duration(micros=1)
[38/50] [abbrv] beam git commit: Use beam.Map rather than beam.ParDo
for PubSub encoding.
Posted by ke...@apache.org.
Use beam.Map rather than beam.ParDo for PubSub encoding.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fb61c540
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fb61c540
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fb61c540
Branch: refs/heads/gearpump-runner
Commit: fb61c540bc15bafb959d7accb7c08f6a681f62ef
Parents: 97957ea
Author: Robert Bradshaw <ro...@google.com>
Authored: Thu Jun 8 15:01:53 2017 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Fri Jun 9 13:00:55 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/io/gcp/pubsub.py | 16 +++-------------
sdks/python/apache_beam/io/gcp/pubsub_test.py | 10 ----------
2 files changed, 3 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/fb61c540/sdks/python/apache_beam/io/gcp/pubsub.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py
index 40326e1..6dc1528 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub.py
@@ -29,7 +29,7 @@ from apache_beam.io.iobase import Read
from apache_beam.io.iobase import Write
from apache_beam.runners.dataflow.native_io import iobase as dataflow_io
from apache_beam.transforms import PTransform
-from apache_beam.transforms import ParDo
+from apache_beam.transforms import Map
from apache_beam.transforms.display import DisplayDataItem
@@ -71,7 +71,7 @@ class ReadStringsFromPubSub(PTransform):
def expand(self, pvalue):
pcoll = pvalue.pipeline | Read(self._source)
pcoll.element_type = bytes
- pcoll = pcoll | 'decode string' >> ParDo(_decodeUtf8String)
+ pcoll = pcoll | 'DecodeString' >> Map(lambda b: b.decode('utf-8'))
pcoll.element_type = unicode
return pcoll
@@ -89,7 +89,7 @@ class WriteStringsToPubSub(PTransform):
self._sink = _PubSubPayloadSink(topic)
def expand(self, pcoll):
- pcoll = pcoll | 'encode string' >> ParDo(_encodeUtf8String)
+ pcoll = pcoll | 'EncodeString' >> Map(lambda s: s.encode('utf-8'))
pcoll.element_type = bytes
return pcoll | Write(self._sink)
@@ -162,16 +162,6 @@ class _PubSubPayloadSink(dataflow_io.NativeSink):
'PubSubPayloadSink is not supported in local execution.')
-def _decodeUtf8String(encoded_value):
- """Decodes a string in utf-8 format from bytes"""
- return encoded_value.decode('utf-8')
-
-
-def _encodeUtf8String(value):
- """Encodes a string in utf-8 format to bytes"""
- return value.encode('utf-8')
-
-
class PubSubSource(dataflow_io.NativeSource):
"""Deprecated: do not use.
http://git-wip-us.apache.org/repos/asf/beam/blob/fb61c540/sdks/python/apache_beam/io/gcp/pubsub_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py b/sdks/python/apache_beam/io/gcp/pubsub_test.py
index cf14e8c..5d3e985 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub_test.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py
@@ -22,8 +22,6 @@ import unittest
import hamcrest as hc
-from apache_beam.io.gcp.pubsub import _decodeUtf8String
-from apache_beam.io.gcp.pubsub import _encodeUtf8String
from apache_beam.io.gcp.pubsub import _PubSubPayloadSink
from apache_beam.io.gcp.pubsub import _PubSubPayloadSource
from apache_beam.io.gcp.pubsub import ReadStringsFromPubSub
@@ -120,14 +118,6 @@ class TestPubSubSink(unittest.TestCase):
hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
-class TestEncodeDecodeUtf8String(unittest.TestCase):
- def test_encode(self):
- self.assertEqual(b'test_data', _encodeUtf8String('test_data'))
-
- def test_decode(self):
- self.assertEqual('test_data', _decodeUtf8String(b'test_data'))
-
-
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
unittest.main()
[44/50] [abbrv] beam git commit: Port ViewOverrideFactory to
SDK-agnostic APIs
Posted by ke...@apache.org.
Port ViewOverrideFactory to SDK-agnostic APIs
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/8c5b57ea
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/8c5b57ea
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/8c5b57ea
Branch: refs/heads/gearpump-runner
Commit: 8c5b57ea8445cd50a35c6dffb460dcf0f426e700
Parents: b4c7716
Author: Kenneth Knowles <kl...@google.com>
Authored: Fri May 26 14:26:55 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Fri Jun 9 19:56:52 2017 -0700
----------------------------------------------------------------------
.../CreatePCollectionViewTranslation.java | 4 +-
.../runners/direct/ViewOverrideFactory.java | 48 ++++++++++++--------
.../direct/ViewEvaluatorFactoryTest.java | 3 +-
.../runners/direct/ViewOverrideFactoryTest.java | 23 ++++++++--
.../beam/sdk/values/PCollectionViews.java | 10 ++++
5 files changed, 62 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/8c5b57ea/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
----------------------------------------------------------------------
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
index aa24909..8fc99b9 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/CreatePCollectionViewTranslation.java
@@ -56,8 +56,8 @@ public class CreatePCollectionViewTranslation {
@Deprecated
public static <ElemT, ViewT> PCollectionView<ViewT> getView(
AppliedPTransform<
- PCollection<ElemT>, PCollectionView<ViewT>,
- PTransform<PCollection<ElemT>, PCollectionView<ViewT>>>
+ PCollection<ElemT>, PCollection<ElemT>,
+ PTransform<PCollection<ElemT>, PCollection<ElemT>>>
application)
throws IOException {
http://git-wip-us.apache.org/repos/asf/beam/blob/8c5b57ea/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
index 06a7388..5dcf016 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ViewOverrideFactory.java
@@ -18,8 +18,9 @@
package org.apache.beam.runners.direct;
+import java.io.IOException;
import java.util.Map;
-import org.apache.beam.runners.core.construction.ForwardingPTransform;
+import org.apache.beam.runners.core.construction.CreatePCollectionViewTranslation;
import org.apache.beam.runners.core.construction.PTransformReplacements;
import org.apache.beam.runners.core.construction.PTransformTranslation.RawPTransform;
import org.apache.beam.runners.core.construction.ReplacementOutputs;
@@ -43,16 +44,30 @@ import org.apache.beam.sdk.values.TupleTag;
*/
class ViewOverrideFactory<ElemT, ViewT>
implements PTransformOverrideFactory<
- PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>> {
+ PCollection<ElemT>, PCollection<ElemT>,
+ PTransform<PCollection<ElemT>, PCollection<ElemT>>> {
@Override
public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>> getReplacementTransform(
AppliedPTransform<
- PCollection<ElemT>, PCollection<ElemT>, CreatePCollectionView<ElemT, ViewT>>
+ PCollection<ElemT>, PCollection<ElemT>,
+ PTransform<PCollection<ElemT>, PCollection<ElemT>>>
transform) {
- return PTransformReplacement.of(
+
+ PCollectionView<ViewT> view;
+ try {
+ view = CreatePCollectionViewTranslation.getView(transform);
+ } catch (IOException exc) {
+ throw new RuntimeException(
+ String.format(
+ "Could not extract %s from transform %s",
+ PCollectionView.class.getSimpleName(), transform),
+ exc);
+ }
+
+ return PTransformReplacement.of(
PTransformReplacements.getSingletonMainInput(transform),
- new GroupAndWriteView<>(transform.getTransform()));
+ new GroupAndWriteView<ElemT, ViewT>(view));
}
@Override
@@ -63,11 +78,11 @@ class ViewOverrideFactory<ElemT, ViewT>
/** The {@link DirectRunner} composite override for {@link CreatePCollectionView}. */
static class GroupAndWriteView<ElemT, ViewT>
- extends ForwardingPTransform<PCollection<ElemT>, PCollection<ElemT>> {
- private final CreatePCollectionView<ElemT, ViewT> og;
+ extends PTransform<PCollection<ElemT>, PCollection<ElemT>> {
+ private final PCollectionView<ViewT> view;
- private GroupAndWriteView(CreatePCollectionView<ElemT, ViewT> og) {
- this.og = og;
+ private GroupAndWriteView(PCollectionView<ViewT> view) {
+ this.view = view;
}
@Override
@@ -77,14 +92,9 @@ class ViewOverrideFactory<ElemT, ViewT>
.setCoder(KvCoder.of(VoidCoder.of(), input.getCoder()))
.apply(GroupByKey.<Void, ElemT>create())
.apply(Values.<Iterable<ElemT>>create())
- .apply(new WriteView<ElemT, ViewT>(og));
+ .apply(new WriteView<ElemT, ViewT>(view));
return input;
}
-
- @Override
- protected PTransform<PCollection<ElemT>, PCollection<ElemT>> delegate() {
- return og;
- }
}
/**
@@ -96,10 +106,10 @@ class ViewOverrideFactory<ElemT, ViewT>
*/
static final class WriteView<ElemT, ViewT>
extends RawPTransform<PCollection<Iterable<ElemT>>, PCollection<Iterable<ElemT>>> {
- private final CreatePCollectionView<ElemT, ViewT> og;
+ private final PCollectionView<ViewT> view;
- WriteView(CreatePCollectionView<ElemT, ViewT> og) {
- this.og = og;
+ WriteView(PCollectionView<ViewT> view) {
+ this.view = view;
}
@Override
@@ -112,7 +122,7 @@ class ViewOverrideFactory<ElemT, ViewT>
@SuppressWarnings("deprecation")
public PCollectionView<ViewT> getView() {
- return og.getView();
+ return view;
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/8c5b57ea/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
index ad1aecc..5bc48b7 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewEvaluatorFactoryTest.java
@@ -66,7 +66,8 @@ public class ViewEvaluatorFactoryTest {
.apply(GroupByKey.<Void, String>create())
.apply(Values.<Iterable<String>>create());
PCollection<Iterable<String>> view =
- concat.apply(new ViewOverrideFactory.WriteView<>(createView));
+ concat.apply(
+ new ViewOverrideFactory.WriteView<String, Iterable<String>>(createView.getView()));
EvaluationContext context = mock(EvaluationContext.class);
TestViewWriter<String, Iterable<String>> viewWriter = new TestViewWriter<>();
http://git-wip-us.apache.org/repos/asf/beam/blob/8c5b57ea/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
index 94728c7..6af9273 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ViewOverrideFactoryTest.java
@@ -18,6 +18,7 @@
package org.apache.beam.runners.direct;
+import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
@@ -36,8 +37,11 @@ import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
+import org.apache.beam.sdk.transforms.ViewFn;
+import org.apache.beam.sdk.transforms.windowing.WindowMappingFn;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PCollectionViews;
@@ -67,7 +71,7 @@ public class ViewOverrideFactoryTest implements Serializable {
factory.getReplacementTransform(
AppliedPTransform
.<PCollection<Integer>, PCollection<Integer>,
- CreatePCollectionView<Integer, List<Integer>>>
+ PTransform<PCollection<Integer>, PCollection<Integer>>>
of(
"foo",
ints.expand(),
@@ -102,7 +106,7 @@ public class ViewOverrideFactoryTest implements Serializable {
factory.getReplacementTransform(
AppliedPTransform
.<PCollection<Integer>, PCollection<Integer>,
- CreatePCollectionView<Integer, List<Integer>>>
+ PTransform<PCollection<Integer>, PCollection<Integer>>>
of(
"foo",
ints.expand(),
@@ -120,8 +124,19 @@ public class ViewOverrideFactoryTest implements Serializable {
"There should only be one WriteView primitive in the graph",
writeViewVisited.getAndSet(true),
is(false));
- PCollectionView replacementView = ((WriteView) node.getTransform()).getView();
- assertThat(replacementView, Matchers.<PCollectionView>theInstance(view));
+ PCollectionView<?> replacementView = ((WriteView) node.getTransform()).getView();
+
+ // replacementView.getPCollection() is null, but that is not a requirement
+ // so not asserted one way or the other
+ assertThat(
+ replacementView.getTagInternal(),
+ equalTo(view.getTagInternal()));
+ assertThat(
+ replacementView.getViewFn(),
+ Matchers.<ViewFn<?, ?>>equalTo(view.getViewFn()));
+ assertThat(
+ replacementView.getWindowMappingFn(),
+ Matchers.<WindowMappingFn<?>>equalTo(view.getWindowMappingFn()));
assertThat(node.getInputs().entrySet(), hasSize(1));
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/8c5b57ea/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
index 5e2e2c3..0c04370 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PCollectionViews.java
@@ -282,6 +282,16 @@ public class PCollectionViews {
}
}));
}
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof ListViewFn;
+ }
+
+ @Override
+ public int hashCode() {
+ return ListViewFn.class.hashCode();
+ }
}
/**
[17/50] [abbrv] beam git commit: Avoid flakiness in data channel for
empty streams.
Posted by ke...@apache.org.
Avoid flakiness in data channel for empty streams.
As empty stream is used as end-of-stream marker, don't ever send
it as the data itself.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4ebebfdb
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4ebebfdb
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4ebebfdb
Branch: refs/heads/gearpump-runner
Commit: 4ebebfdb34de3e209c033de15e32cf67ab346d44
Parents: e066a9d
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Wed Jun 7 23:00:43 2017 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Thu Jun 8 10:06:17 2017 -0700
----------------------------------------------------------------------
.../python/apache_beam/runners/worker/data_plane.py | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/4ebebfdb/sdks/python/apache_beam/runners/worker/data_plane.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/worker/data_plane.py b/sdks/python/apache_beam/runners/worker/data_plane.py
index 5edd0b4..7365db6 100644
--- a/sdks/python/apache_beam/runners/worker/data_plane.py
+++ b/sdks/python/apache_beam/runners/worker/data_plane.py
@@ -167,12 +167,18 @@ class _GrpcDataChannel(DataChannel):
yield data
def output_stream(self, instruction_id, target):
+ # TODO: Return an output stream that sends data
+ # to the Runner once a fixed size buffer is full.
+ # Currently we buffer all the data before sending
+ # any messages.
def add_to_send_queue(data):
- self._to_send.put(
- beam_fn_api_pb2.Elements.Data(
- instruction_reference=instruction_id,
- target=target,
- data=data))
+ if data:
+ self._to_send.put(
+ beam_fn_api_pb2.Elements.Data(
+ instruction_reference=instruction_id,
+ target=target,
+ data=data))
+ # End of stream marker.
self._to_send.put(
beam_fn_api_pb2.Elements.Data(
instruction_reference=instruction_id,
[14/50] [abbrv] beam git commit: This closes #3303
Posted by ke...@apache.org.
This closes #3303
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/e066a9d6
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/e066a9d6
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/e066a9d6
Branch: refs/heads/gearpump-runner
Commit: e066a9d6dfc4d76f2d7d49de9f744b978982bb81
Parents: 86e1fab ccf7344
Author: Thomas Groh <tg...@google.com>
Authored: Thu Jun 8 09:13:57 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Thu Jun 8 09:13:57 2017 -0700
----------------------------------------------------------------------
.../apache/beam/runners/apex/ApexRunner.java | 59 +--
.../translation/ApexPipelineTranslator.java | 16 +-
.../construction/RunnerPCollectionView.java | 8 +
.../apache/beam/runners/direct/DirectGraph.java | 4 +
.../beam/runners/direct/DirectGraphVisitor.java | 22 +-
.../runners/direct/ViewEvaluatorFactory.java | 8 +-
.../runners/direct/ViewOverrideFactory.java | 29 +-
.../beam/runners/direct/DirectGraphs.java | 7 +
.../runners/direct/EvaluationContextTest.java | 5 +-
.../ImmutabilityEnforcementFactoryTest.java | 4 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 1 +
.../runners/direct/TransformExecutorTest.java | 1 +
.../direct/ViewEvaluatorFactoryTest.java | 5 +-
.../runners/direct/ViewOverrideFactoryTest.java | 16 +-
.../direct/WatermarkCallbackExecutorTest.java | 1 +
.../runners/direct/WatermarkManagerTest.java | 1 +
.../runners/flink/CreateStreamingFlinkView.java | 154 ++++++++
.../flink/FlinkStreamingPipelineTranslator.java | 36 +-
.../FlinkStreamingTransformTranslators.java | 8 +-
.../flink/FlinkStreamingViewOverrides.java | 372 -------------------
.../runners/dataflow/BatchViewOverrides.java | 182 +++------
.../runners/dataflow/CreateDataflowView.java | 8 +-
.../dataflow/DataflowPipelineTranslator.java | 11 +-
.../beam/runners/dataflow/DataflowRunner.java | 17 +-
.../dataflow/StreamingViewOverrides.java | 10 +-
.../DataflowPipelineTranslatorTest.java | 6 +-
.../spark/translation/TransformTranslator.java | 50 +--
.../beam/sdk/runners/TransformHierarchy.java | 46 ++-
.../org/apache/beam/sdk/transforms/Combine.java | 17 +-
.../org/apache/beam/sdk/transforms/View.java | 38 +-
.../org/apache/beam/sdk/values/PCollection.java | 12 +
.../beam/sdk/values/PCollectionViews.java | 14 +
.../org/apache/beam/sdk/values/PValueBase.java | 12 -
.../sdk/testing/PCollectionViewTesting.java | 8 +
34 files changed, 458 insertions(+), 730 deletions(-)
----------------------------------------------------------------------
[33/50] [abbrv] beam git commit: This closes #3315
Posted by ke...@apache.org.
This closes #3315
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/e980ae92
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/e980ae92
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/e980ae92
Branch: refs/heads/gearpump-runner
Commit: e980ae921f48b180ffb7fd7dba7eea10ef0f304a
Parents: bc2f97c 7689e43
Author: chamikara@google.com <ch...@google.com>
Authored: Thu Jun 8 15:30:01 2017 -0700
Committer: chamikara@google.com <ch...@google.com>
Committed: Thu Jun 8 15:30:01 2017 -0700
----------------------------------------------------------------------
.gitignore | 2 +-
sdks/python/apache_beam/coders/coders.py | 2 +-
sdks/python/apache_beam/pipeline.py | 4 ++--
sdks/python/apache_beam/portability/__init__.py | 18 +++++++++++++++++
.../apache_beam/portability/runners/__init__.py | 18 +++++++++++++++++
.../portability/runners/api/__init__.py | 21 ++++++++++++++++++++
sdks/python/apache_beam/pvalue.py | 2 +-
sdks/python/apache_beam/runners/api/__init__.py | 21 --------------------
.../runners/dataflow/dataflow_runner.py | 4 ++--
.../apache_beam/runners/pipeline_context.py | 2 +-
.../runners/portability/fn_api_runner.py | 2 +-
.../apache_beam/runners/worker/data_plane.py | 2 +-
.../runners/worker/data_plane_test.py | 2 +-
.../apache_beam/runners/worker/log_handler.py | 2 +-
.../runners/worker/log_handler_test.py | 2 +-
.../apache_beam/runners/worker/sdk_worker.py | 2 +-
.../runners/worker/sdk_worker_main.py | 2 +-
.../runners/worker/sdk_worker_test.py | 2 +-
sdks/python/apache_beam/transforms/core.py | 2 +-
.../python/apache_beam/transforms/ptransform.py | 2 +-
sdks/python/apache_beam/transforms/trigger.py | 2 +-
sdks/python/apache_beam/transforms/window.py | 4 ++--
sdks/python/apache_beam/utils/urns.py | 2 +-
sdks/python/gen_protos.py | 2 +-
sdks/python/run_pylint.sh | 2 +-
25 files changed, 81 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
[12/50] [abbrv] beam git commit: Closes #3317
Posted by ke...@apache.org.
Closes #3317
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/86e1fab6
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/86e1fab6
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/86e1fab6
Branch: refs/heads/gearpump-runner
Commit: 86e1fab69f296247624d7799f151ff25c3b7243a
Parents: 0a0a1bc b5852d2
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Wed Jun 7 22:55:00 2017 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Wed Jun 7 22:55:00 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/io/gcp/pubsub.py | 32 +++++++++++++++-----
sdks/python/apache_beam/io/gcp/pubsub_test.py | 28 +++++++++++++++--
.../runners/dataflow/dataflow_runner.py | 23 ++++++++++----
3 files changed, 67 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
[40/50] [abbrv] beam git commit: This closes #3329
Posted by ke...@apache.org.
This closes #3329
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/33883ed8
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/33883ed8
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/33883ed8
Branch: refs/heads/gearpump-runner
Commit: 33883ed88dfbd122c5a76d98833214044accc251
Parents: d2c4093 261e7df
Author: Thomas Groh <tg...@google.com>
Authored: Fri Jun 9 14:44:42 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Fri Jun 9 14:44:42 2017 -0700
----------------------------------------------------------------------
.../spark/translation/StorageLevelTest.java | 4 +-
.../beam/sdk/runners/TransformHierarchy.java | 79 +++++++-
.../sdk/runners/TransformHierarchyTest.java | 197 +++++++++++++++++++
3 files changed, 274 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[29/50] [abbrv] beam git commit: This closes #3220: Implements
HCatalogIO
Posted by ke...@apache.org.
This closes #3220: Implements HCatalogIO
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/911bfbde
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/911bfbde
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/911bfbde
Branch: refs/heads/gearpump-runner
Commit: 911bfbdef62a7bb750426da9cc59537c4ffd9e1f
Parents: 7c5a70e 349898c
Author: Eugene Kirpichov <ki...@google.com>
Authored: Thu Jun 8 14:07:47 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Jun 8 14:07:47 2017 -0700
----------------------------------------------------------------------
sdks/java/io/hcatalog/pom.xml | 163 ++++++
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 511 +++++++++++++++++++
.../beam/sdk/io/hcatalog/package-info.java | 22 +
.../io/hcatalog/EmbeddedMetastoreService.java | 88 ++++
.../beam/sdk/io/hcatalog/HCatalogIOTest.java | 271 ++++++++++
.../sdk/io/hcatalog/HCatalogIOTestUtils.java | 106 ++++
.../hcatalog/src/test/resources/hive-site.xml | 301 +++++++++++
sdks/java/io/pom.xml | 1 +
8 files changed, 1463 insertions(+)
----------------------------------------------------------------------