You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2016/11/12 02:28:31 UTC
[13/39] incubator-beam git commit: Merge branch 'master' into
apex-runner
Merge branch 'master' into apex-runner
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6a971d6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6a971d6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6a971d6b
Branch: refs/heads/master
Commit: 6a971d6b13a035f74ce1b2d2efc6d37f84a2373c
Parents: c9f1406 7de8d57
Author: Thomas Weise <th...@apache.org>
Authored: Tue Oct 25 09:07:50 2016 -0700
Committer: Thomas Weise <th...@apache.org>
Committed: Tue Oct 25 09:07:50 2016 -0700
----------------------------------------------------------------------
.travis.yml | 10 +-
examples/java/pom.xml | 2 +-
.../beam/examples/DebuggingWordCount.java | 2 +-
.../apache/beam/examples/WindowedWordCount.java | 2 +-
.../org/apache/beam/examples/WordCount.java | 4 +-
.../common/ExampleBigQueryTableOptions.java | 2 +-
...xamplePubsubTopicAndSubscriptionOptions.java | 2 +-
.../common/ExamplePubsubTopicOptions.java | 2 +-
.../beam/examples/complete/AutoComplete.java | 2 +-
.../apache/beam/examples/complete/TfIdf.java | 2 +-
.../examples/complete/TopWikipediaSessions.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 2 +-
.../beam/examples/cookbook/DeDupExample.java | 4 +-
.../beam/examples/cookbook/FilterExamples.java | 2 +-
.../beam/examples/cookbook/JoinExamples.java | 2 +-
.../examples/cookbook/MaxPerKeyExamples.java | 2 +-
.../beam/examples/WindowedWordCountIT.java | 11 +
.../examples/cookbook/BigQueryTornadoesIT.java | 14 +-
examples/java8/pom.xml | 2 +-
.../beam/examples/complete/game/GameStats.java | 2 +-
.../examples/complete/game/HourlyTeamScore.java | 2 +-
.../examples/complete/game/LeaderBoard.java | 2 +-
.../beam/examples/complete/game/UserScore.java | 2 +-
examples/pom.xml | 2 +-
pom.xml | 17 +-
runners/core-java/pom.xml | 2 +-
.../beam/runners/core/AggregatorFactory.java | 39 ++
.../apache/beam/runners/core/DoFnRunner.java | 6 +-
.../apache/beam/runners/core/DoFnRunners.java | 191 ++++--
.../runners/core/ElementByteSizeObservable.java | 5 +-
.../runners/core/GBKIntoKeyedWorkItems.java | 17 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 5 +
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 5 +
.../runners/core/ReduceFnContextFactory.java | 2 +-
.../beam/runners/core/ReduceFnRunner.java | 39 +-
.../beam/runners/core/SimpleDoFnRunner.java | 337 ++++++----
.../beam/runners/core/SimpleOldDoFnRunner.java | 521 +++++++++++++++
.../apache/beam/runners/core/TriggerRunner.java | 247 -------
.../core/UnboundedReadFromBoundedSource.java | 2 +-
.../core/triggers/AfterAllStateMachine.java | 5 +
.../AfterDelayFromFirstElementStateMachine.java | 17 +-
.../core/triggers/AfterEachStateMachine.java | 5 +
.../core/triggers/AfterFirstStateMachine.java | 6 +
.../core/triggers/AfterPaneStateMachine.java | 7 +
.../triggers/AfterWatermarkStateMachine.java | 4 +-
.../runners/core/triggers/FinishedTriggers.java | 8 +-
.../core/triggers/TriggerStateMachine.java | 4 +-
.../triggers/TriggerStateMachineRunner.java | 2 +-
.../core/triggers/TriggerStateMachines.java | 215 +++++++
.../beam/runners/core/ReduceFnRunnerTest.java | 281 ++++----
.../beam/runners/core/ReduceFnTester.java | 157 +++--
.../beam/runners/core/SimpleDoFnRunnerTest.java | 88 ---
.../runners/core/SimpleOldDoFnRunnerTest.java | 88 +++
.../beam/runners/core/SplittableParDoTest.java | 14 +-
.../core/triggers/TriggerStateMachinesTest.java | 199 ++++++
runners/direct-java/pom.xml | 18 +-
.../direct/BoundedReadEvaluatorFactory.java | 3 +-
.../beam/runners/direct/DirectMetrics.java | 7 +
.../beam/runners/direct/DirectRunner.java | 2 +-
.../runners/direct/DoFnLifecycleManager.java | 42 +-
.../runners/direct/ForwardingPTransform.java | 2 +-
.../beam/runners/direct/ParDoEvaluator.java | 4 +-
.../direct/ParDoMultiEvaluatorFactory.java | 6 +-
.../direct/ParDoSingleEvaluatorFactory.java | 5 +-
.../direct/TestStreamEvaluatorFactory.java | 2 +-
.../direct/UnboundedReadEvaluatorFactory.java | 15 +-
.../beam/runners/direct/WatermarkManager.java | 4 +-
.../beam/runners/direct/DirectRunnerTest.java | 32 -
...leManagerRemovingTransformEvaluatorTest.java | 16 +-
.../direct/DoFnLifecycleManagerTest.java | 12 +-
.../direct/DoFnLifecycleManagersTest.java | 48 +-
.../direct/ForwardingPTransformTest.java | 7 +-
.../direct/ParDoMultiEvaluatorFactoryTest.java | 11 +
.../direct/ParDoSingleEvaluatorFactoryTest.java | 11 +
.../beam/runners/direct/SplittableDoFnTest.java | 8 +-
.../UnboundedReadEvaluatorFactoryTest.java | 52 ++
.../runners/direct/WatermarkManagerTest.java | 2 +-
runners/flink/examples/pom.xml | 2 +-
runners/flink/pom.xml | 2 +-
runners/flink/runner/pom.xml | 2 +-
.../flink/FlinkDetachedRunnerResult.java | 76 +++
.../apache/beam/runners/flink/FlinkRunner.java | 9 +-
.../beam/runners/flink/FlinkRunnerResult.java | 11 +-
.../beam/runners/flink/TestFlinkRunner.java | 9 +-
.../functions/FlinkProcessContext.java | 10 +-
runners/google-cloud-dataflow-java/pom.xml | 4 +-
.../dataflow/DataflowPipelineTranslator.java | 133 ++--
.../beam/runners/dataflow/DataflowRunner.java | 20 +-
.../DataflowUnboundedReadFromBoundedSource.java | 4 +-
.../dataflow/internal/ReadTranslator.java | 2 +-
.../options/DataflowPipelineDebugOptions.java | 4 +-
.../options/DataflowPipelineOptions.java | 2 +-
.../DataflowPipelineWorkerPoolOptions.java | 8 +-
.../options/DataflowProfilingOptions.java | 2 +-
.../options/DataflowWorkerLoggingOptions.java | 4 +-
.../beam/runners/dataflow/util/DoFnInfo.java | 30 +-
.../runners/dataflow/util/MonitoringUtil.java | 2 +-
.../beam/runners/dataflow/util/Stager.java | 2 +-
.../DataflowPipelineTranslatorTest.java | 38 +-
.../runners/dataflow/DataflowRunnerTest.java | 6 +-
.../dataflow/util/MonitoringUtilTest.java | 4 +-
runners/pom.xml | 3 +-
runners/spark/pom.xml | 75 ++-
.../runners/spark/SparkPipelineOptions.java | 6 +-
.../apache/beam/runners/spark/SparkRunner.java | 48 +-
.../beam/runners/spark/TestSparkRunner.java | 4 +-
.../beam/runners/spark/examples/WordCount.java | 2 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 6 +-
.../runners/spark/translation/DoFnFunction.java | 74 +--
.../spark/translation/EvaluationContext.java | 6 +-
.../translation/GroupCombineFunctions.java | 235 ++++---
.../spark/translation/MultiDoFnFunction.java | 85 ++-
.../translation/SparkAbstractCombineFn.java | 134 ++++
.../spark/translation/SparkContextFactory.java | 3 +-
.../spark/translation/SparkGlobalCombineFn.java | 260 ++++++++
.../spark/translation/SparkKeyedCombineFn.java | 273 ++++++++
.../spark/translation/SparkProcessContext.java | 160 ++++-
.../spark/translation/TransformTranslator.java | 143 ++--
.../spark/translation/TranslationUtils.java | 28 +-
.../streaming/StreamingEvaluationContext.java | 33 +-
.../streaming/StreamingTransformTranslator.java | 126 ++--
.../runners/spark/util/BroadcastHelper.java | 26 -
.../spark/util/SparkSideInputReader.java | 95 +++
.../apache/beam/runners/spark/DeDupTest.java | 58 --
.../beam/runners/spark/EmptyInputTest.java | 74 ---
.../beam/runners/spark/SimpleWordCountTest.java | 107 ---
.../apache/beam/runners/spark/TfIdfTest.java | 259 --------
.../spark/translation/CombineGloballyTest.java | 101 ---
.../spark/translation/CombinePerKeyTest.java | 78 ---
.../spark/translation/DoFnOutputTest.java | 65 --
.../translation/MultiOutputWordCountTest.java | 174 -----
.../spark/translation/SerializationTest.java | 199 ------
.../spark/translation/SideEffectsTest.java | 20 +-
.../translation/SparkPipelineOptionsTest.java | 42 --
.../translation/TransformTranslatorTest.java | 104 ---
.../translation/WindowedWordCountTest.java | 116 ----
.../streaming/EmptyStreamAssertionTest.java | 10 +-
.../ResumeFromCheckpointStreamingTest.java | 2 +-
.../streaming/SimpleStreamingWordCountTest.java | 1 -
.../streaming/utils/PAssertStreaming.java | 6 +-
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/checkstyle.xml | 9 +-
.../src/main/resources/beam/suppressions.xml | 12 +-
sdks/java/core/pom.xml | 2 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 12 +-
.../org/apache/beam/sdk/PipelineResult.java | 4 +-
.../beam/sdk/annotations/Experimental.java | 4 +-
.../sdk/coders/CannotProvideCoderException.java | 2 +-
.../java/org/apache/beam/sdk/coders/Coder.java | 26 +-
.../apache/beam/sdk/coders/CoderFactory.java | 4 +-
.../apache/beam/sdk/coders/CoderProvider.java | 2 +-
.../apache/beam/sdk/coders/DelegateCoder.java | 4 +-
.../apache/beam/sdk/coders/NullableCoder.java | 6 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 4 +
.../java/org/apache/beam/sdk/io/AvroSource.java | 97 ++-
.../sdk/io/BoundedReadFromUnboundedSource.java | 9 +-
.../apache/beam/sdk/io/CompressedSource.java | 8 +-
.../org/apache/beam/sdk/io/FileBasedSink.java | 14 +-
.../apache/beam/sdk/io/OffsetBasedSource.java | 4 +-
.../java/org/apache/beam/sdk/io/PubsubIO.java | 5 +-
.../main/java/org/apache/beam/sdk/io/Read.java | 4 +-
.../sdk/io/SerializableAvroCodecFactory.java | 2 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 8 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 6 +-
.../org/apache/beam/sdk/options/GcpOptions.java | 8 +-
.../org/apache/beam/sdk/options/GcsOptions.java | 4 +-
.../beam/sdk/options/GoogleApiDebugOptions.java | 2 +-
.../beam/sdk/options/PipelineOptions.java | 12 +-
.../sdk/options/PipelineOptionsFactory.java | 196 ++++--
.../sdk/options/ProxyInvocationHandler.java | 178 ++---
.../apache/beam/sdk/options/ValueProvider.java | 25 +-
.../sdk/runners/PipelineRunnerRegistrar.java | 2 +-
.../beam/sdk/testing/BigqueryMatcher.java | 239 +++++++
.../org/apache/beam/sdk/testing/PAssert.java | 4 +-
.../apache/beam/sdk/transforms/Aggregator.java | 11 +-
.../org/apache/beam/sdk/transforms/Combine.java | 82 ++-
.../beam/sdk/transforms/CombineFnBase.java | 17 +-
.../apache/beam/sdk/transforms/CombineFns.java | 33 +-
.../beam/sdk/transforms/CombineWithContext.java | 3 +-
.../sdk/transforms/DelegatingAggregator.java | 125 ++++
.../org/apache/beam/sdk/transforms/DoFn.java | 135 +++-
.../beam/sdk/transforms/DoFnAdapters.java | 5 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 18 +-
.../beam/sdk/transforms/FlatMapElements.java | 6 +-
.../apache/beam/sdk/transforms/MapElements.java | 8 +-
.../org/apache/beam/sdk/transforms/OldDoFn.java | 301 +++++++--
.../org/apache/beam/sdk/transforms/ParDo.java | 277 +++++---
.../apache/beam/sdk/transforms/Partition.java | 4 +-
.../sdk/transforms/SerializableFunction.java | 2 +-
.../sdk/transforms/display/DisplayData.java | 518 ++++++++++-----
.../sdk/transforms/reflect/DoFnInvokers.java | 21 +-
.../sdk/transforms/reflect/DoFnSignature.java | 280 +++++++-
.../sdk/transforms/reflect/DoFnSignatures.java | 259 +++++++-
.../beam/sdk/transforms/windowing/AfterAll.java | 2 +-
.../windowing/AfterDelayFromFirstElement.java | 15 +
.../sdk/transforms/windowing/AfterEach.java | 2 +-
.../sdk/transforms/windowing/AfterFirst.java | 2 +-
.../sdk/transforms/windowing/AfterPane.java | 7 +
.../AfterSynchronizedProcessingTime.java | 6 +-
.../transforms/windowing/AfterWatermark.java | 12 +-
.../transforms/windowing/DefaultTrigger.java | 2 +-
.../beam/sdk/transforms/windowing/Never.java | 8 +-
.../transforms/windowing/OrFinallyTrigger.java | 17 +-
.../beam/sdk/transforms/windowing/PaneInfo.java | 2 +-
.../sdk/transforms/windowing/Repeatedly.java | 10 +-
.../beam/sdk/transforms/windowing/Trigger.java | 15 +-
.../beam/sdk/transforms/windowing/Window.java | 2 +-
.../beam/sdk/transforms/windowing/WindowFn.java | 2 +-
...AttemptAndTimeBoundedExponentialBackOff.java | 2 +-
.../apache/beam/sdk/util/CredentialFactory.java | 2 +-
.../apache/beam/sdk/util/FinishedTriggers.java | 8 +-
.../apache/beam/sdk/util/IOChannelFactory.java | 2 +-
.../org/apache/beam/sdk/util/PathValidator.java | 6 +-
.../beam/sdk/util/PerKeyCombineFnRunner.java | 26 +-
.../sdk/util/ReifyTimestampAndWindowsDoFn.java | 16 +-
.../apache/beam/sdk/util/ReshuffleTrigger.java | 2 +-
.../org/apache/beam/sdk/util/StringUtils.java | 2 +-
.../java/org/apache/beam/sdk/util/Timer.java | 56 ++
.../apache/beam/sdk/util/TimerInternals.java | 4 +-
.../org/apache/beam/sdk/util/TimerSpec.java | 30 +
.../org/apache/beam/sdk/util/TimerSpecs.java | 41 ++
.../java/org/apache/beam/sdk/util/Timers.java | 10 +-
.../apache/beam/sdk/util/common/Reiterable.java | 2 +-
.../apache/beam/sdk/util/common/Reiterator.java | 2 +-
.../CopyOnAccessInMemoryStateInternals.java | 2 +-
.../beam/sdk/util/state/ReadableState.java | 10 +-
.../beam/sdk/util/state/StateContext.java | 6 +-
.../apache/beam/sdk/values/PCollectionView.java | 8 +-
.../java/org/apache/beam/sdk/values/PInput.java | 6 +-
.../org/apache/beam/sdk/values/POutput.java | 8 +-
.../java/org/apache/beam/sdk/values/PValue.java | 4 +-
.../apache/beam/sdk/coders/AvroCoderTest.java | 4 +-
.../beam/sdk/coders/NullableCoderTest.java | 11 +
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 12 +-
.../org/apache/beam/sdk/io/AvroSourceTest.java | 43 ++
.../io/BoundedReadFromUnboundedSourceTest.java | 4 +-
.../beam/sdk/io/CompressedSourceTest.java | 4 +-
.../beam/sdk/io/OffsetBasedSourceTest.java | 30 +-
.../java/org/apache/beam/sdk/io/ReadTest.java | 10 +-
.../io/SerializableAvroCodecFactoryTest.java | 8 +-
.../java/org/apache/beam/sdk/io/TextIOTest.java | 25 +
.../java/org/apache/beam/sdk/io/WriteTest.java | 6 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 69 +-
.../beam/sdk/options/PipelineOptionsTest.java | 49 +-
.../options/PipelineOptionsValidatorTest.java | 18 +-
.../sdk/options/ProxyInvocationHandlerTest.java | 76 ++-
.../beam/sdk/options/ValueProviderTest.java | 13 +-
.../beam/sdk/testing/BigqueryMatcherTest.java | 176 +++++
.../beam/sdk/transforms/CombineFnsTest.java | 7 +-
.../apache/beam/sdk/transforms/CombineTest.java | 4 +-
.../DoFnDelegatingAggregatorTest.java | 5 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 24 +-
.../transforms/display/DisplayDataMatchers.java | 141 ++--
.../display/DisplayDataMatchersTest.java | 67 +-
.../sdk/transforms/display/DisplayDataTest.java | 367 +++++++----
.../DoFnSignaturesSplittableDoFnTest.java | 6 +-
.../transforms/reflect/DoFnSignaturesTest.java | 645 ++++++++++++++++++-
.../reflect/DoFnSignaturesTestUtils.java | 5 +-
.../sdk/transforms/windowing/WindowTest.java | 4 +-
.../apache/beam/sdk/util/ApiSurfaceTest.java | 24 +-
.../beam/sdk/util/ExecutableTriggerTest.java | 127 ++++
.../sdk/util/FinishedTriggersBitSetTest.java | 55 ++
.../sdk/util/FinishedTriggersProperties.java | 110 ++++
.../beam/sdk/util/FinishedTriggersSetTest.java | 60 ++
.../beam/sdk/util/ReshuffleTriggerTest.java | 67 ++
sdks/java/extensions/join-library/pom.xml | 2 +-
sdks/java/extensions/pom.xml | 2 +-
sdks/java/io/google-cloud-platform/pom.xml | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 12 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 2 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 2 +-
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 39 +-
sdks/java/io/hdfs/pom.xml | 2 +-
sdks/java/io/jdbc/pom.xml | 2 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 70 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 39 +-
sdks/java/io/jms/pom.xml | 2 +-
sdks/java/io/kafka/pom.xml | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 14 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 26 +-
sdks/java/io/kinesis/pom.xml | 2 +-
sdks/java/io/mongodb/pom.xml | 2 +-
sdks/java/io/pom.xml | 2 +-
sdks/java/java8tests/pom.xml | 2 +-
.../PipelineOptionsFactoryJava8Test.java | 8 +-
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 8 +-
.../src/main/java/DebuggingWordCount.java | 2 +-
.../src/main/java/WindowedWordCount.java | 2 +-
.../src/main/java/WordCount.java | 2 +-
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 2 +-
.../resources/projects/basic/reference/pom.xml | 2 +-
sdks/java/microbenchmarks/pom.xml | 2 +-
sdks/java/pom.xml | 2 +-
sdks/pom.xml | 2 +-
298 files changed, 8342 insertions(+), 4136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/6a971d6b/runners/pom.xml
----------------------------------------------------------------------
diff --cc runners/pom.xml
index ff800d1,b2b613f..203b951
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@@ -36,8 -36,8 +36,9 @@@
<module>core-java</module>
<module>direct-java</module>
<module>flink</module>
+ <module>google-cloud-dataflow-java</module>
<module>spark</module>
+ <module>apex</module>
</modules>
<profiles>