You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2016/06/20 22:16:43 UTC
[50/50] [abbrv] incubator-beam git commit: This closes #498
This closes #498
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f166b16b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f166b16b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f166b16b
Branch: refs/heads/python-sdk
Commit: f166b16b811624aa66d54dfe3811a80306d12e9c
Parents: 0e5c662 4b41768
Author: Davor Bonaci <da...@google.com>
Authored: Mon Jun 20 15:14:49 2016 -0700
Committer: Davor Bonaci <da...@google.com>
Committed: Mon Jun 20 15:14:49 2016 -0700
----------------------------------------------------------------------
README.md | 4 +-
examples/java/README.md | 42 +-
examples/java/pom.xml | 22 +-
.../beam/examples/DebuggingWordCount.java | 2 +-
.../apache/beam/examples/MinimalWordCount.java | 4 +-
.../apache/beam/examples/WindowedWordCount.java | 13 +-
.../org/apache/beam/examples/WordCount.java | 21 +-
.../examples/common/DataflowExampleUtils.java | 31 +-
.../beam/examples/complete/AutoComplete.java | 12 +-
.../apache/beam/examples/complete/TfIdf.java | 2 +-
.../examples/complete/TopWikipediaSessions.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 2 +-
.../examples/cookbook/DatastoreWordCount.java | 8 +-
.../beam/examples/cookbook/DeDupExample.java | 2 +-
.../beam/examples/cookbook/FilterExamples.java | 2 +-
.../beam/examples/cookbook/JoinExamples.java | 2 +-
.../examples/cookbook/MaxPerKeyExamples.java | 2 +-
.../org/apache/beam/examples/WordCountTest.java | 6 +-
.../examples/complete/AutoCompleteTest.java | 14 +-
.../cookbook/BigQueryTornadoesTest.java | 8 +-
.../cookbook/CombinePerKeyExamplesTest.java | 4 +-
.../examples/cookbook/FilterExamplesTest.java | 4 +-
.../examples/cookbook/JoinExamplesTest.java | 4 +-
.../cookbook/MaxPerKeyExamplesTest.java | 4 +-
.../examples/cookbook/TriggerExampleTest.java | 4 +-
.../beam/examples/MinimalWordCountJava8.java | 12 +-
.../beam/examples/complete/game/GameStats.java | 7 +-
.../examples/complete/game/HourlyTeamScore.java | 8 +-
.../examples/complete/game/LeaderBoard.java | 2 +-
.../beam/examples/complete/game/UserScore.java | 7 +-
.../examples/MinimalWordCountJava8Test.java | 8 +-
.../complete/game/HourlyTeamScoreTest.java | 7 +-
.../examples/complete/game/UserScoreTest.java | 7 +-
runners/core-java/pom.xml | 13 +
runners/direct-java/pom.xml | 34 +-
.../direct/AbstractModelEnforcement.java | 4 +-
.../direct/BoundedReadEvaluatorFactory.java | 16 +-
.../beam/runners/direct/BundleFactory.java | 8 +-
.../beam/runners/direct/CommittedResult.java | 6 +-
.../beam/runners/direct/CompletionCallback.java | 4 +-
.../runners/direct/DirectExecutionContext.java | 106 +
.../beam/runners/direct/DirectGroupByKey.java | 132 +
.../direct/DirectGroupByKeyOverrideFactory.java | 41 +
.../beam/runners/direct/DirectOptions.java | 101 +
.../beam/runners/direct/DirectRegistrar.java | 57 +
.../beam/runners/direct/DirectRunner.java | 372 ++
.../runners/direct/DirectTimerInternals.java | 84 +
.../runners/direct/EmptyTransformEvaluator.java | 4 +-
.../direct/EncodabilityEnforcementFactory.java | 2 +-
.../beam/runners/direct/EvaluationContext.java | 429 +++
.../beam/runners/direct/EvaluatorKey.java | 4 +-
.../direct/ExecutorServiceParallelExecutor.java | 32 +-
.../runners/direct/FlattenEvaluatorFactory.java | 18 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 127 +
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 186 +
.../ImmutabilityCheckingBundleFactory.java | 4 +-
.../direct/ImmutabilityEnforcementFactory.java | 4 +-
.../direct/ImmutableListBundleFactory.java | 163 +
.../direct/InMemoryWatermarkManager.java | 1420 --------
.../runners/direct/InProcessBundleFactory.java | 161 -
.../direct/InProcessBundleOutputManager.java | 51 -
.../direct/InProcessEvaluationContext.java | 428 ---
.../direct/InProcessExecutionContext.java | 105 -
.../beam/runners/direct/InProcessExecutor.java | 48 -
...rocessGroupAlsoByWindowEvaluatorFactory.java | 127 -
.../runners/direct/InProcessGroupByKey.java | 132 -
...InProcessGroupByKeyOnlyEvaluatorFactory.java | 185 -
.../InProcessGroupByKeyOverrideFactory.java | 41 -
.../direct/InProcessPipelineOptions.java | 101 -
.../runners/direct/InProcessPipelineRunner.java | 370 --
.../beam/runners/direct/InProcessRegistrar.java | 55 -
.../direct/InProcessSideInputContainer.java | 277 --
.../runners/direct/InProcessTimerInternals.java | 84 -
.../direct/InProcessTransformResult.java | 84 -
.../beam/runners/direct/ModelEnforcement.java | 8 +-
.../runners/direct/ModelEnforcementFactory.java | 2 +-
.../beam/runners/direct/ParDoEvaluator.java | 186 +
.../runners/direct/ParDoInProcessEvaluator.java | 186 -
.../direct/ParDoMultiEvaluatorFactory.java | 10 +-
.../direct/ParDoSingleEvaluatorFactory.java | 10 +-
.../direct/PassthroughTransformEvaluator.java | 4 +-
.../beam/runners/direct/PipelineExecutor.java | 48 +
.../beam/runners/direct/SideInputContainer.java | 277 ++
.../runners/direct/StepTransformResult.java | 8 +-
...readLocalInvalidatingTransformEvaluator.java | 2 +-
.../beam/runners/direct/TransformEvaluator.java | 6 +-
.../direct/TransformEvaluatorFactory.java | 4 +-
.../direct/TransformEvaluatorRegistry.java | 12 +-
.../beam/runners/direct/TransformExecutor.java | 16 +-
.../beam/runners/direct/TransformResult.java | 84 +
.../direct/UnboundedReadEvaluatorFactory.java | 161 +-
.../direct/UncommittedBundleOutputManager.java | 51 +
.../runners/direct/ViewEvaluatorFactory.java | 22 +-
.../direct/WatermarkCallbackExecutor.java | 10 +-
.../beam/runners/direct/WatermarkManager.java | 1420 ++++++++
.../runners/direct/WindowEvaluatorFactory.java | 18 +-
.../direct/AvroIOShardedWriteFactoryTest.java | 12 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 14 +-
.../runners/direct/CommittedResultTest.java | 14 +-
.../runners/direct/DirectRegistrarTest.java | 74 +
.../beam/runners/direct/DirectRunnerTest.java | 339 ++
.../direct/DirectTimerInternalsTest.java | 134 +
.../EncodabilityEnforcementFactoryTest.java | 4 +-
.../runners/direct/EvaluationContextTest.java | 545 +++
.../direct/FlattenEvaluatorFactoryTest.java | 16 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 13 +-
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 197 ++
.../ImmutabilityCheckingBundleFactoryTest.java | 6 +-
.../ImmutabilityEnforcementFactoryTest.java | 4 +-
.../direct/ImmutableListBundleFactoryTest.java | 231 ++
.../direct/InMemoryWatermarkManagerTest.java | 1428 --------
.../direct/InProcessBundleFactoryTest.java | 231 --
.../direct/InProcessEvaluationContextTest.java | 545 ---
...ocessGroupByKeyOnlyEvaluatorFactoryTest.java | 196 --
.../direct/InProcessPipelineRegistrarTest.java | 74 -
.../direct/InProcessPipelineRunnerTest.java | 330 --
.../direct/InProcessSideInputContainerTest.java | 520 ---
.../direct/InProcessTimerInternalsTest.java | 134 -
.../direct/KeyedPValueTrackingVisitorTest.java | 7 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 214 ++
.../direct/ParDoInProcessEvaluatorTest.java | 214 --
.../direct/ParDoMultiEvaluatorFactoryTest.java | 36 +-
.../direct/ParDoSingleEvaluatorFactoryTest.java | 36 +-
.../runners/direct/SideInputContainerTest.java | 520 +++
.../direct/TextIOShardedWriteFactoryTest.java | 12 +-
...LocalInvalidatingTransformEvaluatorTest.java | 4 +-
.../runners/direct/TransformExecutorTest.java | 44 +-
.../UnboundedReadEvaluatorFactoryTest.java | 72 +-
.../direct/ViewEvaluatorFactoryTest.java | 8 +-
.../runners/direct/WatermarkManagerTest.java | 1428 ++++++++
.../direct/WindowEvaluatorFactoryTest.java | 20 +-
runners/flink/README.md | 4 +-
.../beam/runners/flink/examples/TFIDF.java | 6 +-
.../beam/runners/flink/examples/WordCount.java | 4 +-
.../flink/examples/streaming/AutoComplete.java | 16 +-
.../flink/examples/streaming/JoinExamples.java | 4 +-
.../examples/streaming/KafkaIOExamples.java | 4 +-
.../KafkaWindowedWordCountExample.java | 4 +-
.../examples/streaming/WindowedWordCount.java | 4 +-
runners/flink/runner/pom.xml | 26 +-
.../beam/runners/flink/FlinkPipelineRunner.java | 180 -
.../apache/beam/runners/flink/FlinkRunner.java | 179 +
.../runners/flink/FlinkRunnerRegistrar.java | 4 +-
.../beam/runners/flink/FlinkRunnerResult.java | 8 +
.../runners/flink/TestFlinkPipelineRunner.java | 80 -
.../beam/runners/flink/TestFlinkRunner.java | 81 +
.../FlinkBatchTransformTranslators.java | 49 +-
.../FlinkBatchTranslationContext.java | 13 +-
.../translation/types/CoderComparator.java | 217 --
.../translation/types/CoderTypeInformation.java | 14 +-
.../translation/types/CoderTypeSerializer.java | 37 +-
.../types/EncodedValueComparator.java | 197 ++
.../types/EncodedValueSerializer.java | 113 +
.../types/EncodedValueTypeInformation.java | 111 +
.../translation/types/KvCoderComperator.java | 259 --
.../types/KvCoderTypeInformation.java | 207 --
.../flink/translation/types/KvKeySelector.java | 51 +
.../streaming/FlinkAbstractParDoWrapper.java | 9 +-
.../streaming/io/UnboundedFlinkSource.java | 6 +-
.../streaming/state/FlinkStateInternals.java | 2 +-
.../flink/EncodedValueComparatorTest.java | 71 +
.../runners/flink/FlinkRunnerRegistrarTest.java | 8 +-
.../beam/runners/flink/FlinkTestPipeline.java | 4 +-
.../beam/runners/flink/PipelineOptionsTest.java | 11 +-
runners/google-cloud-dataflow-java/pom.xml | 11 +
.../BlockingDataflowPipelineRunner.java | 186 -
.../dataflow/BlockingDataflowRunner.java | 186 +
.../dataflow/DataflowJobCancelledException.java | 2 +-
.../dataflow/DataflowJobExecutionException.java | 2 +-
.../dataflow/DataflowJobUpdatedException.java | 2 +-
.../runners/dataflow/DataflowPipelineJob.java | 2 +-
.../dataflow/DataflowPipelineRegistrar.java | 8 +-
.../dataflow/DataflowPipelineRunner.java | 3229 ------------------
.../dataflow/DataflowPipelineRunnerHooks.java | 39 -
.../dataflow/DataflowPipelineTranslator.java | 10 +-
.../beam/runners/dataflow/DataflowRunner.java | 3229 ++++++++++++++++++
.../runners/dataflow/DataflowRunnerHooks.java | 39 +
.../runners/dataflow/internal/package-info.java | 2 +-
.../BlockingDataflowPipelineOptions.java | 6 +-
.../options/DataflowPipelineOptions.java | 4 +-
.../DataflowPipelineWorkerPoolOptions.java | 6 +-
.../options/DataflowWorkerLoggingOptions.java | 79 +-
.../testing/TestDataflowPipelineRunner.java | 271 --
.../dataflow/testing/TestDataflowRunner.java | 271 ++
.../dataflow/util/DataflowTransport.java | 11 -
.../BlockingDataflowPipelineRunnerTest.java | 302 --
.../dataflow/BlockingDataflowRunnerTest.java | 304 ++
.../dataflow/DataflowPipelineRegistrarTest.java | 4 +-
.../dataflow/DataflowPipelineRunnerTest.java | 1415 --------
.../DataflowPipelineTranslatorTest.java | 50 +-
.../runners/dataflow/DataflowRunnerTest.java | 1417 ++++++++
.../runners/dataflow/io/DataflowAvroIOTest.java | 4 +-
.../dataflow/io/DataflowPubsubIOTest.java | 4 +-
.../runners/dataflow/io/DataflowTextIOTest.java | 85 +-
.../testing/TestDataflowPipelineRunnerTest.java | 601 ----
.../testing/TestDataflowRunnerTest.java | 601 ++++
.../DataflowDisplayDataEvaluator.java | 10 +-
.../transforms/DataflowGroupByKeyTest.java | 8 +-
.../dataflow/transforms/DataflowViewTest.java | 8 +-
.../util/DataflowPathValidatorTest.java | 8 +-
runners/spark/pom.xml | 118 +-
.../runners/spark/SparkRunnerRegistrar.java | 3 +-
.../runners/spark/TestSparkPipelineRunner.java | 77 +
.../spark/translation/TransformTranslator.java | 17 +-
.../beam/runners/spark/SimpleWordCountTest.java | 2 +-
.../runners/spark/SparkRunnerRegistrarTest.java | 2 +-
.../spark/translation/SerializationTest.java | 2 +-
.../translation/TransformTranslatorTest.java | 6 +-
.../translation/WindowedWordCountTest.java | 19 +-
sdks/java/core/pom.xml | 3 +
.../apache/beam/sdk/coders/BigDecimalCoder.java | 102 +
.../apache/beam/sdk/coders/BigIntegerCoder.java | 91 +
.../java/org/apache/beam/sdk/io/BigQueryIO.java | 6 +-
.../java/org/apache/beam/sdk/io/PubsubIO.java | 3 +-
.../main/java/org/apache/beam/sdk/io/Read.java | 44 -
.../java/org/apache/beam/sdk/io/TextIO.java | 3 +-
.../beam/sdk/io/range/ByteKeyRangeTracker.java | 8 +
.../beam/sdk/options/DirectPipelineOptions.java | 75 -
.../beam/sdk/options/PipelineOptions.java | 38 +-
.../sdk/runners/DirectPipelineRegistrar.java | 55 -
.../beam/sdk/runners/DirectPipelineRunner.java | 1298 -------
.../apache/beam/sdk/runners/package-info.java | 12 +-
.../apache/beam/sdk/testing/CrashingRunner.java | 33 +-
.../org/apache/beam/sdk/testing/PAssert.java | 206 +-
.../apache/beam/sdk/testing/TestPipeline.java | 5 +-
.../apache/beam/sdk/transforms/Aggregator.java | 2 +-
.../apache/beam/sdk/transforms/DoFnTester.java | 159 +-
.../org/apache/beam/sdk/transforms/Filter.java | 128 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 32 -
.../org/apache/beam/sdk/transforms/ParDo.java | 302 +-
.../org/apache/beam/sdk/transforms/View.java | 24 -
.../sdk/transforms/display/package-info.java | 26 +
.../transforms/windowing/AfterWatermark.java | 14 +-
.../apache/beam/sdk/util/AssignWindowsDoFn.java | 9 +-
.../beam/sdk/util/BigQueryTableRowIterator.java | 4 +-
.../sdk/util/DirectModeExecutionContext.java | 130 -
.../apache/beam/sdk/util/DoFnRunnerBase.java | 3 +-
.../apache/beam/sdk/util/ReduceFnRunner.java | 19 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 4 +-
.../org/apache/beam/sdk/WindowMatchers.java | 5 +
.../beam/sdk/coders/BigDecimalCoderTest.java | 144 +
.../beam/sdk/coders/BigIntegerCoderTest.java | 119 +
.../org/apache/beam/sdk/io/BigQueryIOTest.java | 4 +-
.../io/BoundedReadFromUnboundedSourceTest.java | 6 -
.../beam/sdk/io/CompressedSourceTest.java | 4 +-
.../java/org/apache/beam/sdk/io/TextIOTest.java | 75 +
.../java/org/apache/beam/sdk/io/WriteTest.java | 9 +-
.../sdk/io/range/ByteKeyRangeTrackerTest.java | 84 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 79 +-
.../beam/sdk/options/PipelineOptionsTest.java | 8 -
.../options/PipelineOptionsValidatorTest.java | 15 +
.../runners/DirectPipelineRegistrarTest.java | 71 -
.../sdk/runners/DirectPipelineRunnerTest.java | 221 --
.../beam/sdk/runners/PipelineRunnerTest.java | 35 +-
.../beam/sdk/testing/TestPipelineTest.java | 5 +-
.../apache/beam/sdk/transforms/CombineTest.java | 21 -
.../beam/sdk/transforms/DoFnTesterTest.java | 54 +-
.../apache/beam/sdk/transforms/FilterTest.java | 63 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 219 +-
.../apache/beam/sdk/transforms/ViewTest.java | 29 +-
.../sdk/transforms/join/CoGroupByKeyTest.java | 2 +-
.../windowing/AfterWatermarkTest.java | 19 +
.../sdk/util/GroupAlsoByWindowsProperties.java | 293 +-
.../beam/sdk/util/ReduceFnRunnerTest.java | 68 +
sdks/java/extensions/join-library/pom.xml | 7 +
sdks/java/io/google-cloud-platform/pom.xml | 7 +
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 5 +-
.../sdk/io/gcp/bigtable/BigtableIOTest.java | 2 -
sdks/java/io/hdfs/pom.xml | 7 +
sdks/java/io/kafka/pom.xml | 7 +
sdks/java/java8tests/pom.xml | 7 +
.../beam/sdk/transforms/FilterJava8Test.java | 8 +-
.../beam/sdk/transforms/WithKeysJava8Test.java | 3 +-
.../main/resources/archetype-resources/pom.xml | 19 +
.../src/main/java/DebuggingWordCount.java | 2 +-
.../src/main/java/MinimalWordCount.java | 4 +-
.../src/main/java/WindowedWordCount.java | 2 +-
.../src/main/java/WordCount.java | 2 +-
.../main/java/common/DataflowExampleUtils.java | 25 +-
.../src/test/java/WordCountTest.java | 6 +-
.../main/resources/archetype-resources/pom.xml | 13 +
.../src/main/java/StarterPipeline.java | 4 +-
.../resources/projects/basic/reference/pom.xml | 13 +
.../src/main/java/it/pkg/StarterPipeline.java | 4 +-
sdks/java/pom.xml | 2 +-
testing/travis/test_wordcount.sh | 4 +-
287 files changed, 17190 insertions(+), 18220 deletions(-)
----------------------------------------------------------------------