You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/08/07 23:53:49 UTC
[24/50] [abbrv] beam git commit: Merge branch 'master' of
https://github.com/apache/beam into sync-master
Merge branch 'master' of https://github.com/apache/beam into sync-master
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9e6c9061
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9e6c9061
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9e6c9061
Branch: refs/heads/master
Commit: 9e6c9061c1b78bfd840f2505badac108ce77f4ae
Parents: 99221e7 462335c
Author: manuzhang <ow...@gmail.com>
Authored: Mon Jun 5 10:15:27 2017 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Mon Jun 5 10:15:27 2017 +0800
----------------------------------------------------------------------
.gitignore | 1 +
.../jenkins/common_job_properties.groovy | 6 +-
..._beam_PostCommit_Java_JDKVersionsTest.groovy | 60 +
.../cassandra/LargeITCluster/setup.sh | 21 +
.../cassandra/LargeITCluster/start-up.sh | 22 -
.../cassandra/LargeITCluster/teardown.sh | 1 -
.../cassandra/SmallITCluster/setup.sh | 22 +
.../cassandra/SmallITCluster/start-up.sh | 23 -
.../cassandra/SmallITCluster/teardown.sh | 1 -
.../LargeProductionCluster/setup.sh | 21 +
.../LargeProductionCluster/start-up.sh | 22 -
.../LargeProductionCluster/teardown.sh | 1 -
.../elasticsearch/SmallITCluster/setup.sh | 22 +
.../elasticsearch/SmallITCluster/start-up.sh | 23 -
.../elasticsearch/SmallITCluster/teardown.sh | 1 -
pom.xml | 102 +-
runners/apex/pom.xml | 5 +
.../apache/beam/runners/apex/ApexRunner.java | 126 +-
.../translation/ApexPipelineTranslator.java | 15 +-
.../apex/translation/ParDoTranslator.java | 11 +-
.../apex/translation/TranslationContext.java | 5 +-
.../operators/ApexGroupByKeyOperator.java | 4 +-
.../operators/ApexParDoOperator.java | 10 +-
.../apex/translation/utils/NoOpStepContext.java | 37 +-
runners/core-construction-java/pom.xml | 12 +-
.../core/construction/CoderTranslation.java | 193 ++
.../beam/runners/core/construction/Coders.java | 193 --
.../core/construction/CombineTranslation.java | 125 +
.../construction/ElementAndRestriction.java | 42 +
.../ElementAndRestrictionCoder.java | 88 +
.../core/construction/FlattenTranslator.java | 63 +
.../construction/GroupByKeyTranslation.java | 62 +
.../construction/PCollectionTranslation.java | 98 +
.../runners/core/construction/PCollections.java | 97 -
.../construction/PTransformTranslation.java | 193 ++
.../runners/core/construction/PTransforms.java | 110 -
.../core/construction/ParDoTranslation.java | 540 ++++
.../core/construction/ReadTranslation.java | 242 ++
.../construction/RunnerPCollectionView.java | 88 +
.../core/construction/SdkComponents.java | 26 +-
.../core/construction/SplittableParDo.java | 301 ++
.../TransformPayloadTranslatorRegistrar.java | 29 +
.../core/construction/TriggerTranslation.java | 336 ++
.../runners/core/construction/Triggers.java | 336 --
.../construction/WindowIntoTranslation.java | 109 +
.../core/construction/WindowingStrategies.java | 275 --
.../WindowingStrategyTranslation.java | 371 +++
.../construction/WriteFilesTranslation.java | 152 +
.../core/construction/CoderTranslationTest.java | 165 +
.../runners/core/construction/CodersTest.java | 164 -
.../construction/CombineTranslationTest.java | 130 +
.../ElementAndRestrictionCoderTest.java | 126 +
.../construction/GroupByKeyTranslationTest.java | 44 +
.../PCollectionTranslationTest.java | 203 ++
.../core/construction/PCollectionsTest.java | 189 --
.../construction/PTransformTranslationTest.java | 189 ++
.../core/construction/PTransformsTest.java | 188 --
.../core/construction/ParDoTranslationTest.java | 296 ++
.../core/construction/ReadTranslationTest.java | 179 ++
.../core/construction/SdkComponentsTest.java | 7 +-
.../core/construction/SplittableParDoTest.java | 157 +
.../construction/TriggerTranslationTest.java | 112 +
.../runners/core/construction/TriggersTest.java | 111 -
.../construction/WindowIntoTranslationTest.java | 127 +
.../construction/WindowingStrategiesTest.java | 110 -
.../WindowingStrategyTranslationTest.java | 120 +
.../construction/WriteFilesTranslationTest.java | 186 ++
.../beam/runners/core/BaseExecutionContext.java | 164 -
.../apache/beam/runners/core/DoFnRunners.java | 3 +-
.../runners/core/ElementAndRestriction.java | 42 -
.../core/ElementAndRestrictionCoder.java | 88 -
.../beam/runners/core/ExecutionContext.java | 99 -
.../GroupAlsoByWindowViaWindowSetNewDoFn.java | 4 +-
.../beam/runners/core/ProcessFnRunner.java | 7 +-
.../beam/runners/core/SimpleDoFnRunner.java | 394 +--
.../beam/runners/core/SplittableParDo.java | 602 ----
.../core/SplittableParDoViaKeyedWorkItems.java | 418 +++
.../apache/beam/runners/core/StepContext.java | 33 +
.../core/ElementAndRestrictionCoderTest.java | 127 -
.../beam/runners/core/ReduceFnTester.java | 10 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 4 +-
.../core/SplittableParDoProcessFnTest.java | 515 ++++
.../beam/runners/core/SplittableParDoTest.java | 606 ----
.../runners/core/StatefulDoFnRunnerTest.java | 4 +-
runners/direct-java/pom.xml | 184 +-
.../direct/BoundedReadEvaluatorFactory.java | 14 +-
.../runners/direct/DirectExecutionContext.java | 35 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 9 +-
.../apache/beam/runners/direct/DirectGraph.java | 26 +-
.../beam/runners/direct/DirectGraphVisitor.java | 28 +-
.../beam/runners/direct/DirectGroupByKey.java | 21 +-
.../beam/runners/direct/DirectRunner.java | 14 +-
.../beam/runners/direct/EmptyInputProvider.java | 8 +-
.../beam/runners/direct/EvaluationContext.java | 46 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 26 +-
.../direct/KeyedPValueTrackingVisitor.java | 9 +-
.../beam/runners/direct/ParDoEvaluator.java | 18 +-
.../runners/direct/ParDoEvaluatorFactory.java | 2 +-
.../direct/ParDoMultiOverrideFactory.java | 15 +-
.../runners/direct/ReadEvaluatorFactory.java | 97 +
.../beam/runners/direct/RootInputProvider.java | 7 +-
.../runners/direct/RootProviderRegistry.java | 28 +-
.../apache/beam/runners/direct/SourceShard.java | 33 +
...littableProcessElementsEvaluatorFactory.java | 13 +-
.../direct/StatefulParDoEvaluatorFactory.java | 2 +-
.../direct/TestStreamEvaluatorFactory.java | 28 +-
.../direct/TransformEvaluatorRegistry.java | 130 +-
.../direct/UnboundedReadEvaluatorFactory.java | 31 +-
.../runners/direct/ViewOverrideFactory.java | 12 +-
.../beam/runners/direct/WatermarkManager.java | 19 +-
.../runners/direct/DirectGraphVisitorTest.java | 3 +
.../beam/runners/direct/DirectGraphs.java | 10 +-
.../runners/direct/EvaluationContextTest.java | 22 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 4 +-
.../StatefulParDoEvaluatorFactoryTest.java | 4 +-
.../direct/WindowEvaluatorFactoryTest.java | 10 +
runners/flink/pom.xml | 4 +-
.../flink/FlinkBatchPipelineTranslator.java | 2 +-
.../FlinkPipelineExecutionEnvironment.java | 4 +-
.../runners/flink/FlinkPipelineOptions.java | 11 +
.../apache/beam/runners/flink/FlinkRunner.java | 10 +-
.../flink/FlinkStreamingPipelineTranslator.java | 11 +-
.../FlinkStreamingTransformTranslators.java | 29 +-
.../functions/FlinkDoFnFunction.java | 14 +-
.../functions/FlinkNoOpStepContext.java | 36 +-
.../functions/FlinkStatefulDoFnFunction.java | 10 +-
.../translation/types/CoderTypeSerializer.java | 55 +
.../types/EncodedValueSerializer.java | 18 +-
.../wrappers/streaming/DoFnOperator.java | 111 +-
.../streaming/SplittableDoFnOperator.java | 12 +-
.../state/FlinkBroadcastStateInternals.java | 29 +-
.../streaming/state/FlinkStateInternals.java | 6 +-
.../streaming/UnboundedSourceWrapperTest.java | 37 +
runners/google-cloud-dataflow-java/pom.xml | 124 +-
.../beam/runners/dataflow/DataflowMetrics.java | 2 +-
.../dataflow/DataflowPipelineTranslator.java | 10 +-
.../beam/runners/dataflow/DataflowRunner.java | 2 +-
.../beam/runners/dataflow/ReadTranslator.java | 5 +-
.../beam/runners/dataflow/util/DoFnInfo.java | 25 -
.../runners/dataflow/util/PropertyNames.java | 46 -
.../runners/dataflow/util/RandomAccessData.java | 2 +-
runners/spark/pom.xml | 6 +-
.../apache/beam/runners/spark/SparkRunner.java | 2 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 4 +-
.../spark/translation/MultiDoFnFunction.java | 110 +-
...SparkGroupAlsoByWindowViaOutputBufferFn.java | 4 +-
.../spark/translation/SparkProcessContext.java | 50 +-
.../spark/translation/TransformTranslator.java | 83 +-
.../streaming/StreamingTransformTranslator.java | 4 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 2 +-
.../streaming/TrackStreamingSourcesTest.java | 14 +-
sdks/common/fn-api/pom.xml | 5 +
.../fn-api/src/main/proto/beam_fn_api.proto | 25 +-
.../src/main/proto/beam_runner_api.proto | 43 +-
.../src/main/proto/standard_window_fns.proto | 53 +
.../src/main/resources/beam/findbugs-filter.xml | 7 +
sdks/java/core/pom.xml | 173 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 50 +-
.../apache/beam/sdk/coders/InstantCoder.java | 81 +-
.../org/apache/beam/sdk/io/CountingSource.java | 42 +
.../beam/sdk/io/DefaultFilenamePolicy.java | 151 +-
.../org/apache/beam/sdk/io/FileSystems.java | 6 -
.../java/org/apache/beam/sdk/io/TextIO.java | 8 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 9 +-
.../beam/sdk/runners/TransformHierarchy.java | 6 +-
.../org/apache/beam/sdk/state/StateSpec.java | 53 +
.../org/apache/beam/sdk/state/StateSpecs.java | 53 +-
.../org/apache/beam/sdk/testing/LargeKeys.java | 48 +
.../apache/beam/sdk/testing/StaticWindows.java | 12 +
.../sdk/transforms/ApproximateQuantiles.java | 4 +-
.../org/apache/beam/sdk/transforms/Count.java | 10 +
.../apache/beam/sdk/transforms/Distinct.java | 80 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 12 +-
.../org/apache/beam/sdk/transforms/Max.java | 4 +-
.../org/apache/beam/sdk/transforms/Min.java | 4 +-
.../apache/beam/sdk/transforms/PTransform.java | 8 +-
.../org/apache/beam/sdk/transforms/Sum.java | 30 +
.../org/apache/beam/sdk/transforms/Top.java | 36 +-
.../org/apache/beam/sdk/transforms/View.java | 5 +-
.../sdk/transforms/windowing/BoundedWindow.java | 20 +-
.../transforms/windowing/CalendarWindows.java | 36 +
.../sdk/transforms/windowing/FixedWindows.java | 11 +
.../sdk/transforms/windowing/GlobalWindows.java | 17 +-
.../windowing/IncompatibleWindowException.java | 38 +
.../transforms/windowing/InvalidWindows.java | 11 +
.../beam/sdk/transforms/windowing/Sessions.java | 11 +
.../transforms/windowing/SlidingWindows.java | 11 +
.../beam/sdk/transforms/windowing/Window.java | 6 +-
.../beam/sdk/transforms/windowing/WindowFn.java | 21 +
.../org/apache/beam/sdk/util/ClassPath.java | 8 +-
.../apache/beam/sdk/util/IdentityWindowFn.java | 11 +
.../apache/beam/sdk/coders/AvroCoderTest.java | 4 +-
.../beam/sdk/io/DefaultFilenamePolicyTest.java | 57 +
.../java/org/apache/beam/sdk/io/TextIOTest.java | 10 -
.../sdk/runners/TransformHierarchyTest.java | 2 +-
.../beam/sdk/testing/StaticWindowsTest.java | 12 +
.../transforms/ApproximateQuantilesTest.java | 2 +-
.../beam/sdk/transforms/DistinctTest.java | 130 +-
.../beam/sdk/transforms/GroupByKeyTest.java | 74 +
.../sdk/transforms/GroupIntoBatchesTest.java | 4 +-
.../org/apache/beam/sdk/transforms/MaxTest.java | 2 +-
.../org/apache/beam/sdk/transforms/MinTest.java | 2 +-
.../apache/beam/sdk/transforms/ParDoTest.java | 76 +-
.../org/apache/beam/sdk/transforms/TopTest.java | 10 +-
.../windowing/CalendarWindowsTest.java | 24 +
.../transforms/windowing/FixedWindowsTest.java | 7 +
.../sdk/transforms/windowing/SessionsTest.java | 14 +
.../windowing/SlidingWindowsTest.java | 11 +
.../sdk/transforms/windowing/WindowTest.java | 44 +
.../sdk/util/IdentitySideInputWindowFn.java | 4 +
.../java/org/apache/beam/sdk/util/GcsUtil.java | 8 +-
.../sdk/util/RetryHttpRequestInitializer.java | 12 +-
.../org/apache/beam/sdk/util/GcsUtilTest.java | 16 +-
.../beam/sdk/extensions/joinlibrary/Join.java | 65 +-
.../joinlibrary/OuterFullJoinTest.java | 179 ++
.../beam/fn/harness/fake/FakeStepContext.java | 33 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 36 +-
.../control/ProcessBundleHandlerTest.java | 2 +-
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 4 +-
sdks/java/io/google-cloud-platform/pom.xml | 78 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 8 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 22 +-
.../sdk/io/gcp/bigquery/BigQueryServices.java | 8 +-
.../io/gcp/bigquery/BigQueryServicesImpl.java | 39 +-
.../sdk/io/gcp/bigquery/InsertRetryPolicy.java | 86 +
.../sdk/io/gcp/bigquery/StreamingInserts.java | 35 +-
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 50 +-
.../io/gcp/bigquery/StreamingWriteTables.java | 26 +-
.../beam/sdk/io/gcp/bigquery/WriteResult.java | 27 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 26 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 17 +
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 5 +-
.../io/gcp/spanner/MutationSizeEstimator.java | 98 +
.../beam/sdk/io/gcp/spanner/SpannerIO.java | 350 +++
.../beam/sdk/io/gcp/spanner/package-info.java | 23 +
.../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 6 +-
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 69 +
.../gcp/bigquery/BigQueryServicesImplTest.java | 88 +-
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 11 +-
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 75 +-
.../io/gcp/bigquery/InsertRetryPolicyTest.java | 79 +
.../sdk/io/gcp/datastore/DatastoreV1Test.java | 36 +
.../beam/sdk/io/gcp/datastore/V1ReadIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 15 +-
.../beam/sdk/io/gcp/datastore/V1WriteIT.java | 36 +-
.../gcp/spanner/MutationSizeEstimatorTest.java | 138 +
.../beam/sdk/io/gcp/spanner/SpannerIOTest.java | 244 ++
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 159 +
.../io/hadoop/SerializableConfiguration.java | 18 +-
.../beam/sdk/io/hdfs/HadoopFileSystem.java | 7 +
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 33 +
.../hadoop/inputformat/HadoopInputFormatIO.java | 53 +-
.../inputformat/HadoopInputFormatIOTest.java | 80 +-
.../inputformat/HIFIOWithElasticTest.java | 10 +-
.../org/apache/beam/sdk/io/hbase/HBaseIO.java | 5 +-
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 21 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 5 +-
.../org/apache/beam/sdk/io/jms/JmsIOTest.java | 78 +
sdks/java/io/kafka/pom.xml | 2 +-
.../apache/beam/sdk/io/kafka/ConsumerSpEL.java | 56 +
.../org/apache/beam/sdk/io/kafka/KafkaIO.java | 44 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 122 +-
.../examples-java8/generate-sources.sh | 2 +-
.../main/resources/archetype-resources/pom.xml | 52 +-
.../examples/generate-sources.sh | 2 +-
.../main/resources/archetype-resources/pom.xml | 49 +-
.../main/resources/archetype-resources/pom.xml | 12 +-
.../resources/projects/basic/reference/pom.xml | 12 +-
sdks/python/MANIFEST.in | 1 +
sdks/python/apache_beam/coders/coders.py | 9 +-
.../examples/complete/autocomplete.py | 19 +-
.../examples/complete/autocomplete_test.py | 31 +-
.../examples/complete/estimate_pi.py | 11 +-
.../examples/complete/estimate_pi_test.py | 12 +-
.../examples/complete/game/hourly_team_score.py | 19 +-
.../examples/complete/game/user_score.py | 15 +-
.../complete/juliaset/juliaset/juliaset.py | 44 +-
.../apache_beam/examples/complete/tfidf.py | 21 +-
.../apache_beam/examples/complete/tfidf_test.py | 28 +-
.../examples/complete/top_wikipedia_sessions.py | 12 +-
.../complete/top_wikipedia_sessions_test.py | 9 +-
.../examples/cookbook/bigquery_schema.py | 159 +-
.../examples/cookbook/bigquery_side_input.py | 51 +-
.../cookbook/bigquery_side_input_test.py | 39 +-
.../examples/cookbook/bigquery_tornadoes.py | 33 +-
.../cookbook/bigquery_tornadoes_test.py | 19 +-
.../apache_beam/examples/cookbook/coders.py | 16 +-
.../examples/cookbook/coders_test.py | 14 +-
.../examples/cookbook/custom_ptransform.py | 27 +-
.../examples/cookbook/custom_ptransform_test.py | 11 +-
.../examples/cookbook/datastore_wordcount.py | 20 +-
.../apache_beam/examples/cookbook/filters.py | 21 +-
.../examples/cookbook/group_with_coder.py | 43 +-
.../examples/cookbook/group_with_coder_test.py | 4 +-
.../examples/cookbook/mergecontacts.py | 115 +-
.../examples/cookbook/mergecontacts_test.py | 3 +-
.../examples/cookbook/multiple_output_pardo.py | 72 +-
.../cookbook/multiple_output_pardo_test.py | 2 +-
.../apache_beam/examples/snippets/snippets.py | 576 ++--
.../examples/snippets/snippets_test.py | 402 +--
.../apache_beam/examples/streaming_wordcap.py | 22 +-
.../apache_beam/examples/streaming_wordcount.py | 42 +-
sdks/python/apache_beam/examples/wordcount.py | 1 -
.../apache_beam/examples/wordcount_debugging.py | 55 +-
.../apache_beam/examples/wordcount_minimal.py | 33 +-
.../python/apache_beam/io/filebasedsink_test.py | 16 +-
.../io/gcp/datastore/v1/datastoreio.py | 15 +-
.../io/gcp/datastore/v1/datastoreio_test.py | 16 +
sdks/python/apache_beam/io/gcp/gcsio.py | 2 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 22 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 137 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 57 +-
.../apache_beam/options/value_provider.py | 1 -
sdks/python/apache_beam/pipeline.py | 19 +-
sdks/python/apache_beam/runners/api/__init__.py | 13 +-
.../apache_beam/runners/api/beam_fn_api_pb2.py | 2742 -----------------
.../runners/api/beam_fn_api_pb2_grpc.py | 205 --
.../runners/api/beam_runner_api_pb2.py | 2872 ------------------
.../runners/dataflow/dataflow_runner.py | 49 +-
.../runners/dataflow/dataflow_runner_test.py | 11 +
.../runners/dataflow/internal/dependency.py | 3 +-
.../runners/portability/fn_api_runner.py | 2 +-
.../runners/worker/data_plane_test.py | 2 +-
.../runners/worker/log_handler_test.py | 2 +-
.../runners/worker/operation_specs.py | 11 +-
.../apache_beam/runners/worker/operations.py | 2 +-
.../runners/worker/sdk_worker_test.py | 2 +-
.../apache_beam/transforms/combiners_test.py | 58 +-
sdks/python/apache_beam/transforms/window.py | 57 +-
.../apache_beam/transforms/window_test.py | 147 +-
.../transforms/write_ptransform_test.py | 7 +-
.../typehints/typed_pipeline_test.py | 22 +-
sdks/python/apache_beam/utils/proto_utils.py | 6 +
sdks/python/apache_beam/utils/urns.py | 10 +-
sdks/python/gen_protos.py | 125 +
sdks/python/run_pylint.sh | 4 +-
sdks/python/setup.py | 47 +-
sdks/python/tox.ini | 9 +
338 files changed, 13182 insertions(+), 12472 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/9e6c9061/pom.xml
----------------------------------------------------------------------