You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2017/01/30 23:03:57 UTC
[50/50] [abbrv] beam git commit: This closes #1872
This closes #1872
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c3b97a28
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c3b97a28
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c3b97a28
Branch: refs/heads/master
Commit: c3b97a2878a6ccb7b380cb7724ee0719a1d25d2e
Parents: 847e4e9 2d7ce32
Author: Davor Bonaci <da...@google.com>
Authored: Mon Jan 30 14:59:03 2017 -0800
Committer: Davor Bonaci <da...@google.com>
Committed: Mon Jan 30 14:59:03 2017 -0800
----------------------------------------------------------------------
.gitignore | 10 +
.travis.yml | 21 +-
pom.xml | 20 +
sdks/pom.xml | 2 +
sdks/python/.pylintrc | 164 +
sdks/python/MANIFEST.in | 19 +
sdks/python/README.md | 372 ++
sdks/python/apache_beam/__init__.py | 82 +
sdks/python/apache_beam/coders/__init__.py | 19 +
sdks/python/apache_beam/coders/coder_impl.pxd | 143 +
sdks/python/apache_beam/coders/coder_impl.py | 597 +++
sdks/python/apache_beam/coders/coders.py | 707 +++
sdks/python/apache_beam/coders/coders_test.py | 115 +
.../apache_beam/coders/coders_test_common.py | 355 ++
.../apache_beam/coders/fast_coders_test.py | 37 +
sdks/python/apache_beam/coders/observable.py | 38 +
.../apache_beam/coders/observable_test.py | 57 +
.../coders/proto2_coder_test_messages_pb2.py | 318 ++
.../apache_beam/coders/slow_coders_test.py | 39 +
sdks/python/apache_beam/coders/slow_stream.py | 154 +
.../apache_beam/coders/standard_coders.yaml | 67 +
.../apache_beam/coders/standard_coders_test.py | 136 +
sdks/python/apache_beam/coders/stream.pxd | 63 +
sdks/python/apache_beam/coders/stream.pyx | 215 +
sdks/python/apache_beam/coders/stream_test.py | 169 +
sdks/python/apache_beam/coders/typecoders.py | 186 +
.../apache_beam/coders/typecoders_test.py | 117 +
sdks/python/apache_beam/error.py | 42 +
sdks/python/apache_beam/examples/__init__.py | 16 +
.../apache_beam/examples/complete/__init__.py | 16 +
.../examples/complete/autocomplete.py | 89 +
.../examples/complete/autocomplete_test.py | 52 +
.../examples/complete/estimate_pi.py | 125 +
.../examples/complete/estimate_pi_test.py | 52 +
.../examples/complete/juliaset/__init__.py | 16 +
.../complete/juliaset/juliaset/__init__.py | 16 +
.../complete/juliaset/juliaset/juliaset.py | 123 +
.../complete/juliaset/juliaset/juliaset_test.py | 86 +
.../examples/complete/juliaset/juliaset_main.py | 58 +
.../examples/complete/juliaset/setup.py | 116 +
.../apache_beam/examples/complete/tfidf.py | 208 +
.../apache_beam/examples/complete/tfidf_test.py | 92 +
.../examples/complete/top_wikipedia_sessions.py | 180 +
.../complete/top_wikipedia_sessions_test.py | 62 +
.../apache_beam/examples/cookbook/__init__.py | 16 +
.../examples/cookbook/bigquery_schema.py | 130 +
.../examples/cookbook/bigquery_side_input.py | 123 +
.../cookbook/bigquery_side_input_test.py | 54 +
.../examples/cookbook/bigquery_tornadoes.py | 99 +
.../cookbook/bigquery_tornadoes_test.py | 44 +
.../apache_beam/examples/cookbook/bigshuffle.py | 95 +
.../examples/cookbook/bigshuffle_test.py | 63 +
.../apache_beam/examples/cookbook/coders.py | 101 +
.../examples/cookbook/coders_test.py | 49 +
.../examples/cookbook/combiners_test.py | 74 +
.../examples/cookbook/custom_ptransform.py | 134 +
.../examples/cookbook/custom_ptransform_test.py | 53 +
.../examples/cookbook/datastore_wordcount.py | 256 ++
.../apache_beam/examples/cookbook/filters.py | 107 +
.../examples/cookbook/filters_test.py | 69 +
.../examples/cookbook/group_with_coder.py | 122 +
.../examples/cookbook/group_with_coder_test.py | 89 +
.../examples/cookbook/mergecontacts.py | 133 +
.../examples/cookbook/mergecontacts_test.py | 125 +
.../examples/cookbook/multiple_output_pardo.py | 181 +
.../cookbook/multiple_output_pardo_test.py | 72 +
.../apache_beam/examples/snippets/__init__.py | 16 +
.../apache_beam/examples/snippets/snippets.py | 1142 +++++
.../examples/snippets/snippets_test.py | 758 ++++
.../apache_beam/examples/streaming_wordcap.py | 64 +
.../apache_beam/examples/streaming_wordcount.py | 74 +
sdks/python/apache_beam/examples/wordcount.py | 109 +
.../apache_beam/examples/wordcount_debugging.py | 166 +
.../examples/wordcount_debugging_test.py | 59 +
.../apache_beam/examples/wordcount_it_test.py | 59 +
.../apache_beam/examples/wordcount_minimal.py | 121 +
.../examples/wordcount_minimal_test.py | 59 +
.../apache_beam/examples/wordcount_test.py | 58 +
sdks/python/apache_beam/internal/__init__.py | 16 +
sdks/python/apache_beam/internal/apiclient.py | 639 +++
.../apache_beam/internal/apiclient_test.py | 42 +
sdks/python/apache_beam/internal/auth.py | 185 +
sdks/python/apache_beam/internal/auth_test.py | 44 +
.../apache_beam/internal/clients/__init__.py | 16 +
.../internal/clients/bigquery/__init__.py | 27 +
.../clients/bigquery/bigquery_v2_client.py | 659 +++
.../clients/bigquery/bigquery_v2_messages.py | 1910 ++++++++
.../internal/clients/dataflow/__init__.py | 27 +
.../clients/dataflow/dataflow_v1b3_client.py | 683 +++
.../clients/dataflow/dataflow_v1b3_messages.py | 4173 ++++++++++++++++++
.../clients/dataflow/message_matchers.py | 124 +
.../clients/dataflow/message_matchers_test.py | 69 +
.../internal/clients/storage/__init__.py | 27 +
.../clients/storage/storage_v1_client.py | 1038 +++++
.../clients/storage/storage_v1_messages.py | 1920 ++++++++
sdks/python/apache_beam/internal/json_value.py | 141 +
.../apache_beam/internal/json_value_test.py | 84 +
sdks/python/apache_beam/internal/module_test.py | 62 +
sdks/python/apache_beam/internal/pickler.py | 230 +
.../python/apache_beam/internal/pickler_test.py | 84 +
sdks/python/apache_beam/internal/util.py | 127 +
sdks/python/apache_beam/internal/util_test.py | 61 +
.../python/apache_beam/internal/windmill_pb2.py | 2278 ++++++++++
.../internal/windmill_service_pb2.py | 164 +
sdks/python/apache_beam/io/__init__.py | 31 +
sdks/python/apache_beam/io/avroio.py | 345 ++
sdks/python/apache_beam/io/avroio_test.py | 349 ++
sdks/python/apache_beam/io/bigquery.py | 1081 +++++
sdks/python/apache_beam/io/bigquery_test.py | 806 ++++
sdks/python/apache_beam/io/concat_source.py | 263 ++
.../python/apache_beam/io/concat_source_test.py | 231 +
.../python/apache_beam/io/datastore/__init__.py | 16 +
.../apache_beam/io/datastore/v1/__init__.py | 16 +
.../apache_beam/io/datastore/v1/datastoreio.py | 391 ++
.../io/datastore/v1/datastoreio_test.py | 247 ++
.../io/datastore/v1/fake_datastore.py | 92 +
.../apache_beam/io/datastore/v1/helper.py | 267 ++
.../apache_beam/io/datastore/v1/helper_test.py | 256 ++
.../io/datastore/v1/query_splitter.py | 269 ++
.../io/datastore/v1/query_splitter_test.py | 201 +
sdks/python/apache_beam/io/filebasedsource.py | 329 ++
.../apache_beam/io/filebasedsource_test.py | 708 +++
sdks/python/apache_beam/io/fileio.py | 797 ++++
sdks/python/apache_beam/io/fileio_test.py | 289 ++
sdks/python/apache_beam/io/gcsio.py | 820 ++++
sdks/python/apache_beam/io/gcsio_test.py | 748 ++++
sdks/python/apache_beam/io/iobase.py | 854 ++++
sdks/python/apache_beam/io/pubsub.py | 91 +
sdks/python/apache_beam/io/pubsub_test.py | 62 +
sdks/python/apache_beam/io/range_trackers.py | 493 +++
.../apache_beam/io/range_trackers_test.py | 538 +++
sdks/python/apache_beam/io/source_test_utils.py | 641 +++
.../apache_beam/io/source_test_utils_test.py | 123 +
sdks/python/apache_beam/io/sources_test.py | 111 +
sdks/python/apache_beam/io/textio.py | 318 ++
sdks/python/apache_beam/io/textio_test.py | 537 +++
sdks/python/apache_beam/io/tfrecordio.py | 271 ++
sdks/python/apache_beam/io/tfrecordio_test.py | 389 ++
sdks/python/apache_beam/metrics/__init__.py | 17 +
sdks/python/apache_beam/metrics/cells.py | 315 ++
sdks/python/apache_beam/metrics/cells_test.py | 143 +
sdks/python/apache_beam/metrics/execution.pxd | 31 +
sdks/python/apache_beam/metrics/execution.py | 228 +
.../apache_beam/metrics/execution_test.py | 131 +
sdks/python/apache_beam/metrics/metric.py | 165 +
sdks/python/apache_beam/metrics/metric_test.py | 85 +
sdks/python/apache_beam/metrics/metricbase.py | 81 +
sdks/python/apache_beam/pipeline.py | 442 ++
sdks/python/apache_beam/pipeline_test.py | 399 ++
sdks/python/apache_beam/pvalue.py | 495 +++
sdks/python/apache_beam/pvalue_test.py | 67 +
sdks/python/apache_beam/runners/__init__.py | 29 +
sdks/python/apache_beam/runners/common.pxd | 75 +
sdks/python/apache_beam/runners/common.py | 448 ++
.../apache_beam/runners/dataflow/__init__.py | 16 +
.../runners/dataflow/native_io/__init__.py | 16 +
.../runners/dataflow/native_io/iobase.py | 311 ++
.../apache_beam/runners/dataflow_runner.py | 731 +++
.../apache_beam/runners/dataflow_runner_test.py | 63 +
.../apache_beam/runners/direct/__init__.py | 19 +
.../runners/direct/bundle_factory.py | 102 +
sdks/python/apache_beam/runners/direct/clock.py | 50 +
.../consumer_tracking_pipeline_visitor.py | 59 +
.../consumer_tracking_pipeline_visitor_test.py | 126 +
.../runners/direct/direct_metrics.py | 112 +
.../runners/direct/direct_metrics_test.py | 211 +
.../apache_beam/runners/direct/direct_runner.py | 164 +
.../runners/direct/evaluation_context.py | 281 ++
.../apache_beam/runners/direct/executor.py | 568 +++
.../runners/direct/transform_evaluator.py | 568 +++
.../runners/direct/transform_result.py | 64 +
.../runners/direct/watermark_manager.py | 224 +
sdks/python/apache_beam/runners/runner.py | 351 ++
sdks/python/apache_beam/runners/runner_test.py | 217 +
.../apache_beam/runners/template_runner_test.py | 88 +
.../python/apache_beam/runners/test/__init__.py | 24 +
.../runners/test/test_dataflow_runner.py | 40 +
sdks/python/apache_beam/test_pipeline.py | 163 +
sdks/python/apache_beam/test_pipeline_test.py | 112 +
sdks/python/apache_beam/tests/__init__.py | 16 +
sdks/python/apache_beam/tests/data/README.md | 20 +
.../apache_beam/tests/data/privatekey.p12 | Bin 0 -> 2452 bytes
.../apache_beam/tests/pipeline_verifiers.py | 119 +
.../tests/pipeline_verifiers_test.py | 118 +
sdks/python/apache_beam/tests/test_utils.py | 57 +
sdks/python/apache_beam/transforms/__init__.py | 26 +
.../python/apache_beam/transforms/aggregator.py | 120 +
.../apache_beam/transforms/aggregator_test.py | 77 +
sdks/python/apache_beam/transforms/combiners.py | 595 +++
.../apache_beam/transforms/combiners_test.py | 335 ++
sdks/python/apache_beam/transforms/core.py | 1473 +++++++
.../apache_beam/transforms/cy_combiners.pxd | 92 +
.../apache_beam/transforms/cy_combiners.py | 306 ++
sdks/python/apache_beam/transforms/display.py | 329 ++
.../apache_beam/transforms/display_test.py | 205 +
.../python/apache_beam/transforms/ptransform.py | 742 ++++
.../apache_beam/transforms/ptransform_test.py | 1953 ++++++++
.../python/apache_beam/transforms/sideinputs.py | 214 +
.../apache_beam/transforms/sideinputs_test.py | 337 ++
sdks/python/apache_beam/transforms/timeutil.py | 133 +
sdks/python/apache_beam/transforms/trigger.py | 974 ++++
.../apache_beam/transforms/trigger_test.py | 582 +++
.../transforms/trigger_transcripts.yaml | 224 +
sdks/python/apache_beam/transforms/util.py | 235 +
sdks/python/apache_beam/transforms/window.py | 355 ++
.../apache_beam/transforms/window_test.py | 214 +
.../transforms/write_ptransform_test.py | 126 +
sdks/python/apache_beam/typehints/__init__.py | 22 +
sdks/python/apache_beam/typehints/decorators.py | 533 +++
sdks/python/apache_beam/typehints/opcodes.py | 334 ++
.../apache_beam/typehints/trivial_inference.py | 417 ++
.../typehints/trivial_inference_test.py | 151 +
sdks/python/apache_beam/typehints/typecheck.py | 309 ++
.../typehints/typed_pipeline_test.py | 251 ++
sdks/python/apache_beam/typehints/typehints.py | 1061 +++++
.../apache_beam/typehints/typehints_test.py | 1062 +++++
sdks/python/apache_beam/utils/__init__.py | 22 +
sdks/python/apache_beam/utils/annotations.py | 99 +
.../apache_beam/utils/annotations_test.py | 126 +
sdks/python/apache_beam/utils/counters.pxd | 30 +
sdks/python/apache_beam/utils/counters.py | 183 +
sdks/python/apache_beam/utils/dependency.py | 504 +++
.../python/apache_beam/utils/dependency_test.py | 425 ++
sdks/python/apache_beam/utils/names.py | 82 +
sdks/python/apache_beam/utils/path.py | 47 +
sdks/python/apache_beam/utils/path_test.py | 70 +
.../apache_beam/utils/pipeline_options.py | 528 +++
.../apache_beam/utils/pipeline_options_test.py | 175 +
.../utils/pipeline_options_validator.py | 200 +
.../utils/pipeline_options_validator_test.py | 334 ++
sdks/python/apache_beam/utils/processes.py | 52 +
sdks/python/apache_beam/utils/processes_test.py | 106 +
sdks/python/apache_beam/utils/profiler.py | 69 +
sdks/python/apache_beam/utils/retry.py | 200 +
sdks/python/apache_beam/utils/retry_test.py | 210 +
sdks/python/apache_beam/utils/timestamp.py | 213 +
sdks/python/apache_beam/utils/timestamp_test.py | 168 +
.../python/apache_beam/utils/windowed_value.pxd | 38 +
sdks/python/apache_beam/utils/windowed_value.py | 122 +
.../apache_beam/utils/windowed_value_test.py | 71 +
sdks/python/apache_beam/version.py | 57 +
sdks/python/pom.xml | 169 +
sdks/python/run_postcommit.sh | 102 +
sdks/python/run_pylint.sh | 53 +
sdks/python/setup.cfg | 27 +
sdks/python/setup.py | 146 +
sdks/python/test_config.py | 44 +
sdks/python/tox.ini | 38 +
248 files changed, 64912 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c3b97a28/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c3b97a28/sdks/pom.xml
----------------------------------------------------------------------
diff --cc sdks/pom.xml
index 150e1ae,45d8df0..3d0b893
--- a/sdks/pom.xml
+++ b/sdks/pom.xml
@@@ -33,8 -33,8 +33,9 @@@
<name>Apache Beam :: SDKs</name>
<modules>
+ <module>common</module>
<module>java</module>
+ <module>python</module>
</modules>
<profiles>