You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by bh...@apache.org on 2020/12/08 02:10:22 UTC
[beam] 01/01: Merge remote-tracking branch 'origin/master' into
website-revamp
This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a commit to branch website-revamp
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 8b22fb4b9aa8607313f1e39374a226bc226a349c
Merge: 140a913 31716b0
Author: Brian Hulette <bh...@google.com>
AuthorDate: Mon Dec 7 18:09:14 2020 -0800
Merge remote-tracking branch 'origin/master' into website-revamp
.test-infra/jenkins/PythonTestProperties.groovy | 17 +-
.../jenkins/job_LoadTests_Combine_Flink_Go.groovy | 136 ++++++++++
.../jenkins/job_LoadTests_GBK_Flink_Go.groovy | 220 ++++++++++++++++
.../jenkins/job_LoadTests_coGBK_Flink_Go.groovy | 182 +++++++++++++
.../job_PostCommit_PortableJar_Flink.groovy | 3 +-
.../job_PostCommit_PortableJar_Spark.groovy | 2 +-
...Commit_Python_Chicago_Taxi_Example_Flink.groovy | 8 +-
...ommit_Python_ValidatesContainer_Dataflow.groovy | 9 +-
.../dashboards/Post-Commits_status_dashboard.json | 2 +-
CHANGES.md | 4 +
build.gradle | 13 +-
buildSrc/build.gradle | 4 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 23 +-
dev-support/docker/Dockerfile | 117 +++++++++
dev-support/docker/bashcolors.sh | 91 +++++++
dev-support/docker/beam_env_checks.sh | 121 +++++++++
model/pipeline/src/main/proto/metrics.proto | 13 +
.../DefaultExpansionServiceClientFactory.java | 7 +-
.../core/construction/ExpansionServiceClient.java | 2 +-
.../ExpansionServiceClientFactory.java | 2 +-
.../beam/runners/core/construction/External.java | 2 +-
.../core/construction/PTransformTranslation.java | 1 +
.../beam/runners/core/SideInputHandlerTest.java | 118 +++++----
runners/direct-java/build.gradle | 22 +-
runners/flink/job-server/flink_job_server.gradle | 66 -----
.../worker/windmill/GrpcWindmillServer.java | 14 +-
.../test_flink_uber_jar.sh | 0
runners/portability/test_pipeline_jar.sh | 3 +-
runners/spark/job-server/build.gradle | 32 ---
sdks/go/test/load/build.gradle | 3 +
sdks/go/test/load/cogbk/cogbk.go | 106 ++++++++
sdks/go/test/load/combine/combine.go | 87 +++++++
sdks/go/test/load/group_by_key/group_by_key.go | 89 +++++++
sdks/java/build-tools/beam-linkage-check.sh | 80 +++---
.../apache/beam/sdk/annotations/Experimental.java | 7 -
.../main/java/org/apache/beam/sdk/io/AvroIO.java | 7 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 5 +-
.../main/java/org/apache/beam/sdk/io/TextIO.java | 5 +-
.../java/org/apache/beam/sdk/state/Timers.java | 2 +-
.../apache/beam/sdk/transforms/Deduplicate.java | 3 -
.../java/org/apache/beam/sdk/transforms/DoFn.java | 13 -
.../beam/sdk/transforms/PeriodicSequence.java | 3 -
.../java/org/apache/beam/sdk/transforms/Watch.java | 3 -
.../splittabledofn/ByteKeyRangeTracker.java | 3 -
.../splittabledofn/GrowableOffsetRangeTracker.java | 3 -
.../splittabledofn/HasDefaultTracker.java | 4 -
.../HasDefaultWatermarkEstimator.java | 3 -
.../splittabledofn/ManualWatermarkEstimator.java | 3 -
.../splittabledofn/OffsetRangeTracker.java | 3 -
.../splittabledofn/RestrictionTracker.java | 3 -
.../sdk/transforms/splittabledofn/SplitResult.java | 3 -
.../TimestampObservingWatermarkEstimator.java | 3 -
.../splittabledofn/WatermarkEstimator.java | 3 -
.../splittabledofn/WatermarkEstimators.java | 3 -
.../transforms/splittabledofn/package-info.java | 3 -
sdks/java/extensions/ml/build.gradle | 33 ++-
sdks/java/extensions/sql/build.gradle | 1 +
.../sql/meta/provider/bigtable/BigtableTable.java | 211 +++++++++++++++
.../provider/bigtable/BigtableTableProvider.java | 85 ++++++
.../sql/meta/provider/bigtable/package-info.java | 9 +-
.../BigtableTableCreationFailuresTest.java | 145 +++++++++++
.../provider/bigtable/BigtableTableFlatTest.java | 163 ++++++++++++
.../meta/provider/bigtable/BigtableTableTest.java | 94 +++++++
.../bigtable/BigtableTableWithRowsTest.java | 158 +++++++++++
.../apache/beam/fn/harness/ToStringFnRunner.java | 66 +++++
.../beam/fn/harness/ToStringFnRunnerTest.java | 72 +++++
sdks/java/io/amazon-web-services2/build.gradle | 2 +-
.../beam/sdk/io/aws2/options/AwsOptions.java | 4 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 5 +
.../io/gcp/bigtable/BeamRowToBigtableMutation.java | 123 +++++++++
.../beam/sdk/io/gcp/bigtable/BigtableConfig.java | 15 ++
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 22 ++
.../sdk/io/gcp/bigtable/BigtableRowToBeamRow.java | 176 +++++++++++++
.../io/gcp/bigtable/BigtableRowToBeamRowFlat.java | 108 ++++++++
.../io/gcp/bigtable/BigtableRowToBeamRowFn.java | 46 ++++
.../beam/sdk/io/gcp/bigtable/CellValueParser.java | 106 ++++++++
.../apache/beam/sdk/io/gcp/bigtable/RowUtils.java | 13 +-
.../apache/beam/sdk/io/gcp/healthcare/DicomIO.java | 199 ++++++++++++++
.../sdk/io/gcp/healthcare/HealthcareApiClient.java | 12 +
.../io/gcp/healthcare/HttpHealthcareApiClient.java | 79 ++++++
.../beam/sdk/io/gcp/healthcare/WebPathParser.java | 63 +++++
.../io/gcp/testing/BigtableEmulatorWrapper.java | 75 ++++++
.../beam/sdk/io/gcp/testing/BigtableTestUtils.java | 154 +++++++++++
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 9 +
.../bigtable/BeamRowToBigtableMutationTest.java | 89 +++++++
.../gcp/bigtable/BigtableRowToBeamRowFlatTest.java | 65 +++++
.../io/gcp/bigtable/BigtableRowToBeamRowTest.java | 93 +++++++
.../sdk/io/gcp/bigtable/CellValueParserTest.java | 277 ++++++++++++++++++++
.../apache/beam/sdk/io/gcp/bigtable/TestUtils.java | 109 ++++++++
.../beam/sdk/io/gcp/healthcare/DicomIOReadIT.java | 93 +++++++
.../beam/sdk/io/gcp/healthcare/DicomIOTest.java | 60 +++++
.../sdk/io/gcp/healthcare/WebPathParserTest.java | 48 ++++
.../src/test/resources/DICOM/testDicomFile.dcm | Bin 0 -> 10366 bytes
.../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 1 -
sdks/java/io/kafka/build.gradle | 90 +++++--
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 69 ++++-
sdks/python/apache_beam/coders/coders.py | 16 ++
.../apache_beam/coders/coders_test_common.py | 17 ++
sdks/python/apache_beam/dataframe/io.py | 143 +++++++++-
sdks/python/apache_beam/dataframe/io_test.py | 41 +++
.../apache_beam/examples/kafkataxi/README.md | 4 +-
.../apache_beam/examples/snippets/snippets.py | 4 +
sdks/python/apache_beam/io/gcp/bigquery.py | 273 +++++++++++++------
.../apache_beam/io/gcp/bigquery_read_internal.py | 289 +++++++++++++++++++++
.../apache_beam/io/gcp/bigquery_read_it_test.py | 103 ++++++++
sdks/python/apache_beam/io/gcp/bigquery_test.py | 2 +-
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 26 +-
sdks/python/apache_beam/io/gcp/bigtableio.py | 7 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 114 ++++++++
sdks/python/apache_beam/io/gcp/pubsub_test.py | 123 +++++++++
sdks/python/apache_beam/io/iobase.py | 20 +-
sdks/python/apache_beam/io/restriction_trackers.py | 17 ++
sdks/python/apache_beam/metrics/cells.pxd | 2 +
sdks/python/apache_beam/metrics/cells.py | 15 +-
sdks/python/apache_beam/metrics/cells_test.py | 25 ++
.../python/apache_beam/options/pipeline_options.py | 6 +
sdks/python/apache_beam/options/value_provider.py | 71 ++++-
.../apache_beam/options/value_provider_test.py | 40 +++
sdks/python/apache_beam/runners/common.py | 5 +-
.../Linux/7a35f487b2a5f3a9b9852a8659eeb4bd.png | Bin 700548 -> 703677 bytes
.../runners/portability/flink_runner_test.py | 2 +-
.../portability/fn_api_runner/fn_runner_test.py | 115 +++++++-
.../portability/fn_api_runner/translations.py | 8 +-
.../portability/fn_api_runner/translations_test.py | 31 +++
.../portability/fn_api_runner/worker_handlers.py | 2 +-
.../apache_beam/runners/portability/job_server.py | 3 +-
.../runners/portability/sdk_container_builder.py | 30 ++-
.../runners/portability/spark_runner_test.py | 6 +-
.../apache_beam/runners/worker/bundle_processor.py | 17 ++
.../apache_beam/runners/worker/sdk_worker_test.py | 2 +
sdks/python/apache_beam/transforms/core.py | 32 ++-
.../apache_beam/typehints/sharded_key_type.py | 75 ++++++
.../apache_beam/typehints/sharded_key_type_test.py | 80 ++++++
sdks/python/apache_beam/utils/shared.py | 2 +-
sdks/python/build-requirements.txt | 3 +
sdks/python/setup.py | 2 +-
sdks/python/test-suites/portable/common.gradle | 69 ++++-
start-build-env.sh | 132 ++++++++++
.../www/site/content/en/community/powered-by.md | 2 +
website/www/site/content/en/contribute/_index.md | 16 +-
.../dsls/sql/extensions/create-external-table.md | 132 ++++++++++
.../en/documentation/io/developing-io-java.md | 3 +
.../en/documentation/io/developing-io-overview.md | 80 +++---
.../en/documentation/io/developing-io-python.md | 3 +
.../content/en/documentation/programming-guide.md | 2 +-
.../en/documentation/sdks/python-dependencies.md | 12 +-
.../site/content/en/get-started/quickstart-py.md | 4 +-
.../content/en/get-started/wordcount-example.md | 2 +-
148 files changed, 6607 insertions(+), 549 deletions(-)