You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2021/03/11 04:01:24 UTC
[beam] tag nightly-master updated (485b581 -> 48f9f66)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 485b581 (commit)
to 48f9f66 (commit)
from 485b581 Merge pull request #14175: [BEAM-10402] Turn on nullness checking in DataflowMetrics
add 4069f7b Remove PrettyPrint from job definition
add 4479ff6 Merge pull request #14122 from [BEAM-11904] Remove PrettyPrint from job definition
add 4d9c279 Add __repr__ and disallow __bool__ for DeferredScalar
add 1cd4787 Merge pull request #14181 from TheNeuralBit/deferred-scalard-dunders
add 8d7de5f Allow conversion of non-deferred dataframes to PCollections.
add 265ee7e More tests, expanded docstring.
add 498e9c0 Merge pull request #14170 Allow conversion of non-deferred dataframes to PCollections.
add 45a2193 [BEAM-11946] Use ReadFromKafkaDoFn for KafkaIO.Read by default when beam_fn_api is enabled.
add ae5e1a3 Merge pull request #14168: [BEAM-11946] Use ReadFromKafkaDoFn for KafkaIO.Read by default when beam_fn_api is enabled.
add 7b31eb8 [BEAM-10961] turn on enableStrictDependencies globally
add 6c2c284 Merge pull request #14114 from sonam-vend/enable-global-dep-check
add 79e54b5 [BEAM-10409] Conditionally apply combiner packing.
add 875a509 yapf
add 603fd6c Merge pull request #14161 [BEAM-10409] Use annotations to control combiner packing.
add 6a40d4b Various type inference fixes.
add 382e4c1 fix typo
add 2306bd0 Merge pull request #14167 Various type inference fixes.
add 3a85d9c [BEAM-11613] Updates Dataflow pipelines to always use SDK Harness containers hosted in GCR (#13730)
add 1b1f420 [BEAM-11647] Avoid flake by splitting Go Precommit into 2 phases
add bc8c26c Merge pull request #14134: [BEAM-11647] Avoid flake by splitting Go Precommit into 2 phases
add b101064 [BEAM-11925] Ensure that output expression is partitioned in doctest framework (#14150)
add 9c989cc Fix auto-sharding parameter for BigQuery sink with FILE LOADS
add 63c13d6 Merge pull request #14183 from [BEAM-11772] Fix auto-sharding parameter for BigQuery sink with FILE LOADS
add b08a484 [BEAM-11592] Adding cross-language test using third-party Python dependencies
add 42826cd Merge pull request #13729 from ihji/BEAM-11592
add eeaacf4 [BEAM-11377] Fix credentials regression.
add 1ad764b Merge pull request #14190 from tysonjh/snapshot-fix-fix-fix
add 745dbe63 Add Pubsub RunnerImplementedSource and RunnerImplementedSink which can be translated into PubsubReadPayload and PubsubWritePayload over fnapi
add 48f9f66 Merge pull request #14141 from [BEAM-10861] Add RunnerImplementedSource and RunnerImplementedSink to Pubsub
No new revisions were added by this update.
Summary of changes:
.../jenkins/job_PerformanceTests_KafkaIO_IT.groovy | 2 +-
.../jenkins/job_PostRelease_NightlySnapshot.groovy | 2 +-
build.gradle.kts | 14 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 32 ++-
examples/java/build.gradle | 1 -
examples/kotlin/build.gradle | 2 +-
.../pipeline/src/main/proto/beam_runner_api.proto | 4 +-
release/build.gradle.kts | 1 +
runners/core-construction-java/build.gradle | 1 -
.../core/construction/PTransformTranslation.java | 4 +
.../core/construction/ValidateRunnerXlangTest.java | 24 +++
runners/core-java/build.gradle | 1 -
runners/direct-java/build.gradle | 3 +-
runners/extensions-java/metrics/build.gradle | 1 -
runners/flink/job-server/flink_job_server.gradle | 2 +-
runners/google-cloud-dataflow-java/build.gradle | 2 +
.../examples-streaming/build.gradle | 2 +-
.../examples/build.gradle | 2 +-
.../dataflow/DataflowPipelineTranslator.java | 2 +-
.../beam/runners/dataflow/DataflowRunner.java | 171 +++++----------
.../google-cloud-dataflow-java/worker/build.gradle | 1 +
.../worker/legacy-worker/build.gradle | 1 +
runners/java-fn-execution/build.gradle | 1 -
runners/java-job-service/build.gradle | 1 -
runners/jet/build.gradle | 1 -
runners/local-java/build.gradle | 1 -
runners/portability/java/build.gradle | 2 +-
runners/samza/build.gradle | 1 -
runners/samza/job-server/build.gradle | 1 -
runners/spark/build.gradle | 3 +-
runners/spark/job-server/build.gradle | 1 -
runners/twister2/build.gradle | 1 -
sdks/java/core/build.gradle | 1 -
...etrics.java => UsesPythonExpansionService.java} | 6 +-
sdks/java/expansion-service/build.gradle | 1 -
sdks/java/extensions/euphoria/build.gradle | 1 -
.../google-cloud-platform-core/build.gradle | 1 -
sdks/java/extensions/jackson/build.gradle | 1 -
sdks/java/extensions/join-library/build.gradle | 1 -
sdks/java/extensions/kryo/build.gradle | 1 -
sdks/java/extensions/ml/build.gradle | 1 -
sdks/java/extensions/protobuf/build.gradle | 1 -
sdks/java/extensions/sketching/build.gradle | 1 -
sdks/java/extensions/sorter/build.gradle | 1 -
sdks/java/extensions/sql/build.gradle | 1 -
sdks/java/extensions/sql/datacatalog/build.gradle | 1 -
.../extensions/sql/expansion-service/build.gradle | 1 -
sdks/java/extensions/sql/hcatalog/build.gradle | 1 -
sdks/java/extensions/sql/jdbc/build.gradle | 1 -
sdks/java/extensions/sql/perf-tests/build.gradle | 1 -
.../extensions/sql/udf-test-provider/build.gradle | 1 -
sdks/java/extensions/sql/udf/build.gradle | 1 -
sdks/java/extensions/sql/zetasql/build.gradle | 1 -
sdks/java/extensions/zetasketch/build.gradle | 1 -
sdks/java/fn-execution/build.gradle | 1 -
sdks/java/harness/build.gradle | 1 -
sdks/java/io/amazon-web-services/build.gradle | 1 -
sdks/java/io/amazon-web-services2/build.gradle | 1 -
sdks/java/io/amqp/build.gradle | 2 +-
sdks/java/io/azure/build.gradle | 1 -
sdks/java/io/bigquery-io-perf-tests/build.gradle | 2 +-
sdks/java/io/cassandra/build.gradle | 1 -
sdks/java/io/clickhouse/build.gradle | 1 -
sdks/java/io/common/build.gradle | 2 +-
sdks/java/io/contextualtextio/build.gradle | 1 -
sdks/java/io/debezium/build.gradle | 1 +
.../elasticsearch-tests-2/build.gradle | 1 -
.../elasticsearch-tests-5/build.gradle | 1 -
.../elasticsearch-tests-6/build.gradle | 1 -
.../elasticsearch-tests-7/build.gradle | 1 -
.../elasticsearch-tests-common/build.gradle | 1 -
sdks/java/io/elasticsearch/build.gradle | 2 +-
sdks/java/io/expansion-service/build.gradle | 1 -
sdks/java/io/file-based-io-tests/build.gradle | 2 +-
sdks/java/io/google-cloud-platform/build.gradle | 8 +-
.../expansion-service/build.gradle | 1 -
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 20 +-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 4 +-
.../beam/sdk/io/gcp/pubsub/PubsubMessages.java | 10 +
.../sdk/io/gcp/pubsub/RunnerImplementedSink.java | 68 ++++++
.../pubsub/RunnerImplementedSinkTranslation.java | 87 ++++++++
.../sdk/io/gcp/pubsub/RunnerImplementedSource.java | 83 ++++++++
.../pubsub/RunnerImplementedSourceTranslation.java | 102 +++++++++
.../beam/sdk/io/gcp/pubsub/PubsubClientTest.java | 4 +-
.../RunnerImplementedSinkTranslationTest.java | 120 +++++++++++
.../RunnerImplementedSourceTranslationTest.java | 237 +++++++++++++++++++++
sdks/java/io/hadoop-common/build.gradle | 2 +-
sdks/java/io/hadoop-file-system/build.gradle | 1 -
sdks/java/io/hadoop-format/build.gradle | 1 -
sdks/java/io/hbase/build.gradle | 2 +-
sdks/java/io/hcatalog/build.gradle | 2 +-
sdks/java/io/influxdb/build.gradle | 2 +-
sdks/java/io/jdbc/build.gradle | 1 -
sdks/java/io/jms/build.gradle | 1 -
sdks/java/io/kafka/build.gradle | 1 -
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 16 +-
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 12 +-
sdks/java/io/kinesis/build.gradle | 2 +-
.../java/io/kinesis/expansion-service/build.gradle | 1 -
sdks/java/io/kudu/build.gradle | 1 -
sdks/java/io/mongodb/build.gradle | 2 +-
sdks/java/io/mqtt/build.gradle | 2 +-
sdks/java/io/parquet/build.gradle | 1 -
sdks/java/io/rabbitmq/build.gradle | 2 +-
sdks/java/io/redis/build.gradle | 2 +-
sdks/java/io/snowflake/build.gradle | 1 -
.../io/snowflake/expansion-service/build.gradle | 1 -
sdks/java/io/solr/build.gradle | 1 -
sdks/java/io/splunk/build.gradle | 2 +-
sdks/java/io/synthetic/build.gradle | 1 -
sdks/java/io/thrift/build.gradle | 1 -
sdks/java/io/tika/build.gradle | 2 +-
sdks/java/io/xml/build.gradle | 2 +-
sdks/java/maven-archetypes/examples/build.gradle | 2 +-
.../maven-archetypes/gcp-bom-examples/build.gradle | 2 +-
sdks/java/maven-archetypes/starter/build.gradle | 2 +-
sdks/java/testing/expansion-service/build.gradle | 1 -
sdks/java/testing/jpms-tests/build.gradle | 1 -
sdks/java/testing/kafka-service/build.gradle | 2 +-
sdks/java/testing/load-tests/build.gradle | 1 -
sdks/java/testing/nexmark/build.gradle | 1 -
sdks/java/testing/test-utils/build.gradle | 1 -
sdks/java/testing/watermarks/build.gradle | 1 +
sdks/python/apache_beam/dataframe/convert.py | 49 ++++-
sdks/python/apache_beam/dataframe/convert_test.py | 9 +
sdks/python/apache_beam/dataframe/doctests.py | 46 +++-
sdks/python/apache_beam/dataframe/frame_base.py | 10 +
sdks/python/apache_beam/dataframe/frames.py | 72 ++++++-
sdks/python/apache_beam/dataframe/frames_test.py | 16 ++
.../apache_beam/dataframe/pandas_doctests_test.py | 71 +++++-
sdks/python/apache_beam/dataframe/partitionings.py | 2 +-
.../apache_beam/io/gcp/bigquery_file_loads.py | 13 +-
sdks/python/apache_beam/pipeline.py | 8 +-
sdks/python/apache_beam/portability/python_urns.py | 5 +
.../runners/dataflow/internal/apiclient.py | 50 +++--
.../runners/dataflow/internal/apiclient_test.py | 22 ++
.../runners/portability/artifact_service.py | 3 +-
.../runners/portability/expansion_service_test.py | 30 +++
.../portability/fn_api_runner/translations.py | 49 ++++-
.../portability/fn_api_runner/translations_test.py | 55 ++++-
sdks/python/apache_beam/transforms/core.py | 8 +-
sdks/python/apache_beam/transforms/util.py | 26 ++-
142 files changed, 1382 insertions(+), 325 deletions(-)
copy sdks/java/core/src/main/java/org/apache/beam/sdk/testing/{UsesGaugeMetrics.java => UsesPythonExpansionService.java} (79%)
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/RunnerImplementedSink.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/RunnerImplementedSinkTranslation.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/RunnerImplementedSource.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/RunnerImplementedSourceTranslation.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/RunnerImplementedSinkTranslationTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/RunnerImplementedSourceTranslationTest.java