You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2023/12/08 03:54:13 UTC
(beam) branch nightly-refs/heads/master updated (54c18943bfb -> 85bf3888959)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 54c18943bfb Bump google.golang.org/api from 0.152.0 to 0.153.0 in /sdks (#29628)
add 70926f5da78 Bump go.mongodb.org/mongo-driver from 1.13.0 to 1.13.1 in /sdks (#29657)
add ee5e782c525 Bump actions/setup-go from 4 to 5 (#29653)
add 5d11c20cd4e Add Error Handling to Kafka IO (#29546)
add e620b5aac97 Bump golang.org/x/oauth2 from 0.14.0 to 0.15.0 in /sdks (#29630)
add d2d19ef5dfd Bugfix/extend java precommit timeout (#29664)
add 8d9fd02af5e Bump Java VR interval and shutdown remaining Dataflow VR Jenkins PostCommit (#29647)
add 84d070bae9b Add per-worker keys to stateful BatchElements, plumb through batching options to all model handlers (#29642)
add 33a122cd2ac Add additional log messages and javadoc around reads (#29640)
add 51d71c0c43e Remove experimental note about streaming. (#29652)
add 174e49dc6aa Add cloud profiler dependency to 3.11 containers. (#29651)
add 90dd93f5241 Disable failing mypy check in yaml_io.py (#29666)
add dab5e86b912 Fix for signature exceptions in ptransform_fn.
add 07ebdc6a6f5 Guard entire signature setting.
add e15522f007f Merge pull request #29648 Fix for signature exceptions in ptransform_fn.
add b426dfaf5ad Add the ability to generate markdown documentation from a set of providers. (#29639)
add 74308c39b8b Update to Libraries-Bom 26.28.0
add 6e50c1bbc88 Update libraries-bom java url version to 26.28.0
add be9155205dd Merge pull request #29641 from lqiu96/update-libraries-bom-26.28.0
add 5becfb8ed43 Archive JUnit test result on timeout (#29667)
add 31822734526 Add a pipeline update compatibility version option. (#29140)
add f1a7f71266c Bump the image used by Dataflow runner in dev SDKs. (#29672)
add ed3a58217f4 Fix some lint errors in Python. (#29675)
add 85bf3888959 More efficient map side inputs for small maps. (#29587)
No new revisions were added by this update.
Summary of changes:
...eam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml | 2 +-
...PerformanceTests_BigQueryIO_Batch_Java_Avro.yml | 2 +-
...PerformanceTests_BigQueryIO_Batch_Java_Json.yml | 2 +-
..._PerformanceTests_BigQueryIO_Streaming_Java.yml | 2 +-
...m_PerformanceTests_SQLBigQueryIO_Batch_Java.yml | 2 +-
.github/workflows/beam_PostCommit_Java.yml | 2 +-
.../beam_PostCommit_Java_Avro_Versions.yml | 2 +-
.../beam_PostCommit_Java_BigQueryEarlyRollout.yml | 2 +-
.../workflows/beam_PostCommit_Java_DataflowV1.yml | 2 +-
.../workflows/beam_PostCommit_Java_DataflowV2.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow_ARM.yml | 2 +-
...beam_PostCommit_Java_Examples_Dataflow_Java.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow_V2.yml | 2 +-
...m_PostCommit_Java_Examples_Dataflow_V2_Java.yml | 2 +-
.../beam_PostCommit_Java_Examples_Direct.yml | 2 +-
.../beam_PostCommit_Java_Examples_Flink.yml | 2 +-
.../beam_PostCommit_Java_Examples_Spark.yml | 2 +-
.../beam_PostCommit_Java_Hadoop_Versions.yml | 2 +-
.../beam_PostCommit_Java_IO_Performance_Tests.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Dataflow_Java11.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Dataflow_Java17.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Direct_Java11.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Direct_Java17.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Direct_Java21.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Flink_Java11.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Spark_Java11.yml | 2 +-
.../beam_PostCommit_Java_PVR_Flink_Streaming.yml | 2 +-
.../workflows/beam_PostCommit_Java_PVR_Samza.yml | 2 +-
.../beam_PostCommit_Java_PVR_Spark3_Streaming.yml | 2 +-
.../beam_PostCommit_Java_PVR_Spark_Batch.yml | 2 +-
.github/workflows/beam_PostCommit_Java_Sickbay.yml | 2 +-
...am_PostCommit_Java_ValidatesRunner_Dataflow.yml | 4 +-
..._Java_ValidatesRunner_Dataflow_JavaVersions.yml | 4 +-
...mit_Java_ValidatesRunner_Dataflow_Streaming.yml | 4 +-
...PostCommit_Java_ValidatesRunner_Dataflow_V2.yml | 2 +-
..._Java_ValidatesRunner_Dataflow_V2_Streaming.yml | 2 +-
...beam_PostCommit_Java_ValidatesRunner_Direct.yml | 2 +-
...it_Java_ValidatesRunner_Direct_JavaVersions.yml | 2 +-
.../beam_PostCommit_Java_ValidatesRunner_Flink.yml | 2 +-
...ostCommit_Java_ValidatesRunner_Flink_Java11.yml | 2 +-
.../beam_PostCommit_Java_ValidatesRunner_Samza.yml | 2 +-
.../beam_PostCommit_Java_ValidatesRunner_Spark.yml | 2 +-
...va_ValidatesRunner_SparkStructuredStreaming.yml | 2 +-
...ostCommit_Java_ValidatesRunner_Spark_Java11.yml | 2 +-
...am_PostCommit_Java_ValidatesRunner_Twister2.yml | 2 +-
.../beam_PostCommit_Java_ValidatesRunner_ULR.yml | 2 +-
.github/workflows/beam_PostCommit_SQL.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Direct.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Flink.yml | 2 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 2 +-
...eam_PostCommit_XVR_JavaUsingPython_Dataflow.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Samza.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Spark3.yml | 2 +-
.github/workflows/beam_PreCommit_ItFramework.yml | 2 +-
.github/workflows/beam_PreCommit_Java.yml | 3 +-
...eCommit_Java_Amazon-Web-Services2_IO_Direct.yml | 2 +-
...reCommit_Java_Amazon-Web-Services_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Amqp_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Azure_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Cassandra_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Cdap_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Clickhouse_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Csv_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Debezium_IO_Direct.yml | 2 +-
...beam_PreCommit_Java_ElasticSearch_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Examples_Dataflow.yml | 2 +-
...eam_PreCommit_Java_Examples_Dataflow_Java21.yml | 2 +-
...Commit_Java_File-schema-transform_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Flink_Versions.yml | 2 +-
.../beam_PreCommit_Java_GCP_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Google-ads_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_HBase_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_HCatalog_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Hadoop_IO_Direct.yml | 2 +-
.../workflows/beam_PreCommit_Java_IOs_Direct.yml | 2 +-
.../beam_PreCommit_Java_InfluxDb_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_JDBC_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Jms_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Kafka_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Kinesis_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Kudu_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_MongoDb_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Mqtt_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Neo4j_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_PVR_Flink_Batch.yml | 2 +-
.../beam_PreCommit_Java_PVR_Flink_Docker.yml | 2 +-
.../beam_PreCommit_Java_Parquet_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Pulsar_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_RabbitMq_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Redis_IO_Direct.yml | 2 +-
...am_PreCommit_Java_RequestResponse_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_SingleStore_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Snowflake_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Solr_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Spark3_Versions.yml | 2 +-
.../beam_PreCommit_Java_Splunk_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Thrift_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Tika_IO_Direct.yml | 2 +-
.github/workflows/beam_PreCommit_SQL.yml | 2 +-
.github/workflows/beam_PreCommit_SQL_Java11.yml | 2 +-
.github/workflows/beam_PreCommit_SQL_Java17.yml | 2 +-
.github/workflows/go_tests.yml | 2 +-
.github/workflows/local_env_tests.yml | 4 +-
.github/workflows/python_tests.yml | 2 +-
.github/workflows/tour_of_beam_backend.yml | 2 +-
...PostCommit_Java_ValidatesRunner_Dataflow.groovy | 51 ------
...mit_Java_ValidatesRunner_Dataflow_Java11.groovy | 60 -------
...mit_Java_ValidatesRunner_Dataflow_Java17.groovy | 57 -------
..._Java_ValidatesRunner_Dataflow_Streaming.groovy | 50 ------
...tCommit_Java_ValidatesRunner_Dataflow_V2.groovy | 51 ------
...va_ValidatesRunner_Dataflow_V2_Streaming.groovy | 51 ------
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 45 -----
CHANGES.md | 1 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 10 +-
.../beam/gradle/kafka/KafkaTestUtilities.groovy | 3 +-
.../org/apache/beam/examples/KafkaStreaming.java | 67 ++++++--
.../beam/model/fn_execution/v1/beam_fn_api.proto | 34 +++-
.../job_management/v1/beam_expansion_api.proto | 5 +
.../beam/runners/core/construction/External.java | 1 +
.../fnexecution/control/RemoteExecutionTest.java | 13 +-
sdks/go.mod | 4 +-
sdks/go.sum | 8 +-
.../container/license_scripts/dep_urls_java.yaml | 2 +-
.../apache/beam/sdk/options/StreamingOptions.java | 11 ++
.../sdk/transforms/errorhandling/ErrorHandler.java | 32 +++-
.../sdk/expansion/service/ExpansionService.java | 15 +-
.../sdk/expansion/service/ExpansionServerTest.java | 15 +-
.../beam/fn/harness/state/MultimapSideInput.java | 94 +++++++++--
.../fn/harness/state/FakeBeamFnStateClient.java | 4 +
.../fn/harness/state/MultimapSideInputTest.java | 50 +++++-
.../beam/sdk/io/gcp/spanner/BatchSpannerRead.java | 3 +-
.../beam/sdk/io/gcp/spanner/NaiveSpannerRead.java | 5 +
.../beam/sdk/io/gcp/spanner/ReadOperation.java | 5 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 6 +
sdks/java/io/kafka/kafka-01103/build.gradle | 1 +
sdks/java/io/kafka/kafka-100/build.gradle | 3 +-
sdks/java/io/kafka/kafka-111/build.gradle | 1 +
sdks/java/io/kafka/kafka-201/build.gradle | 1 +
sdks/java/io/kafka/kafka-211/build.gradle | 1 +
sdks/java/io/kafka/kafka-222/build.gradle | 1 +
sdks/java/io/kafka/kafka-231/build.gradle | 1 +
sdks/java/io/kafka/kafka-241/build.gradle | 1 +
sdks/java/io/kafka/kafka-251/build.gradle | 1 +
sdks/java/io/kafka/kafka-integration-test.gradle | 2 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 127 ++++++++++++--
.../KafkaIOReadImplementationCompatibility.java | 1 +
.../org/apache/beam/sdk/io/kafka/KafkaWriter.java | 46 +++--
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 110 +++++++-----
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 8 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 121 ++++++++++---
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 114 +++++++++++--
.../beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java | 168 +++++++++++++++---
.../ml/inference/huggingface_inference.py | 15 ++
.../apache_beam/ml/inference/onnx_inference.py | 18 ++
.../apache_beam/ml/inference/pytorch_inference.py | 10 ++
.../apache_beam/ml/inference/sklearn_inference.py | 10 ++
.../ml/inference/tensorflow_inference.py | 10 ++
.../apache_beam/ml/inference/tensorrt_inference.py | 7 +-
.../ml/inference/vertex_ai_inference.py | 22 ++-
.../apache_beam/ml/inference/xgboost_inference.py | 21 +++
.../python/apache_beam/options/pipeline_options.py | 43 +++++
sdks/python/apache_beam/pipeline_test.py | 8 +-
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
.../runners/portability/expansion_service.py | 15 +-
.../runners/portability/fn_api_runner/execution.py | 21 ++-
.../portability/fn_api_runner/worker_handlers.py | 53 ++++--
.../apache_beam/runners/worker/bundle_processor.py | 49 +++++-
sdks/python/apache_beam/transforms/external.py | 38 +++--
sdks/python/apache_beam/transforms/ptransform.py | 8 +-
sdks/python/apache_beam/transforms/util.py | 29 +++-
sdks/python/apache_beam/typehints/schemas.py | 14 ++
sdks/python/apache_beam/yaml/generate_yaml_docs.py | 188 +++++++++++++++++++++
sdks/python/apache_beam/yaml/standard_io.yaml | 3 +-
sdks/python/apache_beam/yaml/yaml_io.py | 5 +-
sdks/python/apache_beam/yaml/yaml_mapping.py | 22 +++
sdks/python/apache_beam/yaml/yaml_provider.py | 100 ++++++-----
.../container/base_image_requirements_manual.txt | 3 +-
.../container/py310/base_image_requirements.txt | 24 +--
.../container/py311/base_image_requirements.txt | 25 +--
.../container/py38/base_image_requirements.txt | 26 +--
.../container/py39/base_image_requirements.txt | 26 +--
sdks/python/setup.py | 4 +
.../content/en/documentation/runners/direct.md | 4 +
.../site/content/en/documentation/sdks/python.md | 2 +-
185 files changed, 1692 insertions(+), 815 deletions(-)
delete mode 100644 .test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy
delete mode 100644 .test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy
delete mode 100644 .test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java17.groovy
delete mode 100644 .test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.groovy
delete mode 100644 .test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_V2.groovy
delete mode 100644 .test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.groovy
delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
create mode 100644 sdks/python/apache_beam/yaml/generate_yaml_docs.py
mode change 100644 => 100755 sdks/python/apache_beam/yaml/yaml_provider.py