You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2024/02/22 03:49:43 UTC
(beam) branch nightly-refs/heads/master updated (67f930048d3 -> 384c1034cd5)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 67f930048d3 Bump github.com/docker/docker in /sdks (#30308)
add b78a3e1790f fix flaky StreamingEngineClientTest (#30322)
add 6a8c27ef6fb Feature/add error handling for bqio (#30081)
add 9d2fea44b59 Fix XVR test exclusions (#30354)
add 12eee95e741 Bump timeout of PostCommit Java PVR Spark3 Streaming to 3h (#30356)
add bd7ed76281a Add TensorflowHub embeddings to MLTransform (#30289)
add b7b19cce7ae Add some tolerance for skippped images in cleaner (#30294)
add fd886f4fa74 Add the ability to test examples from yaml docs.
add ea1210c1be9 Merge pull request #30198 Add the ability to test examples from yaml docs.
add e341b34ecf0 [YAML] Documentation redirects.
add 5930886acad Merge pull request #30268 [YAML] Documentation redirects.
add 4a407d36236 Also export AsMultiMap view. (#30370)
add 5ec64464e15 Make state cache optional due to a potential performance regression for some users. (#30362)
add 384c1034cd5 add bqio changes to CHANGES.md (#30369)
No new revisions were added by this update.
Summary of changes:
.../trigger_files/beam_PostCommit_Java.json | 0
.../beam_PostCommit_Java_DataflowV1.json | 0
.../beam_PostCommit_Java_PVR_Spark3_Streaming.json | 0
.../beam_PostCommit_Java_PVR_Spark_Batch.json | 0
...ostCommit_Java_ValidatesRunner_Dataflow_V2.json | 0
...Java_ValidatesRunner_Dataflow_V2_Streaming.json | 0
...stCommit_Java_ValidatesRunner_Flink_Java11.json | 0
...a_ValidatesRunner_SparkStructuredStreaming.json | 0
...m_PostCommit_Java_ValidatesRunner_Twister2.json | 0
.../beam_PostCommit_Java_ValidatesRunner_ULR.json | 0
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.json | 0
.../trigger_files/beam_PostCommit_XVR_Flink.json | 0
.../beam_PostCommit_Java_PVR_Spark3_Streaming.yml | 2 +-
.gitignore | 1 +
.../tools/stale_dataflow_prebuilt_image_cleaner.sh | 13 +-
CHANGES.md | 13 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 3 +
runners/direct-java/build.gradle | 6 +-
runners/flink/flink_runner.gradle | 1 +
runners/flink/job-server/flink_job_server.gradle | 3 +-
runners/google-cloud-dataflow-java/build.gradle | 4 +-
.../beam/runners/dataflow/DataflowRunner.java | 6 +-
.../client/grpc/StreamingEngineClient.java | 19 +-
.../client/grpc/StreamingEngineClientTest.java | 115 +++--
runners/jet/build.gradle | 1 +
runners/portability/java/build.gradle | 1 +
runners/samza/build.gradle | 1 +
runners/samza/job-server/build.gradle | 3 +-
runners/spark/job-server/spark_job_server.gradle | 2 +
runners/spark/spark_runner.gradle | 9 +-
runners/twister2/build.gradle | 1 +
.../{NeedsRunner.java => UsesExternalService.java} | 8 +-
.../beam/sdk/testing/UsesJavaExpansionService.java | 2 +-
.../sdk/testing/UsesPythonExpansionService.java | 2 +-
.../errorhandling/ErrorHandlingTestUtils.java | 9 +
.../beam/sdk/io/gcp/bigquery/AvroRowWriter.java | 10 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 153 ++++--
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 358 +++++++++++--
.../sdk/io/gcp/bigquery/BigQueryIOTranslation.java | 23 +
.../sdk/io/gcp/bigquery/BigQueryRowWriter.java | 9 +-
.../BigQueryStorageStreamBundleSource.java | 21 +-
.../gcp/bigquery/BigQueryStorageStreamSource.java | 14 +-
.../io/gcp/bigquery/StorageApiConvertMessages.java | 43 +-
.../beam/sdk/io/gcp/bigquery/StorageApiLoads.java | 86 +++-
.../beam/sdk/io/gcp/bigquery/TableRowWriter.java | 10 +-
.../sdk/io/gcp/bigquery/WriteBundlesToFiles.java | 46 +-
.../gcp/bigquery/WriteGroupedRecordsToFiles.java | 41 +-
.../io/gcp/bigquery/BigQueryIOStorageQueryIT.java | 46 ++
.../gcp/bigquery/BigQueryIOStorageQueryTest.java | 80 ++-
.../io/gcp/bigquery/BigQueryIOStorageReadIT.java | 55 ++
.../io/gcp/bigquery/BigQueryIOTranslationTest.java | 4 +
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 240 +++++++++
.../ml/transforms/embeddings/tensorflow_hub.py | 134 +++++
.../transforms/embeddings/tensorflow_hub_test.py | 176 +++++++
.../python/apache_beam/options/pipeline_options.py | 16 +-
sdks/python/apache_beam/pvalue.py | 1 +
sdks/python/apache_beam/yaml/README.md | 554 +--------------------
sdks/python/apache_beam/yaml/inline_python.md | 185 +------
sdks/python/apache_beam/yaml/readme_test.py | 17 +-
sdks/python/apache_beam/yaml/yaml_combine.md | 149 +-----
sdks/python/apache_beam/yaml/yaml_errors.md | 181 +------
sdks/python/apache_beam/yaml/yaml_mapping.md | 229 +--------
sdks/python/apache_beam/yaml/yaml_mapping_test.py | 6 -
sdks/python/scripts/generate_pydoc.sh | 2 +-
sdks/python/setup.py | 86 ++--
sdks/python/test-suites/tox/py38/build.gradle | 10 +
sdks/python/tox.ini | 14 +
.../en/documentation/sdks/yaml-inline-python.md | 185 +------
68 files changed, 1640 insertions(+), 1769 deletions(-)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Java.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Java_DataflowV1.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Twister2.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_ULR.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json (100%)
copy learning/tour-of-beam/learning-content/io/text-io/text-io-local-write/go-example/myfile.txt => .github/trigger_files/beam_PostCommit_XVR_Flink.json (100%)
copy sdks/java/core/src/main/java/org/apache/beam/sdk/testing/{NeedsRunner.java => UsesExternalService.java} (78%)
create mode 100644 sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py
create mode 100644 sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py