You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2024/08/15 09:47:48 UTC
(beam) branch nightly-refs/heads/master updated (ab81e1fc5e9 -> 1eb416e169d)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from ab81e1fc5e9 Added a data corruption known issue to CHANGES.md and release blogs. (#32166)
add 8ff7f0d75e4 Bump github.com/docker/docker in /sdks (#32176)
add b0f2683cda1 GitHub issue #30257
add 7a4850dcdae Merge pull request #32074: Fixes GitHub issue #30257
add c23e60383bc Add Lineage metrics to Python BigQueryIO (#32116)
add 8fbad485688 Change FnApiDoFnRunner to skip trySplit checkpoint requests if not draining and nothing has yet been claimed by the tracker. (#32044)
add dbd719ba144 [WIP] Gemma Sentiment and Summarization Example Notebook (#32172)
add eaa3b56dbb1 Bump github.com/nats-io/nats.go from 1.36.0 to 1.37.0 in /sdks (#32174)
add 2c492434773 Add Lineage metrics to FileSystems (#32090)
add 3063b557575 Bump to Java11 for GHA runner (#32138)
add 7af43ee4a4a Fix PostCommit Python sklearn on Python3.12 (#32171)
add f1f5f775252 [prism] Skip python tests that require an expansion service at this time. (#32182)
add 1eb416e169d Fix verifyJavaVersion in a few places (#32183)
No new revisions were added by this update.
Summary of changes:
.../actions/setup-environment-action/action.yml | 2 +-
.../arc/images/Dockerfile | 16 +-
.github/trigger_files/beam_PostCommit_Python.json | 3 +-
.github/workflows/README.md | 18 +-
... beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml} | 54 +-
...m_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml} | 52 +-
...beam_PostCommit_Java_Examples_Dataflow_Java.yml | 4 +-
...m_PostCommit_Java_Examples_Dataflow_V2_Java.yml | 4 +-
..._Java_ValidatesRunner_Dataflow_JavaVersions.yml | 6 +-
...it_Java_ValidatesRunner_Direct_JavaVersions.yml | 6 +-
...ostCommit_Java_ValidatesRunner_Flink_Java8.yml} | 21 +-
...ostCommit_Java_ValidatesRunner_Spark_Java8.yml} | 21 +-
.github/workflows/beam_PreCommit_SQL.yml | 2 +-
...SQL_Java11.yml => beam_PreCommit_SQL_Java8.yml} | 22 +-
..._GBK_Dataflow_V2_Batch_2GB_of_100B_records.txt} | 0
...GBK_Dataflow_V2_Batch_2GB_of_100kB_records.txt} | 0
...a_GBK_Dataflow_V2_Batch_2GB_of_10B_records.txt} | 0
...out_4_times_with_2GB_10-byte_records_total.txt} | 0
...out_8_times_with_2GB_10-byte_records_total.txt} | 0
...low_V2_Batch_reiterate_4_times_10kB_values.txt} | 0
...flow_V2_Batch_reiterate_4_times_2MB_values.txt} | 0
..._Dataflow_V2_Streaming_2GB_of_100B_records.txt} | 0
...Dataflow_V2_Streaming_2GB_of_100kB_records.txt} | 0
...K_Dataflow_V2_Streaming_2GB_of_10B_records.txt} | 0
...out_4_times_with_2GB_10-byte_records_total.txt} | 0
...out_8_times_with_2GB_10-byte_records_total.txt} | 0
...V2_Streaming_reiterate_4_times_10kB_values.txt} | 0
..._V2_Streaming_reiterate_4_times_2MB_values.txt} | 0
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 213 ++++---
.../gemma_2_sentiment_and_summarization.ipynb | 625 +++++++++++++++++++++
.../beam/runners/direct/DirectRunnerTest.java | 2 +-
runners/google-cloud-dataflow-java/build.gradle | 3 +
.../examples/build.gradle | 3 +
sdks/go.mod | 5 +-
sdks/go.sum | 10 +-
sdks/java/container/common.gradle | 5 +-
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 1 +
.../org/apache/beam/sdk/io/FileBasedSource.java | 1 +
.../java/org/apache/beam/sdk/io/FileSystem.java | 8 +
.../java/org/apache/beam/sdk/io/FileSystems.java | 11 +
.../sdk/io/ReadAllViaFileBasedSourceTransform.java | 5 +-
.../java/org/apache/beam/sdk/metrics/Lineage.java | 97 +++-
.../sdk/transforms/SerializableComparator.java | 21 +-
.../org/apache/beam/sdk/metrics/LineageTest.java | 59 ++
.../sdk/transforms/SerializableComparatorTest.java | 63 +++
.../sdk/extensions/gcp/storage/GcsFileSystem.java | 11 +
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 57 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 465 ++++++++++++++-
.../apache/beam/sdk/io/aws/s3/S3FileSystem.java | 6 +
.../apache/beam/sdk/io/aws2/s3/S3FileSystem.java | 6 +
.../azure/blobstore/AzureBlobStoreFileSystem.java | 13 +
.../java/org/apache/beam/sdk/io/text/TextIOIT.java | 5 +
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 5 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 3 +-
.../io/gcp/bigquery/BigQueryStorageSourceBase.java | 7 +-
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 3 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 3 +-
.../bigquery/StorageApiWritesShardedRecords.java | 3 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 3 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 3 +-
.../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 6 +-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 15 +-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 7 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 7 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSource.java | 4 +-
.../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +-
.../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +-
.../beam/sdk/io/gcp/pubsub/PubsubClientTest.java | 5 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 4 +-
sdks/java/testing/test-utils/build.gradle | 10 +-
.../testutils/jvmverification/JvmVerification.java | 6 +
.../inference/sklearn_examples_requirements.txt | 7 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 11 +
.../apache_beam/io/gcp/bigquery_file_loads.py | 12 +
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 10 +
.../apache_beam/io/gcp/bigquery_read_internal.py | 7 +
.../io/gcp/bigquery_schema_tools_test.py | 18 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 43 ++
sdks/python/apache_beam/metrics/metric.py | 83 ++-
sdks/python/apache_beam/metrics/metric_test.py | 22 +
.../runners/portability/prism_runner_test.py | 77 +--
81 files changed, 1894 insertions(+), 425 deletions(-)
rename .github/workflows/{beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml => beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml} (78%)
rename .github/workflows/{beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml => beam_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml} (79%)
rename .github/workflows/{beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml => beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml} (89%)
rename .github/workflows/{beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml => beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml} (89%)
rename .github/workflows/{beam_PreCommit_SQL_Java11.yml => beam_PreCommit_SQL_Java8.yml} (89%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt => java_GBK_Dataflow_V2_Batch_2GB_of_100B_records.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt => java_GBK_Dataflow_V2_Batch_2GB_of_100kB_records.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt => java_GBK_Dataflow_V2_Batch_2GB_of_10B_records.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt => java_GBK_Dataflow_V2_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt => java_GBK_Dataflow_V2_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt => java_GBK_Dataflow_V2_Batch_reiterate_4_times_10kB_values.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt => java_GBK_Dataflow_V2_Batch_reiterate_4_times_2MB_values.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt => java_GBK_Dataflow_V2_Streaming_2GB_of_100B_records.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt => java_GBK_Dataflow_V2_Streaming_2GB_of_100kB_records.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt => java_GBK_Dataflow_V2_Streaming_2GB_of_10B_records.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt => java_GBK_Dataflow_V2_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt => java_GBK_Dataflow_V2_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt => java_GBK_Dataflow_V2_Streaming_reiterate_4_times_10kB_values.txt} (100%)
rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt => java_GBK_Dataflow_V2_Streaming_reiterate_4_times_2MB_values.txt} (100%)
create mode 100644 examples/notebooks/beam-ml/gemma_2_sentiment_and_summarization.ipynb
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/metrics/LineageTest.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/SerializableComparatorTest.java