You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2024/03/19 21:39:58 UTC
(beam) branch weekly_update_python_dependencies_1708820977 updated (3b0a585ccb9 -> fb7ba65e223)
This is an automated email from the ASF dual-hosted git repository.
damondouglas pushed a change to branch weekly_update_python_dependencies_1708820977
in repository https://gitbox.apache.org/repos/asf/beam.git
discard 3b0a585ccb9 Update Python Dependencies
add 436f3b223f0 [runners-core-java] restore original SerializablePipelineOptions
add 4982f6ac6e4 [sdks-java-core] add serialVersionUID to PTransform
add 4f13a4e1ea2 [runners-core] add jackson
add 907a4514386 Merge pull request #30403: [flink] #30402 restore upgradability
add d5a2e3b3c9b Bump github.com/aws/aws-sdk-go-v2/config from 1.26.2 to 1.27.4 in /sdks (#30415)
add ffe2dba5320 Implementing lull reporting at bundle level processing (#29882)
add 6406cfe3c9b Remove some uses of ClassLoadingStrategy.Default.INJECTION (#30367)
add 2ae4a28e6fc Force downgrade mpmath to avoid test breakages (#30418)
add 6c3e8ad1af1 [Dataflow Streaming] Start to refactor persistence layer to prepare for direct path (#30265)
add 2a84a20236b [Python] Add doc comment for WriteToText.skip_if_empty (#30409)
add b7a58bf8b2d Ensure flatten windows match (#30410)
add 8fd24b76a19 Bump google.golang.org/grpc from 1.61.0 to 1.62.0 in /sdks (#30434)
add 0a184f449a5 Bump orjson from 3.9.14 to 3.9.15 in /sdks/python/container/py310 (#30426)
add a1d87549f78 Make Monitoring build method public (#30431)
add 5af76b0de1a Remove unused code (#30432)
add 549faba9b98 Roll forward "Read API Source v2 (#25392)" fix data loss (#28778)
add 2b9d958807d Wire error handling into PubSubIO and add initial tests (#30372)
add 2bebb27a64f Fix Python PostCommit BigQuery JSON (#30438)
add 183cc9a2962 [yaml] fix providers doc
add 61c6d09b153 Merge pull request #30433 [yaml] Fix providers doc.
add 09a6fc5c5f3 Fix updating of annotations for explicitly named transforms.
add e6e91b99542 Expand on comment.
add 80d4c852f36 Merge pull request #30380 Fix updating of annotations for explicitly named transforms.
add 2d85be72f31 Fix Avro Version PostCommit
add fbde0ce32c9 Merge pull request #30442: Fix Avro Version PostCommit
add 4f966a85884 Add yaml to programming guide. (#30269)
add 53cae786126 Revise documentation on managing dependencies. (#30450)
add c5e1d106cb5 Support passing credentials from pipeline options into SpannerIO.readChangeStream (#30361)
add 4ea3898e589 Bump cryptography from 42.0.3 to 42.0.4 in /sdks/python/container/py39 (#30373)
add aebb34a94c3 [Dataflow] ActiveWorkRefresh (#30390)
add a20705c20d1 Fix issue creation for grafana alerts (#30424)
add 6cae6b874a5 make the flaky_test_detection script read only be default (#30461)
add 168d06bade5 [Go SDK] filter out TestTestStreamInt16Sequence from flink VR tests. (#30462)
add 935ca9c58d6 fix alerting path for grafana provisioning (#30464)
add be1526b7b57 Register streaming specific options with DataflowPipelineRegistrar (#30474)
add 770c6fe39cc Make windmill service stream max backoff configurable (#30475)
add 4ad8d530916 Add last error time to stream error message (#30476)
add e1fb9ce99a8 Unskip test_user_agent_insert_all (#30455)
add 15d16f706f3 Add instructions on reviewing Python version updates (#30481)
add 2b8a737ebd0 Add test code to overwrite SQL in Beam Python JDBC (#30417)
add 9c3f209aa69 Initial prototype of using Beam transforms directly in a Flink pipeline. (#30332)
add fb58cfc0cad Duet AI data encoding prompts (no links) (#30420)
add ddb9161c412 Update Python Dependencies (#30470)
add 264493bd2d7 adding github workflow for running nightly playground ci (#30454)
add 0e3e796a15e Move credential rotation and gate on beam repo (#30479)
add b776d705997 Duet AI ML data processing prompts (no links) (#30421)
add 6a03f9ba062 Update triggering frequency doc (#30457)
add fa43f82f8a2 Disable BigQueryStorageStreamSource.splitAtFraction when read api v2 used (#30443)
add 6f8f12acd03 Cleanup bigtable open resources on teardown (#30453)
add 94a51619f3d Make the TransformService test suite not pass trivially.
add 1c74cebda6c Merge pull request #30491: Make the TransformService test suite not pass trivially.
add 04c7be3e18e Fix grafana url for alerts and GA Post-Commits Status dashboard (#30468)
add 9ec2d49af98 Duet AI Prompt - Beam YAML (no links) (#30440)
add 407d5077b9f Remove JsonIgore for usePublicIps
add 89366bbb4aa Merge pull request #30484: Remove JsonIgore for usePublicIps
add 3c46415a3ce Document yaml pipeline options (#30490)
add 0440587898c Update Go Version for artifacts to Go v1.21.8 (#30501)
add 3f474535632 Expose JmsIO read receive timeout (#30485)
add b6301b52058 fix: support reading arrays of structs from bigquery with schemas (#30448)
add bc1a458bb7c Bigtable: provide a way to override BigtableDataSettings (#30489)
add 6f5b2197751 Bump cloud.google.com/go/pubsub from 1.34.0 to 1.36.2 in /sdks (#30465)
add 1c7fb4bc8a0 Fix website precommit (#30540)
add 2a10971fb6b Revert "Add test code to overwrite SQL in Beam Python JDBC (#30417)" (#30535)
add f0f1cd37dd1 Bigtable: use InstanceBuilder to dynamic load override class (#30542)
add 8d7adba289b Fix hdfs integration test (#30458)
add 314bd722eea [Python] Vertex AI Feature Store enrichment handler (#30388)
add 1a05f39883f fix playground backend container builds (#30497)
add 363731ebcb8 Add KafkaIO Stress test (#30467)
add ae9cac3c867 Reduce nullness checks in flink adapters.
add d827f7713e2 spotless again
add 77f9e3a85ee Better preconditions.
add 45d5feaad11 Merge pull request #30488 Reduce nullness checks in flink adapters.
add e9bcc37556d Moving to 2.56.0-SNAPSHOT on master branch.
add 827e96d671a Add vertex AI dependency (#30553)
add 1c55117fc33 [BEAM-30531] Automatically execute unbounded pipelines in streaming mode. (#30533)
add 96eabb0a5f5 Add local Run Inference with Gemma notebook (#30532)
add 32f38b92c36 Fixes an upgrade imcompatiblity of BQ read/write transforms
add 2f3fac6e140 Merge pull request #30551: Fixes an upgrade imcompatiblity of BQ read/write transforms
add 9849fcc42c8 Fix internal test failure from PR-30455 (#30546)
add d75210eabe0 Bump InfluxDB PostCommit timeout (#30561)
add 7497495bcf7 [Python] Log dependencies installed in submission environment (#28564)
add 9b71fadd57b Update image name to the one used by the test. (#30566)
add a18c6088457 Do not pre-install beam in tensorrt container (#30552)
add 5342a960af2 Attempt to install staged Beam SDK if one is available, before checking if Beam is installed. (#30565)
add f7fdfa7b348 Duet AI prompts: containers, hints, external calls (no links) (#30435)
add 3f8af4b61b2 Resolve Any environment types before optimization. (#30568)
add afae0215f8f Try fix Dataflow PreCommit on release branch (#30574)
add a391198b5a6 [RRIO]: Add RequestResponseIO examples and documentation on website for the Java SDK (#30430)
add 53e8efa32d7 [yaml] Add Beam YAML Examples (#30003)
add 8e7e8fee387 Add info on large_model flag in LLM page (#30585)
add 3727f4d66a4 Prefer yaml_pipeline in error messages (#30544)
add 00526dd3fa2 Document requirements cache behavior differences. (#30493)
add 31b8a9e3dfe Duet AI Transform service SDK Harness prompts (#30499)
add 022dbfaaa7e Remove Youtube Embeddings from Website
add 59c82319ce5 Merge pull request #30591 from svetakvsundhar/website_update
add a3ea9ef706c Disable remote gradle cache until it is cleaned (#30584)
add e392d7760f7 Bump up beam-master for legacy runner. (#30595)
add e3ee6d2a12c Update Python Dependencies (#30592)
add 792cecba521 Redo spark_runner exclude test (#30589)
add fedca3ce18e Add external transform example (#30597)
add d22a7e783ab Make defaults for optional SchemaTransformProvider methods (#30560)
add 35252b92c0f Duet AI documentation lookup prompts for Beam ML (#30351)
add 0c8b78a9ff0 Remove Anand, add Shunping to reviewers (#30608)
add 97f8dae92f2 Put an explanatory comment in trigger files that exist
add 9a6777ebe7e Merge pull request #30586: Put an explanatory comment in trigger files that exist
add 986f67fd92b Update blog template with known issues link to CHANGES (#30611)
add 5a9a7d2f85b Add a known issue for gcsio in the notes of 2.53.0 and 2.54.0. (#30615)
add 06c76b4e930 Updates the test suite to use the transform service (#30605)
add 17bbe8262b9 Remove MapState and SetState tests that mix v1 and v2 codepaths erroneously
add 8945b8fc144 Remove GroupIntoBatches tests that mix v1 and v2 codepaths erroneously
add bf5551bd118 Remove PubsubSink tests that conflate v1 and v2 codepaths erroneously
add 0a0186de555 Skip v1 translation steps for pipelines that will run on v2
add c79b6e157dd Merge pull request #30604: Skip running Dataflow v1 logic when a job will run on v2
add 289bbc45c56 [Python] Pin keras version for tensorflow inference test (#30614)
add 4ad189db265 Bump github.com/nats-io/nats-server/v2 from 2.10.10 to 2.10.12 in /sdks (#30620)
add e648fd08726 Rename tests to remove them from dataflow test suite (#30623)
add da26f5aa6ba Update header bannder (#30600)
add 8d8f29464fb Properly handle timestamp prefixing of unkown window types.
add 0ba35731aec Add another test.
add 0bfe06b61a4 Merge pull request #30587 Properly handle timestamp prefixing of unkown window types.
add 5697c67421a [runners-flink] #30621 use groupBy for Reshuffle in batch
add 52c0d5a3f86 Merge pull request #30622: [runners-flink] #30621 use groupBy for Reshuffle in batch
add 14d25c3da76 Fix flaky streaming dataflow tests (#30572)
add ef919e2603f Bump google.golang.org/protobuf in /.test-infra/mock-apis (#30628)
add 5b2516a7e45 Add hamcrest as an implementation dependency (#30635)
add 4498f35cec5 update containers (#30636)
add dbbfb80232c Add translator for Flink DataStreams. (#30486)
add 49c7864f0ad add flag for direct path that reads from system properties (#30588)
add c298da550f8 Refactor commit logic out of StreamingDataflowWorker (#30312)
add 3aa78d21f70 Better error message for large elements. (#30639)
add 50c59912bc0 Revert "Implementing lull reporting at bundle level processing (#29882)" (#30648)
add 63aff7e580b Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 in /sdks (#30629)
add c3b3fa62c3a Include byte size of stateKey in estimated weight of WindmillBag, WindmillValue, and WindmillWatermarkHold (#30654)
add 50f33cd786d [Python] Check feature store existence at pipeline construction time (#30668)
add c1c255a0a43 Cache and close windmill grpc channels (#30425)
add 3631a8324f1 Support ValueProvider for _CustomBigQueryStorageSource (#30662)
add 764fcda4e9c update confluent dependency version to 7.6.0 (#30638)
add 71cdde5c5a6 Update Python Dependencies (#30656)
add 4513eaacc04 Use 50 workers for 2GB 10 bytes combine test (#30655)
add a3e5ac86eea Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#30669)
add fb7ba65e223 Revert "Disable remote gradle cache until it is cleaned (#30584)" (#30674)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (3b0a585ccb9)
\
N -- N -- N refs/heads/weekly_update_python_dependencies_1708820977 (fb7ba65e223)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 +
.github/REVIEWERS.yml | 5 +-
.github/trigger_files/beam_PostCommit_Java.json | 0
.../beam_PostCommit_Java_Avro_Versions.json | 3 +
.../beam_PostCommit_Java_DataflowV1.json | 3 +
.../beam_PostCommit_Java_PVR_Spark3_Streaming.json | 3 +
.../beam_PostCommit_Java_PVR_Spark_Batch.json | 3 +
...ostCommit_Java_ValidatesRunner_Dataflow_V2.json | 3 +
...Java_ValidatesRunner_Dataflow_V2_Streaming.json | 3 +
...stCommit_Java_ValidatesRunner_Flink_Java11.json | 3 +
...a_ValidatesRunner_SparkStructuredStreaming.json | 3 +
...m_PostCommit_Java_ValidatesRunner_Twister2.json | 3 +
.../beam_PostCommit_Java_ValidatesRunner_ULR.json | 3 +
.../beam_PostCommit_Python_Examples_Direct.json | 3 +
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.json | 3 +
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 3 +
.../beam_PostCommit_Python_Xlang_IO_Dataflow.json | 3 +
.../beam_PostCommit_TransformService_Direct.json | 3 +
.../trigger_files/beam_PostCommit_XVR_Flink.json | 3 +
.github/workflows/README.md | 1 +
...LoadTests_Python_Combine_Dataflow_Streaming.yml | 2 +-
.../workflows/beam_MetricsCredentialsRotation.yml | 6 +-
.github/workflows/beam_Playground_CI_Nightly.yml | 103 +
.../beam_PostCommit_Java_InfluxDbIO_IT.yml | 2 +-
.github/workflows/flaky_test_detection.yml | 7 +-
...bine_Dataflow_Streaming_2GB_10_byte_records.txt | 2 +-
.github/workflows/update_python_dependencies.yml | 2 +-
.../GA-Post-Commits_status_dashboard.json | 40 +-
.../grafana/provisioning/alerting/flaky_test.yaml | 85 +
.../grafana/provisioning/alerts/flaky_test.yaml | 149 --
.../sync/github/github_runs_prefetcher/README.md | 2 +-
.../github/github_runs_prefetcher/code/main.py | 114 +-
.../sync/github/github_runs_prefetcher/function.tf | 2 +-
.test-infra/mock-apis/go.mod | 2 +-
.test-infra/mock-apis/go.sum | 4 +-
.test-infra/tools/flaky_test_detection.py | 40 +-
.test-infra/tools/requirements.txt | 17 +
CHANGES.md | 10 +
build.gradle.kts | 3 +-
buildSrc/build.gradle.kts | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 12 +-
contributor-docs/release-guide.md | 2 +
examples/java/webapis/build.gradle | 51 +
.../beam/examples/webapis/AdditionalSnippets.java | 50 +
.../beam/examples/webapis/GeminiAIClient.java | 126 ++
.../beam/examples/webapis/GeminiAIExample.java | 183 ++
.../beam/examples/webapis/GeminiAIOptions.java | 35 +-
.../webapis/GenerateContentRequestCoder.java | 48 +
.../webapis/GenerateContentResponseCoder.java | 44 +
.../beam/examples/webapis/HttpImageClient.java | 111 +
.../apache/beam/examples/webapis/ImageRequest.java | 78 +
.../beam/examples/webapis/ImageRequestCoder.java | 48 +
.../beam/examples/webapis/ImageResponse.java | 53 +
.../beam/examples/webapis/ImageResponseCoder.java | 56 +
.../org/apache/beam/examples/webapis/Images.java | 96 +
.../java/org/apache/beam/examples/webapis/Log.java | 56 +
.../examples/webapis/UsingHttpClientExample.java | 68 +
.../beam/examples/webapis/GeminiAIExampleTest.java | 52 +
.../webapis/UsingHttpClientExampleTest.java | 46 +
.../notebooks/beam-ml/run_inference_gemma.ipynb | 1132 ++++++++++
gradle.properties | 4 +-
it/google-cloud-platform/build.gradle | 3 +-
it/kafka/build.gradle | 6 +
.../java/org/apache/beam/it/kafka/KafkaIOST.java | 480 +++++
learning/prompts/README.md | 6 +-
learning/prompts/code-explanation/01_io_kafka.md | 9 +-
learning/prompts/code-explanation/02_io_pubsub.md | 9 +-
.../prompts/code-explanation/03_io_bigquery.md | 9 +-
.../prompts/code-explanation/04_io_bigtable.md | 8 +-
learning/prompts/code-explanation/05_io_spanner.md | 12 +-
.../prompts/code-explanation/06_io_tfrecord.md | 8 +-
learning/prompts/code-explanation/07_io_json.md | 6 +-
learning/prompts/code-explanation/08_io_csv.md | 5 +-
learning/prompts/code-explanation/09_io_avro.md | 7 +-
learning/prompts/code-explanation/10_io_parquet.md | 7 +-
learning/prompts/code-explanation/11_io_jdbc.md | 6 +-
learning/prompts/code-generation/01_io_kafka.md | 4 +-
learning/prompts/code-generation/02_io_pubsub.md | 17 +-
learning/prompts/code-generation/03_io_bigquery.md | 7 +-
learning/prompts/code-generation/04_io_bigtable.md | 8 +-
learning/prompts/code-generation/05_io_spanner.md | 5 +-
learning/prompts/code-generation/06_io_tfrecord.md | 4 +-
learning/prompts/code-generation/07_io_json.md | 7 +-
learning/prompts/code-generation/08_io_csv.md | 6 +-
learning/prompts/code-generation/09_io_avro.md | 5 +-
learning/prompts/code-generation/10_io_parquet.md | 6 +-
learning/prompts/code-generation/11_io_jdbc.md | 7 +-
.../32_data_encoding.md | 15 +
.../33_ai_ml_pipelines.md | 22 +
.../33_coders_data_encoding.md | 35 +
.../34_change_coders_data_encoding.md | 103 +
.../34_ml_data_preprocessing.md | 87 +
.../35_ml_data_exploration.md | 57 +
.../36_ai_kfp_orchestration.md | 51 +
.../37_ai_tfx_orchestration.md | 20 +
.../38_ai_runinference.md | 37 +
.../39_ai_runinference_metrics.md | 38 +
.../40_ai_bigquery_ml.md | 48 +
.../41_ai_model_refresh.md | 42 +
.../42_ai_custom_inference.md | 50 +
.../43_ai_llm_inference.md | 60 +
.../47_batching_for_external_calls.md | 71 +
.../documentation-lookup-nolinks/47_mltransform.md | 72 +
.../48_resource_hints.md | 65 +
.../documentation-lookup-nolinks/48_sdk_harness.md | 17 +
.../49_container_environments.md | 121 ++
.../49_transform_service.md | 34 +
.../documentation-lookup-nolinks/50_beam_yaml.md | 72 +
.../documentation-lookup/33_ai_ml_pipelines.md | 15 +
.../36_ai_kfp_orchestration.md | 51 +
.../37_ai_tfx_orchestration.md | 20 +
.../documentation-lookup/38_ai_runinference.md | 39 +
.../39_ai_runinference_metrics.md | 39 +
.../documentation-lookup/40_ai_bigquery_ml.md | 52 +
.../documentation-lookup/41_ai_model_refresh.md | 42 +
.../documentation-lookup/42_ai_custom_inference.md | 50 +
.../documentation-lookup/43_ai_llm_inference.md | 60 +
.../prompts/documentation-lookup/48_sdk_harness.md | 19 +
.../documentation-lookup/49_transform_service.md | 34 +
playground/backend/containers/java/Dockerfile | 2 +-
playground/backend/containers/python/Dockerfile | 4 +-
runners/core-java/build.gradle | 2 +
.../construction/SerializablePipelineOptions.java | 2 +-
.../runners/core/construction/package-info.java} | 29 +-
.../SerializablePipelineOptionsTest.java | 2 +-
runners/flink/flink_runner.gradle | 2 -
.../FlinkBatchPortablePipelineTranslator.java | 18 +-
.../flink/FlinkBatchTransformTranslators.java | 17 +-
.../runners/flink/FlinkExecutionEnvironments.java | 2 +-
.../beam/runners/flink/FlinkPipelineRunner.java | 28 +-
.../flink/FlinkPortablePipelineTranslator.java | 29 +
.../FlinkStreamingPortablePipelineTranslator.java | 24 +-
.../flink/FlinkStreamingTransformTranslators.java | 2 +-
.../flink/adapter/BeamAdapterCoderUtils.java | 83 +
.../runners/flink/adapter/BeamAdapterUtils.java | 147 ++
.../flink/adapter/BeamFlinkDataSetAdapter.java | 263 +++
.../flink/adapter/BeamFlinkDataStreamAdapter.java | 304 +++
.../beam/runners/flink/adapter/FlinkInput.java | 105 +
.../beam/runners/flink/adapter/FlinkOutput.java | 93 +
.../beam/runners/flink/adapter/package-info.java} | 27 +-
.../translation/functions/FlinkDoFnFunction.java | 2 +-
.../functions/FlinkExecutableStageFunction.java | 2 +-
.../FlinkExecutableStagePruningFunction.java | 2 +-
.../FlinkMergingNonShuffleReduceFunction.java | 2 +-
.../functions/FlinkMultiOutputPruningFunction.java | 2 +-
.../functions/FlinkPartialReduceFunction.java | 2 +-
.../translation/functions/FlinkReduceFunction.java | 2 +-
.../functions/FlinkStatefulDoFnFunction.java | 2 +-
.../translation/types/CoderTypeInformation.java | 2 +-
.../translation/types/CoderTypeSerializer.java | 2 +-
.../translation/wrappers/SourceInputFormat.java | 2 +-
.../wrappers/streaming/DoFnOperator.java | 2 +-
.../streaming/ExecutableStageDoFnOperator.java | 2 +-
.../streaming/KvToByteBufferKeySelector.java | 2 +-
.../streaming/SdfByteBufferKeySelector.java | 2 +-
.../wrappers/streaming/WorkItemKeySelector.java | 2 +-
.../wrappers/streaming/io/DedupingOperator.java | 2 +-
.../streaming/io/UnboundedSourceWrapper.java | 2 +-
.../wrappers/streaming/io/source/FlinkSource.java | 2 +-
.../io/source/bounded/FlinkBoundedSource.java | 2 +-
.../io/source/unbounded/FlinkUnboundedSource.java | 2 +-
.../streaming/stableinput/BufferingDoFnRunner.java | 2 +-
.../state/FlinkBroadcastStateInternals.java | 2 +-
.../streaming/state/FlinkStateInternals.java | 2 +-
.../runners/flink/FlinkPipelineOptionsTest.java | 2 +-
.../flink/adapter/BeamFlinkDataSetAdapterTest.java | 135 ++
.../adapter/BeamFlinkDataStreamAdapterTest.java | 221 ++
.../FlinkBroadcastStateInternalsTest.java | 2 +-
.../flink/streaming/FlinkStateInternalsTest.java | 2 +-
.../translation/types/CoderTypeSerializerTest.java | 2 +-
.../wrappers/streaming/DoFnOperatorTest.java | 2 +-
.../streaming/ExecutableStageDoFnOperatorTest.java | 2 +-
.../wrappers/streaming/WindowDoFnOperatorTest.java | 2 +-
.../stableinput/BufferingDoFnRunnerTest.java | 2 +-
runners/google-cloud-dataflow-java/build.gradle | 4 +-
.../examples/build.gradle | 6 +-
.../dataflow/DataflowPipelineTranslator.java | 6 +-
.../beam/runners/dataflow/DataflowRunner.java | 14 +-
.../options/DataflowPipelineDebugOptions.java | 111 -
.../dataflow/options/DataflowPipelineOptions.java | 1 +
.../options/DataflowPipelineWorkerPoolOptions.java | 2 -
.../options/DataflowStreamingPipelineOptions.java} | 148 +-
.../dataflow/DataflowPipelineTranslatorTest.java | 105 -
.../beam/runners/dataflow/DataflowRunnerTest.java | 29 +-
.../google-cloud-dataflow-java/worker/build.gradle | 12 +
.../dataflow/worker/StreamingDataflowWorker.java | 596 ++----
.../dataflow/worker/WindmillComputationKey.java | 5 +
.../worker/streaming/sideinput/SideInputCache.java | 4 +-
.../streaming/sideinput/SideInputStateFetcher.java | 25 +-
.../worker/windmill/WindmillConnection.java | 5 +
.../windmill/client/AbstractWindmillStream.java | 18 +-
.../client/CloseableStream.java} | 30 +-
.../worker/windmill/client/WindmillStreamPool.java | 7 +
.../client/commits}/Commit.java | 10 +-
.../windmill/client/commits/CompleteCommit.java | 67 +
.../commits/StreamingApplianceWorkCommitter.java | 167 ++
.../commits/StreamingEngineWorkCommitter.java | 233 +++
.../windmill/client/commits/WorkCommitter.java | 54 +
.../windmill/client/grpc/ChannelzServlet.java | 6 +-
.../windmill/client/grpc/GrpcDispatcherClient.java | 4 +-
.../windmill/client/grpc/GrpcWindmillServer.java | 52 +-
.../client/grpc/StreamingEngineClient.java | 25 +-
.../windmill/client/grpc/stubs/ChannelCache.java | 117 ++
...y.java => ChannelCachingRemoteStubFactory.java} | 38 +-
.../grpc/stubs/ChannelCachingStubFactory.java} | 37 +-
.../client/grpc/stubs/IsolationChannel.java | 2 +-
.../client/grpc/stubs/WindmillChannelFactory.java | 2 +-
.../worker/windmill/state/WindmillBag.java | 2 +-
.../worker/windmill/state/WindmillStateCache.java | 35 +-
.../worker/windmill/state/WindmillStateReader.java | 31 +-
.../worker/windmill/state/WindmillValue.java | 2 +-
.../windmill/state/WindmillWatermarkHold.java | 3 +-
.../windmill/work/refresh/ActiveWorkRefresher.java | 107 +
.../work/refresh/ActiveWorkRefreshers.java | 50 +
.../refresh/DispatchedActiveWorkRefresher.java | 68 +
.../worker/DataflowWorkerHarnessHelperTest.java | 9 +-
.../dataflow/worker/FakeWindmillServer.java | 32 +-
.../worker/StreamingDataflowWorkerTest.java | 48 +-
.../worker/StreamingModeExecutionContextTest.java | 2 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 3 +-
.../dataflow/worker/status/DebugCaptureTest.java | 8 +-
.../sideinput/SideInputStateFetcherTest.java | 16 +-
.../StreamingApplianceWorkCommitterTest.java | 140 ++
.../commits/StreamingEngineWorkCommitterTest.java | 308 +++
.../windmill/client/grpc/ChannelzServletTest.java | 10 +-
.../client/grpc/GrpcWindmillServerTest.java | 3 +-
.../client/grpc/StreamingEngineClientTest.java | 31 +-
.../client/grpc/WindmillStreamSenderTest.java | 3 +-
.../client/grpc/stubs/ChannelCacheTest.java | 150 ++
.../windmill/state/WindmillStateCacheTest.java | 2 +-
.../windmill/state/WindmillStateInternalsTest.java | 20 +-
.../windmill/state/WindmillStateReaderTest.java | 8 +-
.../windmill/testing/FakeWindmillStubFactory.java | 27 +-
.../budget/EvenGetWorkBudgetDistributorTest.java | 7 +-
.../work/budget/GetWorkBudgetRefresherTest.java | 84 +-
.../refresh/DispatchedActiveWorkRefresherTest.java | 269 +++
.../beam/runners/jet/JetTransformTranslators.java | 2 +-
.../beam/runners/jet/JetTranslationContext.java | 2 +-
.../runners/jet/processors/AbstractParDoP.java | 2 +-
.../runners/jet/processors/BoundedSourceP.java | 2 +-
.../apache/beam/runners/jet/processors/ParDoP.java | 2 +-
.../runners/jet/processors/StatefulParDoP.java | 2 +-
.../runners/jet/processors/UnboundedSourceP.java | 2 +-
.../beam/runners/jet/processors/WindowGroupP.java | 2 +-
.../runners/samza/adapter/BoundedSourceSystem.java | 2 +-
.../samza/adapter/UnboundedSourceSystem.java | 2 +-
.../SplittableParDoProcessKeyedElementsOp.java | 2 +-
.../runners/samza/translation/ConfigBuilder.java | 2 +-
.../translation/PipelineTranslator.java | 2 +-
.../translation/SparkSessionFactory.java | 2 +-
runners/spark/spark_runner.gradle | 2 +
.../beam/runners/spark/io/SourceDStream.java | 2 +-
.../apache/beam/runners/spark/io/SourceRDD.java | 2 +-
.../runners/spark/io/SparkUnboundedSource.java | 2 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 2 +-
.../runners/spark/stateful/StateSpecFunctions.java | 2 +-
.../spark/translation/EvaluationContext.java | 2 +-
.../spark/translation/MultiDoFnFunction.java | 2 +-
.../runners/spark/translation/SparkCombineFn.java | 2 +-
.../translation/SparkExecutableStageFunction.java | 2 +-
.../SparkGroupAlsoByWindowViaOutputBufferFn.java | 2 +-
.../spark/translation/SparkTranslationContext.java | 2 +-
.../streaming/StreamingTransformTranslator.java | 2 +-
.../spark/translation/SparkCombineFnTest.java | 2 +-
.../SparkExecutableStageFunctionTest.java | 2 +-
.../wrappers/Twister2BoundedSource.java | 2 +-
.../translators/functions/DoFnFunction.java | 2 +-
sdks/go.mod | 109 +-
sdks/go.sum | 230 +--
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/run_with_go_version.sh | 2 +-
sdks/go/test/integration/integration.go | 1 +
.../resources/beam/checkstyle/suppressions.xml | 1 +
.../transforms/SchemaTransformProvider.java | 9 +-
.../transforms/TypedSchemaTransformProvider.java | 20 +-
.../beam/sdk/schemas/utils/AutoValueUtils.java | 10 +-
.../apache/beam/sdk/schemas/utils/POJOUtils.java | 3 +-
.../org/apache/beam/sdk/transforms/PTransform.java | 3 +
.../sdk/util/construction/TransformUpgrader.java | 58 +
.../TypedSchemaTransformProviderTest.java | 24 +-
.../util/construction/TransformUpgraderTest.java | 13 +
sdks/java/extensions/avro/build.gradle | 5 +-
.../avro/schemas/utils/AvroByteBuddyUtils.java | 5 +-
.../extensions/avro/schemas/utils/AvroUtils.java | 2 +-
.../beam/sdk/io/aws2/schemas/AwsSchemaUtils.java | 4 +-
sdks/java/io/google-cloud-platform/build.gradle | 4 +-
.../AvroGenericRecordToStorageApiProto.java | 3 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 187 +-
.../sdk/io/gcp/bigquery/BigQueryIOTranslation.java | 69 +-
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 10 +-
.../io/gcp/bigquery/BigQueryStorageSourceBase.java | 57 +-
.../BigQueryStorageStreamBundleSource.java | 394 ----
.../gcp/bigquery/BigQueryStorageStreamSource.java | 5 +-
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 11 +-
...ueryStorageWriteApiSchemaTransformProvider.java | 5 -
.../io/gcp/bigtable/BigtableConfigTranslator.java | 43 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 17 +
.../BigtableReadSchemaTransformProvider.java | 10 -
.../BigtableWriteSchemaTransformProvider.java | 10 -
.../sdk/io/gcp/pubsub/PreparePubsubWriteDoFn.java | 51 +-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 205 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 15 +-
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 15 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 77 +-
.../MetadataSpannerConfigFactory.java | 6 +
.../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 1 -
.../AvroGenericRecordToStorageApiProtoTest.java | 5 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java | 2 +-
.../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 35 +
...eryIOStorageReadWithStreamBundleSourceTest.java | 2156 --------------------
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 11 +-
.../gcp/bigtable/BigtableConfigTranslatorTest.java | 116 ++
.../beam/sdk/io/gcp/pubsub/PubsubClientTest.java | 2 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 104 +
.../sdk/io/gcp/spanner/SpannerAccessorTest.java | 23 +
.../gcp/spanner/SpannerIOReadChangeStreamTest.java | 143 ++
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 25 +-
sdks/java/io/kafka/build.gradle | 9 +-
sdks/java/io/rrio/build.gradle | 2 -
.../apache/beam/io/requestresponse/ApiIOError.java | 25 +-
.../org/apache/beam/io/requestresponse/Call.java | 4 +-
.../apache/beam/io/requestresponse/Monitoring.java | 2 +-
.../org/apache/beam/io/requestresponse/Quota.java | 69 -
.../beam/io/requestresponse/RequestResponseIO.java | 53 -
.../ThrottleWithExternalResource.java | 418 ----
.../ThrottleWithoutExternalResource.java | 57 -
.../ThrottleWithExternalResourceIT.java | 186 --
.../ThrottleWithExternalResourceTest.java | 77 -
sdks/python/apache_beam/coders/coder_impl.py | 63 +-
sdks/python/apache_beam/coders/coders.py | 28 +
.../apache_beam/coders/coders_test_common.py | 10 +
sdks/python/apache_beam/io/gcp/bigquery.py | 11 +-
.../apache_beam/io/gcp/bigquery_json_it_test.py | 70 +-
.../apache_beam/io/gcp/bigquery_tools_test.py | 58 +-
.../apache_beam/io/gcp/bigtableio_it_test.py | 30 +-
.../io/hdfs_integration_test/docker-compose.yml | 4 +-
sdks/python/apache_beam/io/textio.py | 2 +
.../ml/inference/tensorflow_tests_requirements.txt | 1 +
sdks/python/apache_beam/portability/python_urns.py | 4 +
.../runners/dataflow/dataflow_runner.py | 20 +
.../runners/dataflow/dataflow_runner_test.py | 84 +-
.../runners/portability/expansion_service.py | 7 +
.../runners/portability/flink_runner_test.py | 3 +
.../runners/portability/fn_api_runner/fn_runner.py | 15 +
.../portability/fn_api_runner/fn_runner_test.py | 54 +
.../portability/fn_api_runner/translations.py | 54 +-
.../runners/portability/portable_runner_test.py | 41 +-
.../apache_beam/runners/portability/stager.py | 54 +-
.../apache_beam/runners/portability/stager_test.py | 91 +-
.../apache_beam/runners/worker/data_plane.py | 9 +
sdks/python/apache_beam/testing/test_pipeline.py | 7 +-
sdks/python/apache_beam/transforms/combiners.py | 17 +
sdks/python/apache_beam/transforms/core.py | 8 +
sdks/python/apache_beam/transforms/core_test.py | 34 +
sdks/python/apache_beam/transforms/enrichment.py | 6 +-
.../transforms/enrichment_handlers/bigtable.py | 20 +-
.../enrichment_handlers/utils.py} | 22 +-
.../enrichment_handlers/vertex_ai_feature_store.py | 331 +++
.../vertex_ai_feature_store_it_test.py | 308 +++
.../vertex_ai_feature_store_test.py | 58 +
.../vertex_ai_feature_store_tests_requirement.txt} | 7 +-
.../apache_beam/transforms/environments_test.py | 11 +
sdks/python/apache_beam/transforms/ptransform.py | 9 +
.../apache_beam/transforms/ptransform_test.py | 11 +
.../typehints/native_type_compatibility.py | 3 +
sdks/python/apache_beam/utils/timestamp.py | 17 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/apache_beam/yaml/examples/README.md | 67 +
.../python/apache_beam/yaml/examples}/__init__.py | 0
.../apache_beam/yaml/examples/examples_test.py | 154 ++
.../apache_beam/yaml/examples/regex_matches.yaml | 62 +
.../aggregation/combine_count_minimal.yaml | 62 +
.../aggregation/combine_max_minimal.yaml} | 38 +-
.../aggregation/combine_mean_minimal.yaml} | 38 +-
.../aggregation/combine_min_minimal.yaml} | 38 +-
.../aggregation/combine_multiple_aggregations.yaml | 83 +
.../transforms/aggregation/combine_sum.yaml | 67 +
.../aggregation/combine_sum_minimal.yaml} | 38 +-
.../transforms/aggregation/group_into_batches.yaml | 67 +
.../aggregation/top_largest_per_key.yaml | 57 +
.../aggregation/top_smallest_per_key.yaml | 59 +
.../examples/transforms/elementwise/explode.yaml | 41 +
.../transforms/elementwise/filter_callable.yaml | 52 +
.../elementwise/map_to_fields_callable.yaml} | 31 +-
.../yaml/examples/wordcount_minimal.yaml | 87 +
sdks/python/apache_beam/yaml/main.py | 18 +-
.../apache_beam/yaml/programming_guide_test.py | 410 ++++
sdks/python/apache_beam/yaml/readme_test.py | 6 +-
sdks/python/apache_beam/yaml/yaml_combine.py | 2 +
sdks/python/apache_beam/yaml/yaml_combine_test.py | 26 +
sdks/python/apache_beam/yaml/yaml_provider.py | 2 +
sdks/python/container/boot.go | 32 +-
.../container/py310/base_image_requirements.txt | 57 +-
.../container/py311/base_image_requirements.txt | 54 +-
.../container/py38/base_image_requirements.txt | 63 +-
.../container/py39/base_image_requirements.txt | 61 +-
sdks/python/expansion-service-container/boot.go | 24 +-
.../containers/tensorrt_runinference/README.md | 2 +-
.../tensorrt_runinference/tensor_rt.dockerfile | 3 +-
sdks/python/test-suites/direct/common.gradle | 11 +-
sdks/python/tox.ini | 6 +-
sdks/typescript/package.json | 2 +-
settings.gradle.kts | 1 +
website/www/site/assets/js/sliders/top-banners.js | 10 +-
.../site/content/en/community/youtube-channel.md | 4 +-
.../documentation/io/built-in/google-bigquery.md | 1 +
.../en/documentation/io/built-in/webapis.md | 536 +++++
.../site/content/en/documentation/io/connectors.md | 13 +
.../en/documentation/ml/large-language-modeling.md | 18 +-
.../site/content/en/documentation/ml/overview.md | 2 -
.../content/en/documentation/programming-guide.md | 282 ++-
.../sdks/python-pipeline-dependencies.md | 69 +-
.../www/site/content/en/documentation/sdks/yaml.md | 55 +-
website/www/site/layouts/partials/header.html | 8 +-
.../partials/section-menu/en/documentation.html | 1 +
.../site/layouts/shortcodes/language-switcher.html | 3 +
416 files changed, 14148 insertions(+), 5761 deletions(-)
delete mode 100644 .github/trigger_files/beam_PostCommit_Java.json
create mode 100644 .github/trigger_files/beam_PostCommit_Java_Avro_Versions.json
create mode 100644 .github/trigger_files/beam_PostCommit_TransformService_Direct.json
create mode 100644 .github/workflows/beam_Playground_CI_Nightly.yml
create mode 100644 .test-infra/metrics/grafana/provisioning/alerting/flaky_test.yaml
delete mode 100644 .test-infra/metrics/grafana/provisioning/alerts/flaky_test.yaml
create mode 100644 .test-infra/tools/requirements.txt
create mode 100644 examples/java/webapis/build.gradle
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/AdditionalSnippets.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/GeminiAIClient.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/GeminiAIExample.java
copy runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillComputationKey.java => examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/GeminiAIOptions.java (51%)
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/GenerateContentRequestCoder.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/GenerateContentResponseCoder.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/HttpImageClient.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/ImageRequest.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/ImageRequestCoder.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/ImageResponse.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/ImageResponseCoder.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/Images.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/Log.java
create mode 100644 examples/java/webapis/src/main/java/org/apache/beam/examples/webapis/UsingHttpClientExample.java
create mode 100644 examples/java/webapis/src/test/java/org/apache/beam/examples/webapis/GeminiAIExampleTest.java
create mode 100644 examples/java/webapis/src/test/java/org/apache/beam/examples/webapis/UsingHttpClientExampleTest.java
create mode 100644 examples/notebooks/beam-ml/run_inference_gemma.ipynb
create mode 100644 it/kafka/src/test/java/org/apache/beam/it/kafka/KafkaIOST.java
create mode 100644 learning/prompts/documentation-lookup-nolinks/32_data_encoding.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/33_ai_ml_pipelines.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/33_coders_data_encoding.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/34_change_coders_data_encoding.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/34_ml_data_preprocessing.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/35_ml_data_exploration.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/36_ai_kfp_orchestration.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/37_ai_tfx_orchestration.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/38_ai_runinference.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/39_ai_runinference_metrics.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/40_ai_bigquery_ml.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/41_ai_model_refresh.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/42_ai_custom_inference.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/43_ai_llm_inference.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/47_batching_for_external_calls.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/47_mltransform.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/48_resource_hints.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/48_sdk_harness.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/49_container_environments.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/49_transform_service.md
create mode 100644 learning/prompts/documentation-lookup-nolinks/50_beam_yaml.md
create mode 100644 learning/prompts/documentation-lookup/33_ai_ml_pipelines.md
create mode 100644 learning/prompts/documentation-lookup/36_ai_kfp_orchestration.md
create mode 100644 learning/prompts/documentation-lookup/37_ai_tfx_orchestration.md
create mode 100644 learning/prompts/documentation-lookup/38_ai_runinference.md
create mode 100644 learning/prompts/documentation-lookup/39_ai_runinference_metrics.md
create mode 100644 learning/prompts/documentation-lookup/40_ai_bigquery_ml.md
create mode 100644 learning/prompts/documentation-lookup/41_ai_model_refresh.md
create mode 100644 learning/prompts/documentation-lookup/42_ai_custom_inference.md
create mode 100644 learning/prompts/documentation-lookup/43_ai_llm_inference.md
create mode 100644 learning/prompts/documentation-lookup/48_sdk_harness.md
create mode 100644 learning/prompts/documentation-lookup/49_transform_service.md
rename {sdks/java/core/src/main/java/org/apache/beam/sdk/util => runners/core-java/src/main/java/org/apache/beam/runners/core}/construction/SerializablePipelineOptions.java (98%)
copy runners/{google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillComputationKey.java => core-java/src/main/java/org/apache/beam/runners/core/construction/package-info.java} (50%)
rename {sdks/java/core/src/test/java/org/apache/beam/sdk/util => runners/core-java/src/test/java/org/apache/beam/runners/core}/construction/SerializablePipelineOptionsTest.java (98%)
create mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/adapter/BeamAdapterCoderUtils.java
create mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/adapter/BeamAdapterUtils.java
create mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/adapter/BeamFlinkDataSetAdapter.java
create mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/adapter/BeamFlinkDataStreamAdapter.java
create mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/adapter/FlinkInput.java
create mode 100644 runners/flink/src/main/java/org/apache/beam/runners/flink/adapter/FlinkOutput.java
copy runners/{google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillComputationKey.java => flink/src/main/java/org/apache/beam/runners/flink/adapter/package-info.java} (50%)
create mode 100644 runners/flink/src/test/java/org/apache/beam/runners/flink/adapter/BeamFlinkDataSetAdapterTest.java
create mode 100644 runners/flink/src/test/java/org/apache/beam/runners/flink/adapter/BeamFlinkDataStreamAdapterTest.java
rename runners/google-cloud-dataflow-java/{worker/src/main/java/org/apache/beam/runners/dataflow/worker/options/StreamingDataflowWorkerOptions.java => src/main/java/org/apache/beam/runners/dataflow/options/DataflowStreamingPipelineOptions.java} (53%)
copy runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{WindmillComputationKey.java => windmill/client/CloseableStream.java} (55%)
copy runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{streaming => windmill/client/commits}/Commit.java (81%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/CompleteCommit.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/StreamingApplianceWorkCommitter.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/StreamingEngineWorkCommitter.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/WorkCommitter.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/stubs/ChannelCache.java
rename runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/stubs/{RemoteWindmillStubFactory.java => ChannelCachingRemoteStubFactory.java} (67%)
rename runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{streaming/Commit.java => windmill/client/grpc/stubs/ChannelCachingStubFactory.java} (52%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/ActiveWorkRefresher.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/ActiveWorkRefreshers.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/DispatchedActiveWorkRefresher.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/StreamingApplianceWorkCommitterTest.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/StreamingEngineWorkCommitterTest.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/stubs/ChannelCacheTest.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/DispatchedActiveWorkRefresherTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamBundleSource.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadWithStreamBundleSourceTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIOReadChangeStreamTest.java
delete mode 100644 sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Quota.java
delete mode 100644 sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleWithExternalResource.java
delete mode 100644 sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/ThrottleWithoutExternalResource.java
delete mode 100644 sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/ThrottleWithExternalResourceIT.java
delete mode 100644 sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/ThrottleWithExternalResourceTest.java
copy sdks/python/apache_beam/{version.py => transforms/enrichment_handlers/utils.py} (61%)
create mode 100644 sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store.py
create mode 100644 sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_it_test.py
create mode 100644 sdks/python/apache_beam/transforms/enrichment_handlers/vertex_ai_feature_store_test.py
copy sdks/python/apache_beam/{ml/inference/tensorflow_tests_requirements.txt => transforms/enrichment_handlers/vertex_ai_feature_store_tests_requirement.txt} (93%)
create mode 100644 sdks/python/apache_beam/yaml/examples/README.md
copy {examples/notebooks/notebook_test_scripts => sdks/python/apache_beam/yaml/examples}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/yaml/examples/examples_test.py
create mode 100644 sdks/python/apache_beam/yaml/examples/regex_matches.yaml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_count_minimal.yaml
copy sdks/python/apache_beam/{ml/inference/tensorflow_tests_requirements.txt => yaml/examples/transforms/aggregation/combine_max_minimal.yaml} (50%)
copy sdks/python/apache_beam/{ml/inference/tensorflow_tests_requirements.txt => yaml/examples/transforms/aggregation/combine_mean_minimal.yaml} (50%)
copy sdks/python/apache_beam/{ml/inference/tensorflow_tests_requirements.txt => yaml/examples/transforms/aggregation/combine_min_minimal.yaml} (50%)
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_multiple_aggregations.yaml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum.yaml
copy sdks/python/apache_beam/{ml/inference/tensorflow_tests_requirements.txt => yaml/examples/transforms/aggregation/combine_sum_minimal.yaml} (50%)
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/aggregation/group_into_batches.yaml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_largest_per_key.yaml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_smallest_per_key.yaml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/elementwise/explode.yaml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/elementwise/filter_callable.yaml
copy sdks/python/apache_beam/{version.py => yaml/examples/transforms/elementwise/map_to_fields_callable.yaml} (54%)
create mode 100644 sdks/python/apache_beam/yaml/examples/wordcount_minimal.yaml
create mode 100644 sdks/python/apache_beam/yaml/programming_guide_test.py
create mode 100644 website/www/site/content/en/documentation/io/built-in/webapis.md