You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by de...@apache.org on 2026/06/30 18:46:26 UTC
(beam) branch 20260527_createBOMWorkflow updated (2c807358ca3 -> 8aa8c569494)
This is an automated email from the ASF dual-hosted git repository.
derrickaw pushed a change to branch 20260527_createBOMWorkflow
in repository https://gitbox.apache.org/repos/asf/beam.git
from 2c807358ca3 update workflow logic to only run within 6 days of a release cut
add d2a3cee22e3 Enforce binary-only constraints during early requirements cache attempts (#38688)
add 5096e63ecae Bump docker/setup-qemu-action from 4.0.0 to 4.1.0 (#38718)
add 3cb68c9bd68 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#38719)
add 758042f0580 Bump google.golang.org/api from 0.281.0 to 0.282.0 in /sdks (#38720)
add 789b278172c Bump github.com/aws/smithy-go from 1.25.1 to 1.26.0 in /sdks (#38721)
add 54f7c7dd0d8 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#38722)
add 0302bf5529c Implemented MLTransform generate vocab Dataflow benchmark (#38215)
add cd8b009d9fc [#38708] Add Flink 2.0 to SDK pipeline options validation lists (#38726)
add a0494894c35 up timeout (#38732)
add 3f4bcb01570 remove playground annotations from BatchElementsExample until next release (#38733)
add bdef23f826e Update CHANGES.md after release cut (#38608)
add 3e5a085bd0d Support infer types involving dataclass fields (#38548)
add a32774afe93 [Prism] Fix race condition in element manager (#38734)
add cbab4d7203a Add staged package hashes (#38311)
add d8d4cfc3e11 Updated All Externally Visible Logs/Docs from Runner V2 to Portable Runner (#38532)
add 3d68e9df172 Bump github.com/aws/aws-sdk-go-v2 from 1.41.7 to 1.41.8 in /sdks (#38740)
add dc3b2ccf8e3 Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#38737)
add 09a1e60fa38 fix race condition in sdb-operator (#38698)
add 5702872d9ae Disable build isolation for pip by default. (#38700)
add 07222b4dac6 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#38741)
add 0a7df5c8ba4 Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#38738)
add d9f07c5f6aa Bump setup gradle to allowlisted v6.1.0 (#38745)
add 5a5ef0cc02a Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#38739)
add 6fba3cb34b2 Implement MLTransform One-Hot Encoding benchmark pipeline (#38404)
add 2b5c1131af3 [yaml] - mongodb write normalization (#38376)
add f771827d9f1 Disable gRPC fork support in PortableRunnerTestWithSubprocesses (#38744)
add cbc2c21edad Fix flaky TestDataSampler/GetSamplesForPCollectionsTooManySamples (#38736)
add 44a2ff6146c Re-enable call-args check in yaml_io.py (#38730)
add c31f2327041 Fix Reshuffle handling in Prism to recursively remove nested sub-transforms (#38735)
add 8ac0348beb9 Hugging face model handler #3 (#38696)
add 8364da70a4e Handle empty batches in Python worker counters (#38748)
add 8b07c7a1925 feat(ml): add qdrant ingestion (#38142)
add fe27feb2ab5 [Dataflow Streaming] Enable state tag encoding v2 (#38705)
add a1b0a51efca Bump nanasess/setup-chromedriver from 2 to 3 (#38757)
add 97ae60b0543 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#38758)
add e23bf2d7e0b Bump github.com/golang-cz/devslog from 0.0.15 to 0.0.16 in /sdks (#38759)
add 95c2161fc73 Bump github.com/aws/aws-sdk-go-v2 from 1.41.8 to 1.41.9 in /sdks (#38761)
add 21d13388a94 Bump github.com/tetratelabs/wazero from 1.11.0 to 1.12.0 in /sdks (#38762)
add 6f2c8dd891f Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#38760)
add 2400917d918 Add Worker hashes for Golang (#38754)
add 2ed225f3c4d Updates Beam master container tags for Dataflow (#38766)
add 98ca01f415c [Dataflow Streaming] Prepare BoundedQueueExecutor for MultiKey bundles (#38592)
add c315ceacb5c Fix truncating file handle (#38425)
add 54accc32caf js2py to quickjs (#38473)
add fd0c3e48218 Migrate to Google Cloud Dataflow Client (#37639)
add 4f2d8c0ec2d Adding release-2.74.0-postrelease to protected branches in .asf.yaml
add 8c93ab0efae Update managed-io.md for release 2.74.0-RC1. (#38527)
add ef3d8cccb43 Update Beam website to release 2.74.0 (#38536)
add 13a46356a02 Update Python Dependencies (#38775)
add 7916452e232 Fix internal test configuration for Dataflow client (#38792)
add a83754a5360 Implements the Delta Lake source with support for splitting (#38706)
add bec756f89b8 Fix offlineRepositoryRoot relative to Beam repo root (#38790)
add 41d5dd4f55e Update republish released docker workflow defaults to 2.74.0 (#38796)
add 8346a68b5bc Update beam-master container (#38795)
add 96165fcbc52 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#38801)
add a0c96065833 Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#38804)
add fcdd87f0aad [OpenTelemetry] FNHarness, Dataflow Runner v1 - set open telemetry settings. (#38785)
add 36c8db6c75f [OpenTelemetry] Default open telemetry configuration behind experiment (#38784)
add 8fa601ae949 Bump google.golang.org/api from 0.282.0 to 0.283.0 in /sdks (#38809)
add 2b3df10f21b Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#38810)
add 26b21e964ae Bump github.com/nats-io/nats-server/v2 from 2.14.1 to 2.14.2 in /sdks (#38811)
add db1b852750c Upgrade Google Cloud libraries-bom to 26.83.0 (#38799)
add 3a99f94bd79 feat: Add LogElements transform to Java SDK (#38533)
add 53e4a8c2bfe Move google-auth lower bounds to >= 2.0.0 (#38778)
add dd3a4549d89 Normalize types in dataclass field type resolving (#38797)
add c01ceeab24c [Python] Honor disableCounterMetrics, disableStringSetMetrics, and disableBoundedTrieMetrics experiments (#38749)
add aaf677b16c3 [Dataflow] Fix thread safety of HotKey logger (#38816)
add 0f5b5142c08 Fix BigQuery Storage Write API stream count for bounded writes (#38776)
add 82f84e15ccf Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#38825)
add dd2a86c178f Bump cloud.google.com/go/storage from 1.62.2 to 1.62.3 in /sdks (#38826)
add 4fe3342c71f [OpenTelemetry] Add gcp otel auth extension (#38773)
add e353462e0a8 Revert "remove playground annotations from BatchElementsExample until next re…" (#38830)
add 6c0db120f38 Automate OpenTelemetry license entries in bomupgrader (#38815)
add 66e543997f1 support generics in from row and to row conversions (#37347)
add d9117f423d9 [Dataflow Streaming] [Multi Key] Introduce KeyGroupWorkQueue and integrate with BoundedQueueExecutor (#38767)
add 27341e2bdef Update SpannerIO.java - comment documentation (#38492)
add f88c2609d1a Bump github.com/aws/smithy-go from 1.27.1 to 1.27.2 in /sdks (#38841)
add 34bedad38e3 [Experimental] Fix Yaml Xlang Test Timeout (#38798)
add 52d10870893 Fix parsing of dataflow api endpoint URLs to remove trailing slash (#38847)
add b452ad63fb0 Update Gemini text classification to gemini-2.5-flash (#38839)
add 726933b81db fix flaky Arm postcommit test collection on Python 3.14 (#38844)
add 3732cffb33c [Dataflow Streaming] Fix nullness supression in StreamingModeExecutionContext (#38842)
add 2eff9f19340 Fix ensurepip bundled pip upgrade for Python 3.12+ (#38765)
add 0eac0389a4c Fix cloudML TFT install avoiding pip ResolutionTooDeep (#38781)
add d521df8f1ed use local variable (#38850)
add 0bf63a295cc Fix Dataflow cost benchmark after Dataflow client migration (#38788)
add 59c7a42257d Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#38856)
add 945e6e140d2 Bump golang.org/x/sys from 0.45.0 to 0.46.0 in /sdks (#38857)
add 83a8138b825 Unpin google-cloud-bigtable version (fixes apache#37637) (#38861)
add a9d7d2cb92e Bump distlib from 0.3.9 to 0.4.2 in /sdks/python (#38855)
add 7050d00f5fd Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#38858)
add 9cda5a33300 Bump golang.org/x/text from 0.37.0 to 0.38.0 in /sdks (#38860)
add f9b4ee9cc82 Disable public ip for Dataflow Validates Runner test (#38424)
add 9bcdedda196 increase timeout (#38864)
add 8e8327d6f90 Adds a Github action for running Delta Lake I/O unit tests (#38869)
add fe7dd9e942d Enable code scanning alerts (#38817)
add 0856c22a9e9 Side Input improvements (#38363)
add fcfec9af467 Support global sort and improve UDF overload resolution in Beam SQL (#38832)
add ca0615cc614 Update huggingface model handler info (#38789)
add 3b104b6d3b7 Fix yaml doc generation (#38874)
add ea2fde7a020 Fix race condition in statesampler_fast.pyx (#38851)
add a3cb642db76 Merge pull request #38878: Fix OnWindowExpirationContext.
add 872cb901dca Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#38884)
add 530d0f91b10 Bump google.golang.org/api from 0.283.0 to 0.284.0 in /sdks (#38887)
add d7948d07b80 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#38886)
add be50185053f Bump actions/checkout from 4 to 6 (#38885)
add 6eed130a176 update iceberg add files parameters (#38879)
add 20ac8c4f35a add datadog io normalization and yaml (#38362)
add 6a2dbb67985 Bump torch (#38896)
add 02168d3babe Revert "[Python] Honor disableCounterMetrics, disableStringSetMetrics, and disableBoundedTrieMetrics experiments" (#38901)
add 043556c4b90 Bump torch (#38897)
add 3eb0031274e Bump torch (#38903)
add 081c61ee5b4 Fix Xlang IO Direct installGcpTest uv cache lock (#38862)
add 8051efce4c2 Turn to manual build for Go for CodeQL (#38907)
add 1cf3545e5b7 Fix Dataflow legacy worker abort loop thread death issue (#38894)
add 8e859798a6f Bump cloud.google.com/go/bigtable from 1.48.0 to 1.49.0 in /sdks (#38913)
add 48fe93cb0eb Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#38916)
add 60621107046 Bump golang.org/x/net from 0.55.0 to 0.56.0 in /sdks (#38915)
add 51e35e52440 [IcebergIO] Improve TableCache (#38882)
add c297c0dc4ab Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#38912)
add 61a7368bdb7 Use p310_ml_test (#38890)
add 135a5593441 Replace ClassLoadingStrategy with custom loading strategy (#38906)
add a40685ccc62 Don't preempt snapshot runs (#38904)
add 8ea81a9f289 Bump @grpc/grpc-js from 1.14.3 to 1.14.4 in /sdks/typescript (#38922)
add 6f30f147462 Adds a new agent SKILL for developing new I/O connectors (#38910)
add 782953e3422 Add Delta Lake source to the Java Managed API (#38902)
add afbbc31369b Implement Asynchronous wrapper for DoFn in Java SDK (#38609)
add 8d15c63d551 Updates references to the I/O connector skill (#38923)
add 6a30a3d68de Updates CHANGES.md to mention the Delta Lake source (#38924)
add 03f4f797953 [SQL] Support positional parameters (#38880)
add 5307c7138af Support MAP type in RowJson and fix datetime parsing with spaces (#38865)
add 56a41422765 SQL DDL documentation (#37539)
add a88686c1594 Add DoFnRunner::finishKey() method (#38454)
add 705db250d51 [Dataflow Streaming] Activate SourceState Finalizers before submitting workitem to harness threads (#38921)
add e8249afb1f7 Fix wordcount_rust requirements.txt and documentation (#38877)
add 64bb91099a9 Exclude testSideInputNotReadyTimer from Spark batch suites (#38939)
add 7d642a024a4 Add Gemini RunInference example notebook (#38943)
add 712d981ea79 update agent skills table (#38818)
add 6ffee9feebb [Infra] Add beam_viewer and beam_writer roles for GSoC 2026 participant (#38933)
add ebde16e3713 Add instrumentation for memory profiling in Python SDK (#38853)
add 22ac580f827 [#38059] Fix GCS glob matching to support ** and / in object names (#38099)
add 6c083df5f8e SQL Database DDL & Usability Improvements (#38952)
add 001093e5e0b [mqtt] Add portable MqttIO Read/Write transforms for batch and streaming (revives #32385) (#38493)
add b3a62ee97c7 Upgrade expansion jar to java17 (#38931)
add 6ed0635429a Spanner: Extend SpannerIO to support Spanner Omni mTLS setup (#38928)
add 378e10de4ea ClickHouseIO: Add DateTime64 support for sub-second timestamp precision (#38510)
add 98e29d99fcd Revert "Upgrade expansion jar to java17 (#38931)" (#38965)
add bde545f91e4 Add support for STDDEV_POP and STDDEV_SAMP in VarianceFn (#38871)
add 77348121ee9 Update BEAM_DEV_SDK_CONTAINER_TAG version (#38963)
add a996a9fb610 Bump protobufjs from 8.4.0 to 8.6.0 in /sdks/typescript (#38968)
add 2c4822959e1 Bump form-data from 2.5.5 to 2.5.6 in /sdks/typescript (#38967)
add b7bc937f592 Use Flink 2.0 for load tests (#38279)
add c0cc8897f4a Bump cloud.google.com/go/spanner from 1.91.0 to 1.92.0 in /sdks (#38976)
add 468ae40dc8e Fix AsyncWrapperTest timeout and flakiness issues. (#38970)
add e55a1f7399b fix failure in opentelemetry-gcp-auth-extension jar (#38938)
add 33b34574b3c Bump jsonpickle upper bound. (#38769)
add 2c53494f735 [Dataflow Java] Clarify which portions of DataflowWorkerLoggingOptions are deprecated (#38960)
add 0bc785253eb Fix WriteToPubSub to pass ordering_key to publish() method (#37345)
add d032f0e3ef4 Add drain support for Dataflow and Flink (#38786)
add c608b4da8de Bump google.golang.org/api from 0.284.0 to 0.285.0 in /sdks (#38997)
add 9765e29de68 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#38998)
add 6ccd0c9c861 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#38999)
add b9342767cc5 fix sentence (#38989)
add 597f92530f1 address DOM text reinterpreted as HTML alert (#38949)
add 56d4ffdab31 Align gcpauth extension implementation with ability to turn it off easily while still being added as dependency.
add 2802de39b2e Merge pull request #38846: [OpenTelemetry] Add turning off to gcpauth extension
add ee92a553e5d Disable public IP for Dataflow Python and Go Validate runner tests (#38990)
add 1c182507444 Add support for Apache Flink 2.1.3 (#38961)
add 1d703fe904f Make ModelManager import more robust (#38936)
add 849fcf89c81 [Beam SQL Shell] Add Iceberg deps when using icebergio (#38994)
add a3f9e0e7d96 Add publish_time_field to ReadFromPubSub YAML transform (#38985)
add 15d00559d75 Bump cloud.google.com/go/bigtable from 1.49.0 to 1.50.0 in /sdks (#39012)
add 5c34e1c1c85 Exclude tests for batch-dataset (#39014)
add 9e465a7cfd7 fix parsing error for typescript (#38898)
add c81059fb246 Drop Flink 1.17,1.18 support (#39006)
add 466989d50a4 cancel user code with interruption after timeout (#39018)
add 09c962c4aef Fix Milvus ML PreCommit xdist, container startup (#39007)
add b4e023afece Bump nodemailer from 8.0.5 to 9.0.1 in /scripts/ci/issue-report (#39019)
add 17bed431e05 remove shardedKey (#39024)
add 63bcd3dd2b5 Validate Project of staging bucket (#39008)
add 8ba04741b47 Sanitize logging in FanOutStreamingEngineWorkerHarness (#39029)
add 974ad17b60d Increase CallTest sleep durations to keep test from being flaky due to timeout not being observed. (#39016)
add caba399f96e [Dataflow Java] Fix incorrect warning log when specifying logger override. (#39034)
add 1df57236fe3 Bump github.com/moby/moby/api from 1.54.2 to 1.55.0 in /sdks (#39031)
add fc370c9fdc8 [FnApi Java] Add support for separate named data streams to provide bundle isolation (#38863)
add 630a34c3bfa Bump github.com/moby/moby/client from 0.4.1 to 0.5.0 in /sdks (#39032)
add a21ca397b42 [Iceberg AddFiles] Add NameMapping when creating table (#39022)
add 5334dc1d92a Add Flink 2.2.1 runner support (#38980)
add 06aa37bdffd use proper table naming in iceberg sql (#39035)
add e948ef62041 Roll forward io expansion service to Java17 (#38974)
add eaba570c569 Bump github.com/golang-cz/devslog from 0.0.16 to 0.0.17 in /sdks (#39054)
add 54231e89266 try to fix forward tests for java 17 (#39058)
add 9aed894f5d4 Update pandas version upper bound to support python 3.14 (#39056)
add 9868954ce8c Fix playground precommit test failure (#39062)
add c2474960db7 [yaml] - expand jinja method (#38547)
add d031bddfdc9 Bump actions/checkout from 6 to 7 (#39033)
add 117bb893ca8 Bump google.golang.org/api from 0.285.0 to 0.286.0 in /sdks (#39068)
add 9a9f48b9542 Fix Flink 2.1+ dependency resolution (#39061)
add c77b2d40626 Update Python Dependencies (#39070)
add a467fdc437f Enable precommit yaml xlang test on code push. (#39071)
add f61d0b5eeff Release Note: Rename Dataflow Runner to Dataflow Portable Runner (#38747)
add f29d9af139a init (#39025)
add 447899e46e9 [mqtt] Add Python Xlang Messaging PostCommit with MQTT integration tests (#38966)
add cedfd0bb3e8 [ReduceFnRunner] Fix Prefetches (#39082)
add c3d9a731b42 [Dataflow Streaming] Increase stuck commit invalidation timeout to 1 hour (#39083)
add 24e9ed4e0e9 Bump actions/cache from 4 to 6 (#39081)
add f7f0e1655d5 Bump actions/checkout from 6 to 7 (#39080)
add 0165344ab13 Add UnboundedCountingSource::to for bounded reads from an UnboundedCountingSource (#39084)
add f33feafb2b2 [Dataflow Java Streaming] Add a timeout to how long commits will retry to the service. (#39085)
add 857dfc9a875 [mqtt] Fix streaming xlang IT failing the Messaging PostCommit (#39088)
add 0802263e48e Add a github workflow to publish vllm image. (#39089)
add 8e4ea736213 [Python] Optimize BigQuery copy jobs in file loads using multi-source copy (#38983)
add 5afb93c8c09 [OpenTelemetry] Turn off gcp otel auth extension by default. (#38940)
add 4e9e483a51a Fix Flink XVR for Flink 2
add 49fd8745223 Fix Go Flink VR conf path
add 16acfa8a2c6 Update CHANGES.md and website
add f13ccf7afbe Merge pull request #39077 from Abacn/fix-flink-xvr
add 9e28c9d2150 Disable flaky SDF finalization tests
add 6095fd0de62 Merge pull request #39092 from arunpandianp/disablesdffinalization
add 70b72adffee Fix Install xmllint
add 0aa808dba09 Merge pull request #39094 from apache/cut-release-branch
add cad9f1ebe76 Moving to 2.76.0-SNAPSHOT on master branch.
add beeb715739f Update beam-master (#39095)
add c531095fc40 [GSOC 2026] Adding logic to validate and generate an error if there are key… (#38992)
add 8d252c4659b update changes log for yaml improvements etc (#39093)
add 3fba9950feb Update beam-master container
add e56ed7718a5 Trigger failed postcommit tests.
add 7454aa6a0bf Merge pull request #39100 from shunping/update-python-beam-master
add d71c4aca4f4 Fix race condition in DirectRunner executor shutdown
add 0af28db1142 Apply suggested fix
add 23521069470 Merge pull request #39096 from shunping/fix-flaky-rrio-test
add 96733599e6c Fix JsonToRow swallowing downstream errors when runners fuse transforms. (#39098)
add 8092349ca28 Enable playground tests triggered on cron schedule (#39069)
add 8ecdab8da94 [Iceberg CDC] Import BaseIncrementalChangelogScan (#38823)
add 8c311e45672 [Iceberg CDC] Add utils base, delete reader, and serializable files (#38821)
add c1898ff68b8 Fix pip ResolutionTooDeep (#39017)
add 9d3694165e1 [Python] Add UnboundedSource SDF wrapper (#19137) (#38724)
add 5eefc931b5d remove infra change (#39104)
add eef26ca8b71 Revert "[Python] Optimize BigQuery copy jobs in file loads using multi-source copy" (#39106)
add 0c51b0d288b Write append tables can be asynchronous (#39120)
add b642c5cbd0e Fix typo (#39123)
add 56b5a52294d Bump google-cloud-aiplatform (#39113)
add d77d12a47b2 Update cython requirement from <4,>=3.2.5 to >=3.2.6,<4 in /sdks/python (#39099)
add 5cb2f09298c [Website] Update Python version compatibility table (#38969)
add 698420027d5 Docs: update Python version support in Python quickstart (#39122)
add 8ccd1d70c04 including GitHub Actions to schedule daily audits and Add documentati… (#39116)
add 5a2e98bb712 Fix a flaky test in ApproximateQuantilesTest (#39132)
add e3b4d8c5d0e Fix flaky ML RunInference tests by disabling reshuffle on beam.Create (#39118)
add 3cca8fa3dac [Dataflow Streaming] Access state/timerinternals via StepContext (#38991)
add 2e8f75aaf4b Bump actions/checkout from 6 to 7 (#39147)
add b63ea65d6a3 Bump github.com/aws/smithy-go from 1.27.2 to 1.27.3 in /sdks (#39145)
add 31cd92f5f55 Bump cloud.google.com/go/storage from 1.62.3 to 1.63.0 in /sdks (#39144)
add edd3bfca48e Fix flaky GcsIOIntegrationTest and handle GCS bucket creation race conditions (#39130)
add af005ef679c fix post commit yaml test (#39142)
add 31175f9ab82 fix race condition in statesampler with lock for reset (#39129)
add ccac91ad9b8 add zizmor to existing precommit GHA workflow (#39108)
add f17b370e74a Fix flaky PubSub integration test by deduplicating messages (#39134)
add 402b41c726d Fix ThreadSanitizer data race and failed RPC calls (#39076)
add 961ed7c31ad [IcebergIO] Fix stale openWriters count in RecordWriterManager (#39157)
add f909e8a7af9 Use Flink 2.2 container for load tests
add 984df275bf4 debug only change -- preserve cluster to check error
add 397d9c61a27 Set configs for Flink 2
add a0fd9502818 Refactor Flink Load Tests to use native Dataproc components and Auto Zone
add 3e76ff518fa Remove unnecessary gcloud volume mount from Job Server
add 5d28c414f62 Change memory allocation, use resize feature
add 30a5a00d584 Merge pull request #39154 from apache/flink-load-test-ver
add f9171da59bc Bump github.com/nats-io/nats-server/v2 from 2.14.2 to 2.14.3 in /sdks (#39168)
add aa6942a9853 Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#39169)
add fcd105de015 Bump js-yaml from 3.14.2 to 3.15.0 in /website/www (#39172)
add 24a2ccc830a Bump com.gradle.common-custom-user-data-gradle-plugin (#39166)
add 1c4b2047016 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#39170)
add aec5b5da813 Add embedded NVIDIA Dynamo support to vLLM ModelHandler (#38701)
add 8f60e69177b Fixed DirectRunner PubSub subscriber client lifecycle (#39079)
add ed0a5832eb1 Add open telemetry dependency to dataflow worker (#38941)
add a86e6115270 Retry starting SubprocessServer on failure (#39159)
add e34cc96df4d [yaml] - mongodb read normalization (#38772)
add 8aa8c569494 Merge branch 'master' into 20260527_createBOMWorkflow
No new revisions were added by this update.
Summary of changes:
.agent/skills/README.md | 4 +
.../skills/developing-new-io-connectors/SKILL.md | 347 +
.agent/skills/io-connectors/SKILL.md | 3 +-
.asf.yaml | 2 +
.../test-properties.json | 2 +-
.../actions/setup-environment-action/action.yml | 4 +-
.github/autolabeler.yml | 1 +
.../IO_Iceberg_Integration_Tests.json | 2 +-
.../beam_CloudML_Benchmarks_Dataflow.json | 2 +-
.github/trigger_files/beam_PostCommit_Go.json | 2 +-
.../trigger_files/beam_PostCommit_Go_VR_Flink.json | 2 +-
.../beam_PostCommit_Java_DataflowV2.json | 2 +-
.../beam_PostCommit_Java_Nexmark_Flink.json | 2 +-
....json => beam_PostCommit_Java_Tpcds_Flink.json} | 0
...m_PostCommit_Java_ValidatesRunner_Dataflow.json | 2 +-
..._ValidatesRunner_Dataflow_Streaming_Engine.json | 2 +-
...ostCommit_Java_ValidatesRunner_Dataflow_V2.json | 2 +-
...beam_PostCommit_Java_ValidatesRunner_Spark.json | 18 +-
.github/trigger_files/beam_PostCommit_Python.json | 4 +-
.../beam_PostCommit_Python_Examples_Dataflow.json | 4 +-
...PostCommit_Python_ValidatesRunner_Dataflow.json | 2 +-
.../beam_PostCommit_Python_Versions.json | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.json | 2 +-
..._PostCommit_Python_Xlang_Messaging_Direct.json} | 0
.../trigger_files/beam_PostCommit_XVR_Flink.json | 2 +-
.../trigger_files/beam_PostCommit_XVR_Samza.json | 1 -
.../beam_PostCommit_Yaml_Xlang_Direct.json | 2 +-
...on => beam_PreCommit_Java_Delta_IO_Direct.json} | 0
.../trigger_files/beam_PreCommit_Python_ML.json | 2 +-
.../beam_PreCommit_Yaml_Xlang_Direct.json | 2 +-
.github/workflows/IO_Iceberg_Integration_Tests.yml | 2 +-
.../IO_Iceberg_Integration_Tests_Dataflow.yml | 2 +-
..._Iceberg_Managed_Integration_Tests_Dataflow.yml | 2 +-
.github/workflows/IO_Iceberg_Performance_Tests.yml | 2 +-
.github/workflows/IO_Iceberg_Unit_Tests.yml | 2 +-
.github/workflows/README.md | 4 +
.github/workflows/assign_milestone.yml | 2 +-
.github/workflows/beam_CancelStaleDataflowJobs.yml | 2 +-
.../workflows/beam_CleanUpDataprocResources.yml | 2 +-
.github/workflows/beam_CleanUpGCPResources.yml | 2 +-
.../workflows/beam_CleanUpPrebuiltSDKImages.yml | 2 +-
.../workflows/beam_CloudML_Benchmarks_Dataflow.yml | 2 +-
.../beam_IODatastoresCredentialsRotation.yml | 2 +-
.../beam_Inference_Python_Benchmarks_Dataflow.yml | 27 +-
.../beam_Infrastructure_AuditUnmanagedKeys.yml | 69 +
.../beam_Infrastructure_PolicyEnforcer.yml | 2 +-
.../beam_Infrastructure_SecurityLogging.yml | 2 +-
.../beam_Infrastructure_ServiceAccountKeys.yml | 2 +-
.../beam_Infrastructure_UsersPermissions.yml | 2 +-
.github/workflows/beam_Java_JMH.yml | 2 +-
.../beam_Java_LoadTests_Combine_Smoke.yml | 2 +-
.../beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_CoGBK_Flink_batch.yml | 10 +-
.../beam_LoadTests_Go_Combine_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_Combine_Flink_Batch.yml | 16 +-
.../beam_LoadTests_Go_GBK_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_GBK_Flink_Batch.yml | 16 +-
.../beam_LoadTests_Go_ParDo_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_ParDo_Flink_Batch.yml | 8 +-
.../beam_LoadTests_Go_SideInput_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_SideInput_Flink_Batch.yml | 10 +-
.../beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml | 2 +-
...eam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml | 2 +-
...s_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml | 2 +-
...va_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml | 2 +-
...s_Java_CoGBK_SparkStructuredStreaming_Batch.yml | 2 +-
.../beam_LoadTests_Java_Combine_Dataflow_Batch.yml | 2 +-
...m_LoadTests_Java_Combine_Dataflow_Streaming.yml | 2 +-
...Java_Combine_SparkStructuredStreaming_Batch.yml | 2 +-
.../beam_LoadTests_Java_GBK_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Java_GBK_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml | 2 +-
...LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml | 2 +-
...am_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml | 2 +-
...Tests_Java_GBK_Dataflow_V2_Streaming_Java17.yml | 2 +-
.../workflows/beam_LoadTests_Java_GBK_Smoke.yml | 2 +-
...sts_Java_GBK_SparkStructuredStreaming_Batch.yml | 2 +-
.../beam_LoadTests_Java_ParDo_Dataflow_Batch.yml | 2 +-
...eam_LoadTests_Java_ParDo_Dataflow_Streaming.yml | 2 +-
...s_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml | 2 +-
...va_ParDo_Dataflow_V2_Streaming_JavaVersions.yml | 2 +-
...s_Java_ParDo_SparkStructuredStreaming_Batch.yml | 2 +-
.github/workflows/beam_LoadTests_Java_PubsubIO.yml | 2 +-
.../beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml | 2 +-
...m_LoadTests_Python_CoGBK_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_CoGBK_Flink_Batch.yml | 8 +-
...eam_LoadTests_Python_Combine_Dataflow_Batch.yml | 2 +-
...LoadTests_Python_Combine_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_Combine_Flink_Batch.yml | 18 +-
...am_LoadTests_Python_Combine_Flink_Streaming.yml | 8 +-
...LoadTests_Python_FnApiRunner_Microbenchmark.yml | 2 +-
.../beam_LoadTests_Python_GBK_Dataflow_Batch.yml | 2 +-
...eam_LoadTests_Python_GBK_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_GBK_Flink_Batch.yml | 18 +-
...adTests_Python_GBK_reiterate_Dataflow_Batch.yml | 2 +-
...sts_Python_GBK_reiterate_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_ParDo_Dataflow_Batch.yml | 2 +-
...m_LoadTests_Python_ParDo_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_ParDo_Flink_Batch.yml | 10 +-
...beam_LoadTests_Python_ParDo_Flink_Streaming.yml | 10 +-
...m_LoadTests_Python_SideInput_Dataflow_Batch.yml | 2 +-
.github/workflows/beam_LoadTests_Python_Smoke.yml | 2 +-
.../workflows/beam_MetricsCredentialsRotation.yml | 2 +-
.github/workflows/beam_Metrics_Report.yml | 2 +-
.../workflows/beam_PerformanceTests_AvroIOIT.yml | 2 +-
.../beam_PerformanceTests_AvroIOIT_HDFS.yml | 2 +-
...PerformanceTests_BigQueryIO_Batch_Java_Avro.yml | 2 +-
...PerformanceTests_BigQueryIO_Batch_Java_Json.yml | 2 +-
..._PerformanceTests_BigQueryIO_Streaming_Java.yml | 2 +-
...eam_PerformanceTests_BiqQueryIO_Read_Python.yml | 2 +-
...formanceTests_BiqQueryIO_Write_Python_Batch.yml | 2 +-
.github/workflows/beam_PerformanceTests_Cdap.yml | 2 +-
.../beam_PerformanceTests_Compressed_TextIOIT.yml | 2 +-
...m_PerformanceTests_Compressed_TextIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_HadoopFormat.yml | 2 +-
.github/workflows/beam_PerformanceTests_JDBC.yml | 2 +-
.../workflows/beam_PerformanceTests_Kafka_IO.yml | 2 +-
.../beam_PerformanceTests_ManyFiles_TextIOIT.yml | 2 +-
...am_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_MongoDBIO_IT.yml | 2 +-
.../beam_PerformanceTests_ParquetIOIT.yml | 2 +-
.../beam_PerformanceTests_ParquetIOIT_HDFS.yml | 2 +-
...erformanceTests_PubsubIOIT_Python_Streaming.yml | 2 +-
...m_PerformanceTests_SQLBigQueryIO_Batch_Java.yml | 2 +-
.../beam_PerformanceTests_SingleStoreIO.yml | 4 +-
..._PerformanceTests_SpannerIO_Read_2GB_Python.yml | 2 +-
...manceTests_SpannerIO_Write_2GB_Python_Batch.yml | 2 +-
.../beam_PerformanceTests_SparkReceiver_IO.yml | 2 +-
.../beam_PerformanceTests_TFRecordIOIT.yml | 2 +-
.../beam_PerformanceTests_TFRecordIOIT_HDFS.yml | 2 +-
.../workflows/beam_PerformanceTests_TextIOIT.yml | 2 +-
.../beam_PerformanceTests_TextIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_TextIOIT_Python.yml | 2 +-
...PerformanceTests_WordCountIT_PythonVersions.yml | 2 +-
.../workflows/beam_PerformanceTests_XmlIOIT.yml | 2 +-
.../beam_PerformanceTests_XmlIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_xlang_KafkaIO_Python.yml | 7 +-
.github/workflows/beam_Playground_CI_Nightly.yml | 2 +-
.github/workflows/beam_Playground_Precommit.yml | 5 +-
.github/workflows/beam_PostCommit_Go.yml | 2 +-
.../workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 +-
.github/workflows/beam_PostCommit_Go_VR_Flink.yml | 2 +-
.github/workflows/beam_PostCommit_Go_VR_Spark.yml | 2 +-
.github/workflows/beam_PostCommit_Java.yml | 2 +-
.../beam_PostCommit_Java_Avro_Versions.yml | 2 +-
.../beam_PostCommit_Java_BigQueryEarlyRollout.yml | 2 +-
.../workflows/beam_PostCommit_Java_DataflowV1.yml | 2 +-
.../workflows/beam_PostCommit_Java_DataflowV2.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow_ARM.yml | 2 +-
...beam_PostCommit_Java_Examples_Dataflow_Java.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow_V2.yml | 2 +-
...m_PostCommit_Java_Examples_Dataflow_V2_Java.yml | 2 +-
.../beam_PostCommit_Java_Examples_Direct.yml | 2 +-
.../beam_PostCommit_Java_Examples_Flink.yml | 4 +-
.../beam_PostCommit_Java_Examples_Spark.yml | 2 +-
.../beam_PostCommit_Java_Hadoop_Versions.yml | 2 +-
.../beam_PostCommit_Java_IO_Performance_Tests.yml | 4 +-
.../beam_PostCommit_Java_InfluxDbIO_IT.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Dataflow.yml | 2 +-
...beam_PostCommit_Java_Jpms_Dataflow_Versions.yml | 2 +-
.../workflows/beam_PostCommit_Java_Jpms_Direct.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Direct_Versions.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Flink_Java11.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Spark_Java11.yml | 2 +-
.../beam_PostCommit_Java_Nexmark_Dataflow.yml | 2 +-
.../beam_PostCommit_Java_Nexmark_Dataflow_V2.yml | 2 +-
...am_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml | 2 +-
.../beam_PostCommit_Java_Nexmark_Direct.yml | 2 +-
.../beam_PostCommit_Java_Nexmark_Flink.yml | 7 +-
.../beam_PostCommit_Java_Nexmark_Spark.yml | 2 +-
.../beam_PostCommit_Java_PVR_Flink_Batch.yml | 4 +-
.../beam_PostCommit_Java_PVR_Flink_Streaming.yml | 6 +-
.../beam_PostCommit_Java_PVR_Spark3_Streaming.yml | 2 +-
.../beam_PostCommit_Java_PVR_Spark4_Batch.yml | 2 +-
.../beam_PostCommit_Java_PVR_Spark4_Streaming.yml | 2 +-
.../beam_PostCommit_Java_PVR_Spark_Batch.yml | 2 +-
.../beam_PostCommit_Java_SingleStoreIO_IT.yml | 4 +-
.../beam_PostCommit_Java_Tpcds_Dataflow.yml | 2 +-
.../workflows/beam_PostCommit_Java_Tpcds_Flink.yml | 4 +-
.../workflows/beam_PostCommit_Java_Tpcds_Spark.yml | 2 +-
...am_PostCommit_Java_ValidatesRunner_Dataflow.yml | 2 +-
..._Java_ValidatesRunner_Dataflow_JavaVersions.yml | 2 +-
...mit_Java_ValidatesRunner_Dataflow_Streaming.yml | 2 +-
...a_ValidatesRunner_Dataflow_Streaming_Engine.yml | 2 +-
...atesRunner_Dataflow_Streaming_TagEncodingV2.yml | 2 +-
...PostCommit_Java_ValidatesRunner_Dataflow_V2.yml | 2 +-
..._Java_ValidatesRunner_Dataflow_V2_Streaming.yml | 2 +-
...beam_PostCommit_Java_ValidatesRunner_Direct.yml | 2 +-
...it_Java_ValidatesRunner_Direct_JavaVersions.yml | 2 +-
.../beam_PostCommit_Java_ValidatesRunner_Flink.yml | 6 +-
.../beam_PostCommit_Java_ValidatesRunner_Spark.yml | 2 +-
...beam_PostCommit_Java_ValidatesRunner_Spark4.yml | 2 +-
...va_ValidatesRunner_SparkStructuredStreaming.yml | 2 +-
...am_PostCommit_Java_ValidatesRunner_Twister2.yml | 2 +-
.../beam_PostCommit_Java_ValidatesRunner_ULR.yml | 2 +-
.github/workflows/beam_PostCommit_Javadoc.yml | 2 +-
.../beam_PostCommit_PortableJar_Flink.yml | 2 +-
.../beam_PostCommit_PortableJar_Spark.yml | 2 +-
.github/workflows/beam_PostCommit_Python.yml | 2 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 2 +-
.../beam_PostCommit_Python_Dependency.yml | 2 +-
.../beam_PostCommit_Python_Examples_Dataflow.yml | 2 +-
.../beam_PostCommit_Python_Examples_Direct.yml | 2 +-
.../beam_PostCommit_Python_Examples_Flink.yml | 2 +-
.../beam_PostCommit_Python_Examples_Spark.yml | 2 +-
.../beam_PostCommit_Python_MongoDBIO_IT.yml | 2 +-
.../beam_PostCommit_Python_Nexmark_Direct.yml | 2 +-
.../beam_PostCommit_Python_Portable_Flink.yml | 2 +-
...stCommit_Python_ValidatesContainer_Dataflow.yml | 2 +-
..._Python_ValidatesContainer_Dataflow_With_RC.yml | 2 +-
..._PostCommit_Python_ValidatesRunner_Dataflow.yml | 2 +-
...eam_PostCommit_Python_ValidatesRunner_Flink.yml | 2 +-
...eam_PostCommit_Python_ValidatesRunner_Spark.yml | 2 +-
.../workflows/beam_PostCommit_Python_Versions.yml | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 6 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.yml | 7 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.yml | 6 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.yml | 9 +-
...m_PostCommit_Python_Xlang_Messaging_Direct.yml} | 22 +-
.github/workflows/beam_PostCommit_SQL.yml | 2 +-
.../beam_PostCommit_TransformService_Direct.yml | 9 +-
.github/workflows/beam_PostCommit_Website_Test.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Direct.yml | 6 +-
.github/workflows/beam_PostCommit_XVR_Flink.yml | 8 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 6 +-
...eam_PostCommit_XVR_JavaUsingPython_Dataflow.yml | 6 +-
..._PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml | 6 +-
...eam_PostCommit_XVR_PythonUsingJava_Dataflow.yml | 6 +-
.github/workflows/beam_PostCommit_XVR_Spark3.yml | 6 +-
.../beam_PostCommit_Yaml_Xlang_Direct.yml | 12 +-
.../workflows/beam_PostRelease_NightlySnapshot.yml | 2 +-
.../workflows/beam_PreCommit_CommunityMetrics.yml | 2 +-
.../workflows/beam_PreCommit_Flink_Container.yml | 10 +-
.github/workflows/beam_PreCommit_GHA.yml | 8 +-
.github/workflows/beam_PreCommit_Go.yml | 2 +-
.github/workflows/beam_PreCommit_GoPortable.yml | 2 +-
.github/workflows/beam_PreCommit_GoPrism.yml | 2 +-
.github/workflows/beam_PreCommit_ItFramework.yml | 2 +-
.github/workflows/beam_PreCommit_Java.yml | 2 +-
...eCommit_Java_Amazon-Web-Services2_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Amqp_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Azure_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Cassandra_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Cdap_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Clickhouse_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Csv_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Datadog_IO_Direct.yml | 2 +-
.github/workflows/beam_PreCommit_Java_Dataflow.yml | 2 +-
.../beam_PreCommit_Java_Debezium_IO_Direct.yml | 2 +-
...yml => beam_PreCommit_Java_Delta_IO_Direct.yml} | 34 +-
...beam_PreCommit_Java_ElasticSearch_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Examples_Dataflow.yml | 2 +-
...eam_PreCommit_Java_Examples_Dataflow_Java21.yml | 2 +-
...Commit_Java_File-schema-transform_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Flink_Versions.yml | 2 +-
.../beam_PreCommit_Java_GCP_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Google-ads_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_HBase_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_HCatalog_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Hadoop_IO_Direct.yml | 2 +-
.../workflows/beam_PreCommit_Java_IOs_Direct.yml | 2 +-
.../beam_PreCommit_Java_InfluxDb_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_JDBC_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Jms_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Kafka_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Kudu_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_MongoDb_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Mqtt_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Neo4j_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_PVR_Flink_Batch.yml | 4 +-
.../beam_PreCommit_Java_PVR_Flink_Docker.yml | 4 +-
.../beam_PreCommit_Java_PVR_Prism_Loopback.yml | 2 +-
.../beam_PreCommit_Java_Parquet_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Pulsar_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_RabbitMq_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Redis_IO_Direct.yml | 2 +-
...am_PreCommit_Java_RequestResponse_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_SingleStore_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Snowflake_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Solace_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Solr_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Spark_Versions.yml | 2 +-
.../beam_PreCommit_Java_Splunk_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Thrift_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Tika_IO_Direct.yml | 2 +-
.../workflows/beam_PreCommit_Kotlin_Examples.yml | 2 +-
.../workflows/beam_PreCommit_Portable_Python.yml | 2 +-
.github/workflows/beam_PreCommit_Prism_Python.yml | 2 +-
.github/workflows/beam_PreCommit_Python.yml | 2 +-
.github/workflows/beam_PreCommit_PythonDocker.yml | 4 +-
.github/workflows/beam_PreCommit_PythonDocs.yml | 2 +-
.../workflows/beam_PreCommit_PythonFormatter.yml | 2 +-
.github/workflows/beam_PreCommit_PythonLint.yml | 2 +-
.../workflows/beam_PreCommit_Python_Coverage.yml | 2 +-
.../workflows/beam_PreCommit_Python_Dataframes.yml | 2 +-
.github/workflows/beam_PreCommit_Python_Dill.yml | 2 +-
.../workflows/beam_PreCommit_Python_Examples.yml | 2 +-
.../beam_PreCommit_Python_Integration.yml | 2 +-
.github/workflows/beam_PreCommit_Python_ML.yml | 2 +-
.../workflows/beam_PreCommit_Python_PVR_Flink.yml | 2 +-
.../workflows/beam_PreCommit_Python_Runners.yml | 2 +-
.../workflows/beam_PreCommit_Python_Transforms.yml | 2 +-
.github/workflows/beam_PreCommit_RAT.yml | 2 +-
.github/workflows/beam_PreCommit_SQL.yml | 2 +-
.github/workflows/beam_PreCommit_SQL_Java17.yml | 2 +-
.github/workflows/beam_PreCommit_Spotless.yml | 2 +-
.github/workflows/beam_PreCommit_Typescript.yml | 2 +-
.github/workflows/beam_PreCommit_Website.yml | 2 +-
.../workflows/beam_PreCommit_Website_Stage_GCS.yml | 2 +-
.github/workflows/beam_PreCommit_Whitespace.yml | 2 +-
.../beam_PreCommit_Xlang_Generated_Transforms.yml | 8 +-
.../workflows/beam_PreCommit_Yaml_Xlang_Direct.yml | 10 +-
.github/workflows/beam_Prober_CommunityMetrics.yml | 2 +-
.github/workflows/beam_Publish_BeamMetrics.yml | 2 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 12 +-
.../workflows/beam_Publish_Docker_Snapshots.yml | 6 +-
...Jobs.yml => beam_Publish_Python_VLLM_Image.yml} | 71 +-
.github/workflows/beam_Publish_Website.yml | 4 +-
.../beam_Python_CostBenchmarks_Dataflow.yml | 2 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 2 +-
.github/workflows/beam_Release_NightlySnapshot.yml | 2 +-
.../beam_Release_Python_NightlySnapshot.yml | 2 +-
.../workflows/beam_StressTests_Java_BigQueryIO.yml | 2 +-
.../workflows/beam_StressTests_Java_BigTableIO.yml | 2 +-
.../workflows/beam_StressTests_Java_KafkaIO.yml | 2 +-
.../workflows/beam_StressTests_Java_PubSubIO.yml | 2 +-
.../workflows/beam_StressTests_Java_SpannerIO.yml | 2 +-
.github/workflows/build_release_candidate.yml | 24 +-
.github/workflows/build_runner_image.yml | 2 +-
.github/workflows/build_wheels.yml | 8 +-
.github/workflows/cancel.yml | 2 +-
.github/workflows/code_completion_plugin_tests.yml | 4 +-
.github/workflows/codeql.yml | 134 +
.github/workflows/cut_release_branch.yml | 9 +-
.github/workflows/dask_runner_tests.yml | 4 +-
.../workflows/deploy_release_candidate_pypi.yaml | 2 +-
.github/workflows/finalize_release.yml | 6 +-
.github/workflows/flaky_test_detection.yml | 2 +-
.github/workflows/git_tag_released_version.yml | 2 +-
.github/workflows/go_tests.yml | 2 +-
.github/workflows/issue-tagger.yml | 2 +-
.github/workflows/java_tests.yml | 4 +-
...s_Dataflow_MLTransform_Generate_Vocab_Batch.txt | 42 +
...Dataflow_MLTransform_One_Hot_Encoding_Batch.txt | 36 +
.github/workflows/local_env_tests.yml | 4 +-
.github/workflows/playground_frontend_test.yml | 6 +-
.github/workflows/pr-bot-new-prs.yml | 2 +-
.github/workflows/pr-bot-pr-updates.yml | 2 +-
.github/workflows/pr-bot-prs-needing-attention.yml | 2 +-
.github/workflows/publish_github_release_notes.yml | 4 +-
.github/workflows/python_dependency_tests.yml | 2 +-
.github/workflows/python_tests.yml | 8 +-
.github/workflows/refresh_looker_metrics.yml | 2 +-
.github/workflows/reportGenerator.yml | 2 +-
.../republish_released_docker_containers.yml | 6 +-
.github/workflows/run_perf_alert_tool.yml | 2 +-
.../workflows/run_rc_validation_go_wordcount.yml | 2 +-
.../run_rc_validation_java_mobile_gaming.yml | 2 +-
.../run_rc_validation_java_quickstart.yml | 4 +-
.../run_rc_validation_python_mobile_gaming.yml | 2 +-
.../workflows/run_rc_validation_python_yaml.yml | 2 +-
.github/workflows/tour_of_beam_backend.yml | 2 +-
.../workflows/tour_of_beam_backend_integration.yml | 2 +-
.github/workflows/tour_of_beam_frontend_test.yml | 6 +-
.github/workflows/typescript_tests.yml | 18 +-
.github/workflows/update_python_dependencies.yml | 4 +-
.test-infra/dataproc/flink_cluster.sh | 84 +-
.test-infra/dataproc/init-actions/docker.sh | 103 -
.test-infra/dataproc/init-actions/flink.sh | 217 -
.test-infra/tools/refresh_looker_metrics.py | 4 +-
.test-infra/validate-runner/build.gradle | 13 +
CHANGES.md | 75 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 75 +-
.../org/apache/beam/gradle/Repositories.groovy | 4 +-
examples/java/common.gradle | 2 +
.../notebooks/beam-ml/run_inference_gemini.ipynb | 609 ++
gradle.properties | 6 +-
infra/enforcement/README.md | 12 +-
infra/enforcement/account_keys.py | 103 +-
infra/enforcement/sending.py | 105 +-
infra/iam/users.yml | 8 +-
.../beam/model/fn_execution/v1/beam_fn_api.proto | 38 +-
.../beam/model/pipeline/v1/beam_runner_api.proto | 4 +
.../model/pipeline/v1/external_transforms.proto | 2 +
.../life_cycle/life_cycle_setuper_test.go | 3 +-
release/build.gradle.kts | 2 +-
.../org/apache/beam/runners/core/DoFnRunner.java | 12 +
.../org/apache/beam/runners/core/DoFnRunners.java | 4 +-
.../runners/core/LateDataDroppingDoFnRunner.java | 23 +-
.../apache/beam/runners/core/ReduceFnRunner.java | 3 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 49 +-
.../beam/runners/core/StatefulDoFnRunner.java | 6 +
.../core/LateDataDroppingDoFnRunnerTest.java | 4 +-
.../apache/beam/runners/core/ReduceFnTester.java | 20 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 4 +
.../direct/ExecutorServiceParallelExecutor.java | 28 +-
.../flink/streaming/MemoryStateBackendWrapper.java | 80 -
.../runners/flink/streaming/StreamSources.java | 61 -
runners/flink/2.1/build.gradle | 50 +
.../job-server-container/build.gradle | 0
.../flink/{1.17 => 2.1}/job-server/build.gradle | 9 +-
runners/flink/2.2/build.gradle | 65 +
.../job-server-container/build.gradle | 0
.../flink/{1.18 => 2.2}/job-server/build.gradle | 9 +-
.../runners/flink/FlinkExecutionEnvironments.java | 507 ++
.../wrappers/streaming/DoFnOperator.java | 1786 +++++
.../runners/flink/FlinkPipelineOptionsTest.java | 205 +
runners/flink/job-server/flink_job_server.gradle | 22 +-
.../runners/flink/FlinkDetachedRunnerResult.java | 60 +-
.../beam/runners/flink/FlinkRunnerResult.java | 6 +
.../flink/metrics/DoFnRunnerWithMetricsUpdate.java | 4 +
.../streaming/ExecutableStageDoFnOperator.java | 3 +
.../wrappers/streaming/WindowDoFnOperator.java | 2 +-
.../streaming/stableinput/BufferingDoFnRunner.java | 3 +
.../beam/runners/flink/FlinkRunnerResultTest.java | 92 +
.../flink/streaming/MemoryStateBackendWrapper.java | 28 +-
.../runners/flink/streaming/StreamSources.java | 7 +
.../wrappers/streaming/DoFnOperatorTest.java | 3 +
.../google-cloud-dataflow-java/arm/build.gradle | 2 +-
runners/google-cloud-dataflow-java/build.gradle | 22 +-
.../beam/runners/dataflow/DataflowPipelineJob.java | 67 +-
.../beam/runners/dataflow/DataflowRunner.java | 41 +-
.../options/DataflowStreamingPipelineOptions.java | 19 +-
.../options/DataflowWorkerLoggingOptions.java | 56 +-
.../beam/runners/dataflow/util/PackageUtil.java | 3 +
.../runners/dataflow/DataflowPipelineJobTest.java | 46 +
.../beam/runners/dataflow/DataflowRunnerTest.java | 49 +-
.../runners/dataflow/util/PackageUtilTest.java | 4 +
.../worker/AssignWindowsParDoFnFactory.java | 3 +
.../worker/BatchModeUngroupingParDoFn.java | 3 +
.../CreateIsmShardKeyAndSortKeyDoFnFactory.java | 3 +
.../dataflow/worker/DataflowProcessFnRunner.java | 6 +
.../runners/dataflow/worker/ForwardingParDoFn.java | 6 +
.../dataflow/worker/GroupAlsoByWindowFnRunner.java | 4 +
.../dataflow/worker/GroupAlsoByWindowsParDoFn.java | 8 +-
.../beam/runners/dataflow/worker/HotKeyLogger.java | 10 +-
.../worker/PairWithConstantKeyDoFnFactory.java | 3 +
.../dataflow/worker/PartialGroupByKeyParDoFns.java | 6 +
.../ReifyTimestampAndWindowsParDoFnFactory.java | 3 +
.../dataflow/worker/SimpleDoFnRunnerFactory.java | 12 -
.../runners/dataflow/worker/SimpleParDoFn.java | 534 +-
...impleParDoFn.java => SimpleParDoFnHelpers.java} | 409 +-
.../worker/SplittableProcessFnFactory.java | 20 +-
.../dataflow/worker/StreamingDataflowWorker.java | 29 +-
.../StreamingKeyedWorkItemSideInputDoFnRunner.java | 6 +
.../StreamingKeyedWorkItemSideInputParDoFn.java | 249 +
.../worker/StreamingModeExecutionContext.java | 293 +-
.../StreamingPCollectionViewWriterParDoFn.java | 3 +
.../worker/StreamingSideInputDoFnRunner.java | 47 +-
.../dataflow/worker/StreamingSideInputFetcher.java | 34 +-
.../worker/StreamingSideInputProcessor.java | 132 +
.../worker/ToIsmRecordForMultimapDoFnFactory.java | 3 +
.../dataflow/worker/UserParDoFnFactory.java | 56 +-
.../runners/dataflow/worker/ValuesDoFnFactory.java | 3 +
.../dataflow/worker/WindmillTimerInternals.java | 23 +
.../WorkerCustomSourceOperationExecutor.java | 2 +-
.../logging/DataflowWorkerLoggingInitializer.java | 12 +-
.../streaming/BoundedQueueExecutorWorkHandle.java} | 17 +-
.../dataflow/worker/streaming/ExecutableWork.java | 63 +-
.../runners/dataflow/worker/streaming/Work.java | 71 +
.../FanOutStreamingEngineWorkerHarness.java | 7 +-
.../dataflow/worker/util/BoundedQueueExecutor.java | 197 +-
.../dataflow/worker/util/ExceptionUtils.java | 43 +
.../dataflow/worker/util/KeyGroupWorkQueue.java | 462 ++
.../util/common/worker/FlattenOperation.java | 3 +-
.../worker/util/common/worker/MapTaskExecutor.java | 7 +-
.../worker/util/common/worker/Operation.java | 4 +-
.../worker/util/common/worker/ParDoFn.java | 4 +
.../worker/util/common/worker/ParDoOperation.java | 4 +-
.../worker/util/common/worker/ReadOperation.java | 2 +-
.../worker/SimplePartialGroupByKeyParDoFn.java | 5 +
.../worker/util/common/worker/WorkExecutor.java | 2 +-
.../worker/util/common/worker/WriteOperation.java | 3 +-
.../windmill/client/grpc/GrpcCommitWorkStream.java | 94 +-
.../client/grpc/GrpcWindmillStreamFactory.java | 10 +
.../windmill/state/WindmillStateInternals.java | 17 +
.../work/processing/StreamingCommitFinalizer.java | 2 +-
.../work/processing/StreamingWorkScheduler.java | 23 +-
.../worker/IntrinsicMapTaskExecutorTest.java | 11 +-
.../dataflow/worker/SimpleParDoFnHelpersTest.java | 133 +
.../runners/dataflow/worker/SimpleParDoFnTest.java | 6 +-
.../worker/StreamingDataflowWorkerTest.java | 16 +-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 3 +-
...eamingKeyedWorkItemSideInputDoFnRunnerTest.java | 3 +-
...StreamingKeyedWorkItemSideInputParDoFnTest.java | 488 ++
.../worker/StreamingModeExecutionContextTest.java | 134 +-
.../worker/StreamingSideInputDoFnRunnerTest.java | 4 +
.../worker/StreamingSideInputProcessorTest.java | 215 +
.../dataflow/worker/UserParDoFnFactoryTest.java | 27 +-
.../DataflowWorkerLoggingInitializerTest.java | 44 +-
.../worker/streaming/ActiveWorkStateTest.java | 4 +-
.../streaming/ComputationStateCacheTest.java | 2 +-
.../worker/util/BoundedQueueExecutorTest.java | 257 +-
.../worker/util/KeyGroupWorkQueueTest.java | 504 ++
.../util/common/worker/ExecutorTestUtils.java | 2 +-
.../util/common/worker/MapTaskExecutorTest.java | 11 +-
.../util/common/worker/ParDoOperationTest.java | 8 +-
.../client/grpc/GrpcCommitWorkStreamTest.java | 82 +
.../processing/StreamingCommitFinalizerTest.java | 3 +-
.../failures/WorkFailureProcessorTest.java | 7 +-
.../work/refresh/ActiveWorkRefresherTest.java | 7 +-
.../worker/windmill/src/main/proto/windmill.proto | 9 +-
.../fnexecution/control/SdkHarnessClient.java | 2 +-
.../runners/fnexecution/data/FnDataService.java | 3 +-
.../runners/fnexecution/data/GrpcDataService.java | 13 +-
.../fnexecution/data/GrpcDataServiceTest.java | 2 +-
runners/prism/java/build.gradle | 4 +
runners/spark/job-server/spark_job_server.gradle | 2 +
runners/spark/spark_runner.gradle | 2 +
.../translation/batch/DoFnRunnerFactory.java | 4 +
.../translation/batch/DoFnRunnerWithMetrics.java | 4 +
.../spark/translation/DoFnRunnerWithMetrics.java | 4 +
.../translation/SparkInputDataProcessorTest.java | 4 +
scripts/beam-sql.sh | 75 +-
scripts/ci/issue-report/package-lock.json | 14 +-
scripts/ci/issue-report/package.json | 2 +-
scripts/tools/bomupgrader.py | 53 +-
sdks/go.mod | 88 +-
sdks/go.sum | 190 +-
sdks/go/examples/wasm/README.md | 6 +-
sdks/go/pkg/beam/core/core.go | 2 +-
.../pkg/beam/core/runtime/exec/datasampler_test.go | 13 +-
sdks/go/pkg/beam/io/filesystem/gcs/gcs.go | 88 +-
sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go | 146 +
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 9 +-
sdks/go/pkg/beam/runners/dataflow/dataflow_test.go | 43 +-
.../beam/runners/dataflow/dataflowlib/execute.go | 1 +
.../pkg/beam/runners/dataflow/dataflowlib/job.go | 4 +
.../beam/runners/dataflow/dataflowlib/job_test.go | 16 +
.../prism/internal/engine/elementmanager.go | 7 +-
.../beam/runners/prism/internal/handlerunner.go | 4 +-
.../runners/prism/internal/handlerunner_test.go | 78 +
sdks/go/test/build.gradle | 3 +
.../container/license_scripts/dep_urls_java.yaml | 10 +-
.../java/org/apache/beam/sdk/PipelineResult.java | 13 +
.../sdk/fn/data/BeamFnDataGrpcMultiplexer.java | 2 +-
.../sdk/fn/data/BeamFnDataOutboundAggregator.java | 131 +-
.../org/apache/beam/sdk/io/CountingSource.java | 40 +-
.../sdk/schemas/FieldValueTypeInformation.java | 13 +-
.../beam/sdk/schemas/FromRowUsingCreator.java | 6 +-
.../sdk/schemas/GetterBasedSchemaProvider.java | 98 +-
.../beam/sdk/schemas/utils/AutoValueUtils.java | 9 +-
.../beam/sdk/testing/UsesSideInputsInTimer.java | 23 +-
.../apache/beam/sdk/transforms/AsyncWrapper.java | 781 ++
.../java/org/apache/beam/sdk/transforms/DoFn.java | 16 +
.../org/apache/beam/sdk/transforms/JsonToRow.java | 8 +-
.../apache/beam/sdk/transforms/LogElements.java | 234 +
.../reflect/ByteBuddyDoFnInvokerFactory.java | 12 +
.../beam/sdk/transforms/reflect/DoFnInvoker.java | 17 +
.../beam/sdk/transforms/reflect/DoFnSignature.java | 9 +
.../sdk/transforms/reflect/DoFnSignatures.java | 10 +-
.../java/org/apache/beam/sdk/util/RowJson.java | 65 +-
.../beam/sdk/util/RowJsonValueExtractors.java | 9 +-
.../beam/sdk/util/construction/Environments.java | 1 +
.../construction/SplittableParDoNaiveBounded.java | 6 +
.../main/java/org/apache/beam/sdk/values/Row.java | 6 +-
.../org/apache/beam/sdk/values/RowWithGetters.java | 13 +-
.../fn/data/BeamFnDataOutboundAggregatorTest.java | 135 +-
.../org/apache/beam/sdk/io/CountingSourceTest.java | 36 +
.../beam/sdk/schemas/AutoValueSchemaTest.java | 426 +-
.../beam/sdk/schemas/JavaBeanSchemaTest.java | 167 +-
.../beam/sdk/schemas/JavaFieldSchemaTest.java | 196 +-
.../beam/sdk/schemas/utils/TestJavaBeans.java | 35 +
.../apache/beam/sdk/schemas/utils/TestPOJOs.java | 25 +
.../beam/sdk/transforms/AsyncWrapperTest.java | 941 +++
.../apache/beam/sdk/transforms/JsonToRowTest.java | 26 +
.../beam/sdk/transforms/LogElementsTest.java | 108 +
.../org/apache/beam/sdk/transforms/ParDoTest.java | 153 +
.../sdk/transforms/reflect/DoFnSignaturesTest.java | 8 +-
.../java/org/apache/beam/sdk/util/RowJsonTest.java | 26 +-
sdks/java/expansion-service/container/Dockerfile | 2 +-
.../beam/sdk/extensions/arrow/ArrowConversion.java | 4 +-
.../opentelemetry-gcp-auth-extension/build.gradle | 51 +
.../opentelemetry/gcp/auth/ConfigurableOption.java | 164 +
...GcpAuthAutoConfigurationCustomizerProvider.java | 292 +
.../gcp/auth/GoogleAuthException.java | 69 +
.../opentelemetry/gcp/auth/package-info.java | 13 +-
...uthAutoConfigurationCustomizerProviderTest.java | 1310 ++++
.../extensions/protobuf/ProtoByteBuddyUtils.java | 49 +-
.../meta/provider/iceberg/IcebergMetastore.java | 21 +-
.../sql/meta/provider/iceberg/IcebergTable.java | 19 +-
.../provider/iceberg/IcebergMetastoreTest.java | 13 +
.../meta/provider/iceberg/PubsubToIcebergIT.java | 4 +-
.../sql/src/main/codegen/includes/parserImpls.ftl | 4 +-
.../extensions/sql/impl/CalciteQueryPlanner.java | 93 +-
.../beam/sdk/extensions/sql/impl/UdfImpl.java | 19 +-
.../extensions/sql/impl/parser/SqlDdlNodes.java | 7 +-
.../sdk/extensions/sql/impl/rel/BeamSortRel.java | 58 +-
.../impl/transform/BeamBuiltinAggregations.java | 2 +
.../sql/impl/transform/agg/VarianceFn.java | 38 +-
.../extensions/sql/impl/utils/CalciteUtils.java | 31 +-
.../sql/meta/catalog/InMemoryCatalog.java | 12 +-
.../{BeamSqlAliasTest => BeamSqlAliasTest.java} | 12 +-
.../sdk/extensions/sql/BeamSqlCliDatabaseTest.java | 52 +
.../sql/BeamSqlDslAggregationVarianceTest.java | 43 +-
.../extensions/sql/BeamSqlDslParametersTest.java | 78 +
.../sql/impl/LazyAggregateCombineFnTest.java | 5 +-
.../beam/sdk/extensions/sql/impl/UdfImplTest.java | 65 +
.../extensions/sql/impl/rel/BeamSortRelTest.java | 43 +
.../sql/impl/transform/agg/VarianceFnTest.java | 25 +-
.../sql/impl/utils/CalciteUtilsTest.java | 63 +
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 41 +
.../java/org/apache/beam/fn/harness/FnHarness.java | 66 +-
.../fn/harness/control/ProcessBundleHandler.java | 50 +-
.../beam/fn/harness/data/BeamFnDataClient.java | 28 +-
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 91 +-
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 44 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 2 +-
.../PTransformRunnerFactoryTestContext.java | 12 +-
.../harness/control/ProcessBundleHandlerTest.java | 47 +-
.../fn/harness/data/BeamFnDataGrpcClientTest.java | 41 +-
.../sdk/io/aws2/schemas/AwsSchemaProvider.java | 7 +-
.../apache/beam/sdk/io/aws2/schemas/AwsTypes.java | 16 +
.../beam/sdk/io/clickhouse/ClickHouseIO.java | 3 +
.../beam/sdk/io/clickhouse/ClickHouseWriter.java | 40 +
.../apache/beam/sdk/io/clickhouse/TableSchema.java | 53 +-
.../clickhouse/src/main/javacc/ColumnTypeParser.jj | 27 +
.../beam/sdk/io/clickhouse/ClickHouseIOIT.java | 119 +
.../sdk/io/clickhouse/ClickHouseWriterTest.java | 141 +
.../beam/sdk/io/clickhouse/TableSchemaTest.java | 102 +
.../apache/beam/sdk/io/datadog/DatadogEvent.java | 6 +
.../DatadogWriteSchemaTransformConfiguration.java | 114 +
.../DatadogWriteSchemaTransformProvider.java | 294 +
.../DatadogWriteSchemaTransformProviderTest.java | 535 ++
sdks/java/io/delta/build.gradle | 45 +
.../org/apache/beam/sdk/io/delta/BeamEngine.java | 55 +
.../beam/sdk/io/delta/BeamParquetHandler.java | 376 +
.../beam/sdk/io/delta/CreateReadTasksDoFn.java | 140 +
.../java/org/apache/beam/sdk/io/delta/DeltaIO.java | 92 +-
.../io/delta/DeltaReadSchemaTransformProvider.java | 142 +
.../apache/beam/sdk/io/delta/DeltaReadTask.java | 88 +
.../beam/sdk/io/delta/DeltaReadTaskTracker.java | 56 +
.../apache/beam/sdk/io/delta/DeltaSourceDoFn.java | 455 ++
.../apache/beam/sdk/io/delta/SerializableRow.java | 544 ++
.../beam/sdk/io/delta/SerializableStructType.java | 69 +
.../org/apache/beam/sdk/io/delta/DeltaIOTest.java | 753 +-
.../beam/sdk/io/delta/SerializableRowTest.java | 498 ++
sdks/java/io/expansion-service/build.gradle | 7 +-
...ueryStorageWriteApiSchemaTransformProvider.java | 9 +-
.../providers/BigQueryWriteConfiguration.java | 9 +-
.../beam/sdk/io/gcp/pubsub/ExternalWrite.java | 6 +
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 5 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 13 +-
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 37 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 179 +-
.../io/gcp/spanner/SpannerTransformRegistrar.java | 30 +
.../MetadataSpannerConfigFactory.java | 6 +
...StorageWriteApiSchemaTransformProviderTest.java | 5 +-
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 33 +-
.../beam/sdk/io/gcp/spanner/SpannerTestHelper.java | 110 +
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 110 +-
.../changestreams/it/IntegrationTestEnv.java | 17 +-
.../changestreams/it/SpannerChangeStreamIT.java | 19 +-
...StreamOrderedByTimestampAndTransactionIdIT.java | 10 +-
...nnerChangeStreamOrderedWithinKeyGloballyIT.java | 10 +-
.../it/SpannerChangeStreamOrderedWithinKeyIT.java | 10 +-
.../it/SpannerChangeStreamPlacementTableIT.java | 19 +-
...pannerChangeStreamPlacementTablePostgresIT.java | 10 +-
.../it/SpannerChangeStreamPostgresIT.java | 10 +-
...SpannerChangeStreamTransactionBoundariesIT.java | 10 +-
.../it/SpannerChangeStreamsSchemaTransformIT.java | 4 +
.../org/apache/beam/sdk/io/iceberg/AddFiles.java | 39 +-
.../iceberg/AssignDestinationsAndPartitions.java | 5 +-
.../beam/sdk/io/iceberg/CreateReadTasksDoFn.java | 7 +-
.../beam/sdk/io/iceberg/IcebergCatalogConfig.java | 12 +-
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 2 +-
.../beam/sdk/io/iceberg/IcebergScanConfig.java | 81 +-
.../beam/sdk/io/iceberg/IncrementalScanSource.java | 6 +-
.../apache/beam/sdk/io/iceberg/PartitionUtils.java | 100 +-
.../apache/beam/sdk/io/iceberg/ReadFromTasks.java | 11 +-
.../org/apache/beam/sdk/io/iceberg/ReadUtils.java | 137 +-
.../beam/sdk/io/iceberg/RecordWriterManager.java | 145 +-
.../org/apache/beam/sdk/io/iceberg/ScanSource.java | 3 +-
.../apache/beam/sdk/io/iceberg/ScanTaskReader.java | 4 +-
.../beam/sdk/io/iceberg/SerializableDataFile.java | 100 +-
.../sdk/io/iceberg/SerializableDeleteFile.java | 335 +
.../org/apache/beam/sdk/io/iceberg/TableCache.java | 198 +-
.../beam/sdk/io/iceberg/WatchForSnapshots.java | 7 +-
.../sdk/io/iceberg/WriteDirectRowsToFiles.java | 12 +-
.../sdk/io/iceberg/WriteGroupedRowsToFiles.java | 12 +-
.../io/iceberg/WritePartitionedRowsToFiles.java | 166 +-
.../sdk/io/iceberg/WriteUngroupedRowsToFiles.java | 12 +-
.../beam/sdk/io/iceberg/cdc/CdcOutputUtils.java | 178 +
.../beam/sdk/io/iceberg/cdc/CdcReadUtils.java | 698 ++
.../beam/sdk/io/iceberg/cdc/DeleteReader.java | 309 +
.../io/iceberg/cdc/IcebergCdcMetadataColumns.java | 93 +
.../io/iceberg/cdc/SerializableChangelogTask.java | 282 +
.../beam/sdk/io/iceberg/cdc/package-info.java | 13 +-
.../iceberg/BaseIncrementalChangelogScan.java | 1014 +++
.../main/java/org/apache/iceberg/package-info.java | 13 +-
.../IcebergCdcReadSchemaTransformProviderTest.java | 12 +-
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 83 +-
.../IcebergSchemaTransformTranslationTest.java | 33 +-
.../beam/sdk/io/iceberg/PartitionUtilsTest.java | 68 +
.../apache/beam/sdk/io/iceberg/ReadUtilsTest.java | 59 +-
.../sdk/io/iceberg/RecordWriterManagerTest.java | 121 +-
.../sdk/io/iceberg/SerializableDataFileTest.java | 3 +
.../sdk/io/iceberg/SerializableDeleteFileTest.java | 222 +
.../apache/beam/sdk/io/iceberg/TableCacheTest.java | 126 +
.../beam/sdk/io/iceberg/TestDataWarehouse.java | 2 +-
.../io/iceberg/catalog/IcebergCatalogBaseIT.java | 7 +-
.../beam/sdk/io/iceberg/cdc/CdcReadUtilsTest.java | 381 +
.../beam/sdk/io/iceberg/cdc/DeleteReaderTest.java | 419 +
.../iceberg/cdc/SerializableChangelogTaskTest.java | 256 +
.../io/messaging-expansion-service/build.gradle | 52 +
sdks/java/io/mongodb/build.gradle | 2 +
.../beam/sdk/io/mongodb/AggregationQuery.java | 9 +-
.../MongoDbReadSchemaTransformConfiguration.java | 89 +
.../MongoDbReadSchemaTransformProvider.java | 151 +
.../apache/beam/sdk/io/mongodb/MongoDbUtils.java | 205 +
.../MongoDbWriteSchemaTransformConfiguration.java | 85 +
.../MongoDbWriteSchemaTransformProvider.java | 168 +
.../MongoDbReadSchemaTransformProviderTest.java | 221 +
.../beam/sdk/io/mongodb/MongoDbUtilsTest.java | 258 +
.../MongoDbWriteSchemaTransformProviderTest.java | 166 +
.../java/org/apache/beam/sdk/io/mqtt/MqttIO.java | 7 +
.../io/mqtt/MqttReadSchemaTransformProvider.java | 150 +
.../io/mqtt/MqttWriteSchemaTransformProvider.java | 132 +
.../io/mqtt/MqttSchemaTransformProviderTest.java | 252 +
.../org/apache/beam/sdk/io/parquet/ParquetIO.java | 47 +
.../org/apache/beam/io/requestresponse/Call.java | 1 +
.../apache/beam/io/requestresponse/CallTest.java | 53 +-
.../java/org/apache/beam/sdk/managed/Managed.java | 4 +
sdks/java/testing/nexmark/build.gradle | 15 +
sdks/java/testing/tpcds/build.gradle | 15 +
.../controller-container/Dockerfile | 2 +-
sdks/python/apache_beam/dataframe/frames.py | 7 +-
sdks/python/apache_beam/dataframe/frames_test.py | 37 +-
sdks/python/apache_beam/dataframe/io.py | 15 +-
sdks/python/apache_beam/dataframe/io_test.py | 45 +
.../cookbook/ordered_window_elements/streaming.py | 2 +-
.../anomaly_detection_pipeline/setup.py | 2 +-
.../examples/inference/gemini_image_generation.py | 4 +-
.../inference/gemini_text_classification.py | 4 +-
.../online_clustering/clustering_pipeline/setup.py | 2 +-
.../examples/inference/vllm_text_completion.py | 24 +-
.../apache_beam/examples/kafkataxi/README.md | 2 +-
.../kfp/components/train/requirements.txt | 2 +-
.../apache_beam/examples/ml_transform/README.md | 116 +
.../ml_transform/mltransform_generate_vocab.py | 265 +
.../mltransform_generate_vocab_requirements.txt} | 14 +-
.../mltransform_generate_vocab_test.py | 191 +
.../ml_transform/mltransform_one_hot_encoding.py | 266 +
.../mltransform_one_hot_encoding_test.py | 259 +
.../apache_beam/examples/wordcount_rust/README.md | 4 +-
.../examples/wordcount_rust/requirements.txt | 2 +-
sdks/python/apache_beam/io/external/gcp/pubsub.py | 11 +-
.../io/external/xlang_bigqueryio_it_test.py | 2 +-
.../io/external/xlang_mqttio_it_test.py | 268 +
sdks/python/apache_beam/io/fileio.py | 3 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 3 +-
.../apache_beam/io/gcp/bigquery_file_loads.py | 2 +-
sdks/python/apache_beam/io/gcp/gcsio.py | 61 +-
.../apache_beam/io/gcp/gcsio_integration_test.py | 21 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 132 +
sdks/python/apache_beam/io/gcp/pubsub.py | 48 +-
.../apache_beam/io/gcp/pubsub_integration_test.py | 98 +
sdks/python/apache_beam/io/gcp/pubsub_test.py | 151 +-
sdks/python/apache_beam/io/iobase.py | 14 +-
sdks/python/apache_beam/io/iobase_test.py | 49 +-
sdks/python/apache_beam/io/unbounded_source.py | 996 +++
.../python/apache_beam/io/unbounded_source_test.py | 1380 ++++
sdks/python/apache_beam/metrics/metric.py | 2 +-
sdks/python/apache_beam/ml/inference/base.py | 6 +
sdks/python/apache_beam/ml/inference/base_test.py | 2 +-
.../ml/inference/pytorch_inference_test.py | 5 +-
.../ml/inference/sklearn_inference_test.py | 4 +-
.../ml/inference/tensorflow_inference_test.py | 4 +-
.../ml/inference/tensorrt_inference_test.py | 11 +-
.../inference/test_resources/vllm.dockerfile.old | 14 +-
.../apache_beam/ml/inference/vllm_inference.py | 331 +-
.../ml/inference/vllm_inference_test.py | 157 +
.../ml/rag/enrichment/milvus_search_it_test.py | 1 +
.../ml/rag/ingestion/milvus_search_it_test.py | 1 +
sdks/python/apache_beam/ml/rag/ingestion/qdrant.py | 328 +
.../apache_beam/ml/rag/ingestion/qdrant_it_test.py | 326 +
.../apache_beam/ml/rag/ingestion/qdrant_test.py | 480 ++
sdks/python/apache_beam/ml/rag/test_utils.py | 1 +
.../transforms/mltransform_tests_requirements.txt} | 15 +-
.../python/apache_beam/options/pipeline_options.py | 87 +-
.../apache_beam/options/pipeline_options_test.py | 37 +
.../options/pipeline_options_validator.py | 2 +
.../runners/dataflow/dataflow_metrics.py | 65 +-
.../runners/dataflow/dataflow_metrics_test.py | 314 +-
.../runners/dataflow/dataflow_runner.py | 72 +-
.../runners/dataflow/dataflow_runner_test.py | 59 +-
.../runners/dataflow/internal/apiclient.py | 527 +-
.../runners/dataflow/internal/apiclient_test.py | 328 +-
.../dataflow/internal/clients/dataflow/__init__.py | 34 -
.../clients/dataflow/dataflow_v1b3_client.py | 1318 ----
.../clients/dataflow/dataflow_v1b3_messages.py | 8077 --------------------
.../internal/clients/dataflow/message_matchers.py | 118 -
.../clients/dataflow/message_matchers_test.py | 74 -
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
.../runners/direct/transform_evaluator.py | 113 +-
.../runners/portability/portable_runner_test.py | 32 +-
.../apache_beam/runners/portability/stager.py | 151 +-
.../apache_beam/runners/worker/opcounters.py | 2 +
.../apache_beam/runners/worker/opcounters_test.py | 16 +
.../runners/worker/statesampler_fast.pyx | 19 +-
.../runners/worker/statesampler_test.py | 43 +
.../cloudml/{requirements.txt => constraints.txt} | 34 +-
.../testing/benchmarks/cloudml/requirements.txt | 17 +-
.../mltransform_generate_vocab_benchmark.py | 56 +
.../mltransform_one_hot_encoding_benchmark.py | 197 +
.../testing/load_tests/dataflow_cost_benchmark.py | 48 +-
sdks/python/apache_beam/transforms/async_dofn.py | 4 +-
.../apache_beam/transforms/async_dofn_test.py | 47 +
sdks/python/apache_beam/transforms/core.py | 4 +-
sdks/python/apache_beam/transforms/stats_test.py | 20 +-
sdks/python/apache_beam/transforms/util_test.py | 49 +
sdks/python/apache_beam/typehints/opcodes.py | 8 +
sdks/python/apache_beam/typehints/row_type_test.py | 21 +
.../apache_beam/typehints/trivial_inference.py | 26 +
.../typehints/trivial_inference_test.py | 27 +
sdks/python/apache_beam/typehints/typehints.py | 1 +
sdks/python/apache_beam/utils/subprocess_server.py | 89 +-
.../apache_beam/utils/subprocess_server_test.py | 22 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/apache_beam/yaml/examples/README.md | 4 +
.../yaml/extended_tests/databases/iceberg.yaml | 54 +-
sdks/python/apache_beam/yaml/integration_tests.py | 172 +-
sdks/python/apache_beam/yaml/main.py | 7 +-
sdks/python/apache_beam/yaml/readme_test.py | 16 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 54 +-
.../test_utils/__init__.py} | 3 +-
.../yaml/test_utils/datadog_test_utils.py | 131 +
sdks/python/apache_beam/yaml/tests/datadog.yaml | 63 +
.../apache_beam/yaml/tests/iceberg_add_files.yaml | 3 +
sdks/python/apache_beam/yaml/tests/mongodb.yaml | 74 +
.../yaml/tests/runinference_huggingface.yaml | 62 +
...uninference.yaml => runinference_vertexai.yaml} | 0
sdks/python/apache_beam/yaml/yaml_io.py | 110 +-
sdks/python/apache_beam/yaml/yaml_io_test.py | 103 +
sdks/python/apache_beam/yaml/yaml_mapping.py | 171 +-
sdks/python/apache_beam/yaml/yaml_ml.py | 95 +-
sdks/python/apache_beam/yaml/yaml_provider.py | 11 +-
sdks/python/apache_beam/yaml/yaml_transform.py | 56 +-
.../python/apache_beam/yaml/yaml_transform_test.py | 119 +
sdks/python/apache_beam/yaml/yaml_udf_test.py | 119 +-
sdks/python/build.gradle | 29 +-
sdks/python/container/Dockerfile | 17 +-
.../container/base_image_requirements_manual.txt | 2 +
sdks/python/container/boot.go | 100 +-
.../license_scripts/upgrade_bundled_pip.py | 65 +
.../container/ml/py310/base_image_requirements.txt | 104 +-
.../container/ml/py310/gpu_image_requirements.txt | 140 +-
.../container/ml/py311/base_image_requirements.txt | 108 +-
.../container/ml/py311/gpu_image_requirements.txt | 144 +-
.../container/ml/py312/base_image_requirements.txt | 107 +-
.../container/ml/py312/gpu_image_requirements.txt | 146 +-
.../container/ml/py313/base_image_requirements.txt | 107 +-
sdks/python/container/profiler.go | 336 +
.../container/py310/base_image_requirements.txt | 99 +-
.../container/py311/base_image_requirements.txt | 103 +-
.../container/py312/base_image_requirements.txt | 102 +-
.../container/py313/base_image_requirements.txt | 102 +-
.../container/py314/base_image_requirements.txt | 100 +-
sdks/python/pyproject.toml | 4 +-
sdks/python/pytest.ini | 1 +
sdks/python/scripts/run_integration_test.sh | 11 +-
sdks/python/setup.py | 33 +-
sdks/python/test-suites/dataflow/common.gradle | 46 +-
sdks/python/test-suites/direct/build.gradle | 6 +
sdks/python/test-suites/xlang/build.gradle | 11 +-
sdks/standard_expansion_services.yaml | 15 +
sdks/standard_external_transforms.yaml | 156 +-
sdks/typescript/package-lock.json | 58 +-
sdks/typescript/package.json | 6 +-
sdks/typescript/src/apache_beam/runners/flink.ts | 2 +-
settings.gradle.kts | 4 +-
website/www/site/assets/js/bootstrap.js | 46 +-
website/www/site/assets/js/bootstrap/alert.js | 7 +-
website/www/site/assets/js/bootstrap/carousel.js | 9 +-
website/www/site/assets/js/bootstrap/collapse.js | 6 +-
website/www/site/assets/js/bootstrap/dropdown.js | 7 +-
website/www/site/assets/js/bootstrap/modal.js | 8 +-
website/www/site/assets/js/bootstrap/tooltip.js | 9 +-
website/www/site/assets/js/page-nav.js | 2 +-
website/www/site/config.toml | 2 +-
website/www/site/content/en/blog/beam-2.74.0.md | 70 +
.../site/content/en/documentation/dsls/sql/ddl.md | 344 +
.../content/en/documentation/dsls/sql/overview.md | 2 +
.../documentation/io/built-in/google-bigquery.md | 2 +-
.../en/documentation/io/built-in/iceberg.md | 2 +-
.../site/content/en/documentation/io/managed-io.md | 551 +-
.../site/content/en/documentation/runners/flink.md | 14 +-
.../site/content/en/documentation/sdks/python.md | 6 +-
.../www/site/content/en/documentation/sdks/yaml.md | 34 +
.../www/site/content/en/get-started/downloads.md | 22 +-
.../site/content/en/get-started/quickstart-py.md | 2 +-
website/www/site/content/en/performance/_index.md | 2 +
.../en/performance/mltransformonehot/_index.md | 42 +
.../en/performance/mltransformvocab/_index.md | 37 +
website/www/site/data/performance.yaml | 33 +
.../layouts/partials/section-menu/en/sdks.html | 1 +
website/www/site/layouts/shortcodes/tab.html | 22 +
website/www/yarn.lock | 6 +-
899 files changed, 40592 insertions(+), 15579 deletions(-)
create mode 100644 .agent/skills/developing-new-io-connectors/SKILL.md
copy .github/trigger_files/{beam_PostCommit_Java_Nexmark_Flink.json => beam_PostCommit_Java_Tpcds_Flink.json} (100%)
copy .github/trigger_files/{beam_PostCommit_Python_Xlang_IO_Direct.json => beam_PostCommit_Python_Xlang_Messaging_Direct.json} (100%)
delete mode 100644 .github/trigger_files/beam_PostCommit_XVR_Samza.json
copy .github/trigger_files/{beam_PostCommit_Java_Nexmark_Flink.json => beam_PreCommit_Java_Delta_IO_Direct.json} (100%)
create mode 100644 .github/workflows/beam_Infrastructure_AuditUnmanagedKeys.yml
copy .github/workflows/{beam_PostCommit_Python_Xlang_IO_Direct.yml => beam_PostCommit_Python_Xlang_Messaging_Direct.yml} (84%)
copy .github/workflows/{IO_Iceberg_Unit_Tests.yml => beam_PreCommit_Java_Delta_IO_Direct.yml} (83%)
copy .github/workflows/{beam_CancelStaleDataflowJobs.yml => beam_Publish_Python_VLLM_Image.yml} (55%)
create mode 100644 .github/workflows/codeql.yml
create mode 100644 .github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Generate_Vocab_Batch.txt
create mode 100644 .github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_One_Hot_Encoding_Batch.txt
delete mode 100644 .test-infra/dataproc/init-actions/docker.sh
delete mode 100644 .test-infra/dataproc/init-actions/flink.sh
create mode 100644 examples/notebooks/beam-ml/run_inference_gemini.ipynb
delete mode 100644 runners/flink/1.19/src/test/java/org/apache/beam/runners/flink/streaming/MemoryStateBackendWrapper.java
delete mode 100644 runners/flink/1.19/src/test/java/org/apache/beam/runners/flink/streaming/StreamSources.java
create mode 100644 runners/flink/2.1/build.gradle
rename runners/flink/{1.17 => 2.1}/job-server-container/build.gradle (100%)
rename runners/flink/{1.17 => 2.1}/job-server/build.gradle (80%)
create mode 100644 runners/flink/2.2/build.gradle
rename runners/flink/{1.18 => 2.2}/job-server-container/build.gradle (100%)
rename runners/flink/{1.18 => 2.2}/job-server/build.gradle (80%)
create mode 100644 runners/flink/2.2/src/main/java/org/apache/beam/runners/flink/FlinkExecutionEnvironments.java
create mode 100644 runners/flink/2.2/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
create mode 100644 runners/flink/2.2/src/test/java/org/apache/beam/runners/flink/FlinkPipelineOptionsTest.java
copy runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{SimpleParDoFn.java => SimpleParDoFnHelpers.java} (76%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingKeyedWorkItemSideInputParDoFn.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputProcessor.java
copy runners/{flink/1.18/build.gradle => google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/BoundedQueueExecutorWorkHandle.java} (64%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/ExceptionUtils.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/KeyGroupWorkQueue.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/SimpleParDoFnHelpersTest.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingKeyedWorkItemSideInputParDoFnTest.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputProcessorTest.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/KeyGroupWorkQueueTest.java
create mode 100644 sdks/go/pkg/beam/runners/prism/internal/handlerunner_test.go
copy runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/ParDoFn.java => sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesSideInputsInTimer.java (60%)
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/AsyncWrapper.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/LogElements.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/AsyncWrapperTest.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/LogElementsTest.java
create mode 100644 sdks/java/extensions/opentelemetry-gcp-auth-extension/build.gradle
create mode 100644 sdks/java/extensions/opentelemetry-gcp-auth-extension/src/main/java/org/apache/beam/sdk/extensions/opentelemetry/gcp/auth/ConfigurableOption.java
create mode 100644 sdks/java/extensions/opentelemetry-gcp-auth-extension/src/main/java/org/apache/beam/sdk/extensions/opentelemetry/gcp/auth/GcpAuthAutoConfigurationCustomizerProvider.java
create mode 100644 sdks/java/extensions/opentelemetry-gcp-auth-extension/src/main/java/org/apache/beam/sdk/extensions/opentelemetry/gcp/auth/GoogleAuthException.java
copy runners/flink/1.18/build.gradle => sdks/java/extensions/opentelemetry-gcp-auth-extension/src/main/java/org/apache/beam/sdk/extensions/opentelemetry/gcp/auth/package-info.java (69%)
create mode 100644 sdks/java/extensions/opentelemetry-gcp-auth-extension/src/test/java/org/apache/beam/sdk/extensions/opentelemetry/gcp/auth/GcpAuthAutoConfigurationCustomizerProviderTest.java
rename sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/{BeamSqlAliasTest => BeamSqlAliasTest.java} (92%)
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslParametersTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/impl/UdfImplTest.java
create mode 100644 sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseWriterTest.java
create mode 100644 sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogWriteSchemaTransformConfiguration.java
create mode 100644 sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogWriteSchemaTransformProvider.java
create mode 100644 sdks/java/io/datadog/src/test/java/org/apache/beam/sdk/io/datadog/DatadogWriteSchemaTransformProviderTest.java
create mode 100644 sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/BeamEngine.java
create mode 100644 sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/BeamParquetHandler.java
create mode 100644 sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/CreateReadTasksDoFn.java
create mode 100644 sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/DeltaReadSchemaTransformProvider.java
create mode 100644 sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/DeltaReadTask.java
create mode 100644 sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/DeltaReadTaskTracker.java
create mode 100644 sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/DeltaSourceDoFn.java
create mode 100644 sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/SerializableRow.java
create mode 100644 sdks/java/io/delta/src/main/java/org/apache/beam/sdk/io/delta/SerializableStructType.java
create mode 100644 sdks/java/io/delta/src/test/java/org/apache/beam/sdk/io/delta/SerializableRowTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerTestHelper.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SerializableDeleteFile.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/CdcOutputUtils.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/CdcReadUtils.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/DeleteReader.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/IcebergCdcMetadataColumns.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/SerializableChangelogTask.java
rename runners/flink/1.17/build.gradle => sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/package-info.java (69%)
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java
rename runners/flink/1.18/build.gradle => sdks/java/io/iceberg/src/main/java/org/apache/iceberg/package-info.java (69%)
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/SerializableDeleteFileTest.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/TableCacheTest.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/CdcReadUtilsTest.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/DeleteReaderTest.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/SerializableChangelogTaskTest.java
create mode 100644 sdks/java/io/messaging-expansion-service/build.gradle
create mode 100644 sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbReadSchemaTransformConfiguration.java
create mode 100644 sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbReadSchemaTransformProvider.java
create mode 100644 sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbUtils.java
create mode 100644 sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbWriteSchemaTransformConfiguration.java
create mode 100644 sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbWriteSchemaTransformProvider.java
create mode 100644 sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbReadSchemaTransformProviderTest.java
create mode 100644 sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbUtilsTest.java
create mode 100644 sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbWriteSchemaTransformProviderTest.java
create mode 100644 sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttReadSchemaTransformProvider.java
create mode 100644 sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttWriteSchemaTransformProvider.java
create mode 100644 sdks/java/io/mqtt/src/test/java/org/apache/beam/sdk/io/mqtt/MqttSchemaTransformProviderTest.java
create mode 100644 sdks/python/apache_beam/examples/ml_transform/README.md
create mode 100644 sdks/python/apache_beam/examples/ml_transform/mltransform_generate_vocab.py
copy sdks/python/apache_beam/{version.py => examples/ml_transform/mltransform_generate_vocab_requirements.txt} (67%)
create mode 100644 sdks/python/apache_beam/examples/ml_transform/mltransform_generate_vocab_test.py
create mode 100644 sdks/python/apache_beam/examples/ml_transform/mltransform_one_hot_encoding.py
create mode 100644 sdks/python/apache_beam/examples/ml_transform/mltransform_one_hot_encoding_test.py
create mode 100644 sdks/python/apache_beam/io/external/xlang_mqttio_it_test.py
create mode 100644 sdks/python/apache_beam/io/unbounded_source.py
create mode 100644 sdks/python/apache_beam/io/unbounded_source_test.py
create mode 100644 sdks/python/apache_beam/ml/inference/vllm_inference_test.py
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/qdrant.py
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/qdrant_it_test.py
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/qdrant_test.py
copy sdks/python/apache_beam/{version.py => ml/transforms/mltransform_tests_requirements.txt} (67%)
delete mode 100644 sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/__init__.py
delete mode 100644 sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
delete mode 100644 sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
delete mode 100644 sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers.py
delete mode 100644 sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/message_matchers_test.py
copy sdks/python/apache_beam/testing/benchmarks/cloudml/{requirements.txt => constraints.txt} (53%)
create mode 100644 sdks/python/apache_beam/testing/benchmarks/inference/mltransform_generate_vocab_benchmark.py
create mode 100644 sdks/python/apache_beam/testing/benchmarks/inference/mltransform_one_hot_encoding_benchmark.py
copy sdks/python/apache_beam/{examples/wordcount_rust/requirements.txt => yaml/test_utils/__init__.py} (93%)
create mode 100644 sdks/python/apache_beam/yaml/test_utils/datadog_test_utils.py
create mode 100644 sdks/python/apache_beam/yaml/tests/datadog.yaml
create mode 100644 sdks/python/apache_beam/yaml/tests/mongodb.yaml
create mode 100644 sdks/python/apache_beam/yaml/tests/runinference_huggingface.yaml
rename sdks/python/apache_beam/yaml/tests/{runinference.yaml => runinference_vertexai.yaml} (100%)
create mode 100644 sdks/python/container/license_scripts/upgrade_bundled_pip.py
create mode 100644 sdks/python/container/profiler.go
create mode 100644 website/www/site/content/en/blog/beam-2.74.0.md
create mode 100644 website/www/site/content/en/documentation/dsls/sql/ddl.md
create mode 100644 website/www/site/content/en/performance/mltransformonehot/_index.md
create mode 100644 website/www/site/content/en/performance/mltransformvocab/_index.md