You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tv...@apache.org on 2026/02/20 05:52:46 UTC
(beam) branch tvalentyn/grpc-latest updated (4c4a905f5e9 -> a389563f8b6)
This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a change to branch tvalentyn/grpc-latest
in repository https://gitbox.apache.org/repos/asf/beam.git
discard 4c4a905f5e9 Attempt to unpin GRPC.
add 944eef91344 Upload beam blog. (#36499)
add c22665c5111 Call now() once so start and end have exactly the same base timestamp. (#36574)
add 0d52be60e88 Add image generation code to Gemini Model Handler (#36177)
add 179d4d1ee9c Fix flaky tests (#36579)
add 7e7d866d95a [python] add setup to BigQuery's convert row Map transform (#36502)
add c7d920f26cf Update dev image. (#36582)
add 243d4077319 Integrate lambda name pickling with Cloudpickle (#35904)
add ab892e3dd09 Add logging for credential retrieval failures in GcpCredentialFactory
add db92a3ad0c0 Merge pull request #36415: Add logging for credential retrieval failures in GcpCredentialFactory
add b83c24e4d45 test(spannerio): make batch size validation more flexible for non-deterministic execution (#36584)
add 4c08585626b [IcebergIO] Pass table props to data writers (#36542)
add 0ebf84b6b18 Add ib.collect support for raw records (#36516)
add 38481b58879 Address circular dependencies in Nexmark benchmark suite. (#36513)
add 8bd92b5e376 Fix the soft-delete check and emit soft-delete log warning at most once per bucket. (#36585)
add afeca4ea301 Increase timeouts (#36595)
add 66b7c7476ce Make SpannerChangeStreamPlacementTableIT against Spanner prod. (#36071)
add fdfa6ec6338 Exclude a perma-red test suite beam_PostCommit_XVR_GoUsingJava_Dataflow.yml (#36597)
add 6dedf8f0bab Update REVIEWERS.yml (#36598)
add fe71ab1b47b Add ordered window elements into example folder (#36575)
add ef07e40667b Fix proposal link (#36600)
add f2860fa2fe8 use utils._convert_to_result for huggingface_inference (#36593)
add 05f6f01a33b Force torch to use cpu wheels (#36583)
add 6df5bd04185 Switch to use gcp-io expansion service for YAML BQIO (#36604)
add a846bc92950 Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#36611)
add 1a4298f209c Bump github.com/avast/retry-go/v4 from 4.6.1 to 4.7.0 in /sdks (#36519)
add a3d42aec4e0 Update woodstox package and version (#36613)
add 518b118bcaf Fix vLLM Gemma, add vLLM extra, fix getting throughput (#36451)
add 7bfe0e19974 Do not panic if an element has no windows during PersistBundle. (#36610)
add 64a92b2806d [python] Move histogram metric from internal to public interface/files (#36405)
add 92a40213231 Force logback version in transitive dependencies
add f517b01c502 Merge pull request #36594: Force logback version in transitive dependencies
add 85853a3edf0 proto change
add a6d2b7dabf5 add draining to output builder, encode draining
add 951943e9bba add draining to output builder
add 7ea3109a0e4 default impls
add 3d9e403b3ed comment
add 499039fe37d remove nullable
add cca50bff5aa remove nullable
add f8037f0f5bf remove nullable
add ac99abf3d79 rename field
add fc4b29c4256 rename field
add 2bd497f5d55 Merge pull request #36524: Drain - model part + windowedValue changes
add 4e0dbb9f452 Upgrade joda-time to 2.14.0 (#36626)
add c6d92079b48 fix test (#36632)
add 9612583296a Bump github.com/linkedin/goavro/v2 from 2.14.0 to 2.14.1 in /sdks (#36628)
add 80f4564fbdc Bump github.com/nats-io/nats-server/v2 from 2.12.0 to 2.12.1 in /sdks (#36520)
add 471050c7cde add licences manually (#36635)
add 0f6b605f9e5 Fix a bug in streaming OrderedWindowElements. (#36642)
add 0c10658e9ba [Dataflow Streaming] Intern encoded tags across keys (#36313)
add 949c87f0e20 Cache IntervalWindow hashCode (#36612)
add ecc840cd1e7 [Java SDK] Fix propagation of metrics set during onTimer processing. (#36576)
add 942de54a14c Adding release-2.69.0-postrelease to protected branches in .asf.yaml
add 00516e97235 Update managed-io.md for release 2.69.0-RC2.
add 3deba11b8a2 Merge pull request #36569 from apache/updates_managed_io_docs_2.69.0_rc2
add bcf41e14aba Add missing py313 ML container requirements file (#36647)
add be7acf5eca2 Update Beam website to release 2.69.0
add be47decb542 Update date
add 37ca7d54090 Update breaking changes
add fdc89112719 Merge pull request #36554 from apache/release-269-website
add 5a8e2dc3996 remove empty file (#36648)
add 5fe0d4c084b Add batch version of OrderedWindowElements in examples (#36621)
add 3432480129c Update website for roadmap and version compatibility (#36602)
add 21dbf592f87 Adds Multimap support to JAVA FnApi (#36218)
add c60b8ae8671 Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#36644)
add b33ab6e2e94 Bump cloud.google.com/go/bigquery from 1.70.0 to 1.71.0 in /sdks (#36500)
add b2960c9db3e fix(website): Use standard method for CSP frame-src exceptions (#36653)
add 020335ee23b Fixed KafkaIO Python perf test: use existing kafka-cluster config, fix broker setup, and correct test arguments
add 50477d8bd9b ix KafkaIO perf test assertion logic
add b9ec235e5ec fixed linting
add 54e4dcf737f Fixed the pylint error
add f5416c2bcc7 Fixed the YAPF formatting issue
add 8b2657bf80d fixed kafka stress tests
add 263510afbfd Fix Kafka stress test: remove invalid param
add ab4a80ddd22 fixed formatting
add f8c8dda8ed4 removed invalid params
add 0e4ac08e4b1 Merge pull request #36633 from aIbrahiim/30960-performance-tests-xlang-kafka--python
add 1b34a6d7571 Bump go.mongodb.org/mongo-driver from 1.17.4 to 1.17.6 in /sdks (#36659)
add 86bc45286d1 fixed issue in python precommit (#36561)
add 0697e54a543 Kerberos Authentication for KafkaIO (#36099)
add 1c5f0f3a505 Revert "Force logback version in transitive dependencies" (#36649)
add 55880d361a5 Split ViewTest (#36655)
add 190f21e6e5e Ensure consistent behavior between streaming and batch order window elements (#36661)
add 69685250777 Bump cloud.google.com/go/spanner from 1.86.0 to 1.86.1 in /sdks (#36645)
add 74576946753 Bump cloud.google.com/go/bigtable from 1.39.0 to 1.40.1 in /sdks (#36448)
add 8481373fed5 Fix typo in release notes (#36651)
add 8c626494d3a Bump cloud.google.com/go/datastore from 1.20.0 to 1.21.0 in /sdks (#36627)
add b976b515c4e Change ExecutionStateSampler.ExecutionState to support a scopedActivate method. From benchmarks this is no additional overhead and it is easier to use. (#36646)
add 0fd63981f10 Bump cloud.google.com/go/storage from 1.57.0 to 1.57.1 in /sdks (#36673)
add 42e01c2bd24 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#36674)
add 658b8f6dade Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#36676)
add 47e3088f1c5 Revert "fix(website): Use standard method for CSP frame-src exceptions (#36653)" (#36679)
add 00904948baf Configure GCR auth (#36678)
add 2761883d749 Pin logback 1.5.20 in io-expansion-service (#36669)
add fe07fe76709 Fix test expectations which appear to fail with differences in generated proto classes (#36680)
add 1e51a886f5c Free disk space for snapshot job and use correct requirements file (#36681)
add 79229c1e0fa Update Beam version for republish released docker containers workflow
add 8295cfb28fe Merge pull request #36664 from apache/update-republish
add 6b4dc555a24 Add spannerio vector writer. (#36654)
add 01e1cf6893e [Prism] Fix data race while accesing processing time queue (#36672)
add 330ba8b14c1 Bump actions/setup-node from 5 to 6 (#36589)
add 158e55bd75a Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#36691)
add eba04b2a567 Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#36693)
add d90b4e83017 Use vLLM extra to generate GPU requirements files (#36420)
add d46a0135966 Merge pull request #36425 from reuvenlax/fix_reverse_f_issue
add 26bf1b899dc Update beam-master images for python sdks (#36699)
add 735864d4d5c Update Python Dependencies (#36701)
add 8e0a449aad2 Use JDK 21 and Go 1.25 for Playground CI (#36700)
add 840462b109a Fix PreCommit Java IO Direct Job (#36696)
add f774df0e6af Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#36706)
add 093084a015a Install yaml extra when running yaml tests (#36707)
add 40e2b0d01f2 Only log Kafka commit failures as warning every 10m and if failing for > 10m. (#36685)
add 3a1ae18a8d4 Split some dependencies out of main beam package into existing extras (#36697)
add 1f63118d993 Fix warning message (#36703)
add 89947fded65 fix jsonToRowTest race condition (#36616)
add 6327aab91c5 Multimap streaming (#36714)
add 99a0979c473 Bump github.com/aws/smithy-go from 1.23.1 to 1.23.2 in /sdks (#36715)
add a57872f6946 Fix Docker authentication for Python ValidatesContainer Dataflow With RC tests (#36718)
add ca57641c519 Bump cloud.google.com/go/bigquery from 1.71.0 to 1.72.0 in /sdks (#36675)
add 4455616511e feat(io-gcp-bigquery): Add log for direct read quota exceeded retries (#36717)
add 269706a585c Async Post Load Test Fixes (#36596)
add 6a5923338a1 Add yaml when generating requirements (#36711)
add 107a558a2c9 Support portable decimal types for Beam SQL (#36713)
add b025102b024 When more than one instance of the same PipelineOption subclass is detected, use the first one. (#36704)
add 6d24c3dec3b [Prism] Fix a distribution metric problem when count is zero (#36723)
add 1e40f0663e2 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#36724)
add 0448a5b8062 Bump google.golang.org/api from 0.252.0 to 0.255.0 in /sdks (#36726)
add 2d617c04bc8 Remove Python 3.9 support from Apache Beam (#36665)
add 9ec41f89932 fix pip intall command (#36734)
add 11f9b0ce909 Extract expiring memoizing supplier as a separate utility and run end offset refreshes in the current thread. (#36075)
add c2e72ac2dab Streaming read for BigQuery (#36668)
add ff1d6bb62fe feat(typehints): add Python 3.12 TypeAliasType support (#36709)
add d1327b774cb update beam master tag (#36733)
add bf19899d221 Add timestamp logicaltype (#36705)
add 1d33741c035 Fix Flink IllegalThreadStateException on Java 8 (#36730)
add 09f4963e959 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#36740)
add 83673d84ff1 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#36741)
add b6b65097347 Clean up setup.py to remove old version restrictions (#36731)
add a70f1f3c0e5 Bump github.com/docker/docker in /sdks (#36739)
add d41b50bae94 [Dataflow Streaming] Replace HashBasedTable with HashMap in CachingStateTable (#36743)
add 25e50e73910 Bump github.com/testcontainers/testcontainers-go in /sdks (#36752)
add 5e04af89c6f Fix scio playground (#36748)
add 1b89fcff0ec Split some requirements into extras (#36749)
add d7bb22342ec exclude streaming bigquery it tests on test runners (#36761)
add c750f8a2ecd Utilizes file_interceptor parameter and sets it to get_normalized_path() function. (#36738)
add c7c6a37b83b fix test (#36763)
add 9516397165c Priority to BigQueryProject for executeExtract (#36694)
add 474d651668d Change FakeClock.sleep() to be safely re-entrant in attempt to fix StreamingDataflowWorkerTest flakes. (#36754)
add 03d078f2ad9 Bump golang.org/x/sys from 0.37.0 to 0.38.0 in /sdks (#36766)
add ed6757b2268 Fix Python workflows (#36770)
add 856d506c026 Clickable cards for cases without links (#36755)
add d76263a9575 upgrade pg8000 version (#36760)
add 5ffd998ed80 Bump golang.org/x/oauth2 from 0.32.0 to 0.33.0 in /sdks (#36767)
add ad73e89987e Fix PostCommit XVR GoUsingJava Dataflow job (#36757)
add 8a69cddcbf1 update container java version
add c236996a455 Merge pull request #36762 from derrickaw/updateJavaTagging_20251104
add 659201612f7 Move DataflowRunnerTest.java references to Python 3.10 containers (#36774)
add 11b0579d0c7 Dont use platform separator for docker build paths. (#36777)
add 82fe92a2ba2 Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#36783)
add 2cd7a75a86c Adds SchemaFieldNumber annotations to Iceberg classes that use SchemaCoder
add 296421d0203 Merge pull request #36780: Adds SchemaFieldNumber annotations to Iceberg classes that use the SchemaCoder
add 8e5b84527e1 Bump github.com/tetratelabs/wazero from 1.9.0 to 1.10.1 in /sdks (#36797)
add 4eff1798314 fix CVE-2025-59250 (#36800)
add 4a59cb778e6 Fix Post Commit Transform Service (#36799)
add 83ebe731133 [1/3] sdks/python: refactor Milvus-related utilities as preparation step for Milvus Sink I/O integration (#35708)
add addc06e8200 [Dataflow Streaming] Reuse ByteStringOutputStream buffers in WindmillBag (#36742)
add 8c44a9ee80e Support custom id function in async_dofn (#36779)
add d134f242c78 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#36804)
add de918a653f2 fix(iceberg): Incorrect $partition Metadata in Trino for Iceberg Tables Written via IcebergIO.writeRows with Timestamp Partitioning (#36562)
add ce1b1dcbc59 Fix flaky GCS bucket deletion in test_create_default_bucket (#36786)
add 1aca8519a47 Fix PostCommit_Java_ValidatesRunner_SparkStructuredStreaming (#36810)
add 13949036151 Bring the Python Version update docs into the main repo (#36811)
add 55c5854e6a3 Remove remaining ununsed Python 3.9 artifacts (#36812)
add 49de281005b Convert pipeline.py type comments to type hints (#36801)
add 61b8f416e61 Changes multimap entries Iterable to make a deep copy of pending adds and deletes (#36759)
add db08b7c8c0b Type inference tests (#36776)
add e51998b00ae Fix test_create_default_bucket: check None instead of NotFound (#36814)
add 6ba393a26e9 Added function for deterministic ID for class definitions by hashing. (#36793)
add 63a8be9f023 Handle query returned with empty rows gracefully on bigquery enrichmement (#36791)
add 92888144601 optimize grpc settings (#36528)
add ae2d3b80a4c Bump golang.org/x/net from 0.46.0 to 0.47.0 in /sdks (#36817)
add c8d7ca02867 Fix DinD Action failure (#36813)
add 57320861aa2 Ensure that Operations are aborted when MapTaskExecutor is closed. Add tests around setup/teardown of DoFns (#36631)
add 715a482bf5a Update REVIEWERS.yml (#36828)
add 3af78a7a19d Fix DinD startup failures (#36835)
add d4f5372a67c Improve error message for decoding input stream (#36823)
add ce58563ffd9 update container tag (#36826)
add 53a0d3c31b7 Update BOM to 26.71.0 (#36663)
add 3721c17265e Adds proto fields for overwriting fetch/write state functionality (#36708)
add 914c16eb1d1 Provide more contextual error message for PyCoder (#36825)
add b843f4af3eb Increase the GZip DeflaterOutputStream buffer size to increase performance (#36818)
add a799d2b76ea Bump google.golang.org/grpc from 1.76.0 to 1.77.0 in /sdks (#36843)
add 050369f39d1 make use of generic type info in *Utils classes
add b369dfa3814 make use of generic type info in *Utils classes (#36581)
add 87d204f6038 Fix inconsistent data type in GenericRecord and AvroSchema for AvroWriter (#36839)
add 6d9df8cbc81 Remove 3.9-specific branches in typehinting module (#36732)
add eadbc6e74fc Kerberos auth python (#36211)
add f41cbde78c1 Allow users to specify trusted Avro serializable classes to Dataflow worker (#36809)
add 534e1cb0dbf Add artifactregistry.writer to service account (#36756)
add 0ab2d039f8d Add @Pure annotations to Preconditions
add e84b318ad3c Add @Pure annotations to DoFnSignature
add a33d59420c7 Add @Pure annotations to DoFnSchemaInformation
add 8f5bc796213 Add @Pure annotations to TimerSpec
add edad6545dad Add @Pure annotations to SideInputReader
add 6c99f7ceb71 Add @Pure annotations to LateDataUtils
add f07ca33e355 Fix nullness errors in SimpleDoFnRunner and DoFnInvoker
add 87530a369fb Trivial change to KafkaSourceConsumerFnTest to eliminate warning
add be65cfe8d78 Fix nullness of currentRecordId and currentRecordOffset
add 2a06694cc5e Merge pull request #35029: Fix nullness errors in SimpleDoFnRunner and DoFnInvoker
add f5b4b6d2429 Add back sys import. (#36862)
add 5806b6e4b93 Support Beam MicrosInstant conversion to Avro Timestamp (#36605)
add 614e742a9d0 Bump com.gradle.common-custom-user-data-gradle-plugin (#36856)
add 108225d3797 Add Iceberg CDC support to YAML (#36641)
add 8e8fd1aefc1 Bump golang.org/x/crypto from 0.35.0 to 0.45.0 in /.test-infra/mock-apis (#36865)
add e336419205c Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#36867)
add 8ef3a2446bc Change KafkaIO to default to offset-based deduplication when redistribute is enabled for Dataflow java runner. (#36849)
add 3d5d04b9f39 Enable some advanced state validates runner tests for dataflow legacy and v2. (#36808)
add b0b67bf0796 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#36872)
add 62ffc638bcb Fix DROP IF NOT EXISTS statement
add 4970cba9482 Merge pull request #36863: Fix DROP IF NOT EXISTS statement
add 7f499788a40 Add Iceberg Schema Support for PassThroughLogicalType (#36870)
add b20ccbfe3ef [2/3] sdks/python: sink data with Milvus Search I/O connector (#36729)
add 6d94d5c00ee Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#36882)
add f9f13c331dd enhance python tests (#36852)
add 6a8244815b0 Move Kafka read with redistribute override to Kafka IO package. (#36887)
add 94336fa0187 Make RestrictionTrackers.getProgress unblocking until initial progress successfully returned (#36750)
add c84ccd385c2 callStateMap was accessed from multiple threads without synchronization. changing callStateMap to concurrent hashmap (#36886)
add c8aa39898cf Fix GcsEndpoint serialization in GcsOptions (#36879)
add 876cda39434 [Docs] Add table of contents for contributor docs (#36861)
add 543056a7898 split redis into extra (#36772)
add 388e5e59ff6 Supports Asynchronous Runs in Interactive Beam (#36853)
add a9e2e68e507 [OpenTelemetry] Add OpenTelemetry options to SdkHarnessOptions (#36716)
add 45911229938 Iceberg direct write (#36720)
add 203f45cee86 Support beam:logical_type:micros_instant:v1 in SpannerIo. (#36840)
add db09899eb20 Move to macos-15 (#36900)
add d421c98075c Upgrade HCatalogIO to Hive 4.0.1 (#36901)
add 5b8743b9366 [BEAM-36736] Add state sampling for timer processing in the Python SDK (#36737)
add 2f75792d044 Flink 1.20 support (#36893)
add 436662293b6 Bump github.com/nats-io/nats-server/v2 from 2.12.1 to 2.12.2 in /sdks (#36816)
add c72970d0ee3 Fix CSP issues (#36902)
add baa1751d837 comment out potential race condition until further investigation in enrichment test (#36907)
add 744275030ac Moving to 2.71.0-SNAPSHOT on master branch.
add bdd042509b6 Update CHANGES.md to have fields for 2.71.0 release
add 2ffd443b986 Fix inconsistent handling of Firestore Project and Database ID (#36895)
add 80ba916e94d fix retry scenario for query to table materialization (#36912)
add 7323fb82cb5 split hdfs into extra (#36773)
add dde8937c124 Update CHANGES to mention extras changes (#36875)
add 31612929b02 Re-enable "PostCommit XVR GoUsingJava Dataflow" in cron (#36909)
add a6a8613fca3 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#36922)
add d9c1e4e4f6c Bump cloud.google.com/go/storage from 1.57.1 to 1.57.2 in /sdks (#36844)
add 782a0e7cc26 Revert "Fix CSP issues (#36902)" (#36929)
add 1fbd111b27f Bump golang.org/x/crypto from 0.44.0 to 0.45.0 in /sdks (#36866)
add 43fdd50789d Fix runQuickstartJavaFlinkLocal (#36946)
add dbf4e5a32e9 Update BOM to 26.72.0 (#36919)
add e96c8dda71e Remove dead code. (#36943)
add efed19a6cd0 Fix typo in WriteToKafka docstring: deserializer -> serializer (#36934)
add 0fb3736df78 Enforce version upper bound for scikit-learn. (#36948)
add fa1dad91e53 Fix linkage check script to handle detached HEAD state
add c490d2dbf61 Merge pull request #36924: Fix linkage check script to handle detached HEAD state
add 0592f4fe566 Reflect that dill is no longer a default pickler. (#36903)
add bb8373f1abc Enforce hypothesis version upper bound (#36952)
add f242a88382f Include Beam version details in MongoDB handshake (#36949)
add 1cf077fc957 Bump google.golang.org/api from 0.255.0 to 0.256.0 in /sdks (#36803)
add cbc31be7bbc Reuse secret instead of creating every time (#36955)
add 81a838cd15c propagate offset and record in output builder
add ead63ad5c07 Merge pull request #36926: Offset deduplication - Propagate offset and record in output builder
add 81bb5066bab [Java] Dataflow runner v1 - Propagate drain mode (#36534)
add 79ea2e8562c Add support for sharding while avro write (#36933)
add 21109328f72 [Java] Fix CassandraIO ReadFn to quote column names for reserved keywords (#36459)
add e3afe6207d7 Add new method of generating key for GBEK (#36891)
add b6556b4777b Fix reference leak in ReifyTimestampAndWindowsParDoFnFactory (#36968)
add 67d5477d57f [Elasticsearch] Add support for v9.x (#36670)
add 554add18a2f Finish including Beam version details in MongoDB handshake (#36961)
add 7ca164cc32f Bump google.golang.org/api from 0.256.0 to 0.257.0 in /sdks (#36974)
add 0f7a189705c Improve coder byte calculation methods (#36830)
add ff954f98b29 Avoid creating Duration objects when direclty converting to millis (#36969)
add d40da985458 Remove python 3.9
add d137814862a Add nvidia license
add 065a3aea428 Upgrade pip-licenses
add bfea1e36b30 Remove nvidia requirements
add 5cd9829766c Merge pull request #36954 from apache/fix-push-images
add abe649a8044 Enables persistent state validates runner tests for Dataflow runners. (#36889)
add 54dcb0a4737 Bump github.com/aws/aws-sdk-go-v2 from 1.40.0 to 1.40.1 in /sdks (#36973)
add fd9fe79bd4b Revert "Include Beam version details in MongoDB handshake (#36949)"
add eb9e5b4327c add back import
add 660f061ac7d Merge pull request #36970 from apache/revert-36949-add-metadata
add c9d2e60e676 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#36972)
add 712172dce5a Add arrow conversion support for timestamp nanoseconds logical type. (#36960)
add ce76f81fbbc Add timestamp-nanos avro logical type support in bigquery avro utils. (#36892)
add fc673aae5d8 Reapply CSP fix and correct new js and css references (#36980)
add e24a2a2eeb9 Fix flaky AsyncBatchWriteHandlerTest.correctlyLimitConcurrency (#36975)
add c0a58953470 Fix zero key length panic in Prism (#36983)
add efb37195348 Fix python postcommit
add 82dea870b92 Merge pull request #36977 from apache/users/damccorm/hdfsIT
add b59f3d93ef3 Bump github.com/spf13/cobra from 1.10.1 to 1.10.2 in /sdks (#36987)
add 643282b7dcf Skip spannerio tests that depend on emulator on dataflowrunner
add 186e6acd6bb Fix skips.
add 1d7283e3a03 Fix indent.
add cbda69b94ea Format.
add c6fc698dce3 Format.
add ece2beb0659 Java Native Remote Inference (#36623)
add 1831cbaae95 Fix data race in test. (#36992)
add 9bcae0d5e03 Use HashMap.computeIfAbsent instead of get/put (#36990)
add fc31402d928 Attempt to fix flaky test by not requiring advance to succeed immediately (#36976)
add 7b4a4c9e6f1 Update Python Dependencies (#36994)
add 3ce2abdc505 Bump jws from 4.0.0 to 4.0.1 in /sdks/typescript (#36996)
add 84684aef490 Reduce weighing overhead for caching blocks (#36897)
add b65ec28c93a Temporary disable a few unit tests in SpannerChangeStreamErrorTest (#37003)
add 72e84efb393 Support Vertex Flex API in GeminiModelHandler (#36982)
add 4266ce02938 Update BEAM_DEV_SDK_CONTAINER_TAG. (#37011)
add a6932b60e91 [OpenTelemetry] Add traceparent and tracestate to extended element metadata proto (#36963)
add e5a93388d54 Fix too-many-pings on FnAPI runner under grpc mode (#37013)
add 8ff2c940321 Update spanner client version. (#36995)
add f22d2bc1f16 Save Logical Type and Coder Registry on cloudpickle save main session (#36271)
add 63177cb5db6 Revert "Reduce weighing overhead for caching blocks (#36897)"
add e927f2d02c0 Merge pull request #37019 from apache/revert-36897-optimize_weigh
add 99548bdcc12 Merge branch 'master' into skip-spannertests
add 32345e7cbe9 Merge pull request #37018 from claudevdm/skip-spannertests
add acc8f74b912 Bumpy numpy bounds to include 2.3.X (#36898)
add 118de482203 Bump cloud.google.com/go/storage from 1.57.2 to 1.58.0 in /sdks (#37040)
add 242e9c3317b Free disk space
add d8c5032de61 Merge pull request #37041 from apache/free-disk-space-arm
add 9a0a343d3e3 Bump js-yaml from 3.14.1 to 3.14.2 in /website/www (#36869)
add 99100aaaa70 Improve linkage check script documentation for git references (#36925)
add 7eb62154ba8 Fix nullness in some of Beam SQL (#36890)
add 1e62187ebbf SolaceIO data loss - remove message ack from close and advance as it may lead to data loss during work rebalancing or retry. (#37007)
add e73b24c0d5f Install redis dependencies for enrichment tests
add 2628a38a5c5 Merge pull request #37043 from apache/users/damccorm/testDeps
add 3ea5e1f768d Set keepalive policy for prism grpc server. (#37021)
add 2cf0930e7ae Support specifying database in FirestoreV1ReadFn (resolves #36904) (#36905)
add 585ad4181f9 Revert "Support Vertex Flex API in GeminiModelHandler (#36982)" (#37051)
add 02750862545 [Python] Fix race condition in requirements cache creation (#37042)
add c65c198ac9c Bump github.com/aws/aws-sdk-go-v2/config from 1.32.3 to 1.32.4 in /sdks (#37058)
add 4f97ad90ab5 Use internal DEFAULT_PICKLE_LIB (#37052)
add 40a32d668eb Bump Beartype bounds (#36792)
add cae5347bd3d Bump urllib3 from 2.5.0 to 2.6.0 in /sdks/python/container/ml/py310 (#37028)
add 8afde961490 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#37059)
add 9e8041a95ce Bump golang.org/x/oauth2 from 0.33.0 to 0.34.0 in /sdks (#37056)
add 917cb2ce635 Adds more SchemaFieldNumbers to classes used by Iceberg (#37055)
add a3890c62c1a Bump golang.org/x/net from 0.47.0 to 0.48.0 in /sdks (#37060)
add d119b29da2a [SQL] add SHOW command (#36509)
add 02743314acf Unpin the pip version in setupVirtualenv (#36745)
add 7f47191a37f Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#37068)
add 314d04039bc [Go SDK] Add WithNoOutputTimestamp matching Java SDK. (#37063)
add fc243b7c9d6 Handle cancelled workflow runs in flaky test detection tool (#37072)
add 625625b24a7 Bump nodemailer from 7.0.7 to 7.0.11 in /scripts/ci/issue-report (#36956)
add 48869d24f8c Bump node-forge from 1.3.1 to 1.3.2 in /sdks/typescript (#36920)
add f967bde5fef Bq timestamp schema conversion (#36986)
add 72026d36dd9 Bump js-yaml from 4.1.0 to 4.1.1 in /scripts/ci/pr-bot (#36850)
add 8d6cf28a498 Bump glob from 10.4.5 to 10.5.0 in /scripts/ci/pr-bot (#36851)
add d569788153e Bump urllib3 from 2.5.0 to 2.6.0 in /sdks/python/container/py310 (#37026)
add 0a83ce4e225 Bump urllib3 from 2.5.0 to 2.6.0 in /sdks/python/container/ml/py312 (#37025)
add f7da7230ece Bump urllib3 from 2.5.0 to 2.6.0 in /sdks/python/container/py313 (#37024)
add fe5b3ca3c85 Bump urllib3 from 2.5.0 to 2.6.0 in /sdks/python/container/py312 (#37023)
add ce88469c203 Bump urllib3 from 2.5.0 to 2.6.0 in /sdks/python/container/ml/py313 (#37029)
add f0595a9463a Bump urllib3 from 2.5.0 to 2.6.0 in /sdks/python/container/ml/py311 (#37027)
add e22f33229ea Bump urllib3 from 2.5.0 to 2.6.0 in /sdks/python/container/py311 (#37022)
add af42a1c4e72 Bump commons-cli:commons-cli from 1.9.0 to 1.11.0 (#36802)
add a6bfe01b3c6 Add yaml to requirements (#37073)
add cbcdf1b8cbf Bump transformers (#36250)
add c8f51474ee6 updated tox (#37047)
add dd3de1074eb Update Python Dependencies (#37074)
add 15b50e2a13f Fix clean up image skipped images if there are child path (#36899)
add 576a80232f8 Updates Beam Fn API container used by Dataflow at HEAD (#37088)
add a4b3dfffcf7 Allow using trigger file to trigger yaml xlang workflow. (#37095)
add f9bccb8d36f Warn against DebeziumIO not resilient to worker crash or restart (#37076)
add 8a4a69e6ae9 Repin pip to 2.50.1 to fix hanging yaml xlang workflow (#37092)
add 31a96b6807a Sort the results before assertion (#37097)
add 468f7d5c70a Bump google.golang.org/protobuf from 1.36.10 to 1.36.11 in /sdks (#37105)
add 722dc6daa32 Allow inference args to be passed in for most cases (#37094)
add a13af81f54a [GH Actions] tackle unmaintained Gradle GH actions (#37064)
add 8ef72492b5c convert to playground example
add ccad9761c44 Merge pull request #37098: Convert cogroupbykey static example to a playground example
add 8e70cba6487 Pass options in DaskOptions inheritance hierarchy only for Dask runner (#37101)
add d7cd81a1649 Ignore a few options in doc for jdbc managed ios. (#37111)
add 7e5e3b8875d [Dataflow Streaming] Move timer tag logic to WindmillStateTagUtil (#37006)
add 3516b5d318f [Dataflow Streaming] Fix race in StreamingEngineWorkComitter during stream shutdown (#37104)
add 5089191df93 Adding release-2.70.0-postrelease to protected branches in .asf.yaml
add 973d83faa1a Update managed-io.md for release 2.70.0-RC2.
add ef24543cd5a Merge pull request #37053 from apache/updates_managed_io_docs_2.70.0_rc2
add 4be931a5aa6 Update Beam website to release 2.70.0
add 65f1ade4ce1 Update release date
add 154300711a1 Merge pull request #37062 from apache/release-270-website
add 6415a4b51ec Bump cloud.google.com/go/spanner from 1.86.1 to 1.87.0 in /sdks (#37119)
add 547ab609966 Update github action runner image (#37116)
add f8075337814 Refresh Iceberg Table metadata every 2 minutes instead of before each intermediate file written (#37102)
add d71deb63f3e Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#37118)
add 471ebc4f239 fix: Adapt write() example of KinesisIO to AWS SDK V2 interface (#37122)
add afbbf535530 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#37128)
add 1b4e0c2a560 Modernize terraform config for self-hosted runners (#37127)
add 7a2a5bfeb34 Bump filelock from 3.20.0 to 3.20.1 in /sdks/python/container/ml/py312 (#37124)
add 8cc26a4be1f fix circular PipelineGraph import (#37134)
add ddc497629ec Bump github.com/nats-io/nats.go from 1.47.0 to 1.48.0 in /sdks (#37138)
add c111c8b83ef [KafkaIO] Make ReadFromKafkaDoFn restriction trackers unsplittable (#36935)
add 74874011641 Update Beam version for republish released docker containers workflow (#37141)
add 96d195b2372 Bump github.com/nats-io/nats-server/v2 from 2.12.2 to 2.12.3 in /sdks (#37139)
add 405d2dbb67c Clean up unused Jenkins files (#37148)
add fe9c78a7e8c fix interactive environment circular import (#37147)
add c0fe932a588 Update BOM to 26.73.0 (#37142)
add 5d658c1e867 [Dataflow Streaming] Improve logs (#37152)
add dfe01c0f092 Add WindmillTagEncoding Interface (#37150)
add 423a3c38b06 update py container tag (#37144)
add cc8dc2fe4f2 Add timestamp precision option to bigquery storage read for TIMESTAMP(12) columns. (#37079)
add 39652cf27a0 Address the warning in a DoFn from periodicsequence.py (#37153)
add ff01a52e60e Bump org.javacc.javacc from 3.0.3 to 4.0.3 (#37158)
add 87de10f36cc Add support for kw_only dataclasses (addresses #36978) (#36979)
add b987d37994b Add table information to BigQueryStorageApiInsertError (#36832)
add 3464da5db7e Check that billing project is not null in BigQuerySourceBase (#36788)
add 92913b74903 fix: Add limit to stream partitions query to avoid loading old cell versions (#37099)
add e655f5c5990 Support log level override for direct runner (#37100)
add c5112609071 make pulsar getters public (#37169)
add ca27526d759 Iceberg fileio close (#37168)
add c883631b649 [yaml] - fix windowing link (#37167)
add 358e007df1e Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#37179)
add 71b1e831fd8 fix provider testing issue (#37183)
add 9067220e4c7 Wanting to try out and develop ML examples. Working with pabloem. (#37184)
add dad0287521a fix: handle ResponseException correctly, honor throwWriteExceptions when using RetryConfiguration (#37166)
add 9c4dc4d6b92 Bump google.golang.org/grpc from 1.77.0 to 1.78.0 in /sdks (#37185)
add f7a9b26ab39 Allow yield and return in the same 'process' if return doesn't have a value. (#37190)
add 7b13d427593 Deduplicate MonitoringInfo in PortableMetrics (#37066)
add 63f42c2cbd1 Make TypeScript test reporter optional (#37182)
add 6e4bad2b276 Add Envoy Data Plane dependency (#37196)
add 9e3dd1a2c87 fix CVE-2024-57699 by pinning version (#37200)
add 0eb3626b3a2 Add Albertsons case study (#37201)
add cf8c1029950 Update link URL for Albertsons case study (#37203)
add 1a7205fdf68 Fix publish date (should be published immediately)
add 1cadc5f55dd Fix publish year
add 06764636f28 fix variable names (#37206)
add a3af5d54e25 feat(BigQueryWriteSchemaTransformProvider): add output collection names and error handling check (#36937)
add f7a5bbd4a35 Revert "SolaceIO data loss - remove message ack from close and advance as it may lead to data loss during work rebalancing or retry. " (#37162)
add 78eb4a2a2d7 fix: Enable `create_test` to correctly parse and apply external providers defined in YAML pipeline specifications. (#37216)
add ab14c432dad Support for EnvoyRateLimiter in Beam Python SDK (#37135)
add 44ed498ee5c Add Picosecond timestamp support to storage API WriteTableRows (#37091)
add 19f9eb06fb5 [yaml] - add multi-line windowing json support (#37174)
add 207f9d0e04d Update names.py (#37222)
add 16996bbb480 docs: fix typo and clarify BigQuery error handling method requirements (#37215)
add 716ce6a3d4c Bump aiohttp from 3.13.2 to 3.13.3 in /sdks/python/container/py312 (#37229)
add 34b8eea96f8 Introduce Python Wordcount with Rust Wrapped Functions (#37234)
add bf5c0db53c1 Bump github.com/tetratelabs/wazero from 1.10.1 to 1.11.0 in /sdks (#37160)
add 33195172e78 Fix CHANGES.md format (#37236)
add d55cf588e58 Flink 2 support prerequisites (#37133)
add 9d2abcfa2ba Bump numpy versions (#37238)
add 4777bef1268 Add nanosecond timestamp support for Storage write when writing GenericRecords and Beam Rows (#37235)
add 8a9011382e5 Moving to 2.72.0-SNAPSHOT on master branch.
add 3c3e2024d98 add some debug checks in test case and remove an extra line (#37237)
add 98999f5455d Update CONTRIBUTING.md (#37187)
add 881bbaa0aac DynamicDestinationsHelper.ConstantTimePartitioningClusteringDestinations is parsing per element json configuration for partitioning and clustering which is expensive. Cache the outcome of evaluation so it's done once. (#37014)
add 511036ad997 Revise CHANGES.md for upcoming release (#37243)
add bfff80b6fa4 [Dataflow Streaming] Add WindmillTagEncodingV2. (#37151)
add be924b74637 Update BeamModulePlugin.groovy (#37233)
add 6bedec369f7 Bump cloud.google.com/go/bigtable from 1.40.1 to 1.41.0 in /sdks (#37120)
add 7694450ecc2 Fix CONNECTION_STRING format in tox.ini (#37250)
add d7fa511ee8a [yaml] : add more error handling tests and website example (#37245)
add ef0a03c90b2 Flink 2.0 support (#37175)
add 139724d907c Map TIMSETAMP(12) BQ type -> timestamp-nanos Avro type in default schemafactory (#37257)
add 815041e0a2d Support for RateLimiter in Beam Remote Model Handler (#37218)
add 21c0842a322 Add note about providers to yaml jinja section.
add 38c88a8df5f Merge pull request #37263 Add note about providers to yaml jinja section.
add 8801afdbd32 Update to upstream caller (#37270)
add 1575b298cb8 [Docs] Fix grammar: 'should triggered' -> 'should be triggered' (#37278)
add c481b90f47a Update Python Dependencies (#37280)
add 990276f7271 Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#37282)
add 335e96b5073 [yaml] : Add final expand pipeline ut (#37260)
add 072dd821095 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#37283)
add d0223389f47 update RateLimiter execution function name (#37287)
add be8dfed0530 Bump github.com/aws/aws-sdk-go-v2/config from 1.32.6 to 1.32.7 in /sdks (#37292)
add 697d2992aeb feat: Add support for custom prediction routes in Vertex AI inference (#37155)
add e5cac6c73cb Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#37291)
add 3783f5891db Add step to free disk space in workflow (#37246)
add fdd25a4eeb6 Update trivial inference for Python 3.14 (#37248)
add 13303461281 Bump Python dev container version (#37293)
add 752da38e076 Catch breaking import error (#37295)
add 4dcc27d548e Update users.yml - fix bad role label (#37192)
add aad58648737 Support picosecond tiemstamps when writing GenericRecord and Beam Rows. (#37294)
add 331b08eb940 Add new features to changes.md (#37308)
add daa0f15359c Bump golang.org/x/text from 0.32.0 to 0.33.0 in /sdks (#37301)
add fc39ee67578 Update neo4j resource manager (#37310)
add 097e90ab41a Revert spanner version (#37305)
add 9d751d1b7da Bump golang.org/x/sys from 0.39.0 to 0.40.0 in /sdks (#37269)
add 674763d904f Fix java versions in release guide (#37314)
add ebcf5abcb5f Bump golang.org/x/net from 0.48.0 to 0.49.0 in /sdks (#37302)
add 68e0d668eaf [Python] Fix AttributeError in ExternalTransform.expand by using get_type_hints() (#37299)
add a15de20849a Remove standalone uses of the apitools HttpError class (#37296)
add 3aaf8398095 add back ticks to highlight command to be used (#37321)
add 60f1981e4ef Increase timeout for Python YAML RC validation job (#37322)
add c489cb7237b [Do Not Merge] Test bumping httplib2 upper bound to modern version (#37324)
add 17198be0313 fix: prevent NoSuchElementException in AppendFilesToTables (#37217)
add 5f5fc1f4842 Add LRU cache eviction to CachingStateProvider (#37214)
add 337bb6d12d7 Bump keras from 3.12.0 to 3.13.1 in /sdks/python/container/ml/py310 (#37325)
add cc5309758fc Fix spanner io hang by closing connections (#37327)
add 82ebcb213b9 Pin spanner to old version in gcp bom (#37329)
add 9fba823f2d7 Add Terraform files to deploy Envoy RateLimiter (#37285)
add 600bd612dc2 [Dataflow Streaming] Remove GetWorkBudgetRefresher which is unused and has a flaky test. (#37317)
add fb06f1ba82c Update Python Dependencies (#37361)
add e4f48ad5ba5 Fix typo in workflow name for updating dependencies (#37362)
add 94ce718171e Fix link due to trailing slashes.
add 8a43f5e5b50 Merge pull request #37286 Fix link due to trailing slashes.
add 52e284748f2 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#37367)
add e7c80cd9306 Add more info to CHANGES on logical type issue (#37368)
add 84b467b45a5 Add Datadog IO (#37319)
add f1e4b7468b6 Bump github.com/linkedin/goavro/v2 from 2.14.1 to 2.15.0 in /sdks (#37366)
add 204d3a5bd3f Update Dataflow dev python containers (#37358)
add 7eaf66c8feb Update managed-io.md for release 2.71.0-RC3. (#37333)
add 4bac8785c08 Migrate Cloud Build usage to Python client (#37309)
add 1e6b6755139 Adding release-2.71.0-postrelease to protected branches in .asf.yaml
add 966762c99d8 Update website for 2.71.0 release (#37256)
add 86abab2c2e4 [#37209] Enhance serialization error messages (#37298)
add 94de2a175bd Update BEAM_DEV_SDK_CONTAINER_TAG to latest version (#37373)
add 99d09d429f9 Update default release and RC versions in republish workflow (#37385)
add 34f7d872183 Revert "Use requirements.txt instead of setup.py for juliaset example"
add f26f1369a78 Change setup.py to use setuptools
add ce6a4e5f407 Merge pull request #37337 from apache/users/damccorm/setup_file
add 27ae93b350c notebook (#37393)
add 4b240454948 GCS client library migration in Java SDK - part 1 (#36876)
add b50a0992b95 Remove Read and Write PubsubLite for Yaml (#37383)
add 16993ace395 Update Go version from 1.25.2 to 1.25.6 (#37395)
add 59b5f3a8cbe Bump lodash from 4.17.21 to 4.17.23 in /sdks/typescript (#37380)
add b61995db7ba Bump sentencepiece from 0.1.97 to 0.2.1 in /sdks/python/apache_beam/examples/inference/large_language_modeling (#37394)
add 9b3aa12f2ec remove pubsublite from python sdk (#37401)
add 7a278e2b144 remove pubsublite tests - as depreciated (#37396)
add f819f2ecdb9 Update bom to 26.74.0 (excluding spanner) (#37384)
add 07e580a089a Bump cloud.google.com/go/storage from 1.58.0 to 1.59.1 in /sdks (#37315)
add f14f1ceda50 docs: expand About section with website architecture details (#37242)
add 1b60ce6ff69 Add YAML SQL example using calcite_connection_properties (#37303)
add 7cb1a05cd57 disabled beam metrics report job (#37420)
add d090aab9773 Update BEAM_DEV_SDK_CONTAINER_TAG version (#37400)
add f5af98bc167 Fix website workflows (#37421)
add 911ec6abf0f Add filter() method to PCollection in TypeScript SDK (#37408)
add adc04caf412 Fix gRPC failures in Python PreCommit tests (#37350)
add d0014ff38d2 Simplify encoding kw_only args and test init=False args. (#37231)
add 6a399dff51f Update ChangeStreamDao to query different TVF for postgresSQL based on (#36667)
add f8013e6a6b3 Fix str(WindowedValueCoder) crash when underlying coder isn't KV coder (#37406)
add e4a5eea37a5 Adds a new SchemaCoderPayload proto to use for portable SchemaCoders.
add e12436a9bd3 Merge pull request #37376: Adds a new SchemaCoderPayload proto to use for portable SchemaCoders.
add 1222ae5cf37 Bump python-multipart in /sdks/python/container/ml/py312 (#37426)
add 52b9910f3a7 fix key error for huggingface notebook (#37435)
add 672b888eecc Enable some more tests in runner v2 batch mode. (#37363)
add 47df6aefe88 [Spanner Change Streams] Fix potential data loss issue by ensuring to only claim timestamps that have been fully processed from the restriction tracker. (#37326)
add 0b32d24d96a Update yaml notebook (#37369)
add f7784d8b187 Disable Beam Metrics Report workflow (#37440)
add 928ad7e2aab Bump github.com/nats-io/nats-server/v2 from 2.12.3 to 2.12.4 in /sdks (#37439)
add e26b76b1c22 better formatting output (#37441)
add 61fe95088e6 Bump cloud.google.com/go/storage from 1.59.1 to 1.59.2 in /sdks (#37443)
add 98c980abb40 Bump go.mongodb.org/mongo-driver from 1.17.6 to 1.17.7 in /sdks (#37381)
add 18493cf7fc1 remove website, typescript, etc - pubsublite uses (#37412)
add f947803d5c8 Add exception chaining (#37423)
add 0d7226545f3 Only stage required wheel packages from requirements cache, not entire cache directory (#37360)
add 9fcb17408ee Fix python postcommit (#37447)
add 96edda499ea remove pubsublite from java sdk (#37448)
add b3ee06249c6 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#37444)
add d4015eb395b Remove pubsublite - infra, groovy, checkstyles (#37450)
add cdf48147bdd [RunInference] Add content-aware dynamic batching via element_size_fn… (#37428)
add e7a21ddb5b5 [#37198] Make withBackOffSupplier public to enable bounded retry configuration (#37356)
add d561bc31cb4 Add artifactregistry.admin role (#37456)
add 4b5d149e3a3 Bump github.com/lib/pq from 1.10.9 to 1.11.1 in /sdks (#37454)
add 112685de58d Upgrade google-api-services-dataflow (#37462)
add 501897649dd Upgrade GCP Bom (#37473)
add 0282abbff3b Allow multiprocessshared to spawn process and delete directly with obj (#37112)
add 953f70c5c28 Disable abbrev for yaml.main (#37476)
add 7052f01e45a Add an experiment to disable pip build isolation (#37331)
add e4cd040199d Add LRU cache eviction to CombinePerKeyPrecombineOperator (#37466)
add 4674387e88e Remove pubsublite logs - try example notebook (#37457)
add fd2babbcddb [Python] Add take(n) convenience for PCollection (#37470)
add d1e5849206e fix(dataflow): 'update' infinite loop and nil pointer dereference (#37472)
add b1cf96d3d65 Bump pip from 25.3 to 26.0 in /sdks/python/container/ml/py311 (#37483)
add 1531743ce8d [Dataflow Streaming] Enable state tag encoding v2 based on backend flag (#37464)
add 91418fd69a0 Fix ValidatesContainer Cloud Build test
add 425d82704b8 Update sdk_container_builder for google-cloud-build 3.35 API
add 98cdbc70a59 Added fall back to additionalProperties for older clients
add 1c5d57a0ec2 Merge pull request #37490 from aIbrahiim/fix-postcommit-python-validates-container
add d32bc97290a Revert "Bump pip from 25.3 to 26.0 in /sdks/python/container/ml/py311 (#37483)" (#37494)
add 0aea01394b1 Update java ml module gradle (#37471)
add 9a5de4f46ce [Drain] Propagate drain information to timerData and TImer data structures. mostly noop.
add 1c22136c6e2 Merge pull request #37009: [Drain] TimerData causedByDrain field
add 183ecd36cb4 Add pubsublite IO change (#37489)
add 4e218f0183e Add model manager that automatically manage model across processes (#37113)
add c1ecbb9eeb1 Fix linting/compilation violation (#37505)
add 99e48681a03 Increment PyArrow Bounds (#37503)
add fed1b551c35 Add support for tagged output type hints. (#37434)
add 4e76198555e fix spotless (#37508)
add 5240736a9bc [#37279] Document Ubuntu 24.04 Python version requirements (#37297)
add beec6a3e156 Update logback version to address vulnerability (#37501)
add 15ba4874d72 Add schedule trigger for beam playground precommit (#37518)
add e2be9fd8e36 Plumb custom batch parameters for autosharding from WriteFiles to FileIO. (#37463)
add d6920de30fe Add schedule trigger to Playground PreCommit (#37519)
add f504c5309bc Revert "[Python] Add take(n) convenience for PCollection (#37470)" (#37493)
add dd92b8fc179 [Flink 2.0] Exercise validate runner test for classic Flink runner (#37537)
add 99541ff4336 Fix flakiness in model manager and multiprocessshared tests (#37529)
add 6b1d2cffca2 Align go rc validation inputs with other workflows (#37320)
add 37a9c1a14dd Bump go.mongodb.org/mongo-driver from 1.17.7 to 1.17.8 in /sdks (#37469)
add 6602f1e125f Update RunInference to work with model manager (#37506)
add a0d27de943f Update queuing logic to avoid lock notify contention (#37528)
add b7577e74a6a Bump golang.org/x/oauth2 from 0.34.0 to 0.35.0 in /sdks (#37548)
add f52cf933446 feat: Introduce agent skills for various Apache Beam developments (#37467)
add 4b478a70738 Exercise Flink PVR tests on Flink 2.0 (#37538)
add f96db3733c2 [SQL] Alter Catalog and Table statements (#36571)
add dd0f41d3fe7 Fix _OrderedUnionCoder is_deterministic check. (#37550)
add 25d3a2e46bf Add States for Pause (#37546)
add 257fa5a46d3 Fix Components visibility (#37551)
add 6dcb908e9fe Add back flex API option with improved pydoc (#37491)
add 5142be92511 [Java FnApi] Improve FinalizeCallbackManager to cleanup timeout queue and use recommended ReentrantLock+Condition (#37304)
add f5aa2af52bd Bump pip from 25.3 to 26.0 in /sdks/python/container/ml/py313 (#37485)
add 44eb7f63cdc Add RateLimiter Abstractions (#37547)
add d51ea3afb88 Add access to pipeline options during pipeline construction and translation. (#37522)
add e8920c79a0d Add logic to handle end timestamp of mutable change stream bounded query (#37459)
add 801bf89e3da Parse proto bytes change stream record in ChangeStreamRecordMapper (#37427)
add d9adfa9c34f fix unparse bug (#37562)
add 195cc593590 Update python-pipeline-dependencies.md (#37561)
add 7df489cb7c6 add datadog it support (#37517)
add 91d3b6b1d39 move enum to top level class
add 4a2afe85f9f move enum to sdk core changed boolean causedByDrain in WindowedValue interface to enum. spotless
add aaecfa8f545 Merge pull request #37549: Move CausedByDrain to top level, move types
add 0228126c1b7 Update reviewers (#37572)
add d6390b86f0f Download performance looks to static while building website (#37559)
add 22d1c576272 Add OOM handling for RunInference with model manager (#37557)
add bafb270b48e Update go linter version (#37579)
add cea7923eef6 Use onlyif (#37578)
add 8c9d75567e7 Replace VarInt.getLength with Protobuf's branchless implementation (#36959)
add 9ea7380de85 Refactor update compatibility checks and pipeline options access (#37566)
add 72ce2074caf GCS client library migration in Java SDK - part 2 (#37502)
add cbf36740a53 Add Support for EnvoyRateLimiter Implementation (#37573)
add 695498f03a4 Update testcontainers version (#37591)
add eba1ec36adc Bigtable: skip reading large rows (#37586)
add 0adeae8b771 Optimize data size estimation for proto coder.
add 9a1dedfea25 Use existing _get_nested_size utility.
add 6e81f250977 Merge pull request #37584 Optimize data size estimation for proto coder.
add c7d13c4d56d add aws and azure dependencies (#37602)
add 87aee9a58f6 Upgrade to debezium-io 3.1.3 (#37571)
add 25518f07fe2 Upgrade Spanner to the latest version (#37552)
add c69e8a18601 remove list_prefix deprecated method (#37587)
add e2a2b291921 Fix spelling mistakes in Go comments (#37617)
add 9a60847ad6b Update CHANGES.md for PR 37331 (#37599)
add 8da478be0d7 switch from v19 to v23 (#37620)
add 5849f2e38ea [yaml] : fix validate compatible method (#37588)
add 5d6cb04ea05 [yaml] : switch js2py to pythonmonkey (#37560)
add f2aa7912401 Exercise Python PVR tests on Flink 2.0 (#37313)
add 2ac0fe431e6 [yaml] : fix create different type elements issue (#37585)
add 66134f7107f Fix infer_result_type for pcollection with tags. (#37608)
add 97e5ccf0c6f Upgrade ClickHouseIO to use ClickHouse Java Client V2 (#37611)
add 90253bf5857 Tune Ratelimiter config (#37598)
add 76a53610898 Rename Chunk to EmbeddableItem (#37623)
add a9c14d1698d Add Ubuntu 24.04 runner pools (#37624)
add 0337ecefc61 Fix iceberg nested filter (#37340)
add e13dc0c30cf unpin grpc.
add d1ade2188d1 Relax grpcio constraints even though we know some versions are bad to simplify dependency resolution.
add 891a5acab83 Relax envoy-data-plane constraints.
add a389563f8b6 Set precise version for testing
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (4c4a905f5e9)
\
N -- N -- N refs/heads/tvalentyn/grpc-latest (a389563f8b6)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.agent/skills/README.md | 62 +
.agent/skills/beam-concepts/SKILL.md | 246 ++
.agent/skills/ci-cd/SKILL.md | 188 +
.agent/skills/contributing/SKILL.md | 149 +
.agent/skills/gradle-build/SKILL.md | 224 ++
.agent/skills/io-connectors/SKILL.md | 197 +
.agent/skills/java-development/SKILL.md | 153 +
.agent/skills/license-compliance/SKILL.md | 199 +
.agent/skills/python-development/SKILL.md | 183 +
.agent/skills/runners/SKILL.md | 244 ++
.asf.yaml | 5 +
.github/REVIEWERS.yml | 10 +-
.github/actions/dind-up-action/action.yml | 24 +-
.../test-properties.json | 22 +-
.../actions/setup-environment-action/action.yml | 13 +-
.github/build.gradle | 4 +-
.../gh-actions-self-hosted-runners/arc/README.md | 21 +-
.../arc/environments/beam.env | 80 +-
.github/gh-actions-self-hosted-runners/arc/helm.tf | 23 +-
.../arc/images/Dockerfile | 2 +-
.../gh-actions-self-hosted-runners/arc/provider.tf | 6 +-
...marks_Dataflow.json => beam_PostCommit_Go.json} | 0
.../trigger_files/beam_PostCommit_Go_VR_Flink.json | 2 +-
...m_PostCommit_Java_ValidatesRunner_Dataflow.json | 7 +-
...it_Java_ValidatesRunner_Dataflow_Streaming.json | 8 +-
...ostCommit_Java_ValidatesRunner_Dataflow_V2.json | 8 +-
...Java_ValidatesRunner_Dataflow_V2_Streaming.json | 7 +-
...a_ValidatesRunner_SparkStructuredStreaming.json | 2 -
...json => beam_PostCommit_PortableJar_Flink.json} | 0
.github/trigger_files/beam_PostCommit_Python.json | 4 +-
... beam_PostCommit_Python_Examples_Dataflow.json} | 3 +-
.../beam_PostCommit_Python_Examples_Flink.json | 3 +
...on => beam_PostCommit_Python_MongoDBIO_IT.json} | 0
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.json | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.json | 2 +-
.github/trigger_files/beam_PostCommit_SQL.json | 2 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.json | 4 +-
.../trigger_files/beam_PreCommit_Python_Dill.json | 2 +-
.github/trigger_files/beam_PreCommit_SQL.json | 2 +-
....json => beam_PreCommit_Yaml_Xlang_Direct.json} | 2 +-
.github/workflows/README.md | 3 +
.../workflows/beam_CloudML_Benchmarks_Dataflow.yml | 4 +-
.../workflows/beam_LoadTests_Java_GBK_Smoke.yml | 2 +-
.../beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml | 8 +-
...m_LoadTests_Python_CoGBK_Dataflow_Streaming.yml | 8 +-
.../beam_LoadTests_Python_CoGBK_Flink_Batch.yml | 6 +-
...eam_LoadTests_Python_Combine_Dataflow_Batch.yml | 6 +-
...LoadTests_Python_Combine_Dataflow_Streaming.yml | 6 +-
.../beam_LoadTests_Python_Combine_Flink_Batch.yml | 6 +-
...am_LoadTests_Python_Combine_Flink_Streaming.yml | 4 +-
...LoadTests_Python_FnApiRunner_Microbenchmark.yml | 2 +-
.../beam_LoadTests_Python_GBK_Dataflow_Batch.yml | 10 +-
...eam_LoadTests_Python_GBK_Dataflow_Streaming.yml | 2 +-
...adTests_Python_GBK_reiterate_Dataflow_Batch.yml | 4 +-
...sts_Python_GBK_reiterate_Dataflow_Streaming.yml | 4 +-
.../beam_LoadTests_Python_ParDo_Dataflow_Batch.yml | 8 +-
...m_LoadTests_Python_ParDo_Dataflow_Streaming.yml | 8 +-
.../beam_LoadTests_Python_ParDo_Flink_Batch.yml | 6 +-
...beam_LoadTests_Python_ParDo_Flink_Streaming.yml | 10 +-
...m_LoadTests_Python_SideInput_Dataflow_Batch.yml | 20 +-
.github/workflows/beam_LoadTests_Python_Smoke.yml | 4 +-
.github/workflows/beam_Metrics_Report.yml | 5 +-
...eam_PerformanceTests_BiqQueryIO_Read_Python.yml | 2 +-
...formanceTests_BiqQueryIO_Write_Python_Batch.yml | 2 +-
...erformanceTests_PubsubIOIT_Python_Streaming.yml | 2 +-
..._PerformanceTests_SpannerIO_Read_2GB_Python.yml | 2 +-
...manceTests_SpannerIO_Write_2GB_Python_Batch.yml | 2 +-
.../beam_PerformanceTests_TextIOIT_Python.yml | 2 +-
...PerformanceTests_WordCountIT_PythonVersions.yml | 2 +-
.../beam_PerformanceTests_xlang_KafkaIO_Python.yml | 84 +-
.github/workflows/beam_Playground_CI_Nightly.yml | 7 +-
.github/workflows/beam_Playground_Precommit.yml | 8 +-
.../beam_PostCommit_Java_Examples_Flink.yml | 2 +-
.../beam_PostCommit_Java_Nexmark_Flink.yml | 2 +-
...ml => beam_PostCommit_Java_PVR_Flink_Batch.yml} | 40 +-
.../beam_PostCommit_Java_PVR_Flink_Streaming.yml | 8 +-
.../workflows/beam_PostCommit_Java_Tpcds_Flink.yml | 2 +-
.../beam_PostCommit_Java_ValidatesRunner_Flink.yml | 8 +-
...PostCommit_Java_ValidatesRunner_Flink_Java8.yml | 2 +-
.../beam_PostCommit_PortableJar_Flink.yml | 6 +-
.../beam_PostCommit_PortableJar_Spark.yml | 4 +-
.github/workflows/beam_PostCommit_Python.yml | 2 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 6 +-
.../beam_PostCommit_Python_Dependency.yml | 2 +-
.../beam_PostCommit_Python_Examples_Direct.yml | 2 +-
.../beam_PostCommit_Python_Examples_Flink.yml | 2 +-
.../beam_PostCommit_Python_Examples_Spark.yml | 2 +-
.../beam_PostCommit_Python_Nexmark_Direct.yml | 2 +-
.../beam_PostCommit_Python_Portable_Flink.yml | 11 +-
...stCommit_Python_ValidatesContainer_Dataflow.yml | 19 +-
..._Python_ValidatesContainer_Dataflow_With_RC.yml | 9 +-
..._PostCommit_Python_ValidatesRunner_Dataflow.yml | 2 +-
...eam_PostCommit_Python_ValidatesRunner_Flink.yml | 4 +-
...eam_PostCommit_Python_ValidatesRunner_Samza.yml | 2 +-
...eam_PostCommit_Python_ValidatesRunner_Spark.yml | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.yml | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.yml | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.yml | 2 +-
.../beam_PostCommit_TransformService_Direct.yml | 4 +-
.github/workflows/beam_PostCommit_XVR_Direct.yml | 4 +-
.github/workflows/beam_PostCommit_XVR_Flink.yml | 6 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 6 +-
...eam_PostCommit_XVR_JavaUsingPython_Dataflow.yml | 4 +-
...eam_PostCommit_XVR_PythonUsingJava_Dataflow.yml | 4 +-
.github/workflows/beam_PostCommit_XVR_Samza.yml | 4 +-
.github/workflows/beam_PostCommit_XVR_Spark3.yml | 4 +-
.../beam_PostCommit_Yaml_Xlang_Direct.yml | 2 +-
.../workflows/beam_PreCommit_Flink_Container.yml | 4 +-
...l => beam_PreCommit_Java_Datadog_IO_Direct.yml} | 41 +-
...beam_PreCommit_Java_ElasticSearch_IO_Direct.yml | 3 +-
.../beam_PreCommit_Java_HCatalog_IO_Direct.yml | 15 -
.../workflows/beam_PreCommit_Java_IOs_Direct.yml | 17 +-
.../beam_PreCommit_Java_PVR_Flink_Batch.yml | 2 +-
.../beam_PreCommit_Java_PVR_Flink_Docker.yml | 2 +-
.../workflows/beam_PreCommit_Portable_Python.yml | 4 +-
.github/workflows/beam_PreCommit_Prism_Python.yml | 4 +-
.github/workflows/beam_PreCommit_Python.yml | 2 +-
.github/workflows/beam_PreCommit_PythonDocker.yml | 2 +-
.../workflows/beam_PreCommit_Python_Coverage.yml | 6 +-
.../workflows/beam_PreCommit_Python_Dataframes.yml | 2 +-
.../workflows/beam_PreCommit_Python_Examples.yml | 2 +-
.../beam_PreCommit_Python_Integration.yml | 2 +-
.github/workflows/beam_PreCommit_Python_ML.yml | 6 +-
.../workflows/beam_PreCommit_Python_PVR_Flink.yml | 1 +
.../workflows/beam_PreCommit_Python_Runners.yml | 2 +-
.../workflows/beam_PreCommit_Python_Transforms.yml | 2 +-
.../workflows/beam_PreCommit_Website_Stage_GCS.yml | 12 +-
.../beam_PreCommit_Xlang_Generated_Transforms.yml | 2 +-
.../workflows/beam_PreCommit_Yaml_Xlang_Direct.yml | 6 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 5 +-
.github/workflows/beam_Publish_Website.yml | 8 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 4 +-
.../workflows/beam_StressTests_Java_KafkaIO.yml | 77 +-
.github/workflows/build_release_candidate.yml | 15 +-
.github/workflows/build_wheels.yml | 12 +-
.github/workflows/code_completion_plugin_tests.yml | 4 -
.github/workflows/dask_runner_tests.yml | 3 +-
...Sentiment_Streaming_DistilBert_Base_Uncased.txt | 3 +-
...Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt | 4 +-
...python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt | 2 +-
.../python_CoGBK_Flink_Batch_100b_Single_Key.txt | 2 +-
.../python_CoGBK_Flink_Batch_10kB.txt | 2 +-
...hon_Combine_Flink_Batch_2GB_10_byte_records.txt | 2 +-
.../python_Combine_Flink_Batch_2GB_Fanout_4.txt | 2 +-
.../python_Combine_Flink_Batch_2GB_Fanout_8.txt | 2 +-
...Combine_Flink_Streaming_2GB_10_byte_records.txt | 2 +-
...python_Combine_Flink_Streaming_2GB_Fanout_4.txt | 2 +-
...python_Combine_Flink_Streaming_2GB_Fanout_8.txt | 2 +-
...thon_Combine_Flink_Streaming_small_Fanout_1.txt | 2 +-
...thon_Combine_Flink_Streaming_small_Fanout_2.txt | 2 +-
.../python_GBK_Flink_Batch_2GB_of_100B_records.txt | 2 +-
.../python_GBK_Flink_Batch_2GB_of_10B_records.txt | 2 +-
...nout_4_times_with_2GB_10-byte_records_total.txt | 2 +-
...nout_8_times_with_2GB_10-byte_records_total.txt | 2 +-
...K_Flink_Batch_reiterate_4_times_10kB_values.txt | 2 +-
.../python_ParDo_Flink_Batch_10_Counters.txt | 2 +-
.../python_ParDo_Flink_Batch_10_Iterations.txt | 2 +-
.../python_ParDo_Flink_Batch_200_Iterations.txt | 2 +-
.../python_ParDo_Flink_Streaming_100_Counters.txt | 2 +-
.../python_ParDo_Flink_Streaming_10_Counters.txt | 2 +-
.../python_ParDo_Flink_Streaming_10_Iterations.txt | 2 +-
...python_ParDo_Flink_Streaming_200_Iterations.txt | 2 +-
.../python_ParDo_Flink_Streaming_5_Iterations.txt | 2 +-
.github/workflows/pr-bot-new-prs.yml | 2 +-
.github/workflows/pr-bot-pr-updates.yml | 2 +-
.github/workflows/pr-bot-prs-needing-attention.yml | 2 +-
.github/workflows/python_dependency_tests.yml | 1 -
.github/workflows/python_tests.yml | 3 +-
.github/workflows/refresh_looker_metrics.yml | 6 -
.github/workflows/reportGenerator.yml | 2 +-
.../republish_released_docker_containers.yml | 10 +-
.github/workflows/run_perf_alert_tool.yml | 2 +-
.../workflows/run_rc_validation_go_wordcount.yml | 24 +-
.../run_rc_validation_java_quickstart.yml | 2 +-
.../run_rc_validation_python_mobile_gaming.yml | 2 +-
.../workflows/run_rc_validation_python_yaml.yml | 10 +-
.github/workflows/stale.yml | 1 +
.github/workflows/typescript_tests.yml | 10 +-
.github/workflows/update_python_dependencies.yml | 6 +-
.gitignore | 8 +
.pre-commit-config.yaml | 2 +-
.test-infra/jenkins/Committers.groovy | 109 -
.test-infra/jenkins/CommonJobProperties.groovy | 318 --
.test-infra/jenkins/CommonTestProperties.groovy | 84 -
.test-infra/jenkins/CronJobBuilder.groovy | 52 -
.test-infra/jenkins/Kubernetes.groovy | 165 -
.test-infra/jenkins/LoadTestConfig.groovy | 636 ----
.test-infra/jenkins/LoadTestsBuilder.groovy | 136 -
.test-infra/jenkins/NexmarkBuilder.groovy | 260 --
.../jenkins/NexmarkDatabaseProperties.groovy | 40 -
.../PhraseTriggeringPostCommitBuilder.groovy | 34 -
.test-infra/jenkins/PostcommitJobBuilder.groovy | 94 -
.test-infra/jenkins/PrecommitJobBuilder.groovy | 163 -
.test-infra/jenkins/PythonTestProperties.groovy | 46 -
.test-infra/jenkins/README.md | 26 -
.test-infra/jenkins/TpcdsDatabaseProperties.groovy | 42 -
.test-infra/jenkins/build.gradle | 48 -
.../jenkins/metrics_report/dashboards_parser.py | 115 -
.../jenkins/metrics_report/report_generator.py | 230 --
.../templates/Metrics_Report.template | 141 -
.test-infra/jenkins/metrics_report/tox.ini | 36 -
.../kafka-cluster/03-zookeeper/50pzoo.yml | 121 +-
.../kubernetes/kafka-cluster/05-kafka/50kafka.yml | 156 +-
.test-infra/metrics/influxdb/Dockerfile | 2 +-
.test-infra/metrics/influxdb/gsutil/Dockerfile | 2 +-
.test-infra/metrics/sync/github/Dockerfile | 2 +-
.../github/github_runs_prefetcher/code/main.py | 9 +-
.test-infra/metrics/sync/jenkins/Dockerfile | 2 +-
.test-infra/mock-apis/go.mod | 14 +-
.test-infra/mock-apis/go.sum | 20 +-
.test-infra/mock-apis/poetry.lock | 6 +-
.test-infra/mock-apis/pyproject.toml | 2 +-
.test-infra/tools/python_installer.sh | 2 +-
.../tools/stale_dataflow_prebuilt_image_cleaner.sh | 7 +-
CHANGES.md | 88 +-
CONTRIBUTING.md | 12 +-
build.gradle.kts | 41 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 57 +-
contributor-docs/README.md | 36 +
contributor-docs/code-change-guide.md | 2 +-
contributor-docs/committer-guide.md | 9 -
contributor-docs/python-tips.md | 30 +-
contributor-docs/release-guide.md | 4 +-
.../updating-supported-python-versions.md | 82 +
dev-support/docker/pkglist | 7 +-
examples/java/build.gradle | 2 +
.../apache/beam/examples/RateLimiterSimple.java | 131 +
.../cookbook/BigQueryStreamingTornadoes.java | 209 ++
examples/notebooks/README.md | 13 +-
.../beam-ml/run_inference_huggingface.ipynb | 2 +-
.../notebooks/beam-ml/run_inference_vllm.ipynb | 4 +-
.../spanner_product_catalog_embeddings.ipynb | 2313 ++++++++++++
.../get-started/try-apache-beam-yaml.ipynb | 2924 +++++++++++----
examples/notebooks/patch.py | 2 +-
examples/terraform/envoy-ratelimiter/README.md | 176 +
.../terraform/envoy-ratelimiter/gke.tf | 30 +-
.../terraform/envoy-ratelimiter/network.tf | 10 +-
.../terraform/envoy-ratelimiter/outputs.tf | 12 +-
.../terraform/envoy-ratelimiter/prerequisites.tf | 50 +
.../terraform/envoy-ratelimiter/provider.tf | 39 +-
examples/terraform/envoy-ratelimiter/ratelimit.tf | 371 ++
.../terraform/envoy-ratelimiter/terraform.tfvars | 29 +-
examples/terraform/envoy-ratelimiter/variables.tf | 178 +
gradle.properties | 8 +-
infra/iam/roles/beam_admin.role.yaml | 69 +-
infra/iam/roles/beam_infra_manager.role.yaml | 80 +-
infra/iam/roles/beam_viewer.role.yaml | 108 +-
infra/iam/roles/beam_writer.role.yaml | 22 +-
infra/iam/roles/roles_config.yaml | 2 +-
infra/iam/users.yml | 10 +-
.../apache/beam/it/common/PipelineOperator.java | 16 +-
it/datadog/build.gradle | 50 +
.../beam/it/datadog/DatadogClientFactory.java | 47 +
.../apache/beam/it/datadog/DatadogLogEntry.java | 102 +
.../beam/it/datadog/DatadogResourceManager.java | 335 ++
.../datadog/DatadogResourceManagerException.java | 23 +-
.../it/datadog/DatadogResourceManagerUtils.java | 70 +
.../datadog/conditions/DatadogLogEntriesCheck.java | 91 +
.../beam/it/datadog/conditions}/package-info.java | 4 +-
.../beam/it/datadog/matchers/DatadogAsserts.java | 70 +
.../beam/it/datadog/matchers/package-info.java | 18 +
.../org/apache/beam/it/datadog/package-info.java | 18 +
.../beam/it/datadog/DatadogResourceManagerIT.java | 77 +
.../it/datadog/DatadogResourceManagerTest.java | 188 +
.../datadog/DatadogResourceManagerUtilsTest.java | 67 +
it/google-cloud-platform/build.gradle | 2 -
.../gcp/pubsublite/PubsubliteResourceManager.java | 215 --
.../java/org/apache/beam/it/kafka/KafkaIOST.java | 28 +-
.../apache/beam/it/neo4j/Neo4jResourceManager.java | 41 +-
.../beam/it/neo4j/conditions/Neo4jQueryCheck.java | 8 +-
.../beam/it/neo4j/Neo4jResourceManagerIT.java | 10 +-
.../beam/it/neo4j/Neo4jResourceManagerTest.java | 22 +-
local-env-setup.sh | 4 +-
.../beam/model/fn_execution/v1/beam_fn_api.proto | 52 +-
.../beam/model/pipeline/v1/beam_runner_api.proto | 14 +-
.../apache/beam/model/pipeline/v1/metrics.proto | 10 +-
.../org/apache/beam/model/pipeline/v1/schema.proto | 22 +-
playground/backend/cmd/server/controller.go | 2 +-
playground/backend/cmd/server/server.go | 4 +-
playground/backend/containers/go/Dockerfile | 2 +-
playground/backend/containers/go/build.gradle | 2 +-
playground/backend/containers/scio/Dockerfile | 14 +-
playground/backend/containers/scio/build.gradle | 2 +-
playground/backend/env_setup.sh | 2 +-
.../assets/symbols/java.g.yaml | 39 -
.../cloudbuild/playground_cd_examples.sh | 8 +-
.../cloudbuild/playground_ci_examples.sh | 8 +-
release/build.gradle.kts | 6 +-
release/src/main/Dockerfile | 3 +-
.../python-release/python_release_automation.sh | 2 +-
release/src/main/scripts/run_rc_validation.sh | 8 +-
.../apache/beam/runners/core/LateDataUtils.java | 3 +
...TimeBoundedSplittableProcessElementInvoker.java | 4 +-
.../beam/runners/core/ReduceFnContextFactory.java | 10 +-
.../apache/beam/runners/core/SideInputReader.java | 13 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 169 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 7 +-
.../apache/beam/runners/core/StateNamespaces.java | 8 +
.../org/apache/beam/runners/core/StateTags.java | 17 +
.../apache/beam/runners/core/TimerInternals.java | 63 +-
.../runners/core/InMemoryTimerInternalsTest.java | 73 +-
.../beam/runners/core/KeyedWorkItemCoderTest.java | 4 +-
.../apache/beam/runners/core/ReduceFnTester.java | 10 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 4 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 7 +-
.../beam/runners/core/TimerInternalsTest.java | 55 +-
.../core/triggers/TriggerStateMachineTester.java | 10 +-
.../apache/beam/runners/direct/DirectRunner.java | 3 +-
.../beam/runners/direct/DirectRunnerTest.java | 47 +
.../runners/direct/DirectTimerInternalsTest.java | 19 +-
...cycleManagerRemovingTransformEvaluatorTest.java | 4 +-
.../beam/runners/direct/EvaluationContextTest.java | 7 +-
.../beam/runners/direct/WatermarkManagerTest.java | 114 +-
.../py39 => runners/flink/1.20}/build.gradle | 13 +-
.../job-server-container/build.gradle | 0
.../flink/1.20/job-server}/build.gradle | 19 +-
.../wrappers/streaming/DoFnOperator.java | 1785 +++++++++
runners/flink/2.0/build.gradle | 43 +
.../job-server-container/build.gradle | 0
.../flink/2.0/job-server}/build.gradle | 19 +-
.../flink/FlinkBatchTranslationContext.java | 116 +
.../runners/flink/FlinkExecutionEnvironments.java | 499 +++
.../runners/flink/FlinkMiniClusterEntryPoint.java | 97 +
.../flink/FlinkPipelineExecutionEnvironment.java | 122 +-
.../beam/runners/flink/FlinkPipelineOptions.java | 49 +-
.../beam/runners/flink/FlinkPipelineRunner.java | 9 +-
.../runners/flink/FlinkPipelineTranslator.java | 52 +
.../FlinkStreamingPortablePipelineTranslator.java | 30 +-
.../flink/FlinkStreamingTransformTranslators.java | 17 +-
.../runners/flink/FlinkTransformOverrides.java | 63 +
.../translation/functions/FlinkDoFnFunction.java | 264 ++
.../FlinkExecutableStageContextFactory.java | 63 +
.../functions/FlinkExecutableStageFunction.java | 416 +++
.../FlinkExecutableStagePruningFunction.java | 61 +
.../FlinkMergingNonShuffleReduceFunction.java | 100 +
.../functions/FlinkMultiOutputPruningFunction.java | 64 +
.../functions/FlinkPartialReduceFunction.java | 116 +
.../translation/functions/FlinkReduceFunction.java | 116 +
.../functions/FlinkStatefulDoFnFunction.java | 276 ++
.../functions/ImpulseSourceFunction.java | 117 +
.../translation/types/CoderTypeInformation.java | 142 +
.../translation/types/EncodedValueSerializer.java | 124 +
.../types/EncodedValueTypeInformation.java | 98 +
.../types/UnversionedTypeSerializerSnapshot.java | 85 +
.../wrappers/streaming/DoFnOperator.java | 1785 +++++++++
.../streaming/io/StreamingImpulseSource.java | 85 +
.../wrappers/streaming/io/TestStreamSource.java | 82 +
.../streaming/io/UnboundedSourceWrapper.java | 23 +-
.../state/FlinkBroadcastStateInternals.java | 697 ++++
.../streaming/state/FlinkStateInternals.java | 1851 ++++++++++
.../runners/flink/EncodedValueComparatorTest.java | 69 +
.../flink/FlinkExecutionEnvironmentsTest.java | 73 +-
.../FlinkPipelineExecutionEnvironmentTest.java | 421 +++
.../runners/flink/FlinkPipelineOptionsTest.java | 191 +
.../flink/FlinkRequiresStableInputTest.java | 288 ++
.../apache/beam/runners/flink/FlinkRunnerTest.java | 94 +
.../beam/runners/flink/FlinkSavepointTest.java | 432 +++
.../beam/runners/flink/FlinkSubmissionTest.java | 251 ++
.../apache/beam/runners/flink/ReadSourceTest.java | 92 +
.../adapter/BeamFlinkDataStreamAdapterTest.java | 217 ++
.../flink/streaming/BoundedSourceRestoreTest.java | 199 +
.../flink/streaming/FlinkStateInternalsTest.java | 218 ++
.../flink/streaming/MemoryStateBackendWrapper.java | 38 +-
.../runners/flink/streaming/StreamSources.java | 9 +-
.../functions/FlinkDoFnFunctionTest.java | 109 +
.../FlinkExecutableStageFunctionTest.java | 347 ++
.../functions/FlinkStatefulDoFnFunctionTest.java | 109 +
.../functions/ImpulseSourceFunctionTest.java | 208 ++
.../streaming/io/UnboundedSourceWrapperTest.java | 1027 ++++++
.../stableinput/BufferingDoFnRunnerTest.java | 179 +
.../2.0/src/test/resources/flink-test-config.yaml | 19 +-
runners/flink/flink_runner.gradle | 140 +-
.../flink_job_server_container.gradle | 10 +-
runners/flink/job-server/flink_job_server.gradle | 31 +-
.../flink/FlinkBatchPipelineTranslator.java | 2 +-
.../flink/FlinkPipelineExecutionEnvironment.java | 43 +
.../beam/runners/flink/FlinkPipelineOptions.java | 8 +-
.../beam/runners/flink/FlinkPipelineRunner.java | 4 +-
.../FlinkStreamingPortablePipelineTranslator.java | 21 +-
.../flink/FlinkStreamingTransformTranslators.java | 2 +-
.../flink/metrics/FlinkMetricContainerBase.java | 4 +-
.../translation/types/CoderTypeSerializer.java | 0
.../streaming/io/UnboundedSourceWrapper.java | 2 +-
.../flink/FlinkExecutionEnvironmentsTest.java | 13 +-
.../runners/flink/ReadSourceStreamingTest.java | 7 +-
.../flink/streaming/GroupByNullKeyTest.java | 7 +-
.../flink/streaming/MemoryStateBackendWrapper.java | 0
.../runners/flink/streaming/StreamSources.java | 0
.../flink/streaming/TopWikipediaSessionsTest.java | 7 +-
runners/google-cloud-dataflow-java/build.gradle | 166 +-
.../beam/runners/dataflow/BatchViewOverrides.java | 6 +
.../beam/runners/dataflow/DataflowRunner.java | 38 +-
.../beam/runners/dataflow/util/MonitoringUtil.java | 2 +
.../beam/runners/dataflow/DataflowRunnerTest.java | 92 +-
.../runners/dataflow/util/MonitoringUtilTest.java | 2 +
.../dataflow/worker/DataflowExecutionContext.java | 6 +-
.../worker/IntrinsicMapTaskExecutorFactory.java | 53 +-
.../ReifyTimestampAndWindowsParDoFnFactory.java | 8 +-
.../dataflow/worker/StreamingDataflowWorker.java | 2 +
.../worker/StreamingModeExecutionContext.java | 47 +-
.../dataflow/worker/UngroupedWindmillReader.java | 28 +-
.../dataflow/worker/WindmillKeyedWorkItem.java | 35 +-
.../beam/runners/dataflow/worker/WindmillSink.java | 14 +-
.../dataflow/worker/WindmillTimerInternals.java | 243 +-
.../dataflow/worker/WindowingWindmillReader.java | 9 +-
.../streaming/KeyCommitTooLargeException.java | 4 +-
.../runners/dataflow/worker/streaming/Work.java | 14 +-
.../StreamingEngineComputationConfigFetcher.java | 10 +
.../streaming/config/StreamingGlobalConfig.java | 7 +-
.../harness/SingleSourceWorkerHarness.java | 3 +
.../util/ThreadLocalByteStringOutputStream.java | 103 +
.../dataflow/worker/util/ValueInEmptyWindows.java | 6 +
.../util/common/worker/InternedByteString.java | 77 +
.../worker/util/common/worker/MapTaskExecutor.java | 71 +-
.../commits/StreamingEngineWorkCommitter.java | 24 +-
.../client/grpc/GetWorkResponseChunkAssembler.java | 5 +-
.../client/grpc/GrpcDirectGetWorkStream.java | 1 +
.../windmill/client/grpc/GrpcGetWorkStream.java | 1 +
.../worker/windmill/state/CachingStateTable.java | 221 +-
.../dataflow/worker/windmill/state/IdTracker.java | 3 +-
.../worker/windmill/state/WindmillBag.java | 46 +-
.../windmill/state/WindmillCombiningState.java | 22 +-
.../worker/windmill/state/WindmillMap.java | 30 +-
.../worker/windmill/state/WindmillMultimap.java | 24 +-
.../worker/windmill/state/WindmillOrderedList.java | 20 +-
.../worker/windmill/state/WindmillStateCache.java | 82 +-
.../windmill/state/WindmillStateInternals.java | 27 +-
.../windmill/state/WindmillStateTagUtil.java | 148 -
.../worker/windmill/state/WindmillTagEncoding.java | 134 +
.../windmill/state/WindmillTagEncodingV1.java | 247 ++
.../windmill/state/WindmillTagEncodingV2.java | 406 +++
.../worker/windmill/state/WindmillValue.java | 21 +-
.../windmill/state/WindmillWatermarkHold.java | 30 +-
.../worker/windmill/work/WorkItemReceiver.java | 1 +
.../worker/windmill/work/WorkItemScheduler.java | 2 +
.../work/budget/GetWorkBudgetRefresher.java | 133 -
.../work/processing/StreamingWorkScheduler.java | 17 +-
.../processing/failures/WorkFailureProcessor.java | 22 +-
.../dataflow/worker/FakeWindmillServer.java | 1 +
.../IntrinsicMapTaskExecutorFactoryTest.java | 137 +-
.../runners/dataflow/worker/SimpleParDoFnTest.java | 12 +-
.../worker/StreamingDataflowWorkerTest.java | 138 +-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 30 +-
...reamingGroupAlsoByWindowsReshuffleDoFnTest.java | 9 +-
...eamingKeyedWorkItemSideInputDoFnRunnerTest.java | 4 +-
.../worker/StreamingModeExecutionContextTest.java | 37 +-
.../worker/StreamingSideInputFetcherTest.java | 4 +-
.../dataflow/worker/UserParDoFnFactoryTest.java | 10 +-
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 114 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 2 +
.../worker/streaming/ActiveWorkStateTest.java | 2 +
.../streaming/ComputationStateCacheTest.java | 1 +
...treamingEngineComputationConfigFetcherTest.java | 38 +
.../FanOutStreamingEngineWorkerHarnessTest.java | 1 +
.../harness/WindmillStreamSenderTest.java | 1 +
.../worker/testing/TestCountingSource.java | 5 +
.../worker/util/BoundedQueueExecutorTest.java | 1 +
.../ThreadLocalByteStringOutputStreamTest.java | 68 +
.../util/common/worker/InternedByteStringTest.java | 68 +
.../util/common/worker/MapTaskExecutorTest.java | 39 +
.../StreamingApplianceWorkCommitterTest.java | 1 +
.../commits/StreamingEngineWorkCommitterTest.java | 75 +
.../client/grpc/GrpcDirectGetWorkStreamTest.java | 12 +-
.../client/grpc/GrpcWindmillServerTest.java | 6 +-
.../windmill/state/WindmillStateCacheTest.java | 234 +-
.../windmill/state/WindmillStateInternalsTest.java | 8 +-
.../windmill/state/WindmillStateTagUtilTest.java | 87 -
.../state/WindmillTagEncodingV1Test.java} | 121 +-
.../windmill/state/WindmillTagEncodingV2Test.java | 583 +++
.../work/budget/GetWorkBudgetRefresherTest.java | 127 -
.../failures/WorkFailureProcessorTest.java | 1 +
.../work/refresh/ActiveWorkRefresherTest.java | 1 +
.../beam/runners/portability/PortableMetrics.java | 130 +-
.../runners/portability/PortableRunnerTest.java | 46 +
runners/prism/java/build.gradle | 9 +-
.../samza/runtime/ClassicBundleManager.java | 4 +-
.../samza/runtime/PortableBundleManager.java | 4 +-
.../runners/samza/runtime/KeyedTimerDataTest.java | 8 +-
.../runtime/SamzaTimerInternalsFactoryTest.java | 85 +-
runners/spark/spark_runner.gradle | 2 +-
.../apache/beam/runners/spark/util/TimerUtils.java | 6 +
scripts/beam-sql.sh | 2 +-
scripts/ci/issue-report/package-lock.json | 14 +-
scripts/ci/issue-report/package.json | 2 +-
scripts/ci/pr-bot/package-lock.json | 27 +-
scripts/ci/pr-bot/package.json | 2 +-
scripts/tools/bomupgrader.py | 1 +
sdks/go.mod | 154 +-
sdks/go.sum | 341 +-
sdks/go/examples/wasm/README.md | 2 +-
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 2 +-
sdks/go/pkg/beam/core/timers/timers.go | 10 +
sdks/go/pkg/beam/io/avroio/avroio.go | 118 +-
sdks/go/pkg/beam/io/avroio/avroio_test.go | 108 +-
.../pkg/beam/runners/dataflow/dataflowlib/job.go | 6 +-
.../prism/internal/engine/elementmanager.go | 60 +-
.../runners/prism/internal/engine/teststream.go | 13 +-
sdks/go/pkg/beam/runners/prism/internal/execute.go | 4 +
.../beam/runners/prism/internal/handlerunner.go | 97 +-
.../runners/prism/internal/jobservices/metrics.go | 6 +
.../runners/prism/internal/jobservices/server.go | 5 +
sdks/go/test/build.gradle | 17 +-
sdks/go/test/integration/integration.go | 3 +
sdks/go/test/integration/primitives/timers.go | 114 +-
sdks/go/test/integration/primitives/timers_test.go | 10 +
sdks/go/test/run_validatesrunner_tests.sh | 6 +-
sdks/java/bom/gcp/build.gradle | 2 +-
sdks/java/build-tools/beam-linkage-check.sh | 15 +-
.../resources/beam/checkstyle/suppressions.xml | 4 +-
sdks/java/container/boot.go | 22 +
.../container/license_scripts/dep_urls_java.yaml | 10 +-
.../license_scripts/pull_licenses_java.py | 14 +-
sdks/java/core/build.gradle | 12 +-
.../sdk/fn/splittabledofn/RestrictionTrackers.java | 94 +-
.../beam/sdk/fn/stream/PrefetchableIterables.java | 2 +-
.../java/org/apache/beam/sdk/io/Compression.java | 4 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 66 +
.../apache/beam/sdk/options/SdkHarnessOptions.java | 40 +
.../beam/sdk/schemas/logicaltypes/SqlTypes.java | 4 +
.../beam/sdk/schemas/logicaltypes/Timestamp.java | 165 +
.../beam/sdk/schemas/utils/AutoValueUtils.java | 16 +-
.../beam/sdk/schemas/utils/ByteBuddyUtils.java | 59 +-
.../beam/sdk/schemas/utils/JavaBeanUtils.java | 28 +-
.../apache/beam/sdk/schemas/utils/POJOUtils.java | 138 +-
.../java/org/apache/beam/sdk/state/TimerSpec.java | 2 +
.../java/org/apache/beam/sdk/transforms/DoFn.java | 4 +-
.../beam/sdk/transforms/DoFnSchemaInformation.java | 6 +
.../beam/sdk/transforms/reflect/DoFnInvoker.java | 28 +-
.../beam/sdk/transforms/reflect/DoFnSignature.java | 108 +
.../splittabledofn/GrowableOffsetRangeTracker.java | 2 +-
.../UnsplittableRestrictionTracker.java | 70 +
.../sdk/transforms/windowing/IntervalWindow.java | 20 +-
.../beam/sdk/transforms/windowing/PaneInfo.java | 17 +
.../ExpiringMemoizingSerializableSupplier.java | 101 +
.../beam/sdk/util/GcpHsmGeneratedSecret.java | 191 +
.../org/apache/beam/sdk/util/Preconditions.java | 31 +
.../org/apache/beam/sdk/util/RowJsonUtils.java | 31 +-
.../main/java/org/apache/beam/sdk/util/Secret.java | 45 +-
.../main/java/org/apache/beam/sdk/util/VarInt.java | 10 +-
.../beam/sdk/util/construction/Environments.java | 1 +
.../org/apache/beam/sdk/values/CausedByDrain.java | 11 +-
.../org/apache/beam/sdk/values/OutputBuilder.java | 2 +
.../apache/beam/sdk/values/ValueWithRecordId.java | 17 +
.../org/apache/beam/sdk/values/WindowedValue.java | 2 +
.../org/apache/beam/sdk/values/WindowedValues.java | 163 +-
.../fn/splittabledofn/RestrictionTrackersTest.java | 90 +-
.../java/org/apache/beam/sdk/io/FileIOTest.java | 139 +
.../sdk/schemas/logicaltypes/LogicalTypesTest.java | 267 ++
.../beam/sdk/schemas/utils/POJOUtilsTest.java | 80 -
.../sdk/transforms/GroupByEncryptedKeyTest.java | 80 +
.../apache/beam/sdk/transforms/GroupByKeyIT.java | 112 +
.../transforms/{ViewTest.java => MapViewTest.java} | 884 +----
.../beam/sdk/transforms/ParDoLifecycleTest.java | 11 +-
.../org/apache/beam/sdk/transforms/ParDoTest.java | 67 +
.../org/apache/beam/sdk/transforms/ViewTest.java | 951 -----
.../sdk/transforms/windowing/PaneInfoTest.java | 28 +
.../ExpiringMemoizingSerializableSupplierTest.java | 166 +
.../java/org/apache/beam/sdk/util/SecretTest.java | 18 +-
.../apache/beam/sdk/util/WindowedValueTest.java | 7 +-
.../sdk/util/construction/EnvironmentsTest.java | 3 +
.../beam/sdk/extensions/arrow/ArrowConversion.java | 30 +-
.../sdk/extensions/arrow/ArrowConversionTest.java | 34 +-
sdks/java/extensions/avro/build.gradle | 2 +
.../extensions/avro/schemas/utils/AvroUtils.java | 71 +
.../sdk/extensions/avro/coders/AvroCoderTest.java | 1 +
.../avro/schemas/utils/AvroUtilsTest.java | 117 +
sdks/java/extensions/euphoria/build.gradle | 2 +-
.../google-cloud-platform-core/build.gradle | 3 +
.../extensions/gcp/auth/GcpCredentialFactory.java | 5 +
.../sdk/extensions/gcp/options/GcsOptions.java | 14 +-
.../beam/sdk/extensions/gcp/util/GcsUtil.java | 1438 ++------
.../gcp/util/{GcsUtil.java => GcsUtilV1.java} | 102 +-
.../beam/sdk/extensions/gcp/util/GcsUtilV2.java | 325 ++
.../sdk/extensions/gcp/util/gcsfs/GcsPath.java | 15 +
.../sdk/extensions/gcp/GcpCoreApiSurfaceTest.java | 18 +-
.../beam/sdk/extensions/gcp/util/GcsUtilIT.java | 6 +-
.../gcp/util/GcsUtilParameterizedIT.java | 300 ++
.../beam/sdk/extensions/gcp/util/GcsUtilTest.java | 120 +-
.../sdk/extensions/gcp/util/gcsfs/GcsPathTest.java | 20 +
sdks/java/extensions/kafka-factories/build.gradle | 46 +
.../kafka/factories/FileAwareFactoryFn.java | 258 ++
.../kafka/factories/KerberosConsumerFactoryFn.java | 150 +
.../extensions/kafka/factories}/package-info.java | 4 +-
.../kafka/factories/FileAwareFactoryFnTest.java | 268 ++
.../factories/KerberosConsumerFactoryFnTest.java | 242 ++
sdks/java/extensions/kryo/build.gradle | 2 +-
sdks/java/extensions/ml/build.gradle | 1 +
.../src/test/proto/proto3_schema_messages.proto | 47 +-
sdks/java/extensions/sql/build.gradle | 4 -
.../extensions/sql/expansion-service/build.gradle | 1 -
sdks/java/extensions/sql/hcatalog/build.gradle | 4 +-
sdks/java/extensions/sql/iceberg/build.gradle | 2 +
.../provider/iceberg/IcebergAlterTableOps.java | 50 +
.../sql/meta/provider/iceberg/IcebergCatalog.java | 56 +-
.../meta/provider/iceberg/IcebergMetastore.java | 16 +-
.../iceberg/BeamSqlCliIcebergAlterTest.java | 231 ++
.../extensions/sql/jdbc/BeamSqlLineShowTest.java | 306 ++
.../extensions/sql/src/main/codegen/config.fmpp | 14 +
.../sql/src/main/codegen/includes/parserImpls.ftl | 311 +-
.../sdk/extensions/sql/impl/BeamCalciteSchema.java | 33 +-
.../sql/impl/BeamCalciteSchemaFactory.java | 12 +-
.../sdk/extensions/sql/impl/BeamCalciteTable.java | 15 +-
.../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 30 +-
.../sql/impl/BeamSystemDbMetadataSchema.java | 111 +
.../sdk/extensions/sql/impl/BeamSystemSchema.java | 155 +
.../sql/impl/BeamSystemTableMetadataSchema.java | 126 +
.../extensions/sql/impl/CatalogManagerSchema.java | 34 +-
.../sdk/extensions/sql/impl/CatalogSchema.java | 4 +
.../sdk/extensions/sql/impl/JdbcConnection.java | 38 +-
.../beam/sdk/extensions/sql/impl/JdbcDriver.java | 23 +-
.../extensions/sql/impl/ScalarFunctionImpl.java | 8 +-
.../extensions/sql/impl/TVFSlidingWindowFn.java | 25 +-
.../extensions/sql/impl/TableResolutionUtils.java | 33 +-
.../beam/sdk/extensions/sql/impl/UdfImpl.java | 22 +-
.../sql/impl/parser/SqlAlterCatalog.java | 159 +
.../extensions/sql/impl/parser/SqlAlterTable.java | 254 ++
.../extensions/sql/impl/parser/SqlDdlNodes.java | 43 +-
.../sql/impl/parser/SqlDropDatabase.java | 6 +-
.../extensions/sql/impl/parser/SqlDropObject.java | 6 +-
.../extensions/sql/impl/parser/SqlDropTable.java | 6 +-
.../sdk/extensions/sql/impl/rel/BeamCalcRel.java | 5 +
.../beam/sdk/extensions/sql/meta/SystemTables.java | 182 +
.../sdk/extensions/sql/meta/catalog/Catalog.java | 7 +
.../sql/meta/catalog/InMemoryCatalog.java | 14 +-
.../sql/meta/provider/AlterTableOps.java | 45 +
.../sql/meta/provider/FullNameTableProvider.java | 7 +-
.../sql/meta/provider/ReadOnlyTableProvider.java | 7 +-
.../sql/meta/provider/TableProvider.java | 10 +-
.../pubsublite/PubsubLiteSubscriptionTable.java | 73 -
.../pubsublite/PubsubLiteTableProvider.java | 210 --
.../provider/pubsublite/PubsubLiteTopicTable.java | 72 -
.../sql/meta/provider/pubsublite/RowHandler.java | 171 -
.../sql/meta/provider/test/AlterTestTableOps.java | 101 +
.../sql/meta/provider/test/TestTableProvider.java | 21 +
.../sql/meta/store/InMemoryMetaStore.java | 11 +
.../sdk/extensions/sql/BeamSqlCliCatalogTest.java | 93 +
.../sdk/extensions/sql/BeamSqlCliDatabaseTest.java | 4 +-
.../beam/sdk/extensions/sql/BeamSqlCliTest.java | 249 ++
.../sdk/extensions/sql/impl/JdbcDriverTest.java | 2 +-
.../pubsublite/PubsubLiteTableProviderTest.java | 239 --
.../meta/provider/pubsublite/RowHandlerTest.java | 237 --
.../control/ExecutionStateSamplerBenchmark.java | 17 +
.../fn/harness/control/ExecutionStateSampler.java | 19 +
.../fn/harness/control/FinalizeBundleHandler.java | 182 +-
.../fn/harness/control/ProcessBundleHandler.java | 11 +-
.../harness/data/PCollectionConsumerRegistry.java | 89 +-
.../harness/data/PTransformFunctionRegistry.java | 5 +-
.../beam/fn/harness/state/FnApiStateAccessor.java | 121 +-
.../beam/fn/harness/state/MultimapUserState.java | 147 +-
.../harness/control/ExecutionStateSamplerTest.java | 140 +-
.../harness/control/FinalizeBundleHandlerTest.java | 49 +
.../harness/control/ProcessBundleHandlerTest.java | 134 +
.../harness/logging/BeamFnLoggingClientTest.java | 6 +-
.../fn/harness/state/MultimapUserStateTest.java | 129 +
.../apache/beam/sdk/io/aws2/kinesis/KinesisIO.java | 4 +-
.../io/aws2/common/AsyncBatchWriteHandlerTest.java | 2 +-
.../org/apache/beam/sdk/io/cassandra/ReadFn.java | 10 +
.../beam/sdk/io/cassandra/CassandraIOTest.java | 374 ++
sdks/java/io/clickhouse/build.gradle | 7 +-
.../beam/sdk/io/clickhouse/ClickHouseIO.java | 393 +-
.../sdk/io/clickhouse/ClickHouseJdbcUrlParser.java | 261 ++
.../apache/beam/sdk/io/clickhouse/TableSchema.java | 8 +-
.../beam/sdk/io/clickhouse/AtomicInsertTest.java | 28 +-
.../beam/sdk/io/clickhouse/BaseClickHouseTest.java | 140 +-
.../ClickHouseIOJdbcBackwardCompatibilityTest.java | 98 +
.../ClickHouseIOPropertyMergingTest.java | 211 ++
.../beam/sdk/io/clickhouse/ClickHouseIOTest.java | 142 +-
.../io/clickhouse/ClickHouseJdbcUrlParserTest.java | 341 ++
sdks/java/io/components/build.gradle | 5 +
.../io/components/deadletterqueue/DLQRouter.java | 21 +-
.../components/deadletterqueue}/package-info.java | 4 +-
.../deadletterqueue/sinks/ThrowingSink.java | 3 +-
.../deadletterqueue/sinks/package-info.java} | 9 +-
.../components/ratelimiter/EnvoyRateLimiter.java} | 33 +-
.../ratelimiter/EnvoyRateLimiterContext.java | 65 +
.../ratelimiter/EnvoyRateLimiterFactory.java | 239 ++
.../sdk/io/components/ratelimiter/RateLimiter.java | 41 +
.../ratelimiter/RateLimiterClientCache.java | 103 +
.../ratelimiter/RateLimiterContext.java} | 13 +-
.../components/ratelimiter/RateLimiterFactory.java | 57 +
.../components/ratelimiter/RateLimiterOptions.java | 68 +
.../io/components/ratelimiter}/package-info.java | 4 +-
.../components/throttling/ThrottlingSignaler.java | 11 +-
.../io/components/throttling}/package-info.java | 4 +-
.../components/deadletterqueue/DLQRouterTest.java | 30 +-
.../ratelimiter/EnvoyRateLimiterTest.java | 168 +
.../ratelimiter/RateLimiterClientCacheTest.java | 115 +
.../ratelimiter/RateLimiterOptionsTest.java | 81 +
sdks/java/io/{components => datadog}/build.gradle | 35 +-
.../apache/beam/sdk/io/datadog/DatadogEvent.java | 97 +
.../beam/sdk/io/datadog/DatadogEventCoder.java | 94 +
.../beam/sdk/io/datadog/DatadogEventPublisher.java | 330 ++
.../sdk/io/datadog/DatadogEventSerializer.java | 45 +
.../beam/sdk/io/datadog/DatadogEventWriter.java | 521 +++
.../org/apache/beam/sdk/io/datadog/DatadogIO.java | 236 ++
.../beam/sdk/io/datadog/DatadogWriteError.java | 77 +
.../sdk/io/datadog/DatadogWriteErrorCoder.java | 88 +
.../apache/beam/sdk/io/datadog/package-info.java | 19 +-
.../beam/sdk/io/datadog/DatadogEventCoderTest.java | 65 +
.../sdk/io/datadog/DatadogEventPublisherTest.java | 176 +
.../sdk/io/datadog/DatadogEventSerializerTest.java | 95 +
.../beam/sdk/io/datadog/DatadogEventTest.java | 73 +
.../sdk/io/datadog/DatadogEventWriterTest.java | 566 +++
.../apache/beam/sdk/io/datadog/DatadogIOTest.java | 172 +
.../sdk/io/datadog/DatadogWriteErrorCoderTest.java | 61 +
.../beam/sdk/io/datadog/DatadogWriteErrorTest.java | 66 +
sdks/java/io/debezium/build.gradle | 8 +-
.../io/debezium/expansion-service/build.gradle | 2 +-
.../org/apache/beam/io/debezium/DebeziumIO.java | 5 +
.../io/debezium/DebeziumIOMySqlConnectorIT.java | 4 +-
.../debezium/DebeziumIOPostgresSqlConnectorIT.java | 4 +-
.../io/debezium/KafkaSourceConsumerFnTest.java | 2 +-
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 6 +
.../elasticsearch-tests-9}/build.gradle | 45 +-
.../sdk/io/elasticsearch/ElasticsearchIOIT.java | 219 ++
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 10 +-
.../src/test/resources/clientkeystore | Bin
.../elasticsearch/ElasticsearchIOTestCommon.java | 64 +-
.../io/elasticsearch/ElasticsearchIOTestUtils.java | 42 +
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 32 +-
sdks/java/io/expansion-service/build.gradle | 6 +
sdks/java/io/google-ads/build.gradle | 4 +-
.../apache/beam/sdk/io/googleads/GoogleAdsIO.java | 6 +-
.../{GoogleAdsV19.java => GoogleAdsV23.java} | 56 +-
.../sdk/io/googleads/DummyRateLimitPolicy.java | 4 +-
.../beam/sdk/io/googleads/GoogleAdsIOTest.java | 40 +-
.../io/googleads/MockGoogleAdsClientFactory.java | 10 +-
sdks/java/io/google-cloud-platform/build.gradle | 25 +-
.../beam/sdk/io/gcp/bigquery/AppendClientInfo.java | 1 +
.../AvroGenericRecordToStorageApiProto.java | 107 +-
.../io/gcp/bigquery/BeamRowToStorageApiProto.java | 52 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 118 +-
.../bigquery/BigQueryDynamicReadDescriptor.java | 101 +
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 375 +-
.../sdk/io/gcp/bigquery/BigQueryIOTranslation.java | 18 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 1 +
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 10 +-
.../bigquery/BigQueryStorageApiInsertError.java | 18 +-
.../BigQueryStorageApiInsertErrorCoder.java | 11 +-
.../gcp/bigquery/BigQueryStorageQuerySource.java | 65 +-
.../io/gcp/bigquery/BigQueryStorageSourceBase.java | 67 +-
.../gcp/bigquery/BigQueryStorageStreamSource.java | 21 +
.../gcp/bigquery/BigQueryStorageTableSource.java | 40 +-
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 197 +-
.../gcp/bigquery/DynamicDestinationsHelpers.java | 47 +-
.../beam/sdk/io/gcp/bigquery/RowWriterFactory.java | 20 +-
.../io/gcp/bigquery/StorageApiConvertMessages.java | 8 +-
.../StorageApiDynamicDestinationsBeamRow.java | 13 +-
...StorageApiDynamicDestinationsGenericRecord.java | 12 +-
.../StorageApiDynamicDestinationsProto.java | 19 +-
.../StorageApiDynamicDestinationsTableRow.java | 15 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 23 +-
.../bigquery/StorageApiWritesShardedRecords.java | 13 +-
.../io/gcp/bigquery/TableRowToStorageApiProto.java | 1011 ++++--
.../TimestampPrecision.java} | 13 +-
.../BigQueryWriteSchemaTransformProvider.java | 12 +-
.../providers/PortableBigQueryDestinations.java | 2 +-
.../io/gcp/bigtable/BigtableServiceFactory.java | 7 +-
.../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 31 +-
.../changestreams/dao/MetadataTableDao.java | 21 +-
.../sdk/io/gcp/firestore/FirestoreOptions.java | 5 +-
.../FirestoreStatefulComponentFactory.java | 23 +-
.../beam/sdk/io/gcp/firestore/FirestoreV1.java | 331 +-
.../sdk/io/gcp/firestore/FirestoreV1ReadFn.java | 114 +-
.../sdk/io/gcp/firestore/FirestoreV1WriteFn.java | 4 +-
.../io/gcp/pubsublite/CloudPubsubTransforms.java | 104 -
.../sdk/io/gcp/pubsublite/PublisherOptions.java | 45 -
.../beam/sdk/io/gcp/pubsublite/PubsubLiteIO.java | 131 -
.../PubsubLiteReadSchemaTransformProvider.java | 544 ---
.../PubsubLiteWriteSchemaTransformProvider.java | 466 ---
.../sdk/io/gcp/pubsublite/SubscriberOptions.java | 46 -
.../gcp/pubsublite/UuidDeduplicationOptions.java | 92 -
.../gcp/pubsublite/internal/AddUuidsTransform.java | 50 -
.../pubsublite/internal/BlockingCommitterImpl.java | 55 -
.../pubsublite/internal/CheckpointMarkImpl.java | 77 -
.../gcp/pubsublite/internal/CloserReference.java | 68 -
.../io/gcp/pubsublite/internal/DlqProvider.java | 79 -
.../internal/ExternalTransformConfig.java | 103 -
.../internal/ExternalTransformRegistrarImpl.java | 39 -
.../internal/InitialOffsetReaderImpl.java | 51 -
.../internal/LimitingTopicBacklogReader.java | 81 -
.../pubsublite/internal/ManagedFactoryImpl.java | 60 -
.../internal/MemoryBufferedSubscriber.java | 48 -
.../internal/MemoryBufferedSubscriberImpl.java | 158 -
.../gcp/pubsublite/internal/MemoryLimiterImpl.java | 93 -
.../pubsublite/internal/OffsetByteProgress.java | 33 -
.../pubsublite/internal/OffsetByteRangeCoder.java | 63 -
.../internal/OffsetByteRangeTracker.java | 104 -
.../internal/PerServerSubscriberCache.java | 34 -
.../internal/PerSubscriptionPartitionSdf.java | 131 -
.../io/gcp/pubsublite/internal/ProtoFromBytes.java | 55 -
.../io/gcp/pubsublite/internal/ProtoToBytes.java | 39 -
.../pubsublite/internal/PublisherAssembler.java | 117 -
.../io/gcp/pubsublite/internal/PubsubLiteSink.java | 85 -
.../io/gcp/pubsublite/internal/ServiceCache.java | 79 -
.../pubsublite/internal/SubscribeTransform.java | 227 --
.../pubsublite/internal/SubscriberAssembler.java | 179 -
.../pubsublite/internal/SubscriptionPartition.java | 35 -
.../internal/SubscriptionPartitionCoder.java | 66 -
.../internal/SubscriptionPartitionLoader.java | 179 -
.../internal/SubscriptionPartitionProcessor.java | 28 -
.../SubscriptionPartitionProcessorImpl.java | 76 -
.../pubsublite/internal/TopicBacklogReader.java | 39 -
.../internal/TopicBacklogReaderImpl.java | 57 -
.../internal/TopicBacklogReaderSettings.java | 64 -
.../pubsublite/internal/TrackerWithProgress.java | 24 -
.../pubsublite/internal/UnboundedReaderImpl.java | 148 -
.../pubsublite/internal/UnboundedSourceImpl.java | 121 -
.../beam/sdk/io/gcp/pubsublite/internal/Uuid.java | 59 -
.../sdk/io/gcp/pubsublite/internal/UuidCoder.java | 49 -
.../internal/UuidDeduplicationTransform.java | 53 -
.../io/gcp/pubsublite/internal/package-info.java | 24 -
.../beam/sdk/io/gcp/pubsublite/package-info.java | 24 -
.../beam/sdk/io/gcp/spanner/MutationUtils.java | 59 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 5 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 77 +-
.../beam/sdk/io/gcp/spanner/StructUtils.java | 37 +-
.../changestreams/action/ActionFactory.java | 6 +-
.../action/QueryChangeStreamAction.java | 115 +-
.../spanner/changestreams/dao/ChangeStreamDao.java | 20 +-
.../changestreams/dao/ChangeStreamResultSet.java | 31 +
.../gcp/spanner/changestreams/dao/DaoFactory.java | 12 +-
.../dofn/ReadChangeStreamPartitionDoFn.java | 5 +-
.../mapper/ChangeStreamRecordMapper.java | 13 +-
.../sdk/io/gcp/testing/FakeDatasetService.java | 11 +-
.../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 1 -
.../AvroGenericRecordToStorageApiProtoTest.java | 154 +
.../gcp/bigquery/BeamRowToStorageApiProtoTest.java | 84 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java | 171 +-
.../io/gcp/bigquery/BigQueryIODynamicQueryIT.java | 169 +
.../io/gcp/bigquery/BigQueryIODynamicReadIT.java | 185 +
.../bigquery/BigQueryIODynamicReadTableRowIT.java | 176 +
.../io/gcp/bigquery/BigQueryIODynamicReadTest.java | 786 ++++
.../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 644 ++++
.../io/gcp/bigquery/BigQueryIOTranslationTest.java | 22 +
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 544 ++-
.../BigQueryStorageApiInsertErrorCoderTest.java | 121 +
.../io/gcp/bigquery/BigQueryTimestampPicosIT.java | 539 +++
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 312 +-
.../gcp/bigquery/TableRowToStorageApiProtoIT.java | 170 +
.../bigquery/TableRowToStorageApiProtoTest.java | 565 ++-
...gQueryFileLoadsSchemaTransformProviderTest.java | 23 +-
.../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 41 +
.../io/gcp/bigtable/BigtableServiceImplTest.java | 3 +-
.../changestreams/dao/MetadataTableDaoTest.java | 70 +
.../gcp/firestore/BaseFirestoreV1ReadFnTest.java | 4 +-
.../gcp/firestore/BaseFirestoreV1WriteFnTest.java | 8 +-
.../FirestoreV1FnBatchGetDocumentsTest.java | 32 +-
...storeV1FnBatchWriteWithDeadLetterQueueTest.java | 2 +-
.../FirestoreV1FnBatchWriteWithSummaryTest.java | 2 +-
.../FirestoreV1FnListCollectionIdsTest.java | 30 +-
.../firestore/FirestoreV1FnListDocumentsTest.java | 30 +-
.../firestore/FirestoreV1FnPartitionQueryTest.java | 34 +-
.../gcp/firestore/FirestoreV1FnRunQueryTest.java | 28 +-
.../sdk/io/gcp/firestore/it/BaseFirestoreIT.java | 60 +-
.../beam/sdk/io/gcp/pubsublite/ReadWriteIT.java | 379 --
.../pubsublite/internal/AddUuidsTransformTest.java | 97 -
.../internal/BlockingCommitterImplTest.java | 67 -
.../internal/CheckpointMarkImplTest.java | 67 -
.../gcp/pubsublite/internal/FakeSerializable.java | 59 -
.../internal/MemoryBufferedSubscriberImplTest.java | 180 -
.../internal/OffsetByteRangeTrackerTest.java | 135 -
.../internal/PerSubscriptionPartitionSdfTest.java | 205 --
.../gcp/pubsublite/internal/PubsubLiteDlqTest.java | 565 ---
.../pubsublite/internal/PubsubLiteSinkTest.java | 195 -
.../internal/PubsubLiteWriteDlqTest.java | 273 --
.../internal/SubscriptionPartitionLoaderTest.java | 91 -
.../SubscriptionPartitionProcessorImplTest.java | 146 -
.../internal/TopicBacklogReaderImplTest.java | 99 -
.../internal/UnboundedReaderImplTest.java | 204 --
.../internal/UuidDeduplicationTransformTest.java | 149 -
.../beam/sdk/io/gcp/spanner/MutationUtilsTest.java | 33 +
.../gcp/spanner/SpannerIOReadChangeStreamTest.java | 292 +-
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 7 +-
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 7 +-
.../beam/sdk/io/gcp/spanner/StructUtilsTest.java | 39 +
.../SpannerChangeStreamErrorTest.java | 38 +
.../action/QueryChangeStreamActionTest.java | 432 ++-
.../changestreams/dao/ChangeStreamDaoTest.java | 97 +
.../dao/ChangeStreamResultSetTest.java | 60 +
.../dofn/ReadChangeStreamPartitionDoFnTest.java | 25 +-
.../changestreams/it/IntegrationTestEnv.java | 7 +-
.../it/SpannerChangeStreamPlacementTableIT.java | 16 +-
...annerChangeStreamPlacementTablePostgresIT.java} | 162 +-
.../mapper/ChangeStreamRecordMapperTest.java | 105 +
sdks/java/io/hcatalog/build.gradle | 19 +-
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 2 +-
.../io/hcatalog/test/EmbeddedMetastoreService.java | 17 +-
sdks/java/io/iceberg/build.gradle | 4 +
sdks/java/io/iceberg/hive/build.gradle | 11 +-
.../beam/sdk/io/iceberg/AppendFilesToTables.java | 47 +-
.../apache/beam/sdk/io/iceberg/BundleLifter.java | 170 +
.../beam/sdk/io/iceberg/FileWriteResult.java | 3 +
.../apache/beam/sdk/io/iceberg/FilterUtils.java | 30 +-
.../beam/sdk/io/iceberg/IcebergCatalogConfig.java | 103 +-
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 19 +-
.../apache/beam/sdk/io/iceberg/IcebergUtils.java | 25 +-
.../IcebergWriteSchemaTransformProvider.java | 11 +
.../apache/beam/sdk/io/iceberg/PartitionUtils.java | 37 +
.../org/apache/beam/sdk/io/iceberg/ReadTask.java | 2 +
.../beam/sdk/io/iceberg/ReadTaskDescriptor.java | 2 +
.../apache/beam/sdk/io/iceberg/RecordWriter.java | 60 +-
.../beam/sdk/io/iceberg/RecordWriterManager.java | 74 +-
.../beam/sdk/io/iceberg/SerializableDataFile.java | 17 +-
.../apache/beam/sdk/io/iceberg/SnapshotInfo.java | 10 +
...owsToFiles.java => WriteDirectRowsToFiles.java} | 86 +-
.../sdk/io/iceberg/WriteGroupedRowsToFiles.java | 9 +-
.../beam/sdk/io/iceberg/WriteToDestinations.java | 173 +-
.../sdk/io/iceberg/WriteUngroupedRowsToFiles.java | 9 +-
.../sdk/io/iceberg/AppendFilesToTablesTest.java | 98 +
.../beam/sdk/io/iceberg/BundleLifterTest.java | 99 +
.../beam/sdk/io/iceberg/FilterUtilsTest.java | 22 +-
.../IcebergReadSchemaTransformProviderTest.java | 64 +
.../beam/sdk/io/iceberg/IcebergUtilsTest.java | 103 +
.../sdk/io/iceberg/RecordWriterManagerTest.java | 169 +-
.../io/iceberg/catalog/IcebergCatalogBaseIT.java | 19 +
.../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 28 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 47 +
.../kafka/KafkaReadWithRedistributeOverride.java | 73 +
.../beam/sdk/io/kafka/KafkaUnboundedReader.java | 22 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 123 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 31 +
.../KafkaReadWithRedistributeOverrideTest.java | 133 +
.../apache/beam/sdk/io/pulsar/PulsarMessage.java | 12 +-
.../beam/io/requestresponse/RequestResponseIO.java | 6 +-
.../io/requestresponse/RequestResponseIOTest.java | 55 +
sdks/java/io/xml/build.gradle | 4 +-
.../src/main/resources/available_configs.yaml | 37 +-
.../{io/xml => ml/inference/openai}/build.gradle | 38 +-
.../ml/inference/openai/OpenAIModelHandler.java | 163 +
.../sdk/ml/inference/openai/OpenAIModelInput.java | 64 +
.../ml/inference/openai/OpenAIModelParameters.java | 111 +
.../ml/inference/openai/OpenAIModelResponse.java | 63 +
.../sdk/ml/inference/openai}/package-info.java | 4 +-
.../ml/inference/openai/OpenAIModelHandlerIT.java | 459 +++
.../inference/openai/OpenAIModelHandlerTest.java | 463 +++
.../{io/xml => ml/inference/remote}/build.gradle | 38 +-
.../beam/sdk/ml/inference/remote/BaseInput.java} | 11 +-
.../sdk/ml/inference/remote/BaseModelHandler.java | 66 +
.../ml/inference/remote/BaseModelParameters.java} | 30 +-
.../sdk/ml/inference/remote/BaseResponse.java} | 17 +-
.../sdk/ml/inference/remote/PredictionResult.java | 53 +
.../sdk/ml/inference/remote/RemoteInference.java | 173 +
.../beam/sdk/ml/inference/remote/RetryHandler.java | 97 +
.../sdk/ml/inference/remote}/package-info.java | 4 +-
.../ml/inference/remote/RemoteInferenceTest.java | 608 ++++
sdks/python/.isort.cfg | 58 +
sdks/python/.pylintrc | 15 +-
sdks/python/apache_beam/__init__.py | 5 +-
sdks/python/apache_beam/coders/coder_impl.py | 57 +-
sdks/python/apache_beam/coders/coders.py | 37 +-
.../coders/coders_property_based_test.py | 2 +-
sdks/python/apache_beam/coders/coders_test.py | 2 +-
.../apache_beam/coders/coders_test_common.py | 225 +-
sdks/python/apache_beam/coders/typecoders.py | 26 +-
sdks/python/apache_beam/dataframe/expressions.py | 3 +-
sdks/python/apache_beam/dataframe/io.py | 2 +-
sdks/python/apache_beam/dataframe/io_it_test.py | 7 +-
sdks/python/apache_beam/dataframe/io_test.py | 7 +-
.../complete/juliaset/juliaset/juliaset_test_it.py | 6 +-
.../examples/complete/juliaset/juliaset_main.py | 9 +-
.../examples/complete/juliaset/requirements.txt | 17 -
.../examples/complete/juliaset/setup.py | 125 +
.../cookbook/ordered_window_elements}/__init__.py | 0
.../cookbook/ordered_window_elements/batch.py | 522 +++
.../cookbook/ordered_window_elements/batch_test.py | 333 ++
.../cookbook/ordered_window_elements/streaming.py | 638 ++++
.../ordered_window_elements/streaming_test.py | 397 ++
...lassification.py => gemini_image_generation.py} | 49 +-
.../inference/gemini_text_classification.py | 12 +-
.../large_language_modeling/requirements.txt | 6 +-
.../examples/inference/rate_limiter_vertex_ai.py | 85 +
.../apache_beam/examples/rate_limiter_simple.py | 93 +
.../transforms/elementwise/enrichment_test.py | 70 +-
.../apache_beam/examples/wordcount_rust/README.md | 51 +
.../wordcount_rust/requirements.txt} | 8 +-
.../wordcount_rust/word_processing/Cargo.lock | 234 ++
.../wordcount_rust/word_processing/Cargo.toml} | 17 +-
.../wordcount_rust/word_processing/pyproject.toml} | 15 +-
.../wordcount_rust/word_processing/src/lib.rs} | 38 +-
.../examples/wordcount_rust/wordcount_rust.py | 86 +
.../internal/cloudpickle/cloudpickle.py | 107 +-
.../apache_beam/internal/cloudpickle_pickler.py | 83 +-
.../internal/cloudpickle_pickler_test.py | 44 +
.../apache_beam/internal/code_object_pickler.py | 90 +-
.../internal/code_object_pickler_test.py | 27 +-
sdks/python/apache_beam/internal/gcp/auth.py | 2 +-
sdks/python/apache_beam/internal/metrics/cells.py | 165 -
.../apache_beam/internal/metrics/cells_test.py | 77 -
sdks/python/apache_beam/internal/metrics/metric.py | 44 -
.../apache_beam/internal/metrics/metric_test.py | 130 -
sdks/python/apache_beam/internal/module_test.py | 7 +
sdks/python/apache_beam/internal/pickler.py | 25 +-
sdks/python/apache_beam/internal/pickler_test.py | 44 +
sdks/python/apache_beam/io/__init__.py | 1 +
sdks/python/apache_beam/io/aws/s3io.py | 28 +-
sdks/python/apache_beam/io/aws/s3io_test.py | 36 +-
.../io/azure/blobstoragefilesystem_test.py | 2 +-
sdks/python/apache_beam/io/azure/blobstorageio.py | 32 +-
.../apache_beam/io/components/rate_limiter.py | 256 ++
.../apache_beam/io/components/rate_limiter_test.py | 143 +
.../io/external/xlang_bigqueryio_it_test.py | 37 +-
.../io/external/xlang_debeziumio_it_test.py | 2 +-
.../io/external/xlang_jdbcio_it_test.py | 2 +-
.../io/external/xlang_kafkaio_perf_test.py | 10 +-
sdks/python/apache_beam/io/filebasedsink.py | 14 +
sdks/python/apache_beam/io/filebasedsource_test.py | 2 +-
sdks/python/apache_beam/io/filesystem_test.py | 1 +
sdks/python/apache_beam/io/gcp/__init__.py | 1 +
sdks/python/apache_beam/io/gcp/bigquery.py | 37 +-
.../apache_beam/io/gcp/bigquery_file_loads.py | 4 +-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 17 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 13 +-
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 14 +-
.../apache_beam/io/gcp/bigquery_tools_test.py | 6 +-
sdks/python/apache_beam/io/gcp/bigtableio.py | 3 +-
.../apache_beam/io/gcp/bigtableio_it_test.py | 4 +-
sdks/python/apache_beam/io/gcp/bigtableio_test.py | 9 +-
.../io/gcp/datastore/v1new/datastoreio.py | 11 +-
.../io/gcp/datastore/v1new/query_splitter_test.py | 3 +-
.../io/gcp/datastore/v1new/types_test.py | 1 +
.../apache_beam/io/gcp/experimental/spannerio.py | 14 +-
.../io/gcp/experimental/spannerio_read_it_test.py | 5 +-
.../gcp/experimental/spannerio_read_perf_test.py | 1 +
.../io/gcp/experimental/spannerio_test.py | 44 +-
.../io/gcp/experimental/spannerio_write_it_test.py | 5 +-
.../gcp/experimental/spannerio_write_perf_test.py | 1 +
sdks/python/apache_beam/io/gcp/gcsio.py | 28 +-
.../apache_beam/io/gcp/gcsio_integration_test.py | 25 +-
sdks/python/apache_beam/io/gcp/gcsio_retry_test.py | 6 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 13 +-
.../io/gcp/healthcare/dicomio_integration_test.py | 9 +-
.../io/gcp/internal/clients/bigquery/__init__.py | 1 +
.../apache_beam/io/gcp/pubsub_io_perf_test.py | 1 +
.../apache_beam/io/gcp/pubsublite/external.py | 115 -
.../apache_beam/io/gcp/pubsublite/proto_api.py | 106 -
.../io/gcp/tests/xlang_spannerio_it_test.py | 134 +-
sdks/python/apache_beam/io/hadoopfilesystem.py | 11 +-
.../python/apache_beam/io/hadoopfilesystem_test.py | 7 +
.../hdfs_integration_test/hdfs_integration_test.sh | 2 +-
sdks/python/apache_beam/io/kafka.py | 51 +-
sdks/python/apache_beam/io/mongodbio.py | 15 +-
sdks/python/apache_beam/io/parquetio.py | 1 +
sdks/python/apache_beam/io/parquetio_test.py | 2 +-
sdks/python/apache_beam/io/requestresponse.py | 11 +-
.../apache_beam/io/requestresponse_it_test.py | 1 +
sdks/python/apache_beam/io/requestresponse_test.py | 1 +
sdks/python/apache_beam/io/textio_test.py | 4 +-
sdks/python/apache_beam/io/tfrecordio.py | 18 +-
sdks/python/apache_beam/io/tfrecordio_test.py | 7 +-
sdks/python/apache_beam/metrics/cells.py | 131 +
sdks/python/apache_beam/metrics/cells_test.py | 51 +
sdks/python/apache_beam/metrics/execution.py | 6 +-
sdks/python/apache_beam/metrics/metric.py | 44 +
sdks/python/apache_beam/metrics/metric_test.py | 38 +-
.../python/apache_beam/metrics/monitoring_infos.py | 2 +-
.../apache_beam/metrics/monitoring_infos_test.py | 4 +-
.../apache_beam/ml/anomaly/detectors/__init__.py | 4 +-
.../ml/anomaly/detectors/pyod_adapter.py | 2 +-
.../ml/anomaly/detectors/pyod_adapter_test.py | 3 +-
sdks/python/apache_beam/ml/gcp/cloud_dlp_test.py | 3 +-
.../apache_beam/ml/gcp/recommendations_ai_test.py | 1 +
.../ml/gcp/recommendations_ai_test_it.py | 1 +
.../apache_beam/ml/gcp/videointelligenceml_test.py | 3 +-
.../ml/gcp/videointelligenceml_test_it.py | 4 +-
sdks/python/apache_beam/ml/gcp/visionml_test.py | 3 +-
sdks/python/apache_beam/ml/gcp/visionml_test_it.py | 4 +-
sdks/python/apache_beam/ml/inference/base.py | 234 +-
sdks/python/apache_beam/ml/inference/base_test.py | 317 +-
.../apache_beam/ml/inference/gemini_inference.py | 84 +-
.../ml/inference/gemini_inference_it_test.py | 25 +
.../ml/inference/gemini_inference_test.py | 30 +-
.../ml/inference/gemini_tests_requirements.txt | 3 +-
.../ml/inference/huggingface_inference.py | 117 +-
.../ml/inference/huggingface_inference_test.py | 35 +-
.../apache_beam/ml/inference/model_manager.py | 792 ++++
.../ml/inference/model_manager_it_test.py | 191 +
.../apache_beam/ml/inference/model_manager_test.py | 629 ++++
.../apache_beam/ml/inference/onnx_inference.py | 35 +-
.../ml/inference/onnx_inference_it_test.py | 1 +
.../ml/inference/onnx_inference_test.py | 9 +-
.../apache_beam/ml/inference/pytorch_inference.py | 71 +-
.../ml/inference/pytorch_inference_it_test.py | 3 +-
.../ml/inference/pytorch_inference_test.py | 7 +-
.../apache_beam/ml/inference/sklearn_inference.py | 67 +-
.../ml/inference/tensorflow_inference.py | 74 +-
.../ml/inference/tensorflow_inference_it_test.py | 1 +
.../ml/inference/tensorflow_inference_test.py | 6 +-
.../apache_beam/ml/inference/tensorrt_inference.py | 43 +-
.../ml/inference/tensorrt_inference_test.py | 7 +-
.../vertex_ai_custom_prediction/Dockerfile | 21 +
.../vertex_ai_custom_prediction/README.md | 103 +
.../vertex_ai_custom_prediction/echo_server.py | 47 +-
.../ml/inference/test_resources/vllm.dockerfile | 14 +-
.../ml/inference/vertex_ai_inference.py | 85 +-
.../ml/inference/vertex_ai_inference_it_test.py | 47 +
.../ml/inference/vertex_ai_inference_test.py | 75 +-
.../apache_beam/ml/inference/vllm_inference.py | 54 +-
.../ml/inference/vllm_tests_requirements.txt | 2 +-
.../apache_beam/ml/inference/xgboost_inference.py | 30 +-
.../ml/inference/xgboost_inference_it_test.py | 3 +-
.../apache_beam/ml/rag/chunking/langchain_test.py | 7 +-
sdks/python/apache_beam/ml/rag/embeddings/base.py | 66 +-
.../apache_beam/ml/rag/embeddings/base_test.py | 10 +-
.../apache_beam/ml/rag/embeddings/huggingface.py | 11 +-
.../apache_beam/ml/rag/embeddings/vertex_ai.py | 11 +-
.../ml/rag/embeddings/vertex_ai_test.py | 1 +
.../ml/rag/enrichment/bigquery_vector_search.py | 129 +-
.../enrichment/bigquery_vector_search_it_test.py | 6 +-
.../apache_beam/ml/rag/enrichment/milvus_search.py | 170 +-
.../ml/rag/enrichment/milvus_search_it_test.py | 457 +--
.../ml/rag/enrichment/milvus_search_test.py | 25 +-
.../python/apache_beam/ml/rag/ingestion/alloydb.py | 2 +-
sdks/python/apache_beam/ml/rag/ingestion/base.py | 34 +-
.../apache_beam/ml/rag/ingestion/bigquery.py | 30 +-
.../apache_beam/ml/rag/ingestion/cloudsql.py | 2 +-
.../apache_beam/ml/rag/ingestion/milvus_search.py | 350 ++
.../ml/rag/ingestion/milvus_search_it_test.py | 635 ++++
.../ml/rag/ingestion/milvus_search_test.py | 123 +
sdks/python/apache_beam/ml/rag/ingestion/mysql.py | 15 +-
.../apache_beam/ml/rag/ingestion/mysql_common.py | 129 +-
.../apache_beam/ml/rag/ingestion/postgres.py | 14 +-
.../ml/rag/ingestion/postgres_common.py | 236 +-
.../python/apache_beam/ml/rag/ingestion/spanner.py | 648 ++++
.../ml/rag/ingestion/spanner_it_test.py | 606 ++++
sdks/python/apache_beam/ml/rag/test_utils.py | 413 +++
sdks/python/apache_beam/ml/rag/types.py | 94 +-
sdks/python/apache_beam/ml/rag/utils.py | 224 ++
.../ml/transforms/embeddings/huggingface_test.py | 8 +-
.../ml/transforms/embeddings/open_ai.py | 7 +-
.../ml/transforms/embeddings/tensorflow_hub.py | 3 +-
.../transforms/embeddings/tensorflow_hub_test.py | 4 +-
.../ml/transforms/embeddings/vertex_ai.py | 28 +-
.../ml/transforms/embeddings/vertex_ai_test.py | 9 +-
sdks/python/apache_beam/ml/transforms/handlers.py | 18 +-
.../apache_beam/ml/transforms/handlers_test.py | 7 +-
sdks/python/apache_beam/ml/transforms/tft.py | 5 +-
sdks/python/apache_beam/ml/transforms/utils.py | 2 +-
.../python/apache_beam/options/pipeline_options.py | 96 +-
.../options/pipeline_options_context.py | 65 +
.../options/pipeline_options_context_test.py | 335 ++
.../apache_beam/options/pipeline_options_test.py | 82 +-
sdks/python/apache_beam/pipeline.py | 448 +--
sdks/python/apache_beam/pipeline_test.py | 4 +-
sdks/python/apache_beam/pvalue.py | 18 +-
sdks/python/apache_beam/runners/__init__.py | 3 +-
sdks/python/apache_beam/runners/common.py | 6 +-
.../python/apache_beam/runners/dask/dask_runner.py | 2 +-
.../runners/dask/transform_evaluator.py | 3 +-
.../runners/dataflow/dataflow_runner.py | 5 +-
.../runners/dataflow/dataflow_runner_test.py | 63 +-
.../runners/dataflow/internal/apiclient.py | 2 +-
.../runners/dataflow/internal/apiclient_test.py | 4 +-
.../internal/clients/cloudbuild/__init__.py | 33 -
.../clients/cloudbuild/cloudbuild_v1_client.py | 1703 ---------
.../clients/cloudbuild/cloudbuild_v1_messages.py | 3836 --------------------
.../dataflow/internal/clients/dataflow/__init__.py | 3 +-
.../apache_beam/runners/dataflow/internal/names.py | 3 +-
.../runners/dataflow/ptransform_overrides.py | 2 +-
.../apache_beam/runners/direct/direct_runner.py | 13 +-
.../runners/direct/evaluation_context.py | 3 +-
.../runners/direct/transform_evaluator.py | 5 +-
.../runners/interactive/augmented_pipeline.py | 2 +-
.../interactive/background_caching_job_test.py | 1 +
.../runners/interactive/cache_manager.py | 2 +-
.../runners/interactive/caching/cacheable.py | 1 +
.../dataproc/dataproc_cluster_manager.py | 3 +-
.../runners/interactive/display/display_manager.py | 1 +
.../interactive/display/pcoll_visualization.py | 3 +-
.../runners/interactive/display/pipeline_graph.py | 7 +-
.../src/yaml/EmojiMap.ts | 2 -
.../runners/interactive/interactive_beam.py | 129 +-
.../runners/interactive/interactive_beam_test.py | 476 +++
.../runners/interactive/interactive_environment.py | 27 +-
.../interactive/interactive_environment_test.py | 41 +
.../runners/interactive/interactive_runner.py | 2 +-
.../runners/interactive/pipeline_instrument.py | 2 +-
.../interactive/pipeline_instrument_test.py | 4 +-
.../runners/interactive/recording_manager.py | 478 ++-
.../runners/interactive/recording_manager_test.py | 500 +++
.../runners/interactive/sql/beam_sql_magics.py | 7 +-
.../interactive/sql/beam_sql_magics_test.py | 2 +-
.../apache_beam/runners/interactive/utils.py | 4 +
.../apache_beam/runners/interactive/utils_test.py | 15 +-
.../python/apache_beam/runners/pipeline_context.py | 1 +
.../runners/portability/flink_runner_test.py | 29 +-
.../portability/fn_api_runner/worker_handlers.py | 22 +-
.../runners/portability/sdk_container_builder.py | 87 +-
.../portability/sdk_container_builder_test.py | 26 +
.../apache_beam/runners/portability/stager.py | 63 +-
.../apache_beam/runners/portability/stager_test.py | 96 +-
sdks/python/apache_beam/runners/runner.py | 4 +-
.../apache_beam/runners/worker/bundle_processor.py | 7 +-
.../apache_beam/runners/worker/channel_factory.py | 10 +-
.../apache_beam/runners/worker/operations.pxd | 1 +
.../apache_beam/runners/worker/operations.py | 24 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 70 +-
.../runners/worker/statesampler_test.py | 185 +
.../testing/analyzers/perf_analysis_test.py | 4 +-
.../testing/analyzers/perf_analysis_utils.py | 2 +-
.../benchmarks/cloudml/pipelines/workflow.py | 5 +-
.../pytorch_image_classification_benchmarks.py | 3 +-
.../benchmarks/inference/vllm_gemma_benchmarks.py | 2 +-
.../benchmarks/nexmark/models/auction_bid.py | 3 +-
.../benchmarks/nexmark/models/nexmark_model.py | 27 +-
.../testing/benchmarks/nexmark/nexmark_util.py | 18 -
.../testing/metric_result_matchers_test.py | 2 +-
.../apache_beam/testing/pipeline_verifiers.py | 6 +-
.../apache_beam/testing/pipeline_verifiers_test.py | 18 +-
.../apache_beam/tools/coders_microbenchmark.py | 2 +-
sdks/python/apache_beam/transforms/async_dofn.py | 86 +-
.../apache_beam/transforms/async_dofn_test.py | 73 +-
sdks/python/apache_beam/transforms/core.py | 77 +-
sdks/python/apache_beam/transforms/core_it_test.py | 57 +
sdks/python/apache_beam/transforms/core_test.py | 25 +-
.../transforms/enrichment_handlers/bigquery.py | 38 +-
.../enrichment_handlers/bigquery_it_test.py | 147 +-
.../enrichment_handlers/bigtable_it_test.py | 1 +
.../transforms/enrichment_handlers/cloudsql.py | 4 +-
.../enrichment_handlers/cloudsql_it_test.py | 31 +-
.../enrichment_handlers/cloudsql_test.py | 21 +-
.../enrichment_handlers/feast_feature_store.py | 3 +-
.../feast_feature_store_it_test.py | 6 +-
.../feast_feature_store_test.py | 6 +-
.../vertex_ai_feature_store_it_test.py | 6 +-
.../vertex_ai_feature_store_test.py | 7 +-
.../enrichment_tests_requirements.txt} | 2 +-
sdks/python/apache_beam/transforms/external.py | 24 +-
.../python/apache_beam/transforms/external_test.py | 22 +-
.../apache_beam/transforms/periodicsequence.py | 10 +-
sdks/python/apache_beam/transforms/ptransform.py | 54 +-
.../apache_beam/transforms/ptransform_test.py | 150 +
sdks/python/apache_beam/transforms/trigger_test.py | 1 +
sdks/python/apache_beam/transforms/userstate.py | 1 +
sdks/python/apache_beam/transforms/util.py | 196 +-
sdks/python/apache_beam/transforms/util_test.py | 207 +-
sdks/python/apache_beam/transforms/window_test.py | 13 +
sdks/python/apache_beam/typehints/__init__.py | 4 +-
sdks/python/apache_beam/typehints/decorators.py | 184 +-
.../apache_beam/typehints/decorators_test.py | 125 +
.../typehints/native_type_compatibility.py | 23 +-
.../typehints/native_type_compatibility_test.py | 20 +-
sdks/python/apache_beam/typehints/opcodes.py | 24 +-
.../typehints/pytorch_type_compatibility.py | 1 +
.../typehints/pytorch_type_compatibility_test.py | 1 +
sdks/python/apache_beam/typehints/schemas.py | 59 +-
.../typehints/tagged_output_typehints_test.py | 440 +++
.../apache_beam/typehints/trivial_inference.py | 35 +-
sdks/python/apache_beam/typehints/typehints.py | 6 +-
.../python/apache_beam/typehints/typehints_test.py | 14 +-
sdks/python/apache_beam/utils/interactive_utils.py | 1 +
.../apache_beam/utils/multi_process_shared.py | 239 +-
.../apache_beam/utils/multi_process_shared_test.py | 227 +-
sdks/python/apache_beam/utils/processes.py | 24 +-
sdks/python/apache_beam/utils/subprocess_server.py | 16 +-
sdks/python/apache_beam/version.py | 2 +-
.../sql/sql_calcite_connection_properties.yaml | 51 +
.../yaml/extended_tests/data/enrichment.yaml | 78 +-
.../yaml/extended_tests/databases/iceberg.yaml | 54 +-
sdks/python/apache_beam/yaml/integration_tests.py | 2 +-
sdks/python/apache_beam/yaml/json_utils.py | 37 +-
sdks/python/apache_beam/yaml/json_utils_test.py | 124 +
sdks/python/apache_beam/yaml/main.py | 6 +-
sdks/python/apache_beam/yaml/main_test.py | 6 +
sdks/python/apache_beam/yaml/standard_io.yaml | 69 +-
sdks/python/apache_beam/yaml/tests/create.yaml | 18 +
sdks/python/apache_beam/yaml/tests/sql.yaml | 23 +
sdks/python/apache_beam/yaml/yaml_io.py | 5 +-
sdks/python/apache_beam/yaml/yaml_io_test.py | 6 +
sdks/python/apache_beam/yaml/yaml_mapping.py | 249 +-
sdks/python/apache_beam/yaml/yaml_mapping_test.py | 6 +
sdks/python/apache_beam/yaml/yaml_ml.py | 10 +
sdks/python/apache_beam/yaml/yaml_provider.py | 43 +-
.../apache_beam/yaml/yaml_provider_unit_test.py | 13 +
sdks/python/apache_beam/yaml/yaml_testing.py | 45 +-
sdks/python/apache_beam/yaml/yaml_testing_test.py | 73 +
sdks/python/apache_beam/yaml/yaml_transform.py | 15 +
.../python/apache_beam/yaml/yaml_transform_test.py | 228 +-
.../apache_beam/yaml/yaml_transform_unit_test.py | 30 +
sdks/python/apache_beam/yaml/yaml_udf_test.py | 21 +-
sdks/python/conftest.py | 39 +-
sdks/python/container/Dockerfile | 4 +-
sdks/python/container/boot.go | 39 +
sdks/python/container/common.gradle | 27 +-
.../container/license_scripts/dep_urls_py.yaml | 7 +
.../container/ml/py310/base_image_requirements.txt | 279 +-
...requirements.txt => gpu_image_requirements.txt} | 252 +-
.../container/ml/py311/base_image_requirements.txt | 279 +-
...requirements.txt => gpu_image_requirements.txt} | 252 +-
.../container/ml/py312/base_image_requirements.txt | 275 +-
...requirements.txt => gpu_image_requirements.txt} | 254 +-
...equirements.txt => base_image_requirements.txt} | 211 +-
.../container/ml/py39/base_image_requirements.txt | 247 --
.../container/ml/py39/ml_image_requirements.txt | 248 --
sdks/python/container/piputil.go | 14 +
.../container/py310/base_image_requirements.txt | 161 +-
.../container/py311/base_image_requirements.txt | 161 +-
.../container/py312/base_image_requirements.txt | 158 +-
.../container/py313/base_image_requirements.txt | 164 +-
.../container/py39/base_image_requirements.txt | 199 -
sdks/python/container/run_generate_requirements.sh | 17 +-
sdks/python/container/run_validatescontainer.sh | 5 +
sdks/python/expansion-service-container/Dockerfile | 7 +-
.../expansion-service-container/build.gradle | 4 +-
sdks/python/gen_managed_doc.py | 3 +-
sdks/python/gen_xlang_wrappers.py | 1 -
sdks/python/mypy.ini | 2 +-
sdks/python/pyproject.toml | 2 +-
sdks/python/scripts/run_pylint.sh | 5 +-
sdks/python/scripts/run_pytest.sh | 2 +-
sdks/python/setup.cfg | 3 -
sdks/python/setup.py | 77 +-
sdks/python/test-suites/dataflow/common.gradle | 5 +-
sdks/python/test-suites/dataflow/py39/build.gradle | 24 -
sdks/python/test-suites/direct/py39/build.gradle | 24 -
sdks/python/test-suites/gradle.properties | 26 +-
sdks/python/test-suites/portable/common.gradle | 66 +-
sdks/python/test-suites/portable/py39/build.gradle | 26 -
sdks/python/test-suites/tox/common.gradle | 5 +-
sdks/python/test-suites/tox/py310/build.gradle | 195 +
sdks/python/test-suites/tox/py39/build.gradle | 236 --
sdks/python/tox.ini | 101 +-
sdks/standard_expansion_services.yaml | 3 -
sdks/typescript/package-lock.json | 63 +-
sdks/typescript/package.json | 2 +-
sdks/typescript/reporterConfig.js | 17 +-
sdks/typescript/src/apache_beam/io/index.ts | 1 -
sdks/typescript/src/apache_beam/io/pubsublite.ts | 55 -
sdks/typescript/src/apache_beam/pvalue.ts | 49 +
sdks/typescript/src/apache_beam/runners/flink.ts | 2 +-
.../typescript/src/apache_beam/worker/operators.ts | 44 +-
sdks/typescript/src/apache_beam/worker/state.ts | 135 +-
sdks/typescript/test/primitives_test.ts | 45 +
sdks/typescript/test/state_provider_test.ts | 328 ++
settings.gradle.kts | 85 +-
website/Dockerfile | 39 +-
website/README.md | 13 +-
website/build.gradle | 55 +
.../site/assets/css/fontawesome/release-v5.4.1.css | 5 +
website/www/site/assets/css/roboto/roboto.css | 35 +
.../site/assets/css/swiper@8/swiper-bundle.min.css | 13 +
.../www/site/assets/js/jquery/jquery-2.2.4.min.js | 4 +
website/www/site/assets/scss/_case_study.scss | 3 +
website/www/site/assets/scss/_global.sass | 1 -
website/www/site/config.toml | 2 +-
website/www/site/content/en/blog/beam-2.69.0.md | 86 +
website/www/site/content/en/blog/beam-2.70.0.md | 53 +
website/www/site/content/en/blog/beam-2.71.0.md | 53 +
.../en/blog/gsoc-25-jupyterlab-extensions.md | 74 +
.../www/site/content/en/case-studies/albertsons.md | 203 ++
.../en/documentation/dsls/sql/calcite/overview.md | 27 +
.../dsls/sql/calcite/scalar-functions.md | 9 +-
.../en/documentation/dsls/sql/zetasql/overview.md | 2 +-
.../site/content/en/documentation/io/managed-io.md | 1211 +++++-
.../site/content/en/documentation/io/testing.md | 8 -
.../content/en/documentation/programming-guide.md | 4 +-
.../en/documentation/runtime/environments.md | 3 +-
.../www/site/content/en/documentation/sdks/java.md | 29 +
.../content/en/documentation/sdks/java/euphoria.md | 2 +
.../sdks/python-pipeline-dependencies.md | 28 +-
.../site/content/en/documentation/sdks/python.md | 51 +-
.../content/en/documentation/sdks/yaml-errors.md | 58 +
.../www/site/content/en/documentation/sdks/yaml.md | 2 +
.../transforms/python/aggregation/cogroupbykey.md | 14 +-
.../www/site/content/en/get-started/downloads.md | 34 +-
website/www/site/content/en/roadmap/_index.md | 19 +-
.../www/site/content/en/security/CVE-2020-1929.md | 16 -
website/www/site/data/authors.yml | 3 +
website/www/site/data/en/quotes.yaml | 5 +
website/www/site/data/performance.yaml | 11 +-
website/www/site/layouts/case-studies/list.html | 4 +-
website/www/site/layouts/partials/head.html | 12 +-
.../www/site/layouts/partials/head_homepage.html | 12 +-
.../shortcodes/flink_java_pipeline_options.html | 19 +-
.../shortcodes/flink_python_pipeline_options.html | 19 +-
.../site/layouts/shortcodes/performance_looks.html | 21 +-
website/www/site/static/.htaccess | 5 +-
.../gsoc-25-jupyterlab-extensions/Yaml_main.png | Bin 0 -> 305413 bytes
.../images/case-study/albertsons/ankurraj.jpeg | Bin 0 -> 27941 bytes
.../case-study/albertsons/mohammedjawedkhan.jpeg | Bin 0 -> 5515 bytes
.../images/case-study/albertsons/utkarshparekh.png | Bin 0 -> 1239692 bytes
.../images/case-study/albertsons/vinaydesai.jpeg | Bin 0 -> 42897 bytes
.../static/images/logos/powered-by/albertsons.jpg | Bin 0 -> 192436 bytes
website/www/yarn.lock | 6 +-
1387 files changed, 78168 insertions(+), 33030 deletions(-)
create mode 100644 .agent/skills/README.md
create mode 100644 .agent/skills/beam-concepts/SKILL.md
create mode 100644 .agent/skills/ci-cd/SKILL.md
create mode 100644 .agent/skills/contributing/SKILL.md
create mode 100644 .agent/skills/gradle-build/SKILL.md
create mode 100644 .agent/skills/io-connectors/SKILL.md
create mode 100644 .agent/skills/java-development/SKILL.md
create mode 100644 .agent/skills/license-compliance/SKILL.md
create mode 100644 .agent/skills/python-development/SKILL.md
create mode 100644 .agent/skills/runners/SKILL.md
copy .github/trigger_files/{beam_CloudML_Benchmarks_Dataflow.json => beam_PostCommit_Go.json} (100%)
copy .github/trigger_files/{beam_PostCommit_Java_Examples_Dataflow_ARM.json => beam_PostCommit_PortableJar_Flink.json} (100%)
copy .github/trigger_files/{beam_PreCommit_SQL.json => beam_PostCommit_Python_Examples_Dataflow.json} (85%)
create mode 100644 .github/trigger_files/beam_PostCommit_Python_Examples_Flink.json
copy .github/trigger_files/{beam_PostCommit_Java_Examples_Dataflow_ARM.json => beam_PostCommit_Python_MongoDBIO_IT.json} (100%)
copy .github/trigger_files/{beam_PreCommit_Python_Dill.json => beam_PreCommit_Yaml_Xlang_Direct.json} (83%)
copy .github/workflows/{beam_PreCommit_Java_PVR_Flink_Batch.yml => beam_PostCommit_Java_PVR_Flink_Batch.yml} (74%)
copy .github/workflows/{beam_PreCommit_Java_HCatalog_IO_Direct.yml => beam_PreCommit_Java_Datadog_IO_Direct.yml} (74%)
delete mode 100644 .test-infra/jenkins/Committers.groovy
delete mode 100644 .test-infra/jenkins/CommonJobProperties.groovy
delete mode 100644 .test-infra/jenkins/CommonTestProperties.groovy
delete mode 100644 .test-infra/jenkins/CronJobBuilder.groovy
delete mode 100644 .test-infra/jenkins/Kubernetes.groovy
delete mode 100644 .test-infra/jenkins/LoadTestConfig.groovy
delete mode 100644 .test-infra/jenkins/LoadTestsBuilder.groovy
delete mode 100644 .test-infra/jenkins/NexmarkBuilder.groovy
delete mode 100644 .test-infra/jenkins/NexmarkDatabaseProperties.groovy
delete mode 100644 .test-infra/jenkins/PhraseTriggeringPostCommitBuilder.groovy
delete mode 100644 .test-infra/jenkins/PostcommitJobBuilder.groovy
delete mode 100644 .test-infra/jenkins/PrecommitJobBuilder.groovy
delete mode 100644 .test-infra/jenkins/PythonTestProperties.groovy
delete mode 100644 .test-infra/jenkins/README.md
delete mode 100644 .test-infra/jenkins/TpcdsDatabaseProperties.groovy
delete mode 100644 .test-infra/jenkins/build.gradle
delete mode 100644 .test-infra/jenkins/metrics_report/dashboards_parser.py
delete mode 100644 .test-infra/jenkins/metrics_report/report_generator.py
delete mode 100644 .test-infra/jenkins/metrics_report/templates/Metrics_Report.template
delete mode 100644 .test-infra/jenkins/metrics_report/tox.ini
create mode 100644 contributor-docs/README.md
create mode 100644 contributor-docs/updating-supported-python-versions.md
create mode 100644 examples/java/src/main/java/org/apache/beam/examples/RateLimiterSimple.java
create mode 100644 examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryStreamingTornadoes.java
create mode 100644 examples/notebooks/beam-ml/spanner_product_catalog_embeddings.ipynb
create mode 100644 examples/terraform/envoy-ratelimiter/README.md
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRange.java => examples/terraform/envoy-ratelimiter/gke.tf (54%)
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReader.java => examples/terraform/envoy-ratelimiter/network.tf (76%)
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/BlockingCommitter.java => examples/terraform/envoy-ratelimiter/outputs.tf (75%)
create mode 100644 examples/terraform/envoy-ratelimiter/prerequisites.tf
rename .test-infra/jenkins/InfluxDBCredentialsHelper.groovy => examples/terraform/envoy-ratelimiter/provider.tf (55%)
create mode 100644 examples/terraform/envoy-ratelimiter/ratelimit.tf
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ApiServices.java => examples/terraform/envoy-ratelimiter/terraform.tfvars (59%)
create mode 100644 examples/terraform/envoy-ratelimiter/variables.tf
create mode 100644 it/datadog/build.gradle
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogClientFactory.java
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogLogEntry.java
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogResourceManager.java
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryLimiter.java => it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogResourceManagerException.java (67%)
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogResourceManagerUtils.java
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/conditions/DatadogLogEntriesCheck.java
copy it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite => datadog/src/main/java/org/apache/beam/it/datadog/conditions}/package-info.java (87%)
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/matchers/DatadogAsserts.java
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/matchers/package-info.java
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/package-info.java
create mode 100644 it/datadog/src/test/java/org/apache/beam/it/datadog/DatadogResourceManagerIT.java
create mode 100644 it/datadog/src/test/java/org/apache/beam/it/datadog/DatadogResourceManagerTest.java
create mode 100644 it/datadog/src/test/java/org/apache/beam/it/datadog/DatadogResourceManagerUtilsTest.java
delete mode 100644 it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite/PubsubliteResourceManager.java
rename {sdks/python/container/distroless/py39 => runners/flink/1.20}/build.gradle (82%)
copy runners/flink/{1.17 => 1.20}/job-server-container/build.gradle (100%)
rename {sdks/python/container/py39 => runners/flink/1.20/job-server}/build.gradle (62%)
create mode 100644 runners/flink/1.20/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
create mode 100644 runners/flink/2.0/build.gradle
copy runners/flink/{1.17 => 2.0}/job-server-container/build.gradle (100%)
rename {sdks/python/container/ml/py39 => runners/flink/2.0/job-server}/build.gradle (62%)
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/FlinkBatchTranslationContext.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/FlinkExecutionEnvironments.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/FlinkMiniClusterEntryPoint.java
copy runners/flink/{ => 2.0}/src/main/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironment.java (67%)
copy runners/flink/{ => 2.0}/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java (90%)
copy runners/flink/{ => 2.0}/src/main/java/org/apache/beam/runners/flink/FlinkPipelineRunner.java (95%)
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/FlinkPipelineTranslator.java
copy runners/flink/{ => 2.0}/src/main/java/org/apache/beam/runners/flink/FlinkStreamingPortablePipelineTranslator.java (97%)
copy runners/flink/{ => 2.0}/src/main/java/org/apache/beam/runners/flink/FlinkStreamingTransformTranslators.java (99%)
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunction.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageContextFactory.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunction.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStagePruningFunction.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMergingNonShuffleReduceFunction.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkMultiOutputPruningFunction.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkPartialReduceFunction.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkReduceFunction.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunction.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/functions/ImpulseSourceFunction.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeInformation.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueSerializer.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueTypeInformation.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/types/UnversionedTypeSerializerSnapshot.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/StreamingImpulseSource.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/TestStreamSource.java
copy runners/flink/{ => 2.0}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapper.java (95%)
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkBroadcastStateInternals.java
create mode 100644 runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/state/FlinkStateInternals.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/EncodedValueComparatorTest.java
copy runners/flink/{ => 2.0}/src/test/java/org/apache/beam/runners/flink/FlinkExecutionEnvironmentsTest.java (89%)
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/FlinkPipelineExecutionEnvironmentTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/FlinkPipelineOptionsTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/FlinkRequiresStableInputTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/FlinkRunnerTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/FlinkSubmissionTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/ReadSourceTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/adapter/BeamFlinkDataStreamAdapterTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/streaming/BoundedSourceRestoreTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/streaming/FlinkStateInternalsTest.java
copy runners/flink/{1.17 => 2.0}/src/test/java/org/apache/beam/runners/flink/streaming/MemoryStateBackendWrapper.java (73%)
copy runners/flink/{1.17 => 2.0}/src/test/java/org/apache/beam/runners/flink/streaming/StreamSources.java (85%)
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/translation/functions/FlinkDoFnFunctionTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/translation/functions/FlinkExecutableStageFunctionTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/translation/functions/FlinkStatefulDoFnFunctionTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/translation/functions/ImpulseSourceFunctionTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/UnboundedSourceWrapperTest.java
create mode 100644 runners/flink/2.0/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/stableinput/BufferingDoFnRunnerTest.java
rename .test-infra/jenkins/metrics_report/requirements.txt => runners/flink/2.0/src/test/resources/flink-test-config.yaml (70%)
rename runners/flink/{1.17 => }/src/main/java/org/apache/beam/runners/flink/translation/types/CoderTypeSerializer.java (100%)
rename runners/flink/{1.17 => }/src/test/java/org/apache/beam/runners/flink/streaming/MemoryStateBackendWrapper.java (100%)
rename runners/flink/{1.17 => }/src/test/java/org/apache/beam/runners/flink/streaming/StreamSources.java (100%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/ThreadLocalByteStringOutputStream.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/common/worker/InternedByteString.java
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateTagUtil.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillTagEncoding.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillTagEncodingV1.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillTagEncodingV2.java
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetRefresher.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/ThreadLocalByteStringOutputStreamTest.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/common/worker/InternedByteStringTest.java
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateTagUtilTest.java
rename runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/{WindmillTimerInternalsTest.java => windmill/state/WindmillTagEncodingV1Test.java} (56%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillTagEncodingV2Test.java
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetRefresherTest.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/Timestamp.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/UnsplittableRestrictionTracker.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExpiringMemoizingSerializableSupplier.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcpHsmGeneratedSecret.java
rename .test-infra/jenkins/JavaTestProperties.groovy => sdks/java/core/src/main/java/org/apache/beam/sdk/values/CausedByDrain.java (84%)
copy sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/{ViewTest.java => MapViewTest.java} (54%)
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/util/ExpiringMemoizingSerializableSupplierTest.java
copy sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/{GcsUtil.java => GcsUtilV1.java} (95%)
create mode 100644 sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilV2.java
create mode 100644 sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilParameterizedIT.java
create mode 100644 sdks/java/extensions/kafka-factories/build.gradle
create mode 100644 sdks/java/extensions/kafka-factories/src/main/java/org/apache/beam/sdk/extensions/kafka/factories/FileAwareFactoryFn.java
create mode 100644 sdks/java/extensions/kafka-factories/src/main/java/org/apache/beam/sdk/extensions/kafka/factories/KerberosConsumerFactoryFn.java
copy {it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite => sdks/java/extensions/kafka-factories/src/main/java/org/apache/beam/sdk/extensions/kafka/factories}/package-info.java (85%)
create mode 100644 sdks/java/extensions/kafka-factories/src/test/java/org/apache/beam/sdk/extensions/kafka/factories/FileAwareFactoryFnTest.java
create mode 100644 sdks/java/extensions/kafka-factories/src/test/java/org/apache/beam/sdk/extensions/kafka/factories/KerberosConsumerFactoryFnTest.java
create mode 100644 sdks/java/extensions/sql/iceberg/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergAlterTableOps.java
create mode 100644 sdks/java/extensions/sql/iceberg/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/BeamSqlCliIcebergAlterTest.java
create mode 100644 sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineShowTest.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSystemDbMetadataSchema.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSystemSchema.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSystemTableMetadataSchema.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlAlterCatalog.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlAlterTable.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/SystemTables.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/AlterTableOps.java
delete mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteSubscriptionTable.java
delete mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTableProvider.java
delete mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTopicTable.java
delete mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/RowHandler.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/AlterTestTableOps.java
delete mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/PubsubLiteTableProviderTest.java
delete mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/RowHandlerTest.java
create mode 100644 sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseJdbcUrlParser.java
create mode 100644 sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIOJdbcBackwardCompatibilityTest.java
create mode 100644 sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIOPropertyMergingTest.java
create mode 100644 sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseJdbcUrlParserTest.java
copy {it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite => sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/deadletterqueue}/package-info.java (86%)
rename sdks/java/io/{google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReader.java => components/src/main/java/org/apache/beam/sdk/io/components/deadletterqueue/sinks/package-info.java} (79%)
rename sdks/java/io/{google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PerServerPublisherCache.java => components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/EnvoyRateLimiter.java} (51%)
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/EnvoyRateLimiterContext.java
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/EnvoyRateLimiterFactory.java
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiter.java
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterClientCache.java
rename sdks/java/{extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/pubsublite/package-info.java => io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterContext.java} (74%)
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterFactory.java
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterOptions.java
copy {it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite => sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter}/package-info.java (86%)
copy {it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite => sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/throttling}/package-info.java (87%)
create mode 100644 sdks/java/io/components/src/test/java/org/apache/beam/sdk/io/components/ratelimiter/EnvoyRateLimiterTest.java
create mode 100644 sdks/java/io/components/src/test/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterClientCacheTest.java
create mode 100644 sdks/java/io/components/src/test/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterOptionsTest.java
copy sdks/java/io/{components => datadog}/build.gradle (50%)
create mode 100644 sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogEvent.java
create mode 100644 sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogEventCoder.java
create mode 100644 sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogEventPublisher.java
create mode 100644 sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogEventSerializer.java
create mode 100644 sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogEventWriter.java
create mode 100644 sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogIO.java
create mode 100644 sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogWriteError.java
create mode 100644 sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogWriteErrorCoder.java
rename .test-infra/jenkins/NoPhraseTriggeringPostCommitBuilder.groovy => sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/package-info.java (64%)
create mode 100644 sdks/java/io/datadog/src/test/java/org/apache/beam/sdk/io/datadog/DatadogEventCoderTest.java
create mode 100644 sdks/java/io/datadog/src/test/java/org/apache/beam/sdk/io/datadog/DatadogEventPublisherTest.java
create mode 100644 sdks/java/io/datadog/src/test/java/org/apache/beam/sdk/io/datadog/DatadogEventSerializerTest.java
create mode 100644 sdks/java/io/datadog/src/test/java/org/apache/beam/sdk/io/datadog/DatadogEventTest.java
create mode 100644 sdks/java/io/datadog/src/test/java/org/apache/beam/sdk/io/datadog/DatadogEventWriterTest.java
create mode 100644 sdks/java/io/datadog/src/test/java/org/apache/beam/sdk/io/datadog/DatadogIOTest.java
create mode 100644 sdks/java/io/datadog/src/test/java/org/apache/beam/sdk/io/datadog/DatadogWriteErrorCoderTest.java
create mode 100644 sdks/java/io/datadog/src/test/java/org/apache/beam/sdk/io/datadog/DatadogWriteErrorTest.java
copy sdks/java/io/{xml => elasticsearch-tests/elasticsearch-tests-9}/build.gradle (51%)
create mode 100644 sdks/java/io/elasticsearch-tests/elasticsearch-tests-9/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
copy sdks/java/io/elasticsearch-tests/{elasticsearch-tests-8 => elasticsearch-tests-9}/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java (97%)
copy sdks/java/io/elasticsearch-tests/{elasticsearch-tests-7 => elasticsearch-tests-9}/src/test/resources/clientkeystore (100%)
rename sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/{GoogleAdsV19.java => GoogleAdsV23.java} (93%)
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryDynamicReadDescriptor.java
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/{pubsublite/internal/BlockingCommitter.java => bigquery/TimestampPrecision.java} (81%)
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/CloudPubsubTransforms.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PublisherOptions.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteIO.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteReadSchemaTransformProvider.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteWriteSchemaTransformProvider.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriberOptions.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/UuidDeduplicationOptions.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/AddUuidsTransform.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/BlockingCommitterImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/CheckpointMarkImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/CloserReference.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/DlqProvider.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ExternalTransformConfig.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ExternalTransformRegistrarImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/InitialOffsetReaderImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/LimitingTopicBacklogReader.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ManagedFactoryImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryBufferedSubscriber.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryBufferedSubscriberImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryLimiterImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteProgress.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeCoder.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeTracker.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PerServerSubscriberCache.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PerSubscriptionPartitionSdf.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ProtoFromBytes.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ProtoToBytes.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PublisherAssembler.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteSink.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ServiceCache.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscribeTransform.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscriberAssembler.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscriptionPartition.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscriptionPartitionCoder.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscriptionPartitionLoader.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscriptionPartitionProcessor.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscriptionPartitionProcessorImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/TopicBacklogReader.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/TopicBacklogReaderImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/TopicBacklogReaderSettings.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/TrackerWithProgress.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/UnboundedReaderImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/UnboundedSourceImpl.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/Uuid.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/UuidCoder.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/UuidDeduplicationTransform.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/package-info.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/package-info.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIODynamicQueryIT.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIODynamicReadIT.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIODynamicReadTableRowIT.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIODynamicReadTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageApiInsertErrorCoderTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTimestampPicosIT.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/ReadWriteIT.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/AddUuidsTransformTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/BlockingCommitterImplTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/CheckpointMarkImplTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/FakeSerializable.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryBufferedSubscriberImplTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/OffsetByteRangeTrackerTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PerSubscriptionPartitionSdfTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteDlqTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteSinkTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteWriteDlqTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscriptionPartitionLoaderTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscriptionPartitionProcessorImplTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/TopicBacklogReaderImplTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/UnboundedReaderImplTest.java
delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/UuidDeduplicationTransformTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/ChangeStreamDaoTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/ChangeStreamResultSetTest.java
copy sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/{SpannerChangeStreamPlacementTableIT.java => SpannerChangeStreamPlacementTablePostgresIT.java} (60%)
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/BundleLifter.java
copy sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/{WriteGroupedRowsToFiles.java => WriteDirectRowsToFiles.java} (58%)
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/AppendFilesToTablesTest.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/BundleLifterTest.java
create mode 100644 sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadWithRedistributeOverride.java
create mode 100644 sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadWithRedistributeOverrideTest.java
copy sdks/java/{io/xml => ml/inference/openai}/build.gradle (55%)
create mode 100644 sdks/java/ml/inference/openai/src/main/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelHandler.java
create mode 100644 sdks/java/ml/inference/openai/src/main/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelInput.java
create mode 100644 sdks/java/ml/inference/openai/src/main/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelParameters.java
create mode 100644 sdks/java/ml/inference/openai/src/main/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelResponse.java
copy {it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite => sdks/java/ml/inference/openai/src/main/java/org/apache/beam/sdk/ml/inference/openai}/package-info.java (87%)
create mode 100644 sdks/java/ml/inference/openai/src/test/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelHandlerIT.java
create mode 100644 sdks/java/ml/inference/openai/src/test/java/org/apache/beam/sdk/ml/inference/openai/OpenAIModelHandlerTest.java
copy sdks/java/{io/xml => ml/inference/remote}/build.gradle (61%)
copy sdks/java/{core/src/main/java/org/apache/beam/sdk/state/TimerSpec.java => ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseInput.java} (76%)
create mode 100644 sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseModelHandler.java
rename sdks/java/{io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscriptionPartitionProcessorFactory.java => ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseModelParameters.java} (52%)
rename sdks/java/{io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/ManagedFactory.java => ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseResponse.java} (69%)
create mode 100644 sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/PredictionResult.java
create mode 100644 sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/RemoteInference.java
create mode 100644 sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/RetryHandler.java
rename {it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/pubsublite => sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote}/package-info.java (87%)
create mode 100644 sdks/java/ml/inference/remote/src/test/java/org/apache/beam/sdk/ml/inference/remote/RemoteInferenceTest.java
create mode 100644 sdks/python/.isort.cfg
delete mode 100644 sdks/python/apache_beam/examples/complete/juliaset/requirements.txt
create mode 100644 sdks/python/apache_beam/examples/complete/juliaset/setup.py
copy {examples/notebooks/notebook_test_scripts => sdks/python/apache_beam/examples/cookbook/ordered_window_elements}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/examples/cookbook/ordered_window_elements/batch.py
create mode 100644 sdks/python/apache_beam/examples/cookbook/ordered_window_elements/batch_test.py
create mode 100644 sdks/python/apache_beam/examples/cookbook/ordered_window_elements/streaming.py
create mode 100644 sdks/python/apache_beam/examples/cookbook/ordered_window_elements/streaming_test.py
copy sdks/python/apache_beam/examples/inference/{gemini_text_classification.py => gemini_image_generation.py} (72%)
create mode 100644 sdks/python/apache_beam/examples/inference/rate_limiter_vertex_ai.py
create mode 100644 sdks/python/apache_beam/examples/rate_limiter_simple.py
create mode 100644 sdks/python/apache_beam/examples/wordcount_rust/README.md
copy sdks/python/apache_beam/{io/gcp/pubsublite/__init__.py => examples/wordcount_rust/requirements.txt} (86%)
create mode 100644 sdks/python/apache_beam/examples/wordcount_rust/word_processing/Cargo.lock
copy sdks/python/apache_beam/{io/gcp/pubsublite/__init__.py => examples/wordcount_rust/word_processing/Cargo.toml} (74%)
rename sdks/python/apache_beam/{io/gcp/pubsublite/__init__.py => examples/wordcount_rust/word_processing/pyproject.toml} (69%)
copy sdks/{go/pkg/beam/core/core.go => python/apache_beam/examples/wordcount_rust/word_processing/src/lib.rs} (53%)
create mode 100644 sdks/python/apache_beam/examples/wordcount_rust/wordcount_rust.py
delete mode 100644 sdks/python/apache_beam/internal/metrics/cells.py
delete mode 100644 sdks/python/apache_beam/internal/metrics/cells_test.py
delete mode 100644 sdks/python/apache_beam/internal/metrics/metric_test.py
create mode 100644 sdks/python/apache_beam/io/components/rate_limiter.py
create mode 100644 sdks/python/apache_beam/io/components/rate_limiter_test.py
delete mode 100644 sdks/python/apache_beam/io/gcp/pubsublite/external.py
delete mode 100644 sdks/python/apache_beam/io/gcp/pubsublite/proto_api.py
create mode 100644 sdks/python/apache_beam/ml/inference/model_manager.py
create mode 100644 sdks/python/apache_beam/ml/inference/model_manager_it_test.py
create mode 100644 sdks/python/apache_beam/ml/inference/model_manager_test.py
create mode 100644 sdks/python/apache_beam/ml/inference/test_resources/vertex_ai_custom_prediction/Dockerfile
create mode 100644 sdks/python/apache_beam/ml/inference/test_resources/vertex_ai_custom_prediction/README.md
copy .test-infra/mock-apis/pyproject.toml => sdks/python/apache_beam/ml/inference/test_resources/vertex_ai_custom_prediction/echo_server.py (60%)
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/milvus_search.py
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/milvus_search_it_test.py
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/milvus_search_test.py
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/spanner.py
create mode 100644 sdks/python/apache_beam/ml/rag/ingestion/spanner_it_test.py
create mode 100644 sdks/python/apache_beam/ml/rag/test_utils.py
create mode 100644 sdks/python/apache_beam/ml/rag/utils.py
create mode 100644 sdks/python/apache_beam/options/pipeline_options_context.py
create mode 100644 sdks/python/apache_beam/options/pipeline_options_context_test.py
delete mode 100644 sdks/python/apache_beam/runners/dataflow/internal/clients/cloudbuild/__init__.py
delete mode 100644 sdks/python/apache_beam/runners/dataflow/internal/clients/cloudbuild/cloudbuild_v1_client.py
delete mode 100644 sdks/python/apache_beam/runners/dataflow/internal/clients/cloudbuild/cloudbuild_v1_messages.py
copy sdks/python/apache_beam/{ml/inference/gemini_tests_requirements.txt => transforms/enrichment_tests_requirements.txt} (97%)
create mode 100644 sdks/python/apache_beam/typehints/tagged_output_typehints_test.py
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/sql/sql_calcite_connection_properties.yaml
rename sdks/python/container/ml/py310/{ml_image_requirements.txt => gpu_image_requirements.txt} (54%)
rename sdks/python/container/ml/py311/{ml_image_requirements.txt => gpu_image_requirements.txt} (54%)
rename sdks/python/container/ml/py312/{ml_image_requirements.txt => gpu_image_requirements.txt} (53%)
rename sdks/python/container/ml/py313/{ml_image_requirements.txt => base_image_requirements.txt} (57%)
delete mode 100644 sdks/python/container/ml/py39/base_image_requirements.txt
delete mode 100644 sdks/python/container/ml/py39/ml_image_requirements.txt
delete mode 100644 sdks/python/container/py39/base_image_requirements.txt
delete mode 100644 sdks/python/test-suites/dataflow/py39/build.gradle
delete mode 100644 sdks/python/test-suites/direct/py39/build.gradle
delete mode 100644 sdks/python/test-suites/portable/py39/build.gradle
delete mode 100644 sdks/python/test-suites/tox/py39/build.gradle
delete mode 100644 sdks/typescript/src/apache_beam/io/pubsublite.ts
create mode 100644 sdks/typescript/test/state_provider_test.ts
create mode 100644 website/www/site/assets/css/fontawesome/release-v5.4.1.css
create mode 100644 website/www/site/assets/css/roboto/roboto.css
create mode 100644 website/www/site/assets/css/swiper@8/swiper-bundle.min.css
create mode 100644 website/www/site/assets/js/jquery/jquery-2.2.4.min.js
create mode 100644 website/www/site/content/en/blog/beam-2.69.0.md
create mode 100644 website/www/site/content/en/blog/beam-2.70.0.md
create mode 100644 website/www/site/content/en/blog/beam-2.71.0.md
create mode 100644 website/www/site/content/en/blog/gsoc-25-jupyterlab-extensions.md
create mode 100644 website/www/site/content/en/case-studies/albertsons.md
delete mode 100644 website/www/site/content/en/security/CVE-2020-1929.md
create mode 100644 website/www/site/static/images/blog/gsoc-25-jupyterlab-extensions/Yaml_main.png
create mode 100644 website/www/site/static/images/case-study/albertsons/ankurraj.jpeg
create mode 100644 website/www/site/static/images/case-study/albertsons/mohammedjawedkhan.jpeg
create mode 100644 website/www/site/static/images/case-study/albertsons/utkarshparekh.png
create mode 100644 website/www/site/static/images/case-study/albertsons/vinaydesai.jpeg
create mode 100644 website/www/site/static/images/logos/powered-by/albertsons.jpg