You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2026/04/28 20:07:38 UTC
(beam) branch dependabot/gradle/gradle-wrapper-9.3.1 updated (d1c26f96ec7 -> 6bbf8c38e38)
This is an automated email from the ASF dual-hosted git repository.
dependabot[bot] pushed a change to branch dependabot/gradle/gradle-wrapper-9.3.1
in repository https://gitbox.apache.org/repos/asf/beam.git
discard d1c26f96ec7 Bump gradle-wrapper from 8.14.3 to 9.3.1
add d4015eb395b Remove pubsublite - infra, groovy, checkstyles (#37450)
add cdf48147bdd [RunInference] Add content-aware dynamic batching via element_size_fn… (#37428)
add e7a21ddb5b5 [#37198] Make withBackOffSupplier public to enable bounded retry configuration (#37356)
add d561bc31cb4 Add artifactregistry.admin role (#37456)
add 4b5d149e3a3 Bump github.com/lib/pq from 1.10.9 to 1.11.1 in /sdks (#37454)
add 112685de58d Upgrade google-api-services-dataflow (#37462)
add 501897649dd Upgrade GCP Bom (#37473)
add 0282abbff3b Allow multiprocessshared to spawn process and delete directly with obj (#37112)
add 953f70c5c28 Disable abbrev for yaml.main (#37476)
add 7052f01e45a Add an experiment to disable pip build isolation (#37331)
add e4cd040199d Add LRU cache eviction to CombinePerKeyPrecombineOperator (#37466)
add 4674387e88e Remove pubsublite logs - try example notebook (#37457)
add fd2babbcddb [Python] Add take(n) convenience for PCollection (#37470)
add d1e5849206e fix(dataflow): 'update' infinite loop and nil pointer dereference (#37472)
add b1cf96d3d65 Bump pip from 25.3 to 26.0 in /sdks/python/container/ml/py311 (#37483)
add 1531743ce8d [Dataflow Streaming] Enable state tag encoding v2 based on backend flag (#37464)
add 91418fd69a0 Fix ValidatesContainer Cloud Build test
add 425d82704b8 Update sdk_container_builder for google-cloud-build 3.35 API
add 98cdbc70a59 Added fall back to additionalProperties for older clients
add 1c5d57a0ec2 Merge pull request #37490 from aIbrahiim/fix-postcommit-python-validates-container
add d32bc97290a Revert "Bump pip from 25.3 to 26.0 in /sdks/python/container/ml/py311 (#37483)" (#37494)
add 0aea01394b1 Update java ml module gradle (#37471)
add 9a5de4f46ce [Drain] Propagate drain information to timerData and TImer data structures. mostly noop.
add 1c22136c6e2 Merge pull request #37009: [Drain] TimerData causedByDrain field
add 183ecd36cb4 Add pubsublite IO change (#37489)
add 4e218f0183e Add model manager that automatically manage model across processes (#37113)
add c1ecbb9eeb1 Fix linting/compilation violation (#37505)
add 99e48681a03 Increment PyArrow Bounds (#37503)
add fed1b551c35 Add support for tagged output type hints. (#37434)
add 4e76198555e fix spotless (#37508)
add 5240736a9bc [#37279] Document Ubuntu 24.04 Python version requirements (#37297)
add beec6a3e156 Update logback version to address vulnerability (#37501)
add 15ba4874d72 Add schedule trigger for beam playground precommit (#37518)
add e2be9fd8e36 Plumb custom batch parameters for autosharding from WriteFiles to FileIO. (#37463)
add d6920de30fe Add schedule trigger to Playground PreCommit (#37519)
add f504c5309bc Revert "[Python] Add take(n) convenience for PCollection (#37470)" (#37493)
add dd92b8fc179 [Flink 2.0] Exercise validate runner test for classic Flink runner (#37537)
add 99541ff4336 Fix flakiness in model manager and multiprocessshared tests (#37529)
add 6b1d2cffca2 Align go rc validation inputs with other workflows (#37320)
add 37a9c1a14dd Bump go.mongodb.org/mongo-driver from 1.17.7 to 1.17.8 in /sdks (#37469)
add 6602f1e125f Update RunInference to work with model manager (#37506)
add a0d27de943f Update queuing logic to avoid lock notify contention (#37528)
add b7577e74a6a Bump golang.org/x/oauth2 from 0.34.0 to 0.35.0 in /sdks (#37548)
add f52cf933446 feat: Introduce agent skills for various Apache Beam developments (#37467)
add 4b478a70738 Exercise Flink PVR tests on Flink 2.0 (#37538)
add f96db3733c2 [SQL] Alter Catalog and Table statements (#36571)
add dd0f41d3fe7 Fix _OrderedUnionCoder is_deterministic check. (#37550)
add 25d3a2e46bf Add States for Pause (#37546)
add 257fa5a46d3 Fix Components visibility (#37551)
add 6dcb908e9fe Add back flex API option with improved pydoc (#37491)
add 5142be92511 [Java FnApi] Improve FinalizeCallbackManager to cleanup timeout queue and use recommended ReentrantLock+Condition (#37304)
add f5aa2af52bd Bump pip from 25.3 to 26.0 in /sdks/python/container/ml/py313 (#37485)
add 44eb7f63cdc Add RateLimiter Abstractions (#37547)
add d51ea3afb88 Add access to pipeline options during pipeline construction and translation. (#37522)
add e8920c79a0d Add logic to handle end timestamp of mutable change stream bounded query (#37459)
add 801bf89e3da Parse proto bytes change stream record in ChangeStreamRecordMapper (#37427)
add d9adfa9c34f fix unparse bug (#37562)
add 195cc593590 Update python-pipeline-dependencies.md (#37561)
add 7df489cb7c6 add datadog it support (#37517)
add 91d3b6b1d39 move enum to top level class
add 4a2afe85f9f move enum to sdk core changed boolean causedByDrain in WindowedValue interface to enum. spotless
add aaecfa8f545 Merge pull request #37549: Move CausedByDrain to top level, move types
add 0228126c1b7 Update reviewers (#37572)
add d6390b86f0f Download performance looks to static while building website (#37559)
add 22d1c576272 Add OOM handling for RunInference with model manager (#37557)
add bafb270b48e Update go linter version (#37579)
add cea7923eef6 Use onlyif (#37578)
add 8c9d75567e7 Replace VarInt.getLength with Protobuf's branchless implementation (#36959)
add 9ea7380de85 Refactor update compatibility checks and pipeline options access (#37566)
add 72ce2074caf GCS client library migration in Java SDK - part 2 (#37502)
add cbf36740a53 Add Support for EnvoyRateLimiter Implementation (#37573)
add 695498f03a4 Update testcontainers version (#37591)
add eba1ec36adc Bigtable: skip reading large rows (#37586)
add 0adeae8b771 Optimize data size estimation for proto coder.
add 9a1dedfea25 Use existing _get_nested_size utility.
add 6e81f250977 Merge pull request #37584 Optimize data size estimation for proto coder.
add c7d13c4d56d add aws and azure dependencies (#37602)
add 87aee9a58f6 Upgrade to debezium-io 3.1.3 (#37571)
add 25518f07fe2 Upgrade Spanner to the latest version (#37552)
add c69e8a18601 remove list_prefix deprecated method (#37587)
add e2a2b291921 Fix spelling mistakes in Go comments (#37617)
add 9a60847ad6b Update CHANGES.md for PR 37331 (#37599)
add 8da478be0d7 switch from v19 to v23 (#37620)
add 5849f2e38ea [yaml] : fix validate compatible method (#37588)
add 5d6cb04ea05 [yaml] : switch js2py to pythonmonkey (#37560)
add f2aa7912401 Exercise Python PVR tests on Flink 2.0 (#37313)
add 2ac0fe431e6 [yaml] : fix create different type elements issue (#37585)
add 66134f7107f Fix infer_result_type for pcollection with tags. (#37608)
add 97e5ccf0c6f Upgrade ClickHouseIO to use ClickHouse Java Client V2 (#37611)
add 90253bf5857 Tune Ratelimiter config (#37598)
add 76a53610898 Rename Chunk to EmbeddableItem (#37623)
add a9c14d1698d Add Ubuntu 24.04 runner pools (#37624)
add 0337ecefc61 Fix iceberg nested filter (#37340)
add 1ee2c717f3e Don't prematurely close FileIO (#37626)
add bbaace8b6be update changes for 2.72 - yaml etc (#37627)
add 0699f3594e6 Add post submit test for dataflow streaming with windmill tag encoding v2 (#37603)
add bab2374552d Add Pause and Paused states (#37581)
add 3729c75d027 Improve RateLimiter Config
add 11586e46c39 add flag
add 9ac321abca9 fix readme
add 46b27eee882 Merge pull request #37630: Improve RateLimiter Config in terraform example
add b46a6352ba9 Add HTTPS detection for ClickHouse JDBC URL parsing (#37635)
add 51eec3c8658 Update to latest go version (#37638)
add 30bdbafcc95 feat: Improve YamlUtils.dumpAsMap error handling (#37468)
add f2765cab203 Fix bom upgrader and upgrade to latest (#37636)
add 808d23877bd Fix DoFnInvoker cache collision for generic types (#37355)
add 8dc5da5f500 Skip test for dataflow (#37612)
add e87bb292545 Bigtable: add an option to pass in row filter text proto for use in template (#37632)
add 74fc46c404d GCS client library migration in Java SDK - part 2b (#37592)
add 9d528015cb6 Moving to 2.73.0-SNAPSHOT on master branch.
add f73bd6a0919 Update CHANGES.md
add cecc2a6ae0c Add image embedding support to ml/rag (#37628)
add 9690853b873 Bump golang.org/x/net from 0.49.0 to 0.50.0 in /sdks (#37558)
add 1164f941f9c Bump filippo.io/edwards25519 from 1.1.0 to 1.1.1 in /sdks (#37644)
add 41779ad88d3 [Dataflow Streaming] Update windmill timer clasification logic to work with windmill state tag encoding v2 (#37604)
add 120ceab62a7 Fix renaming of embeddable_to_dict_fn (#37646)
add e41b9b6ba69 runQuickstartJavaFlinkLocal on Flink 2.0 (#37649)
add 872c671b15b Update cibuildwheel dependency (#37504)
add b9d48fa1750 Fix type checking failure with Python 3.10+ union pipe syntax (int | None) (#37645)
add dfb3f872192 [yaml] : add jinja inheritance example (#37601)
add 59922a390e0 Fix breaking changes from renaming Chunk to EmbeddableItem (#37651)
add 87c50c67e37 [Python] Support large pipeline options via file (#37379)
add 1052216416f Migrate remaining checkNotNull to checkArgumentNotNull/checkStateNotNull in options package #18719 (#37654)
add 9ff96dfd329 Update Golang - DebeziumIO 3.1.3 (#37667)
add caa18faf4ff Adds triggerring configuration to KafkaIO eos. (#37648)
add cb7caaad2ca [Dataflow Streaming] Remove dead code around redundant experiments (#37684)
add d614058ca95 Bump go.mongodb.org/mongo-driver from 1.17.8 to 1.17.9 in /sdks (#37657)
add 9bfb7ab8c42 Fix resource leak in KafkaIO GCS truststore file download (#37681)
add 4fe61733f77 Update Python Dependencies (#37678)
add bc35bef5d8f fix some python deps (#37668)
add 3a41311a1f9 unpin grpc. (#37633)
add 26b18c6e6f1 Revert "[yaml] : switch js2py to pythonmonkey (#37560)" (#37665)
add 7095c880f34 [Dataflow Streaming] Mark support for windmill state tag encoding v2 (#37683)
add ace4a54a15a dataframes -> dataframe
add 71c46ef01e4 Merge pull request #37697 from apache/fix-dask
add 451be0e289d Bump github.com/aws/smithy-go from 1.24.0 to 1.24.1 in /sdks (#37695)
add 1541d027c52 Remove test-only functions in GcsUtil and change related unit tests to use V1. (#37597)
add 6a1618eca88 fix(python): Register all output pcollections of a transform rather than only ones that happened to be accessed in DoOutputsTuple (#37556)
add 015b6417f97 Improve PR Bot reviewer assignment / pinging
add 6cab6eb17a8 Merge pull request #37663: Improve PR Bot reviewer assignment / pinging
add eac11fed1d4 Simply wheel validation logic. (#37687)
add d503869c450 Bump lodash from 4.17.21 to 4.17.23 in /website/www
add df59030c9ba Merge pull request #37403: Bump lodash from 4.17.21 to 4.17.23 in /website/www
add f47f81cf244 [Python] Update dev SDK container tag for Python SDK (#37661)
add 2341d3f7b7d Remove relative change threshold condition (#37643)
add 80b49029c64 Fix Playground Frontend Test job (#37642)
add 9ee4fe64f2b [yaml] : add available yaml io connectors to website (#37605)
add e755b53b5a5 Add an instruction for using Docker buildx when building containers. (#37706)
add 27a5fca9cca Moves expectMessage to before creating BeamFnLoggingClient since exception can be thrown inside createAndStart method.
add 6b647c7e3d6 Merge pull request #37692: Moves expectMessage to before creating BeamFnLoggingClient
add 940a8ca5db9 Bigtable: clean up client side metrics flag
add 193db392a93 Merge pull request #37596: Bigtable: clean up client side metrics flag
add caf661798f7 Fix PreCommit tests involving docker-in-docker after Milvus dependency rearrangements (#37711)
add 06dd48e9313 Revert "Update cibuildwheel dependency (#37504)" (#37713)
add 72027e11d1f Fix DebeziumIO resuming from worker restart (#37689)
add c51e244bb95 revert 'Optimize data size estimation for proto coder.' (#37709)
add 2403014afce Stabilize test_check_standard_external_transforms_config_in_sync (#37717)
add 9eb3a18ae70 fix conflict (#37594)
add 32d6bce8c23 Enable pickling main by reference option within CloudpickleConfig (#37554)
add 5c2c3530411 Add Pyrefly configuration for Beam Python (#37700)
add e9f4c99206a fix Python 3.12 PostCommit by installing setuptools (#37719)
add 67c3183913a Revise title and header in YAML documentation (#37722)
add 9524b56c148 Add observability (#37716)
add 30518c840bf [Stateful] Implement length-aware keying to minimize padding in BatchElements (Part 2/3) (#37565)
add 5bee123ce8e Add monitoring for model manager (#37703)
add 4b3dcf26142 Fix GroupBy snippet tests for issue #30778 (#37672)
add 342de16b2f2 Fix Python PostCommit Flink runner log spam switching to simple logger (#37720)
add af81a0c1635 Fix ml_base dependency (#37721)
add 5a6f763da54 [Drain] OnTimer - propagate caused by drain bit up to DoFnRunner (#37012)
add fcbb0340800 Bump go.opentelemetry.io/otel/sdk from 1.38.0 to 1.40.0 in /sdks (#37730)
add e761581b30d Reduce Fn Runner changes
add 4739405cdbc Merge pull request #37727: [Drain] Reduce Fn Runner changes
add 1430241b6fe Pin aircompressor. (#37737)
add 3197d88ef5c Update java containers. (#37733)
add c9ab4b6ae8d Add default_pickle_library_override delegation to InteractiveRunner (#37752)
add e5db1b3c63e [Dataflow Java Streaming] Reset state using finally blocks instead of catching Exception, in cases where it may otherwise corrupt datastructures if an OutOfMemoryError is thrown. (#37746)
add e1e0637d989 [ErrorProne] Fix FormatStringShouldUsePlaceholders string concat when Preconditions is used (#37745)
add eb5f9950c4a [Direct Runner] Fix test failure and logging configuration by ensuring overridden Loggers are not garbaged collected (#37763)
add 7ba0efd4d24 [ErrorProne] Enable `MutablePublicArray` Error Prone Check (#37761)
add 38b416cfb63 update java container version (#37756)
add d42111cd019 Merge pull request #37748: Where a Throwable is being caught, take care to rethrow OutOfMemoryError
add 505ac1a9b22 Fix MissingSummary errorprone Warnings Across Codebase
add e7a97c26893 Fix MissingSummary errorprone Warnings Across Codebase (#37759)
add 96306945ad6 refactor: Remove redundant protected access modifiers and enable the `ProtectedMembersInFinalClass` ErrorProne check. (#37757)
add dfc0bd12021 Enable UseCorrectAssertInTests ErrorProne Check (#37766)
add 298f3090920 Fix InlineFormatString Error Prone checks (#37768)
add e28ed02b9b8 Add Apache Beam installation to Dockerfile
add 08af69c65f6 Install setuptools for playground cd examples
add f682a85fbbe Merge pull request #37619 from apache/playground-interactive
add 828914840df [Dataflow Java Streaming] Ensure that OutOfMemoryError triggers JVM shutdown instead of being retried. This matches MemoryMonitor detection of gc thrashing and prevents workers from remaining in broken state that may be fixed by restarting (#37749)
add b42c992c7d4 [ErrorProne] Fix BadInstanceof and enable check (#37753)
add 676c998dec7 Enable StringCharset Error Prone Check (#37767)
add 94c39272196 refactor: replace lambda expressions with method references
add 52c6603a8fe Merge pull request #37765: [ErrorProne] Fix UnnecessaryLambda Error Prone issues
add 500996fbf99 Remove LongDoubleConversion from ErrorProne disabledChecks
add 1317aa0b777 Fix LongDoubleConversion warnings
add d2fead7c692 Merge pull request #37769: [ErrorProne] Enable LongDoubleConversion check
add 8ee8b10b7f1 Add more ParquetIo write options (#37740)
add b6bc9040954 Merge pull request #37715: Disable combiner lifting only for count triggers
add 927ee2c4835 fix: Correct malformed Javadoc tags and update Error Prone configuration. (#37755)
add 78061b82a24 If the partition count or kafka IO size is large, then skip committin… (#37510)
add 4d9e7fc77f7 Fix flaky TextIOWriteTest by loosening the shard count. Records may end up in the same shard as other records as it is random. Simplify the test to use Iterables instead of arrays.
add 95d848167db Merge pull request #37798: Fix flaky TextIOWriteTest by loosening the shard count
add 590684f81b0 Exclude vertex_ai_postcommit tests from embeddings dep suite
add 417922c2f04 centralize skipping vertex ai test
add 81fe00cff1e Fix pylint vertex ai test
add d64c21493d5 remove RUN_VERTEX_AI_TESTS flag
add 0f805b56d3b Use pytest.importorskip for Vertex AI tests & remove vertex_ai_skip helper
add ea473d22b2b Merge pull request #37725 from aIbrahiim/fix-python-postcommit-dependency-1
add 9b915fd18f6 Remove logging in tfrecordio.py (#37794)
add b203f53a128 Move FileIO close from RecordWriter to RecordWriterManager (#37782)
add 6c42cc107f2 [ErrorProne] Enable BadImport ErrorProne check and fix violations (#37760)
add d10374b755c add py313 dep (#37799)
add d27dc82c568 [IcebergIO] Add ITs for RESTCatalog using BLMS (#35360)
add 5034e40fa0f [Dataflow Streaming] Remove nullness suppression of StreamingDataflowWorker (#37797)
add d6759cf7dd6 [Dataflow Streaming] Add a pipeline option to skip input elements that cannot be decoded successfully (#37762)
add 78b22bc11f2 Use p310_ml_test
add eccfdbc213a Merge pull request #37800 from apache/fix-python-ml-3-10
add 084c4da7314 Revert "fix(python): Register all output pcollections of a transform rather t…" (#37801)
add 6c67a938edd Bump minimatch from 3.1.2 to 3.1.5 in /sdks/typescript (#37729)
add 57ab2b96113 [Dataflow Java Runner] Add support for sending logs directly to Cloud Logging (#37662)
add 7d756c2e109 update container version (#37811)
add e6fcdd72ed6 [Java][Debezium] Fix NPE in debeziumRecordInstant for DELETE events (#37795)
add 8e0736ab304 Loosen GRPC requirements. (#37817)
add 9829d6d3cf9 Merge pull request #37792: [ErrorProne] Fix AutoValueBoxedValues warnings across the codebase
add edaeae93065 update python container tag (#37812)
add 2ebe33d976a fix non-breaking vulnerabilities (#37826)
add 9f5904b2b59 Support inferring schemas from Python dataclasses (#37728)
add 9d73d769ff0 Only pull license for selected tests and publishing container (#37827)
add cf536ea06d6 Fix 36181 cloudml benchmarks job (#37803)
add a0674a97db6 [ErrorProne] Fix InvalidParam, InvalidInlineTag, InvalidBlockTag, and InvalidLink checks (#37773)
add d6bc507d769 Skip intermediate python wheels on pull request trigger (#37832)
add 630f32ada54 [Drain] Expose drain to dofn processElement and onTimer (#37825)
add 87d7bba644f docs: Clarify JAVA_HOME setup and initial build expectations for all OSes (#37415)
add 892fa0a9357 Fix Run in Colab thumbnails blocked by CSP (#37838)
add e497c1e9b43 Partly Revert #37832 (#37837)
add 0705b4376ec apt-get upgrade in container images to install security fixes. (#37841)
add ab566197fd2 upgrade vllm for vuln (#37843)
add 02d8f56a1e8 [Dataflow Streaming] Remove rolled out Windmill isolated channels flag (#37844)
add 3b41e8b4320 Remove rolled out Windmill separate heartbeat stream flag (#37845)
add b92c436ee64 Use Python 3.11 and cibuildwheel 3.4.0 for wheel builds (#37850)
add d6d2433827d Fix python postcommit 30513 (#37849)
add 5c5c710441c Revert "upgrade vllm for vuln" (#37856)
add 47bab7e7597 [SpannerIO] Add low-latency configuration in Spanner Change Streams (#37718)
add 0f6c88e0d4b Pin zookeeper in expansion service (#37859)
add b9a7585d7f4 [ErrorProne] Enable StaticAssignmentInConstructor check (#37786)
add 3b472aaa9d3 [ErrorProne] Enable UnsafeReflectiveConstructionCast check (#37784)
add 914e425b26d Fix #21317: Replace colons with underscores in window filenames for Windows compatibility (#37821)
add 85c1b88ce5a [Dataflow Streaming] Create a separate option to control commit threads with direct path. (#37848)
add 9292e1d8140 [Dataflow Streaming] Prevent commit threads from sharing commit streams (#37847)
add 487696c9fd1 [Dataflow Logging handler] Change @LazyInit to AtomicBoolean (#37846)
add e1e2c3ba85c [DebeziumIO] Implement startOffset & offset persistence for Kafka (#28248) (#37750)
add 4c95230392c [typescript] - fix rest of vulnerabilities (#37852)
add 39c7b46aecb Regenerate beam container image dependencies to include newer version of grpc. (#37822)
add 4aff88ea91a Revert "[Dataflow Streaming] Prevent commit threads from sharing commit streams" (#37873)
add 65179691f25 Fix maven archetype dependency for io-components
add 41b513e210c Merge pull request #37871 from Abacn/fix-maven-archtype
add 90c2ab28a26 [Gemini] Fix bad-unpacking pyrefly error (#37828)
add 0fb490f927f [ErrorProne] Enable EmptyCatch Error Prone check and document ignored exceptions (#37754)
add babf20c99dc Adds validates runner post commit tests for Java streaming worker on streaming engine (#37872)
add 1007f73c8e4 Update python dev container tag (#37853)
add fa9d19063ba SDF draining in dataflow runner v1 (#37831)
add 8c48120697e Enable error_prone UnnecessaryLongToIntConversion check (#37783)
add fe29172781f Updated URLs in notebooks: Folders changed from `/tour-of-beam/` to (#37876)
add 7b33e1cb50b [Java] Improve GroupIntoBatches display data to show batching parameters (#37865)
add ae6d624c2b6 Add a configuration to wait for session creation (#37625)
add d51177b9cee Add a check to make sure GitHub action workflows are listed in readme (#37883)
add fee107ada1f Fix vLLM Gemma benchmark by updating transformers version
add d48108d6f83 Merge pull request #37874 from aIbrahiim/fix-30644-inference-python-benchmark
add 2c99a01f327 Bump google.golang.org/grpc from 1.78.0 to 1.79.3 in /sdks (#37889)
add 180f3a6bdb5 Attempt to fix flaky GrpcLoggingServiceTest (#37891)
add 153875eaf24 Fix #37833: Use named logger instead of root logger in transforms/util.py (#37857)
add 2209bd60122 Adds bundleFinalizer support to Dataflow non-portable worker. (#37723)
add b3d2b06b2a8 [ErrorProne] Fix UnnecessaryMethodReference Check Findings (#37785)
add c05733a3a4d Handle some cases during infer schema from dataclass (#37855)
add d16f5d18d2a [ErrorProne] Fix EqualsGetClass warnings and enable Error Prone check (#37758)
add 7b065555cd0 Fix flakiness in tests by increasing timeout and changing windowing type. (#37887)
add c10028b2583 Fix UnusedVariable warnings across codebase (#37770)
add 48935a83720 Resolve Error Prone NonCanonicalType warnings across the codebase (#37772)
add 4163dc752ac [Error Prone] Fix Slf4j formatting errors and enable Slf4jFormatShouldBeConst check (#37771)
add 4221266ffab [ErrorProne] enable JavaUtilDate, TimeUnitConversionChecker, InvalidThrows checks (#37791)
add 0afbdf684fb Update Go Version to 1.26.1 (#37897)
add 7620a935280 Fix #37736: Allow composite transforms to use implicit input chaining (#37861)
add 181995db2c6 Upgrade Docker setup actions to version 4 (#37914)
add 5139ca67b64 Sync clickhouse resource manager client version with IO (#37918)
add 0e7dd4f3cf7 Add version to beam-sdks-java-io-components
add 866d6f73ddb Merge pull request #37920 from apache/fix-maven-archetypes
add d9a6159d97f Pin docker actions to sha for now
add 0cd35ec0927 Merge pull request #37922 from apache/users/damccorm/shas
add 982c9bd8044 Pin version for setup-qemu-action (#37924)
add ca3421606ee Add Quick Start section to README (#37909)
add 65a896b9780 Fix #37894: Update yaml doc for composite transform changes (#37901)
add 600b7d9050e Pin gcp actions (#37926)
add 45c2863c0a3 Pin to SHA for GCP GitHub Actions
add 90a7db1c094 Merge pull request #37927 from apache/gcp-actions
add 3316ef93182 [Website] Add missing I/O connectors to built-in connectors table (#37903)
add 7c82e73bb1e [Python][Java] Add support for record headers in WriteToKafka (Fixes #27033) (#37458)
add 79c52affb1d Updates YAML Database transforms to use managed I/O by default (#37923)
add cf3d6ed3e52 [ErrorProne] Enable UnnecessaryParentheses validation and fix all violations (#37913)
add 4b448b069f4 Bump Java dev containers (#37931)
add 7187ff67ee8 Handle Flink 2 job server tag in releases and SDK snapshot (#37939)
add 3aad50b99e8 [Security] Bump ActiveMQ from 5.14.5 to 5.19.2 (#37944)
add 012f825245d [Java] Optimize beam pubsub message to serialized pubsub protobuf message (#37896)
add 8e90c36ff1b [Java] Optimize StreamUtils to avoid copying bytes (#37912)
add b6d8f428e29 Iceberg Add Files (#37701)
add 683ecebe3af Fix typo in arc readme (#37948)
add 7431dea675b [#37930][Runners] Reap child processes in ProcessManager to prevent zombie accumulation (#37932)
add 08685524465 Pin setuptools below 82 for TensorFlow Hub pkg_resources compatibility (#37929)
add 40951024bfe Iceberg AddFiles -- handle FileNotFound exceptions (#37952)
add 97da2d69121 Update changes.md (#37936)
add dcb1b756e7d Add OrderedListState support in Go SDK (#37629)
add 2eb71e989b4 Bump picomatch from 2.2.2 to 2.3.2 in /website/www (#37955)
add f8dbba156db Improve docstring for PGBKCVOperation (#37946)
add 299f71d6957 Address no-access pyrefly error (#37888)
add 0e827993098 Fix flaky SpannerChangeStreamOrderedByTimestampAndTransactionId IT (#37950)
add 661add48ae8 Fix version of docker github action (#37970)
add 4b60dae6c9a [yaml] Iceberg add files yaml transform and test (#37938)
add 512c0faedf8 ClickHouseIO - align the RM with the test containers upgrade (#37971)
add d2aed60042e [Security] Bump PostgreSQL JDBC Driver from 42.2.16 to 42.7.10 (#37941)
add 5a65f6ed00c Exposed source window to AssignContext in window_mapping_fn for side inputs (#36722)
add be7ac47c2d5 Add Public Website documentation for RateLimiter (#37949)
add 030460c7fa1 Filter TestOrderedListState for dataflow runner v2 (#37983)
add 72749a2a2bd Some terraform changes to make arc updates work (#37985)
add 19e7c0f2b17 [yaml] - add bigquery destination test (#37787)
add 97c2faf919d Bump picomatch from 2.3.1 to 2.3.2 in /sdks/typescript (#37978)
add 7bda560ebe8 Update Beam website to release 2.72.0 (#37869)
add dc075f3641b Update managed-io.md for release 2.72.0-RC3.
add 755819493e3 Merge pull request #37915 from apache/updates_managed_io_docs_2.72.0_rc3
add 43f1c49d4ac [yaml] Remove test_ReadFromBigQuery_ExternalJavaProvider_0 deselection (#37986)
add c1755e37a12 Bump handlebars from 4.7.7 to 4.7.9 in /sdks/typescript (#37988)
add c4638a39dc7 Migrate pilot workflows to ubuntu-24.04 self-hosted pools (#37995)
add eaf02846d4a Upgrade Snowflake JDBC client to v4 (#37885)
add 12f00acc067 Revert "Migrate pilot workflows to ubuntu-24.04 self-hosted pools (#37995)" (#37998)
add 7d159c77e6e Implemented ML Pipeline Continuous new table rows RunInference (#37647)
add 01b2f219559 Improve AlloyDB notebook setup instructions by clarifying database creation (#37793)
add c0954b797c4 add batch import yaml IT (#37976)
add 6d3b17db06a Adding release-2.72.0-postrelease to protected branches in .asf.yaml
add 47694885673 Add build-essential to self-hosted ARC runner image (#38000)
add e6c9faaf9d9 Add Iceberg Quickstart page (#37739)
add 71405b81be1 Update default version in republish_released_docker_containers.yml (#38010)
add 6f1f242296f Migrate pilot workflows to ubuntu-24.04 self-hosted runners (#38006)
add 1b523ef1d90 Allow sliding windows + combine with hot key fanout (#37899)
add d9cdd6fdf6d [Python] Ad BigQuery Change History source. (#37724)
add e08e9d56e5e Adjust schedule for long-running Java ValidatesRunner Dataflow Streaming variants (#38001)
add 001fc988a7e use pre-1.7.1 implementation (#38016)
add 4da3e553750 Add Python 3.14 support (#37500)
add 0b6a93f486c [#37994] Fix NullPointerException in Spark Runner with multiple outputs and serialization (#38011)
add f4f422c59df [Integration] Expose length-aware batching in all ModelHandler subclasses (#37945)
add 16aee7998a9 Update Python workflows to Ubuntu-24.04 runners, regenerate requirements files (#38020)
add 1fb308859ef Add GNU time to ARC runner image (#38022)
add f456b70ecfd Add ADK model handler (#37917)
add 24f02bb2841 Remove 3.14 runs from ML Tests (#38023)
add ee02db885b8 [Iceberg AddFiles] Use thread pool to process incoming datafiles (#38018)
add b39a95f5d8d Improve bq cdc scalability (#38015)
add b090e22064e Allow tagged types in with_exception_handling. (#37590)
add d301a673218 Fix lint (#38032)
add 5d97107143b adding new metadata skill
add be364d5f8ed add license.
add 4f473d87f7a Merge pull request #37868 from stankiewicz/branch_skill_metadata
add 1ae923e60d5 Set python 3.14 version where needed (#38030)
add d83275adc61 remove void
add 3249e2e334a outstanding jdbc violation
add fae0219d692 cleanup
add 9ea861238d1 Merge pull request #37902 from stankiewicz/error_prone_voidUsed
add d9fa6082ba8 [BigQueryIO] Fix Storage Read API crash on empty tables in streaming pipelines.
add 3fa8b0ca5f4 Update sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageSourceBase.java
add bfdac1aaa90 add member variable to keep track of source split result
add b7c87f23ce8 Merge pull request #38009 from stankiewicz/fix_bq_empty_table
add a05482e92d9 - adds new outputWindowedValue(WV) public interface with implementations - add tests to cover metadata propagation across parDos in single and mulitple stages for direct runner - plumbs metadata propagation, where it was lost previously.
add 36d2b4832f3 fix npe in ReduceRunner..
add 80c7a49e743 find npe
add 7a5c9822318 review feedback - remove duplicated code and unused variables
add 1c53cb26cd1 Merge pull request #37851 from stankiewicz/metadata_propagation
add 24f562d84f6 Downgrade Postgres driver to 42.6.2
add 7fdee986a2a Merge pull request #38024 from Abacn/downgrade-postgres
add 295dc1c57be Move Python PostCommits to Ubuntu-24.04 Runners (#38031)
add ceed48c0781 Fix Portable PreCommit on 3.14 (#38033)
add 090133198e5 Add a FailoverChannel wrapper on top of IsolationChannel to maintain a set of primary and failover channel. (#37840)
add bb8e6bcbab9 added GCP auth to Java Dataflow V2 streaming validates runner (#38027)
add 84a28e7e084 Update Python container workflows for parallelization
add c8e45e79c69 Merge pull request #38034 from jrmccluskey/314container
add f7bdf517ccb Fix vLLM worker (#38008)
add 9baebbbae92 Bump lodash from 4.17.23 to 4.18.1 in /website/www (#38042)
add 303a0768f13 Fix 3.14 distroless build, snapshot publishing (#38048)
add 67259e0f1e4 Add PubSub RateLimiter example (#37992)
add db7c5391fbb Bump requests from 2.32.4 to 2.33.0 in /infra/enforcement (#37975)
add bb438338f45 fix vllm lint (#38051)
add f348cd0c4fd [BigQuery] Handle 409 Conflict in BigqueryClient dataset and table creation
add 47bebc33cad [BigQuery] Handle 409 Conflict in BigqueryClient testing client for dataset and table creation
add a7c65ff32b5 Remove 3.14 ML container reference from snapshot action (#38049)
add f7022b5a25b Update Google BOM to 26.79.0
add 2505e4e7fd9 Merge pull request #38021 from kennknowles/update-google-bom
add 8f9c005a5c0 Moving to 2.74.0-SNAPSHOT on master branch.
add 97359441166 Update CHANGES.md for 2.74.0
add 08857d2210b Use ubuntu 24.04 for PostCommit TransformService Direct
add 5f5b03c0d2a Merge pull request #38054 from apache/fix-transform-service
add 60723cde2c0 Bump Python FnAPI beam-master container (#38041)
add 190898cd743 Avoid cloning live dev venv for yaml expansion in CI (#38063)
add b9b7a6729cd Enable SpannerChangeStreamPlacementTableIT and update IntegrationTestEnv cleanup (#38052)
add d0ec8d1e40d fix MetadataPropagationTest.java
add 5ff3611614b Merge pull request #38065 from apanich/patch-6
add 3cc224fb8f1 increase timeout via posargs (#37989)
add 392b869ef86 Fix py314:mongodbioIT (#38062)
add a03784efc93 Fix BigQueryEnrichmentHandler batch handling for duplicate keys (#38040)
add 5932587d20e Add CHANGES.md note for BigQueryEnrichmentHandler duplicate-key batch fix (#38068)
add 211cd6662d2 Run python_dependency_tests during release validation. (#38070)
add 718ff4ec15d Fix Prism global window side input visibility in GetSideData (#38067)
add c1e44fb0258 portable date java and model changes (#38077)
add 6756b1223d0 Fix type in dask runner tox target
add 3875b217aae Merge pull request #38076 from Abacn/fixdask
add 08caae8e328 Bump node-forge from 1.3.2 to 1.4.0 in /sdks/typescript
add 63823a80d53 Merge pull request #37987: Bump node-forge from 1.3.2 to 1.4.0 in /sdks/typescript
add ee3ea237e89 Use python version argument (#38066)
add 28a0cb14ebd Fix GrpcLoggingServiceTest hang on client error (#38082)
add 5fc49f97e99 Ensure LazyFlinkSourceSplitEnumerator handle splits happens after initialized
add 9fe425ffb56 Merge pull request #38083 from Abacn/fix-flink-race
add f7c280c77d2 add waitForReady to logging stream and control stream to handle sdk starting before the runner. (#37982)
add d4ff738d503 Add explicit warning if cython is not available in setup.py (#37953)
add b05b7f3ccca Moves finalize ids out of WorkItem proto and into AssembledWorkItem class. (#37954)
add d1d546976d3 [Java] Add ByteStringOutputStream.isEmpty() and reduce ByteStringOutputStream.size() calls (#37947)
add 31448481939 Optimize ApproximateQuantiles by removing unused context and adding registrations. (#38085)
add c68f520d273 [Iceberg AddFiles] Allow very large batch import (#38039)
add 37ae1ffc176 [Java Portable SDK] Configure JVM so that it exits upon OutOfMemoryError originating in JVM. (#37764)
add 219cac9fae8 Fix Recommendations AI catalog outputs and sklearn pins for 3.14 (#38069)
add a5f000fca5c Fix concurrency for pull_request_target triggered workflows (#38093)
add 8e40d4450b7 compare argument type in a null-friendly way (#38092)
add 7ae7d248fa8 small dep upgrade to avoid vulns (#38091)
add 532dd85dab8 [ML] Add AnthropicModelHandler for direct Claude inference (#37956)
add 46052504435 Fix NameError when pydot is not installed in Beam Playground (#38074)
add 47031efee65 Make MinRowCountForPageSizeCheck available with ValueProvider (#38114)
add a1f260be744 add AWS to IO expansion service (#38111)
add 923ed03ce5a Fix table-row benchmark argument file paths (#38128)
add 4d4da5723c7 Fix ML flakes (#38088)
add 5ee67c6c0bb Bump go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp (#38112)
add 60c8a23b43d Improve the Javadoc for encode/decode methods in coder methods in coder for better clarity on context behaviour
add 760905140cb Apply spotless formatting
add 18395a2a3cc Fix minimal javadoc wording for coder encode decode
add 88ebf83177f Fix serial warning in starter and example archetype
add 379a7f03acf add PicoWordCount Example (Minimal Beam wordcount pipeline)
add cf2eebe615a Fix formmatting (spotless)
add c632f800abb Remove unintended StarterPipeline changes from PicoWordCount PR
add 78b770b920b Address review: use @Element instead of ProcessContext
add 357b7206f82 Merge pull request #38029 from Subramanya-Veeregowda/pico-wordcount-example
add b296b40e3fc Revert "Fix NameError when pydot is not installed in Beam Playground (#38074)" (#38134)
add bb06d9b7040 Fix flaky ViewTest which would fail encoding null value if there were no elements between early trigger and on-time global window firing.
add 91e1b065749 Merge pull request #38129 from scwhittle/flaky_view
add f938385d471 Skip clean up versioned images (#38019)
add e82fc4713f3 Changes cleanup thread to forceExecute. (#38073)
add 6dd599cf3f6 Pin google-adk to 1.17.0 for YAML/Xlang precommit dependency resolution (#38090)
add b4c45719fa6 Migrate self-hosted workflows from ubuntu-20.04 to ubuntu-24.04 (#38133)
add 09c166b8c5c Improved comments and readability in minimal_wordcount example (#37908)
add 379d2124e12 [ErrorProne] Enable JavaTimeDefaultTimeZone check (#37775)
add dfe6bf351fd fix preCommit Spotless rsync install (#38160)
add e1f02622f36 fixed redis tests with testcontainers (#38157)
add 53f8367adf6 Portable Date Type (Python changes) (#38078)
add d89dcacd7c2 migrate MongoDB IO tests from flapdoodle to Testcontainers (#38161)
add cd384e28db3 fix use configured Python in pullLicenses script (#38162)
add 80c4c7a8dd0 Fix spotless in examples:java:iceberg (#38166)
add 549aa066a90 Add rsync to self-hosted ARC runner Docker image (#38164)
add 4ba599c3ffb [Drain] Expose drain to dofn processElement and onTimer - add missing implementation for SDF and WindowExpiration processContext
add 3f5ce80418b Merge pull request #38119 from stankiewicz/fix_drain
add cd1a0998c52 Support annotationlib in native_type_compatibility (#38188)
add f3bd329bbf0 Update go version to 1.26.2 (#38131)
add 98f5b339bab Add mutable change stream in IntegrateionTestEnv.java for postgresql (#38057)
add 5655186de73 Retry transient NOT_FOUND on Storage API default stream
add a056c5b4991 Merge pull request #38132 from aIbrahiim/fix-gcp-io-default-stream
add 0471bb5e935 Change TriggerState finished bitset coder to a SentinelBitSetCoder (#38139)
add 48a6ceb7a57 [Dataflow Streaming] Fix spammy log during GrpcGetDataStream physical stream half-closing (#38158)
add 77ff67063f5 Install openssh-client
add 0bdc149b9b9 Merge pull request #38195 from apache/fix-flink-container
add e6c0796d2c2 Fix Lint PreCommit
add 56c8de59b13 skip spanner assignment too
add 09bc761f57b remove unused ignore
add 2e1edb08232 Merge pull request #38191 from jrmccluskey/assignmentSkip
add 4ee32cddb4c [Java Harness] Reapply optimization to use Caches.weigh on objects within cached blocks (#37964)
add a60a71e238a [Iceberg AddFiles] Attach name-mapping to table (#38189)
add 9a9d241b341 [ErrorProne] Enable BigDecimalEquals check and fix violations (#38193)
add c7e54f81e57 ErrorProne - enable NarrowCalculation check
add b2e97dee8c2 Merge pull request #38192 from stankiewicz/errorprone_NarrowCalculation
add d6fd0698769 [Iceberg] Support nested column paths in keep/drop configuration (#37516)
add ae536b6f0f8 fixed PythonDocker rerun matrix gating and gh setup (#38199)
add 30690fcf82d ignore test temporarily as enabling metadata propagation may break other tests.
add ff68119b42a Merge pull request #38213 from stankiewicz/ignore_metadata_propagation_test
add 3a19ef136fb Enable error-prone checks: ClassInitializationDeadlock, Finalize, NullableTypeParameter, ThreadPriorityCheck (#37788)
add 1f83bf05321 [yaml]: add kafka parameter explicitly and uncover kafka test (#36603)
add fb48a421441 Increase timeout for BQ early rollout job (#38216)
add 688c692702b Bump pytest from 8.4.2 to 9.0.3 in /sdks/python/container/py310 (#38180)
add c8ba2d875be Fix setup python environment
add 0e8953cb9cb Merge pull request #38217 from apache/fix-flink-cluster
add e71ba541497 Add ExceptionInfo type for with_exception_handling (#38173)
add a7c7786645c Migrate Common Linting Tooling to Rust-Based Packages (#38107)
add 9766c4a126d Bump Pytest Bounds (#38218)
add 9ba9960bc6e Use verbose mode when uploading pypi artifacts (#38223)
add c166bbe066c Fix CVE-2026-34477 (#38210)
add e841064660a Bump protobufjs and google-gax in /sdks/typescript (#38224)
add 621354f992b [ErrorProne] Enable UndefinedEquals check (#38197)
add f300e59fff9 ClickHouseIO - Improve user agent (#37815)
add 4e24a9c69f1 [IcebergIO AddFiles] Apply name mapping at commit time (#38232)
add 86ca83e81dd Exclude 3.14 ML containers from push314 (#38221)
add 1cb2a711aa6 Update Google BOM dependencies. (#38239)
add e5c4c1ed162 [Spark Runner] Prepare Spark 3 structured-streaming to shared base, adopt Flink-style version overrides (#38233)
add 0f48ffcbb27 Bump nltk from 3.9.3 to 3.9.4 in /sdks/python/container/ml/py312 (#38227)
add ac161e8d229 Pin numbus-jose-jwt to 9.37.4 (#38187)
add 00e825e71ea Update hbase version to fix netty CVEs. (#38241)
add ce20f7345b5 Remove widget state data in run_inference_gemma.ipynb (#38248)
add 79fb3600f97 [Python] Expand SDF by default in PortableRunner (#37965)
add 2ee8b6b667d Improve Javadoc for Coder class and clarify Context usage (#38235)
add cacf8f43fc9 Bump lodash from 4.17.23 to 4.18.1 in /sdks/typescript (#38043)
add bc6dffee9d3 Bump flatted from 3.4.1 to 3.4.2 in /sdks/typescript (#37906)
add e696acdfad0 Bump github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream in /sdks (#38101)
add 25370f50b39 Bump github.com/nats-io/nats-server/v2 from 2.12.4 to 2.12.6 in /sdks (#37940)
add b4e1d57901e Install python (#38253)
add 0cdd1b49f3b Use static shared catalog and remove premature FileIO close (#38149)
add a615657d5a4 Allow Python user type pass through Beam SQL (#38206)
add 4151dded54f remove processContext usage across examples (java and kotlin) (#37937)
add 3ea73d2ba60 [ErrorProne] enable AutoValueSubclassLeaked, EscapedEntity, JUnitIncompatibleType, EqualsUnsafeCast (#38256)
add 357862aee37 [runners-flink] Bump FlinkRequiresStableInputTest timeout to 120s (#38273)
add 4cdcf04bd78 Add connectivity type tracking to endpoint consumption logic (#38249)
add 5a5f402d6a2 Implement pluggable Lineage in Java SDK (#36781)
add de1cc7d02bd improve Maven badge readability in README (#37910)
add 2f499a34f2c [runners-flink] Use index-based version comparison in flink_runner.gradle (#38272)
add efd2abfe4e8 [ErrorProne] Update comment in BeamModulePlugin.groovy (#38281)
add a4c7944ea68 Add PreCommit Java Dataflow workflow (#38094) (#38124)
add 6106b308996 [BigQueryIO] Include insert error in log message if TableRow parsing fails (#38231)
add e01f711413f bump beam container versions (#38285)
add d225abf7f30 Bump pytest from 8.4.2 to 9.0.3 in /sdks/python/container/py314 (#38179)
add 7597370249d [Enrichment] Expose max_batch_duration_secs in SQL/BQ handlers (#38243) (#38244)
add a375d6c608d FileBasedSinkTest failure (#38284)
add e126106eaba Fix ML presubmits. (#38289)
add b56d94f4334 Automaticaly update BigQuery table schema when unknown fields are seen. (#38058)
add 448422bde99 Fix data race in JsonPayloadSerializerProvider. (#38265)
add f87ee765642 Go validation runner test script fixes (#38283)
add 11cc79e66dd Allow SpannerIO.readChangeStream() to query & union a list of change stream TVF's (#38167)
add 9caf1a6f5cb Update changes (#38293)
add 106e4493873 Pin transformers/tokenizers for ML tests (#38291)
add 9e1c7a8f210 Fix PreCommit Java Dataflow non-portable runner (#38290)
add 48139dd7333 [IcebergIO] Support sort order on dynamic table creation (#38269)
add 3776a9992dd Increase timeouts (#38301)
add e51fb74dbbf Fix LineagePluginTest on runners other than Direct runner (#38296)
add 29b168035b1 Revert #38256 partially (AutoValueSubclassLeaked)
add 4fc480e2c2b Supress warnings
add b4e2c8ca633 Merge pull request #38292 from apache/fix-hadoop-format
add 2f300cc87b7 update beam python container (#38298)
add 7712cf3a460 feat(ml): add stateless bundle-local size-aware batching and benchmark (#37532)
add 725583ef839 Bump pytest from 6.2.5 to 9.0.3 in /playground/infrastructure
add 43e9ea96713 Merge pull request #38165 from apache/dependabot/pip/playground/infrastructure/pytest-9.0.3
add ac7704f068c Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks
add bfa7fb9406d Merge pull request #38102 from apache/dependabot/go_modules/sdks/github.com/aws/aws-sdk-go-v2/service/s3-1.97.3
add 0d1d2476795 Enhance documentation for stateful conversations (#38247)
add 99a72bce645 Use registered type for Row (#38108)
add 6ca2f4ebfe9 Bump Docker PreCommit timeout, address errant setuptools warning (#38305)
add 61fda334aa8 Update Flink Version Compatibility (#38277)
add 062fab3181c Bump nodemailer from 7.0.11 to 8.0.5 in /scripts/ci/issue-report
add 33d6040b61d Merge pull request #38106 from apache/dependabot/npm_and_yarn/scripts/ci/issue-report/nodemailer-8.0.5
add a00d5ab772b Bump picomatch from 2.3.1 to 2.3.2 in /scripts/ci/pr-bot
add 4c8d8522050 Merge pull request #37979 from apache/dependabot/npm_and_yarn/scripts/ci/pr-bot/picomatch-2.3.2
add 6bbf8c38e38 Bump gradle-wrapper from 8.14.3 to 9.5.0
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (d1c26f96ec7)
\
N -- N -- N refs/heads/dependabot/gradle/gradle-wrapper-9.3.1 (6bbf8c38e38)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.agent/skills/README.md | 62 +
.agent/skills/adding-new-metadata/SKILL.md | 119 +
.agent/skills/beam-concepts/SKILL.md | 246 ++
.agent/skills/beam-dofn-modernizer/SKILL.md | 207 ++
.agent/skills/ci-cd/SKILL.md | 188 ++
.agent/skills/contributing/SKILL.md | 149 ++
.agent/skills/gradle-build/SKILL.md | 244 ++
.agent/skills/io-connectors/SKILL.md | 197 ++
.agent/skills/java-development/SKILL.md | 153 ++
.agent/skills/license-compliance/SKILL.md | 199 ++
.agent/skills/python-development/SKILL.md | 189 ++
.agent/skills/runners/SKILL.md | 244 ++
.asf.yaml | 3 +
.github/ACTIONS.md | 6 +-
.github/REVIEWERS.yml | 1 -
.../gradle-command-self-hosted-action/action.yml | 2 +-
.github/actions/rerun-job-action/action.yml | 6 +-
.../test-properties.json | 14 +-
.../actions/setup-environment-action/action.yml | 4 +-
.github/build.gradle | 6 +
.github/codecov.yml | 1 +
.../gh-actions-self-hosted-runners/arc/README.md | 6 +-
.../arc/environments/beam.env | 74 +-
.github/gh-actions-self-hosted-runners/arc/gke.tf | 13 +
.github/gh-actions-self-hosted-runners/arc/helm.tf | 10 +-
.../arc/images/Dockerfile | 8 +-
.../IO_Iceberg_Integration_Tests.json | 2 +-
...Iceberg_Managed_Integration_Tests_Dataflow.json | 2 +-
.../trigger_files/beam_PostCommit_Go_VR_Flink.json | 3 +-
.github/trigger_files/beam_PostCommit_Java.json | 4 +-
.../beam_PostCommit_Java_Jpms_Flink_Java11.json | 2 +-
...m_PostCommit_Java_ValidatesRunner_Dataflow.json | 3 +-
...it_Java_ValidatesRunner_Dataflow_Streaming.json | 2 +-
...ValidatesRunner_Dataflow_Streaming_Engine.json} | 2 +-
...esRunner_Dataflow_Streaming_TagEncodingV2.json} | 2 +-
...eam_PostCommit_Java_ValidatesRunner_Direct.json | 1 +
...beam_PostCommit_Java_ValidatesRunner_Flink.json | 1 +
...beam_PostCommit_Java_ValidatesRunner_Samza.json | 3 +-
...beam_PostCommit_Java_ValidatesRunner_Spark.json | 3 +-
...json => beam_PostCommit_PortableJar_Flink.json} | 0
.github/trigger_files/beam_PostCommit_Python.json | 4 +-
.../beam_PostCommit_Python_Examples_Flink.json | 3 +
...am_PostCommit_Python_ValidatesRunner_Flink.json | 3 +-
...am_PostCommit_Python_ValidatesRunner_Spark.json | 3 +-
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.json | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.json | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.json | 2 +-
.github/trigger_files/beam_PostCommit_SQL.json | 2 +-
.../trigger_files/beam_PostCommit_XVR_Flink.json | 3 +-
.../trigger_files/beam_PostCommit_XVR_Spark3.json | 4 +-
.../beam_PostCommit_Yaml_Xlang_Direct.json | 2 +-
.../trigger_files/beam_PreCommit_Python_Dill.json | 2 +-
.../trigger_files/beam_PreCommit_Python_ML.json | 4 +
.github/trigger_files/beam_PreCommit_SQL.json | 2 +-
.github/workflows/IO_Iceberg_Integration_Tests.yml | 4 +-
.../IO_Iceberg_Integration_Tests_Dataflow.yml | 4 +-
..._Iceberg_Managed_Integration_Tests_Dataflow.yml | 4 +-
.github/workflows/IO_Iceberg_Performance_Tests.yml | 4 +-
.github/workflows/IO_Iceberg_Unit_Tests.yml | 4 +-
.github/workflows/README.md | 37 +-
.github/workflows/beam_CancelStaleDataflowJobs.yml | 2 +-
.../workflows/beam_CleanUpDataprocResources.yml | 2 +-
.github/workflows/beam_CleanUpGCPResources.yml | 9 +-
.../workflows/beam_CleanUpPrebuiltSDKImages.yml | 2 +-
.../workflows/beam_CloudML_Benchmarks_Dataflow.yml | 4 +-
.../beam_IODatastoresCredentialsRotation.yml | 2 +-
.../beam_Inference_Python_Benchmarks_Dataflow.yml | 28 +-
.../beam_Infrastructure_PolicyEnforcer.yml | 4 +-
.../beam_Infrastructure_SecurityLogging.yml | 4 +-
.../beam_Infrastructure_ServiceAccountKeys.yml | 4 +-
.../beam_Infrastructure_UsersPermissions.yml | 6 +-
.github/workflows/beam_Java_JMH.yml | 4 +-
.../beam_Java_LoadTests_Combine_Smoke.yml | 2 +-
.../beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_CoGBK_Flink_batch.yml | 6 +-
.../beam_LoadTests_Go_Combine_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_Combine_Flink_Batch.yml | 8 +-
.../beam_LoadTests_Go_GBK_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_GBK_Flink_Batch.yml | 6 +-
.../beam_LoadTests_Go_ParDo_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_ParDo_Flink_Batch.yml | 6 +-
.../beam_LoadTests_Go_SideInput_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_SideInput_Flink_Batch.yml | 6 +-
.../beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml | 2 +-
...eam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml | 2 +-
...s_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml | 2 +-
...va_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml | 2 +-
...s_Java_CoGBK_SparkStructuredStreaming_Batch.yml | 2 +-
.../beam_LoadTests_Java_Combine_Dataflow_Batch.yml | 2 +-
...m_LoadTests_Java_Combine_Dataflow_Streaming.yml | 2 +-
...Java_Combine_SparkStructuredStreaming_Batch.yml | 2 +-
.../beam_LoadTests_Java_GBK_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Java_GBK_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml | 2 +-
...LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml | 2 +-
...am_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml | 2 +-
...Tests_Java_GBK_Dataflow_V2_Streaming_Java17.yml | 2 +-
.../workflows/beam_LoadTests_Java_GBK_Smoke.yml | 2 +-
...sts_Java_GBK_SparkStructuredStreaming_Batch.yml | 2 +-
.../beam_LoadTests_Java_ParDo_Dataflow_Batch.yml | 2 +-
...eam_LoadTests_Java_ParDo_Dataflow_Streaming.yml | 2 +-
...s_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml | 2 +-
...va_ParDo_Dataflow_V2_Streaming_JavaVersions.yml | 2 +-
...s_Java_ParDo_SparkStructuredStreaming_Batch.yml | 2 +-
.github/workflows/beam_LoadTests_Java_PubsubIO.yml | 2 +-
.../beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml | 2 +-
...m_LoadTests_Python_CoGBK_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_CoGBK_Flink_Batch.yml | 2 +-
...eam_LoadTests_Python_Combine_Dataflow_Batch.yml | 2 +-
...LoadTests_Python_Combine_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_Combine_Flink_Batch.yml | 2 +-
...am_LoadTests_Python_Combine_Flink_Streaming.yml | 2 +-
...LoadTests_Python_FnApiRunner_Microbenchmark.yml | 2 +-
.../beam_LoadTests_Python_GBK_Dataflow_Batch.yml | 2 +-
...eam_LoadTests_Python_GBK_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_GBK_Flink_Batch.yml | 2 +-
...adTests_Python_GBK_reiterate_Dataflow_Batch.yml | 2 +-
...sts_Python_GBK_reiterate_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_ParDo_Dataflow_Batch.yml | 2 +-
...m_LoadTests_Python_ParDo_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_ParDo_Flink_Batch.yml | 2 +-
...beam_LoadTests_Python_ParDo_Flink_Streaming.yml | 2 +-
...m_LoadTests_Python_SideInput_Dataflow_Batch.yml | 2 +-
.github/workflows/beam_LoadTests_Python_Smoke.yml | 2 +-
.../workflows/beam_MetricsCredentialsRotation.yml | 2 +-
.github/workflows/beam_Metrics_Report.yml | 2 +-
.../workflows/beam_PerformanceTests_AvroIOIT.yml | 2 +-
.../beam_PerformanceTests_AvroIOIT_HDFS.yml | 2 +-
...PerformanceTests_BigQueryIO_Batch_Java_Avro.yml | 2 +-
...PerformanceTests_BigQueryIO_Batch_Java_Json.yml | 2 +-
..._PerformanceTests_BigQueryIO_Streaming_Java.yml | 2 +-
...eam_PerformanceTests_BiqQueryIO_Read_Python.yml | 2 +-
...formanceTests_BiqQueryIO_Write_Python_Batch.yml | 2 +-
.github/workflows/beam_PerformanceTests_Cdap.yml | 2 +-
.../beam_PerformanceTests_Compressed_TextIOIT.yml | 2 +-
...m_PerformanceTests_Compressed_TextIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_HadoopFormat.yml | 2 +-
.github/workflows/beam_PerformanceTests_JDBC.yml | 2 +-
.../workflows/beam_PerformanceTests_Kafka_IO.yml | 2 +-
.../beam_PerformanceTests_ManyFiles_TextIOIT.yml | 2 +-
...am_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_MongoDBIO_IT.yml | 2 +-
.../beam_PerformanceTests_ParquetIOIT.yml | 2 +-
.../beam_PerformanceTests_ParquetIOIT_HDFS.yml | 2 +-
...erformanceTests_PubsubIOIT_Python_Streaming.yml | 2 +-
...m_PerformanceTests_SQLBigQueryIO_Batch_Java.yml | 2 +-
.../beam_PerformanceTests_SingleStoreIO.yml | 2 +-
..._PerformanceTests_SpannerIO_Read_2GB_Python.yml | 2 +-
...manceTests_SpannerIO_Write_2GB_Python_Batch.yml | 2 +-
.../beam_PerformanceTests_SparkReceiver_IO.yml | 2 +-
.../beam_PerformanceTests_TFRecordIOIT.yml | 2 +-
.../beam_PerformanceTests_TFRecordIOIT_HDFS.yml | 2 +-
.../workflows/beam_PerformanceTests_TextIOIT.yml | 2 +-
.../beam_PerformanceTests_TextIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_TextIOIT_Python.yml | 2 +-
...PerformanceTests_WordCountIT_PythonVersions.yml | 2 +-
.../workflows/beam_PerformanceTests_XmlIOIT.yml | 2 +-
.../beam_PerformanceTests_XmlIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_xlang_KafkaIO_Python.yml | 2 +-
.github/workflows/beam_Playground_CI_Nightly.yml | 2 +-
.github/workflows/beam_Playground_Precommit.yml | 10 +-
.github/workflows/beam_PostCommit_Go.yml | 6 +-
.../workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 6 +-
.github/workflows/beam_PostCommit_Go_VR_Flink.yml | 4 +-
.github/workflows/beam_PostCommit_Go_VR_Samza.yml | 4 +-
.github/workflows/beam_PostCommit_Go_VR_Spark.yml | 4 +-
.github/workflows/beam_PostCommit_Java.yml | 4 +-
.../beam_PostCommit_Java_Avro_Versions.yml | 4 +-
.../beam_PostCommit_Java_BigQueryEarlyRollout.yml | 6 +-
.../workflows/beam_PostCommit_Java_DataflowV1.yml | 4 +-
.../workflows/beam_PostCommit_Java_DataflowV2.yml | 4 +-
.../beam_PostCommit_Java_Examples_Dataflow.yml | 4 +-
.../beam_PostCommit_Java_Examples_Dataflow_ARM.yml | 6 +-
...beam_PostCommit_Java_Examples_Dataflow_Java.yml | 4 +-
.../beam_PostCommit_Java_Examples_Dataflow_V2.yml | 4 +-
...m_PostCommit_Java_Examples_Dataflow_V2_Java.yml | 4 +-
.../beam_PostCommit_Java_Examples_Direct.yml | 4 +-
.../beam_PostCommit_Java_Examples_Flink.yml | 4 +-
.../beam_PostCommit_Java_Examples_Spark.yml | 4 +-
.../beam_PostCommit_Java_Hadoop_Versions.yml | 4 +-
.../beam_PostCommit_Java_IO_Performance_Tests.yml | 4 +-
.../beam_PostCommit_Java_InfluxDbIO_IT.yml | 4 +-
.../beam_PostCommit_Java_Jpms_Dataflow.yml | 4 +-
...beam_PostCommit_Java_Jpms_Dataflow_Versions.yml | 4 +-
.../workflows/beam_PostCommit_Java_Jpms_Direct.yml | 4 +-
.../beam_PostCommit_Java_Jpms_Direct_Versions.yml | 4 +-
.../beam_PostCommit_Java_Jpms_Flink_Java11.yml | 4 +-
.../beam_PostCommit_Java_Jpms_Spark_Java11.yml | 4 +-
.../beam_PostCommit_Java_Nexmark_Dataflow.yml | 4 +-
.../beam_PostCommit_Java_Nexmark_Dataflow_V2.yml | 4 +-
...am_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml | 4 +-
.../beam_PostCommit_Java_Nexmark_Direct.yml | 4 +-
.../beam_PostCommit_Java_Nexmark_Flink.yml | 4 +-
.../beam_PostCommit_Java_Nexmark_Spark.yml | 4 +-
.../beam_PostCommit_Java_PVR_Flink_Batch.yml | 25 +-
.../beam_PostCommit_Java_PVR_Flink_Streaming.yml | 12 +-
.../workflows/beam_PostCommit_Java_PVR_Samza.yml | 4 +-
.../beam_PostCommit_Java_PVR_Spark3_Streaming.yml | 4 +-
.../beam_PostCommit_Java_PVR_Spark_Batch.yml | 4 +-
.../beam_PostCommit_Java_SingleStoreIO_IT.yml | 4 +-
.../beam_PostCommit_Java_Tpcds_Dataflow.yml | 4 +-
.../workflows/beam_PostCommit_Java_Tpcds_Flink.yml | 4 +-
.../workflows/beam_PostCommit_Java_Tpcds_Spark.yml | 4 +-
...am_PostCommit_Java_ValidatesRunner_Dataflow.yml | 4 +-
..._Java_ValidatesRunner_Dataflow_JavaVersions.yml | 4 +-
...mit_Java_ValidatesRunner_Dataflow_Streaming.yml | 6 +-
..._ValidatesRunner_Dataflow_Streaming_Engine.yml} | 21 +-
...tesRunner_Dataflow_Streaming_TagEncodingV2.yml} | 24 +-
...PostCommit_Java_ValidatesRunner_Dataflow_V2.yml | 4 +-
..._Java_ValidatesRunner_Dataflow_V2_Streaming.yml | 13 +-
...beam_PostCommit_Java_ValidatesRunner_Direct.yml | 4 +-
...it_Java_ValidatesRunner_Direct_JavaVersions.yml | 4 +-
.../beam_PostCommit_Java_ValidatesRunner_Flink.yml | 12 +-
...PostCommit_Java_ValidatesRunner_Flink_Java8.yml | 4 +-
.../beam_PostCommit_Java_ValidatesRunner_Samza.yml | 4 +-
.../beam_PostCommit_Java_ValidatesRunner_Spark.yml | 4 +-
...va_ValidatesRunner_SparkStructuredStreaming.yml | 4 +-
...PostCommit_Java_ValidatesRunner_Spark_Java8.yml | 4 +-
...am_PostCommit_Java_ValidatesRunner_Twister2.yml | 4 +-
.../beam_PostCommit_Java_ValidatesRunner_ULR.yml | 4 +-
.github/workflows/beam_PostCommit_Javadoc.yml | 4 +-
.../beam_PostCommit_PortableJar_Flink.yml | 4 +-
.../beam_PostCommit_PortableJar_Spark.yml | 4 +-
.github/workflows/beam_PostCommit_Python.yml | 6 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 12 +-
.../beam_PostCommit_Python_Dependency.yml | 6 +-
.../beam_PostCommit_Python_Examples_Dataflow.yml | 8 +-
.../beam_PostCommit_Python_Examples_Direct.yml | 6 +-
.../beam_PostCommit_Python_Examples_Flink.yml | 6 +-
.../beam_PostCommit_Python_Examples_Spark.yml | 6 +-
.../beam_PostCommit_Python_MongoDBIO_IT.yml | 10 +-
.../beam_PostCommit_Python_Nexmark_Direct.yml | 4 +-
.../beam_PostCommit_Python_Portable_Flink.yml | 9 +-
...stCommit_Python_ValidatesContainer_Dataflow.yml | 10 +-
..._Python_ValidatesContainer_Dataflow_With_RC.yml | 10 +-
..._PostCommit_Python_ValidatesRunner_Dataflow.yml | 6 +-
...eam_PostCommit_Python_ValidatesRunner_Flink.yml | 8 +-
...eam_PostCommit_Python_ValidatesRunner_Samza.yml | 6 +-
...eam_PostCommit_Python_ValidatesRunner_Spark.yml | 6 +-
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 12 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.yml | 6 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.yml | 9 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.yml | 6 +-
.github/workflows/beam_PostCommit_SQL.yml | 4 +-
.../beam_PostCommit_TransformService_Direct.yml | 8 +-
.github/workflows/beam_PostCommit_Website_Test.yml | 4 +-
.github/workflows/beam_PostCommit_XVR_Direct.yml | 6 +-
.github/workflows/beam_PostCommit_XVR_Flink.yml | 6 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 6 +-
...eam_PostCommit_XVR_JavaUsingPython_Dataflow.yml | 6 +-
..._PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml | 8 +-
...eam_PostCommit_XVR_PythonUsingJava_Dataflow.yml | 8 +-
.github/workflows/beam_PostCommit_XVR_Samza.yml | 6 +-
.github/workflows/beam_PostCommit_XVR_Spark3.yml | 6 +-
.../beam_PostCommit_Yaml_Xlang_Direct.yml | 6 +-
.../workflows/beam_PostRelease_NightlySnapshot.yml | 2 +-
.../workflows/beam_PreCommit_CommunityMetrics.yml | 6 +-
.../workflows/beam_PreCommit_Flink_Container.yml | 4 +-
.github/workflows/beam_PreCommit_GHA.yml | 4 +-
.github/workflows/beam_PreCommit_Go.yml | 4 +-
.github/workflows/beam_PreCommit_GoPortable.yml | 9 +-
.github/workflows/beam_PreCommit_GoPrism.yml | 4 +-
.github/workflows/beam_PreCommit_ItFramework.yml | 4 +-
.github/workflows/beam_PreCommit_Java.yml | 6 +-
...eCommit_Java_Amazon-Web-Services2_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Amqp_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Azure_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Cassandra_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Cdap_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Clickhouse_IO_Direct.yml | 11 +-
.../beam_PreCommit_Java_Csv_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Datadog_IO_Direct.yml | 4 +-
...it_SQL.yml => beam_PreCommit_Java_Dataflow.yml} | 71 +-
.../beam_PreCommit_Java_Debezium_IO_Direct.yml | 4 +-
...beam_PreCommit_Java_ElasticSearch_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Examples_Dataflow.yml | 5 +-
...eam_PreCommit_Java_Examples_Dataflow_Java21.yml | 5 +-
...Commit_Java_File-schema-transform_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Flink_Versions.yml | 4 +-
.../beam_PreCommit_Java_GCP_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Google-ads_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_HBase_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_HCatalog_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Hadoop_IO_Direct.yml | 4 +-
.../workflows/beam_PreCommit_Java_IOs_Direct.yml | 4 +-
.../beam_PreCommit_Java_InfluxDb_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_JDBC_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Jms_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Kafka_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Kudu_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_MongoDb_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Mqtt_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Neo4j_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_PVR_Flink_Batch.yml | 4 +-
.../beam_PreCommit_Java_PVR_Flink_Docker.yml | 4 +-
.../beam_PreCommit_Java_PVR_Prism_Loopback.yml | 4 +-
.../beam_PreCommit_Java_Parquet_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Pulsar_IO_Direct.yml | 6 +-
.../beam_PreCommit_Java_RabbitMq_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Redis_IO_Direct.yml | 4 +-
...am_PreCommit_Java_RequestResponse_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_SingleStore_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Snowflake_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Solace_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Solr_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Spark3_Versions.yml | 4 +-
.../beam_PreCommit_Java_Splunk_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Thrift_IO_Direct.yml | 4 +-
.../beam_PreCommit_Java_Tika_IO_Direct.yml | 4 +-
.../workflows/beam_PreCommit_Kotlin_Examples.yml | 4 +-
.../workflows/beam_PreCommit_Portable_Python.yml | 6 +-
.github/workflows/beam_PreCommit_Prism_Python.yml | 6 +-
.github/workflows/beam_PreCommit_Python.yml | 11 +-
.github/workflows/beam_PreCommit_PythonDocker.yml | 18 +-
.github/workflows/beam_PreCommit_PythonDocs.yml | 4 +-
.../workflows/beam_PreCommit_PythonFormatter.yml | 4 +-
.github/workflows/beam_PreCommit_PythonLint.yml | 4 +-
.../workflows/beam_PreCommit_Python_Coverage.yml | 37 +-
.../workflows/beam_PreCommit_Python_Dataframes.yml | 6 +-
.github/workflows/beam_PreCommit_Python_Dill.yml | 4 +-
.../workflows/beam_PreCommit_Python_Examples.yml | 6 +-
.../beam_PreCommit_Python_Integration.yml | 6 +-
.github/workflows/beam_PreCommit_Python_ML.yml | 10 +-
.../workflows/beam_PreCommit_Python_PVR_Flink.yml | 13 +-
.../workflows/beam_PreCommit_Python_Runners.yml | 6 +-
.../workflows/beam_PreCommit_Python_Transforms.yml | 6 +-
.github/workflows/beam_PreCommit_RAT.yml | 4 +-
.github/workflows/beam_PreCommit_SQL.yml | 4 +-
.github/workflows/beam_PreCommit_SQL_Java17.yml | 4 +-
.github/workflows/beam_PreCommit_SQL_Java8.yml | 4 +-
.github/workflows/beam_PreCommit_Spotless.yml | 6 +-
.github/workflows/beam_PreCommit_Typescript.yml | 4 +-
.github/workflows/beam_PreCommit_Website.yml | 4 +-
.../workflows/beam_PreCommit_Website_Stage_GCS.yml | 18 +-
.github/workflows/beam_PreCommit_Whitespace.yml | 4 +-
.../beam_PreCommit_Xlang_Generated_Transforms.yml | 6 +-
.../workflows/beam_PreCommit_Yaml_Xlang_Direct.yml | 4 +-
.github/workflows/beam_Prober_CommunityMetrics.yml | 2 +-
.github/workflows/beam_Publish_BeamMetrics.yml | 2 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 9 +-
.../workflows/beam_Publish_Docker_Snapshots.yml | 8 +-
.github/workflows/beam_Publish_Website.yml | 10 +-
.../beam_Python_CostBenchmarks_Dataflow.yml | 2 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 8 +-
.github/workflows/beam_Release_NightlySnapshot.yml | 2 +-
.../beam_Release_Python_NightlySnapshot.yml | 2 +-
.../workflows/beam_StressTests_Java_BigQueryIO.yml | 2 +-
.../workflows/beam_StressTests_Java_BigTableIO.yml | 2 +-
.../workflows/beam_StressTests_Java_KafkaIO.yml | 2 +-
.../workflows/beam_StressTests_Java_PubSubIO.yml | 2 +-
.../workflows/beam_StressTests_Java_SpannerIO.yml | 2 +-
.github/workflows/build_release_candidate.yml | 11 +-
.github/workflows/build_runner_image.yml | 8 +-
.github/workflows/build_wheels.yml | 32 +-
.github/workflows/dask_runner_tests.yml | 2 +-
.../workflows/deploy_release_candidate_pypi.yaml | 4 +-
.github/workflows/finalize_release.yml | 14 +-
.github/workflows/flaky_test_detection.yml | 2 +-
.github/workflows/go_tests.yml | 2 +-
.github/workflows/java_tests.yml | 4 +-
.../beam_CloudML_Benchmarks_Dataflow_arguments.txt | 2 +-
...chmarks_Dataflow_Table_Row_Inference_Batch.txt} | 36 +-
...hmarks_Dataflow_Table_Row_Inference_Stream.txt} | 38 +-
...Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt | 1 +
.github/workflows/python_dependency_tests.yml | 3 +-
.github/workflows/python_tests.yml | 9 +-
.github/workflows/refresh_looker_metrics.yml | 6 +-
.../republish_released_docker_containers.yml | 13 +-
.github/workflows/run_perf_alert_tool.yml | 2 +-
.../workflows/run_rc_validation_go_wordcount.yml | 24 +-
.../run_rc_validation_java_mobile_gaming.yml | 2 +-
.../run_rc_validation_java_quickstart.yml | 4 +-
.../run_rc_validation_python_mobile_gaming.yml | 4 +-
.../workflows/run_rc_validation_python_yaml.yml | 4 +-
.github/workflows/stale.yml | 1 +
.github/workflows/typescript_tests.yml | 6 +-
.github/workflows/update_python_dependencies.yml | 5 +-
.gitignore | 7 +
.pre-commit-config.yaml | 13 +-
.test-infra/tools/refresh_looker_metrics.py | 3 +-
.../tools/stale_dataflow_prebuilt_image_cleaner.sh | 4 +-
CHANGES.md | 86 +-
CONTRIBUTING.md | 10 +-
README.md | 21 +-
build.gradle.kts | 20 +-
.../org/apache/beam/gradle/BeamDockerPlugin.groovy | 7 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 115 +-
contributor-docs/release-guide.md | 2 +-
dev-support/docker/Dockerfile | 2 +-
examples/java/build.gradle | 27 +
.../examples/iceberg/IcebergBatchWriteExample.java | 15 +-
.../IcebergRestCatalogStreamingWriteExample.java | 2 +-
.../iceberg/snippets/IcebergBeamSchemaAndRow.java | 84 +
.../beam/examples/iceberg/snippets/Quickstart.java | 157 ++
.../beam/examples/SchemaTransformExample.java | 8 +-
.../apache/beam/examples/SqlTransformExample.java | 8 +-
.../beam/examples/ApproximateQuantilesExample.java | 8 +-
.../beam/examples/CoCombineTransformExample.java | 17 +-
.../apache/beam/examples/CoGroupByKeyExample.java | 8 +-
.../org/apache/beam/examples/CombineExample.java | 8 +-
.../org/apache/beam/examples/CountExample.java | 8 +-
.../apache/beam/examples/CountPerKeyExample.java | 8 +-
.../org/apache/beam/examples/CreateExample.java | 8 +-
.../apache/beam/examples/DebuggingWordCount.java | 13 +-
.../org/apache/beam/examples/DistinctExample.java | 8 +-
.../beam/examples/FlatMapElementsExample.java | 8 +-
.../beam/examples/GroupIntoBatchesExample.java | 8 +-
.../beam/examples/KafkaPassengerCountJson.java | 19 +-
.../org/apache/beam/examples/KafkaStreaming.java | 31 +-
.../apache/beam/examples/KafkaWordCountAvro.java | 9 +-
.../apache/beam/examples/KafkaWordCountJson.java | 9 +-
.../java/org/apache/beam/examples/KeysExample.java | 8 +-
.../org/apache/beam/examples/KvSwapExample.java | 8 +-
.../org/apache/beam/examples/LatestExample.java | 8 +-
.../apache/beam/examples/MapElementsExample.java | 8 +-
.../java/org/apache/beam/examples/MaxExample.java | 8 +-
.../org/apache/beam/examples/MaxPerKeyExample.java | 8 +-
.../java/org/apache/beam/examples/MeanExample.java | 8 +-
.../apache/beam/examples/MeanPerKeyExample.java | 8 +-
.../java/org/apache/beam/examples/MinExample.java | 8 +-
.../org/apache/beam/examples/MinPerKeyExample.java | 8 +-
.../org/apache/beam/examples/PartitionExample.java | 8 +-
.../org/apache/beam/examples/PicoWordCount.java | 56 +
.../beam/examples/RateLimitedPubSubReader.java | 135 ++
.../apache/beam/examples/RateLimiterSimple.java | 135 ++
.../org/apache/beam/examples/RegexExample.java | 8 +-
.../org/apache/beam/examples/SampleExample.java | 8 +-
.../java/org/apache/beam/examples/SumExample.java | 8 +-
.../org/apache/beam/examples/SumPerKeyExample.java | 8 +-
.../org/apache/beam/examples/ToStringExample.java | 8 +-
.../java/org/apache/beam/examples/TopExample.java | 8 +-
.../org/apache/beam/examples/ValuesExample.java | 8 +-
.../java/org/apache/beam/examples/ViewExample.java | 19 +-
.../org/apache/beam/examples/WindowExample.java | 8 +-
.../beam/examples/complete/AutoComplete.java | 60 +-
.../examples/complete/StreamingWordExtract.java | 16 +-
.../org/apache/beam/examples/complete/TfIdf.java | 75 +-
.../examples/complete/TopWikipediaSessions.java | 30 +-
.../beam/examples/complete/TrafficMaxLaneFlow.java | 30 +-
.../beam/examples/complete/TrafficRoutes.java | 48 +-
.../transforms/DataProtectors.java | 21 +-
.../transforms/io/TokenizationBigQueryIO.java | 7 +-
.../transforms/io/TokenizationBigTableIO.java | 6 +-
.../datatokenization/utils/CsvConverters.java | 115 +-
.../datatokenization/utils/ErrorConverters.java | 30 +-
.../datatokenization/utils/SchemasUtils.java | 5 +-
.../beam/examples/complete/game/GameStats.java | 70 +-
.../examples/complete/game/HourlyTeamScore.java | 6 +-
.../beam/examples/complete/game/LeaderBoard.java | 28 +-
.../examples/complete/game/StatefulTeamScore.java | 26 +-
.../beam/examples/complete/game/UserScore.java | 16 +-
.../examples/complete/game/injector/Injector.java | 6 +-
.../complete/game/utils/WriteToBigQuery.java | 22 +-
.../examples/complete/game/utils/WriteToText.java | 21 +-
.../game/utils/WriteWindowedToBigQuery.java | 15 +-
.../complete/kafkatopubsub/KafkaToPubsub.java | 10 +-
.../cookbook/BigQueryStreamingTornadoes.java | 23 +-
.../beam/examples/cookbook/BigQueryTornadoes.java | 18 +-
.../examples/cookbook/CombinePerKeyExamples.java | 14 +-
.../beam/examples/cookbook/FilterExamples.java | 31 +-
.../beam/examples/cookbook/JoinExamples.java | 32 +-
.../beam/examples/cookbook/MaxPerKeyExamples.java | 17 +-
.../cookbook/MinimalBigQueryTornadoes.java | 18 +-
.../beam/examples/cookbook/TriggerExample.java | 47 +-
.../apache/beam/examples/snippets/Snippets.java | 40 +-
.../examples/subprocess/ExampleEchoPipeline.java | 10 +-
.../subprocess/kernel/SubProcessIOFiles.java | 9 +-
.../subprocess/kernel/SubProcessKernel.java | 10 +-
.../subprocess/utils/CallingSubProcessUtils.java | 2 +-
.../beam/examples/subprocess/utils/FileUtils.java | 25 +-
.../apache/beam/examples/WindowedWordCountIT.java | 7 +-
.../java/org/apache/beam/examples/WordCountIT.java | 6 +-
.../org/apache/beam/examples/complete/TfIdfIT.java | 6 +-
.../examples/complete/TopWikipediaSessionsIT.java | 7 +-
.../beam/examples/cookbook/DistinctExampleIT.java | 6 +-
.../beam/examples/cookbook/JoinExamplesIT.java | 6 +-
.../beam/examples/cookbook/TriggerExampleTest.java | 8 +-
.../subprocess/ExampleEchoPipelineTest.java | 10 +-
.../ReadFromTwitterDoFn.java | 5 +-
.../twitterstreamgenerator/TwitterConfig.java | 2 +-
.../twitterstreamgenerator/TwitterConnection.java | 1 +
.../ReadFromTwitterDoFnTest.java | 1 +
.../apache/beam/examples/webapis/ImageRequest.java | 2 +-
.../beam/examples/kotlin/DebuggingWordCount.kt | 12 +-
.../examples/kotlin/cookbook/BigQueryTornadoes.kt | 15 +-
.../kotlin/cookbook/CombinePerKeyExamples.kt | 13 +-
.../examples/kotlin/cookbook/FilterExamples.kt | 30 +-
.../beam/examples/kotlin/cookbook/JoinExamples.kt | 31 +-
.../examples/kotlin/cookbook/MaxPerKeyExamples.kt | 15 +-
.../examples/kotlin/cookbook/TriggerExample.kt | 42 +-
.../beam/examples/kotlin/snippets/Snippets.kt | 13 +-
examples/notebooks/README.md | 13 +-
.../alloydb_product_catalog_embeddings.ipynb | 270 ++-
.../bigquery_vector_ingestion_and_search.ipynb | 2 +-
.../notebooks/beam-ml/run_inference_gemma.ipynb | 698 ------
.../learn_beam_windowing_by_doing.ipynb | 2 +-
.../get-started/try-apache-beam-yaml.ipynb | 353 +--
.../interactive-overview/getting-started.ipynb | 6 +-
.../reading-and-writing-data.ipynb | 4 +-
examples/terraform/envoy-ratelimiter/README.md | 63 +-
examples/terraform/envoy-ratelimiter/deploy.sh | 66 +
examples/terraform/envoy-ratelimiter/gke.tf | 4 +-
.../terraform/envoy-ratelimiter/prerequisites.tf | 1 +
examples/terraform/envoy-ratelimiter/ratelimit.tf | 156 +-
examples/terraform/envoy-ratelimiter/variables.tf | 40 +-
gradle.properties | 8 +-
gradle/wrapper/gradle-wrapper.jar | Bin 46175 -> 48462 bytes
gradle/wrapper/gradle-wrapper.properties | 4 +-
gradlew | 2 +-
gradlew.bat | 31 +-
infra/enforcement/requirements.txt | 2 +-
infra/iam/roles/beam_admin.role.yaml | 69 +-
infra/iam/roles/beam_infra_manager.role.yaml | 80 +-
infra/iam/roles/beam_viewer.role.yaml | 108 +-
infra/iam/roles/beam_writer.role.yaml | 22 +-
infra/iam/roles/roles_config.yaml | 2 +-
infra/iam/users.yml | 3 +-
it/clickhouse/build.gradle | 7 +-
.../it/clickhouse/ClickHouseResourceManager.java | 35 +-
.../clickhouse/conditions/ClickHouseRowsCheck.java | 4 +-
.../clickhouse/ClickHouseResourceManagerTest.java | 5 +-
.../org/apache/beam/it/common/TestProperties.java | 6 +-
.../beam/it/common/utils/IORedirectUtil.java | 2 +-
it/datadog/build.gradle | 50 +
.../beam/it/datadog/DatadogClientFactory.java | 47 +
.../apache/beam/it/datadog/DatadogLogEntry.java | 102 +
.../beam/it/datadog/DatadogResourceManager.java | 335 +++
.../datadog/DatadogResourceManagerException.java | 16 +-
.../it/datadog/DatadogResourceManagerUtils.java | 70 +
.../datadog/conditions/DatadogLogEntriesCheck.java | 91 +
.../beam/it/datadog/conditions}/package-info.java | 4 +-
.../beam/it/datadog/matchers/DatadogAsserts.java | 70 +
.../beam/it/datadog/matchers/package-info.java | 18 +
.../org/apache/beam/it/datadog/package-info.java | 18 +
.../beam/it/datadog/DatadogResourceManagerIT.java | 77 +
.../it/datadog/DatadogResourceManagerTest.java | 188 ++
.../datadog/DatadogResourceManagerUtilsTest.java | 67 +
.../java/org/apache/beam/it/gcp/LoadTestBase.java | 4 +-
.../beam/it/gcp/artifacts/utils/ArtifactUtils.java | 4 +-
.../gcp/bigquery/conditions/BigQueryRowsCheck.java | 4 +-
.../it/gcp/dataflow/DefaultPipelineLauncher.java | 2 +-
.../beam/it/gcp/dataflow/DirectRunnerClient.java | 4 +-
.../beam/it/gcp/datastore/DatastoreUtils.java | 4 +-
.../gcp/datastream/DatastreamResourceManager.java | 18 +-
.../org/apache/beam/it/gcp/pubsub/PubsubUtils.java | 4 +-
.../gcp/pubsub/conditions/PubsubMessagesCheck.java | 4 +-
.../apache/beam/it/gcp/bigquery/BigQueryIOLT.java | 8 +-
.../apache/beam/it/gcp/bigquery/BigQueryIOST.java | 4 +-
.../beam/it/gcp/bigquery/BigQueryStreamingLT.java | 8 +-
.../apache/beam/it/gcp/bigtable/BigTableIOLT.java | 8 +-
.../apache/beam/it/gcp/spanner/SpannerIOLT.java | 6 +-
.../apache/beam/it/gcp/spanner/SpannerIOST.java | 4 +-
.../apache/beam/it/gcp/storage/FileBasedIOLT.java | 8 +-
.../beam/it/jdbc/AbstractJDBCResourceManager.java | 2 +-
.../java/org/apache/beam/it/kafka/KafkaIOLT.java | 12 +-
.../java/org/apache/beam/it/kafka/KafkaIOST.java | 4 +-
.../mongodb/conditions/MongoDBDocumentsCheck.java | 4 +-
.../it/splunk/conditions/SplunkEventsCheck.java | 4 +-
.../beam/it/truthmatchers/ListAccumulator.java | 2 +-
local-env-setup.sh | 4 +-
.../org/apache/beam/model/pipeline/v1/schema.proto | 7 +
playground/backend/cmd/server/controller.go | 2 +-
playground/backend/cmd/server/server.go | 4 +-
playground/backend/containers/python/Dockerfile | 4 +-
playground/backend/env_setup.sh | 2 +-
playground/frontend/build.gradle | 3 +-
.../integration_test/embedded_run_test.dart | 2 +-
.../frontend/playground_components/pubspec.yaml | 6 +-
.../lib/src/widget_tester.dart | 13 +-
.../playground_components_dev/pubspec.yaml | 4 +-
playground/frontend/pubspec.yaml | 4 +-
.../cloudbuild/playground_cd_examples.sh | 9 +-
.../cloudbuild/playground_ci_examples.sh | 2 +-
playground/infrastructure/requirements.txt | 2 +-
release/build.gradle.kts | 2 +-
.../python-release/python_release_automation.sh | 2 +-
.../org/apache/beam/runners/core/DoFnRunner.java | 7 +-
.../runners/core/LateDataDroppingDoFnRunner.java | 7 +-
...TimeBoundedSplittableProcessElementInvoker.java | 47 +-
.../apache/beam/runners/core/ProcessFnRunner.java | 4 +-
.../runners/core/PushbackSideInputDoFnRunner.java | 7 +-
.../org/apache/beam/runners/core/ReduceFn.java | 3 +
.../beam/runners/core/ReduceFnContextFactory.java | 48 +-
.../apache/beam/runners/core/ReduceFnRunner.java | 52 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 107 +-
.../core/SimplePushbackSideInputDoFnRunner.java | 7 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 33 +-
.../beam/runners/core/StatefulDoFnRunner.java | 15 +-
.../apache/beam/runners/core/TimerInternals.java | 63 +-
.../apache/beam/runners/core/WindowMatchers.java | 52 +-
.../construction/SerializablePipelineOptions.java | 2 +-
.../beam/runners/core/metrics/BoundedTrieData.java | 6 +-
.../beam/runners/core/metrics/MetricsLogger.java | 6 +-
.../core/metrics/MonitoringInfoMetricName.java | 2 +-
.../runners/core/metrics/SimpleExecutionState.java | 2 +-
.../runners/core/metrics/SimpleStateRegistry.java | 2 +-
.../core/serialization/SentinelBitSetCoder.java | 81 +
.../core/triggers/AfterWatermarkStateMachine.java | 2 +-
.../core/triggers/TriggerStateMachineRunner.java | 4 +-
.../core/InMemoryMultimapSideInputViewTest.java | 13 +-
.../runners/core/InMemoryTimerInternalsTest.java | 73 +-
.../beam/runners/core/KeyedWorkItemCoderTest.java | 4 +-
.../runners/core/MergingActiveWindowSetTest.java | 6 +-
...BoundedSplittableProcessElementInvokerTest.java | 8 +-
.../apache/beam/runners/core/ReduceFnTester.java | 10 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 19 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 26 +-
.../runners/core/SplittableParDoProcessFnTest.java | 63 +-
.../beam/runners/core/StatefulDoFnRunnerTest.java | 3 +-
.../beam/runners/core/TimerInternalsTest.java | 55 +-
.../serialization/SentinelBitSetCoderTest.java | 150 ++
.../core/triggers/TriggerStateMachineTester.java | 10 +-
.../apache/beam/runners/direct/DirectMetrics.java | 4 +-
.../apache/beam/runners/direct/DirectRunner.java | 12 +-
.../beam/runners/direct/DirectTimerInternals.java | 4 +-
...LifecycleManagerRemovingTransformEvaluator.java | 2 +-
.../apache/beam/runners/direct/ParDoEvaluator.java | 5 +-
.../beam/runners/direct/WatermarkManager.java | 5 +-
.../runners/direct/WriteWithShardingFactory.java | 2 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 11 +-
.../beam/runners/direct/DirectRegistrarTest.java | 4 +-
.../runners/direct/DirectTimerInternalsTest.java | 19 +-
...cycleManagerRemovingTransformEvaluatorTest.java | 6 +-
.../runners/direct/DoFnLifecycleManagerTest.java | 2 +-
.../runners/direct/DoFnLifecycleManagersTest.java | 4 +-
.../beam/runners/direct/EvaluationContextTest.java | 7 +-
.../direct/UnboundedReadEvaluatorFactoryTest.java | 44 +-
.../beam/runners/direct/WatermarkManagerTest.java | 114 +-
.../runners/flink/streaming/StreamSources.java | 2 +-
.../wrappers/streaming/DoFnOperator.java | 13 +-
.../runners/flink/FlinkExecutionEnvironments.java | 4 +-
.../FlinkStreamingPortablePipelineTranslator.java | 31 +-
.../functions/FlinkStatefulDoFnFunction.java | 3 +-
.../translation/types/CoderTypeInformation.java | 2 +-
.../wrappers/streaming/DoFnOperator.java | 13 +-
.../state/FlinkBroadcastStateInternals.java | 8 +-
.../streaming/state/FlinkStateInternals.java | 16 +-
.../flink/FlinkRequiresStableInputTest.java | 7 +-
.../runners/flink/streaming/StreamSources.java | 2 +-
.../functions/ImpulseSourceFunctionTest.java | 3 +-
runners/flink/flink_runner.gradle | 15 +-
.../flink_job_server_container.gradle | 3 +
.../runners/flink/FlinkExecutionEnvironments.java | 2 +-
.../flink/FlinkPortableClientEntryPoint.java | 3 +-
.../flink/FlinkStreamingPipelineTranslator.java | 2 +-
.../FlinkStreamingPortablePipelineTranslator.java | 26 +-
.../flink/FlinkStreamingTranslationContext.java | 2 +-
.../flink/PipelineTranslationModeOptimizer.java | 2 +-
.../flink/adapter/BeamFlinkDataStreamAdapter.java | 3 +-
.../flink/metrics/DoFnRunnerWithMetricsUpdate.java | 14 +-
.../runners/flink/metrics/MetricsAccumulator.java | 1 +
.../functions/FlinkSideInputReader.java | 2 +-
.../functions/FlinkStatefulDoFnFunction.java | 3 +-
.../translation/types/CoderTypeInformation.java | 2 +-
.../translation/types/CoderTypeSerializer.java | 2 +-
.../translation/utils/LookupPipelineVisitor.java | 2 +-
.../wrappers/streaming/DoFnOperator.java | 22 +-
.../streaming/ExecutableStageDoFnOperator.java | 11 +-
.../io/source/LazyFlinkSourceSplitEnumerator.java | 17 +-
.../unbounded/FlinkUnboundedSourceReader.java | 2 +-
.../streaming/stableinput/BufferedElements.java | 10 +-
.../streaming/stableinput/BufferingDoFnRunner.java | 4 +-
.../state/FlinkBroadcastStateInternals.java | 8 +-
.../streaming/state/FlinkStateInternals.java | 16 +-
.../flink/FlinkRequiresStableInputTest.java | 15 +-
.../beam/runners/flink/ReadSourcePortableTest.java | 2 +
.../flink/metrics/FlinkMetricContainerTest.java | 8 +-
.../flink/streaming/GroupByWithNullValuesTest.java | 1 +
.../functions/ImpulseSourceFunctionTest.java | 3 +-
.../wrappers/streaming/DoFnOperatorTest.java | 4 +-
.../streaming/ExecutableStageDoFnOperatorTest.java | 13 +-
.../wrappers/streaming/FlinkKeyUtilsTest.java | 3 +-
runners/google-cloud-dataflow-java/build.gradle | 83 +-
.../beam/runners/dataflow/BatchViewOverrides.java | 5 +-
.../beam/runners/dataflow/DataflowPipelineJob.java | 1 +
.../dataflow/DataflowPipelineTranslator.java | 90 +-
.../beam/runners/dataflow/DataflowRunner.java | 32 +-
.../runners/dataflow/GroupIntoBatchesOverride.java | 12 +
.../runners/dataflow/internal/CustomSources.java | 2 +-
.../options/DataflowStreamingPipelineOptions.java | 23 +-
.../options/DataflowWorkerLoggingOptions.java | 74 +
.../beam/runners/dataflow/util/MonitoringUtil.java | 3 +
.../beam/runners/dataflow/util/PackageUtil.java | 11 +-
.../dataflow/DataflowPipelineRegistrarTest.java | 4 +-
.../dataflow/DataflowPipelineTranslatorTest.java | 144 +-
.../beam/runners/dataflow/DataflowRunnerTest.java | 5 +-
.../dataflow/PrimitiveParDoSingleFactoryTest.java | 2 +-
.../runners/dataflow/util/CloudObjectsTest.java | 4 +-
.../runners/dataflow/util/MonitoringUtilTest.java | 2 +
.../google-cloud-dataflow-java/worker/build.gradle | 7 +-
.../dataflow/worker/ApplianceShuffleReader.java | 1 +
.../dataflow/worker/ApplianceShuffleWriter.java | 1 +
.../dataflow/worker/BatchDataflowWorker.java | 5 +-
.../dataflow/worker/BatchModeExecutionContext.java | 6 +-
.../beam/runners/dataflow/worker/ConcatReader.java | 2 +-
.../worker/DataflowElementExecutionTracker.java | 8 +-
.../dataflow/worker/DataflowExecutionContext.java | 6 +-
.../dataflow/worker/DataflowProcessFnRunner.java | 4 +-
.../dataflow/worker/DoFnInstanceManagers.java | 10 +-
.../dataflow/worker/GroupAlsoByWindowFnRunner.java | 10 +-
.../dataflow/worker/GroupAlsoByWindowsParDoFn.java | 2 +
.../MetricsToPerStepNamespaceMetricsConverter.java | 11 +-
.../beam/runners/dataflow/worker/PubsubReader.java | 42 +-
.../beam/runners/dataflow/worker/PubsubSink.java | 9 +-
.../runners/dataflow/worker/SimpleParDoFn.java | 3 +-
.../worker/SplittableProcessFnFactory.java | 5 +-
.../dataflow/worker/StreamingDataflowWorker.java | 137 +-
.../StreamingKeyedWorkItemSideInputDoFnRunner.java | 4 +-
.../worker/StreamingModeExecutionContext.java | 125 +-
.../worker/StreamingSideInputDoFnRunner.java | 4 +-
.../worker/StreamingStepMetricsContainer.java | 5 +-
.../dataflow/worker/UngroupedWindmillReader.java | 48 +-
.../dataflow/worker/WindmillKeyedWorkItem.java | 120 +-
.../worker/WindmillReaderIteratorBase.java | 59 +-
.../beam/runners/dataflow/worker/WindmillSink.java | 84 +-
.../dataflow/worker/WindmillTimerInternals.java | 41 +-
...bFactoryFactory.java => WindmillTimerType.java} | 8 +-
.../dataflow/worker/WindowingWindmillReader.java | 55 +-
.../runners/dataflow/worker/WorkUnitClient.java | 2 +-
.../dataflow/worker/WorkerCustomSources.java | 10 +-
.../worker/WorkerPipelineOptionsFactory.java | 4 +-
.../dataflow/worker/counters/CounterFactory.java | 2 +-
.../logging/DataflowWorkerLoggingHandler.java | 782 ++++--
.../logging/DataflowWorkerLoggingInitializer.java | 180 +-
.../JulHandlerPrintStreamAdapterFactory.java | 11 +-
.../dataflow/worker/status/DebugCapture.java | 2 +
.../StreamingEngineComputationConfigFetcher.java | 12 +-
.../streaming/config/StreamingGlobalConfig.java | 7 +-
.../FanOutStreamingEngineWorkerHarness.java | 42 +-
.../harness/SingleSourceWorkerHarness.java | 10 +
.../runners/dataflow/worker/util/JfrInterop.java | 2 +
.../dataflow/worker/util/MemoryMonitor.java | 2 +-
.../util/ThreadLocalByteStringOutputStream.java | 2 +-
.../dataflow/worker/util/ValueInEmptyWindows.java | 5 +-
.../common/worker/BatchingShuffleEntryReader.java | 2 +-
.../common/worker/CachingShuffleBatchReader.java | 4 +-
.../worker/util/common/worker/ShuffleEntry.java | 8 +-
.../util/common/worker/WorkProgressUpdater.java | 10 +-
.../worker/windmill/WindmillEndpoints.java | 23 +
.../worker/windmill/WindmillServerBase.java | 1 +
.../windmill/client/AbstractWindmillStream.java | 49 +-
.../client/ResettableThrowingStreamObserver.java | 17 +-
.../client/getdata/ApplianceGetDataClient.java | 4 +-
.../client/grpc/GetWorkResponseChunkAssembler.java | 14 +-
.../client/grpc/GrpcDirectGetWorkStream.java | 1 +
.../windmill/client/grpc/GrpcDispatcherClient.java | 50 +-
.../windmill/client/grpc/GrpcGetDataStream.java | 19 +-
.../windmill/client/grpc/GrpcGetWorkStream.java | 1 +
.../client/grpc/GrpcGetWorkerMetadataStream.java | 18 +-
.../windmill/client/grpc/GrpcWindmillServer.java | 3 +-
.../grpc/observers/DirectStreamObserver.java | 2 +-
.../client/grpc/stubs/FailoverChannel.java | 443 ++++
.../grpc/stubs/WindmillStubFactoryFactory.java | 2 +-
.../grpc/stubs/WindmillStubFactoryFactoryImpl.java | 19 +-
.../dataflow/worker/windmill/state/RangeCoder.java | 4 +-
.../worker/windmill/state/WindmillMultimap.java | 4 +-
.../worker/windmill/state/WindmillTagEncoding.java | 24 +-
.../windmill/state/WindmillTagEncodingV1.java | 90 +-
.../windmill/state/WindmillTagEncodingV2.java | 46 +-
.../worker/windmill/state/WindmillTimerData.java | 22 +-
.../worker/windmill/work/WorkItemReceiver.java | 1 +
.../worker/windmill/work/WorkItemScheduler.java | 2 +
.../processing/ComputationWorkExecutorFactory.java | 14 +-
.../work/processing/StreamingCommitFinalizer.java | 154 +-
.../work/processing/StreamingWorkScheduler.java | 34 +-
.../processing/failures/WorkFailureProcessor.java | 118 +-
.../work/refresh/StreamPoolHeartbeatSender.java | 27 -
.../worker/CombineValuesFnFactoryTest.java | 2 +-
.../dataflow/worker/DefaultParDoFnFactoryTest.java | 2 +-
.../dataflow/worker/DoFnInstanceManagersTest.java | 3 +-
.../dataflow/worker/FakeWindmillServer.java | 1 +
.../runners/dataflow/worker/ShuffleSinkTest.java | 3 +-
.../runners/dataflow/worker/SimpleParDoFnTest.java | 111 +-
.../worker/StreamingDataflowWorkerTest.java | 251 +-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 101 +-
...eamingKeyedWorkItemSideInputDoFnRunnerTest.java | 4 +-
.../worker/StreamingModeExecutionContextTest.java | 39 +-
.../worker/StreamingSideInputFetcherTest.java | 4 +-
.../dataflow/worker/TestShuffleReaderTest.java | 4 +-
.../worker/UngroupedShuffleReaderTest.java | 2 +-
.../dataflow/worker/UserParDoFnFactoryTest.java | 10 +-
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 119 +-
.../worker/WindmillReaderIteratorBaseTest.java | 42 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 19 +-
.../logging/DataflowWorkerLoggingHandlerTest.java | 505 +++-
.../DataflowWorkerLoggingInitializerTest.java | 111 +-
.../JulHandlerPrintStreamAdapterFactoryTest.java | 4 +-
.../worker/status/WorkerStatusPagesTest.java | 9 +-
.../config/FixedGlobalConfigHandleTest.java | 5 +-
...treamingEngineComputationConfigFetcherTest.java | 38 +
.../StreamingGlobalConfigHandleImplTest.java | 17 +-
.../FanOutStreamingEngineWorkerHarnessTest.java | 162 +-
.../harness/SingleSourceWorkerHarnessTest.java | 15 +-
.../harness/WindmillStreamSenderTest.java | 1 +
.../worker/CachingShuffleBatchReaderTest.java | 2 +-
.../client/grpc/GrpcDirectGetWorkStreamTest.java | 5 +
.../client/grpc/GrpcDispatcherClientTest.java | 141 --
.../grpc/GrpcGetWorkerMetadataStreamTest.java | 40 -
.../client/grpc/GrpcWindmillServerTest.java | 4 +-
.../client/grpc/stubs/FailoverChannelTest.java | 575 +++++
.../client/grpc/stubs/IsolationChannelTest.java | 64 +-
.../windmill/client/grpc/stubs/NoopClientCall.java | 72 +
.../windmill/state/WindmillStateInternalsTest.java | 5 +-
.../windmill/state/WindmillTagEncodingV1Test.java | 50 +-
.../windmill/state/WindmillTagEncodingV2Test.java | 78 +-
.../testing/FakeWindmillStubFactoryFactory.java | 2 +-
.../processing/StreamingCommitFinalizerTest.java | 188 ++
.../failures/WorkFailureProcessorTest.java | 22 +-
.../work/refresh/ActiveWorkRefresherTest.java | 8 +-
.../refresh/StreamPoolHeartbeatSenderTest.java | 84 -
.../worker/windmill/src/main/proto/windmill.proto | 27 +-
.../artifact/ArtifactStagingService.java | 4 +-
.../control/DefaultJobBundleFactory.java | 4 +
.../control/FnApiControlClientPoolService.java | 2 +-
.../control/ProcessBundleDescriptors.java | 2 +-
...renceCountingExecutableStageContextFactory.java | 6 +-
.../fnexecution/control/TimerReceiverFactory.java | 3 +-
.../environment/EmbeddedEnvironmentFactory.java | 2 +
.../fnexecution/environment/ProcessManager.java | 20 +-
.../fnexecution/logging/Slf4jLogWriter.java | 1 +
.../fnexecution/status/WorkerStatusClient.java | 6 +-
.../translation/PipelineTranslatorUtils.java | 3 +-
.../artifact/ArtifactStagingServiceTest.java | 3 +-
.../control/DefaultJobBundleFactoryTest.java | 11 +-
.../fnexecution/control/RemoteExecutionTest.java | 8 +-
.../logging/GrpcLoggingServiceTest.java | 117 +-
.../runners/fnexecution/wire/CommonCoderTest.java | 6 +-
.../runners/jobsubmission/InMemoryJobService.java | 26 +-
.../beam/runners/jobsubmission/JobInvocation.java | 16 +-
.../PortablePipelineJarCreatorTest.java | 4 +-
.../apache/beam/runners/jet/JetPipelineResult.java | 2 +-
.../org/apache/beam/runners/jet/JetRunner.java | 2 +-
.../java/org/apache/beam/runners/jet/Utils.java | 4 +-
.../runners/jet/processors/StatefulParDoP.java | 3 +-
.../apache/beam/runners/prism/PrismExecutor.java | 6 +-
.../apache/beam/runners/prism/PrismLocator.java | 2 +-
.../beam/runners/prism/PrismPipelineResult.java | 2 +-
.../beam/runners/prism/PrismRegistrarTest.java | 4 +-
.../beam/runners/samza/SamzaPipelineResult.java | 2 +-
.../beam/runners/samza/SamzaPipelineRunner.java | 2 +-
.../org/apache/beam/runners/samza/SamzaRunner.java | 2 +-
.../runners/samza/adapter/BoundedSourceSystem.java | 2 +-
.../samza/adapter/UnboundedSourceSystem.java | 2 +-
.../samza/container/ContainerCfgLoader.java | 4 +-
.../samza/metrics/DoFnRunnerWithMetrics.java | 13 +-
.../samza/metrics/SamzaTransformMetrics.java | 3 +-
.../runners/samza/runtime/AsyncDoFnRunner.java | 9 +-
.../samza/runtime/ClassicBundleManager.java | 13 +-
.../apache/beam/runners/samza/runtime/DoFnOp.java | 3 +-
.../runtime/DoFnRunnerWithKeyedInternals.java | 13 +-
.../beam/runners/samza/runtime/KeyedTimerData.java | 2 +-
.../beam/runners/samza/runtime/OpMessage.java | 2 +-
.../samza/runtime/PortableBundleManager.java | 4 +-
.../beam/runners/samza/runtime/PortableDoFnOp.java | 3 +-
.../runners/samza/runtime/SamzaDoFnRunners.java | 7 +-
.../samza/runtime/SamzaStoreStateInternals.java | 4 +-
.../runners/samza/translation/ConfigBuilder.java | 8 +-
.../translation/ParDoBoundMultiTranslator.java | 6 +-
.../translation/SplittableParDoTranslators.java | 2 +-
.../samza/translation/TranslationContext.java | 7 +-
.../TestSamzaRunnerWithTransformMetrics.java | 4 +-
.../metrics/TestSamzaTransformMetricsRegistry.java | 4 +-
.../samza/runtime/ClassicBundleManagerTest.java | 2 +-
.../runners/samza/runtime/KeyedTimerDataTest.java | 8 +-
.../runtime/SamzaTimerInternalsFactoryTest.java | 90 +-
.../beam/runners/samza/util/FutureUtilsTest.java | 9 +-
runners/spark/3/build.gradle | 2 +-
runners/spark/spark_runner.gradle | 112 +-
.../beam/runners/spark/SparkPipelineRunner.java | 2 +-
.../spark/stateful/SparkStateInternals.java | 2 +-
.../SparkStructuredStreamingPipelineOptions.java | 0
.../SparkStructuredStreamingPipelineResult.java | 0
.../SparkStructuredStreamingRunner.java | 0
.../SparkStructuredStreamingRunnerRegistrar.java | 0
.../structuredstreaming/examples/WordCount.java | 0
.../io/BoundedDatasetFactory.java | 0
.../spark/structuredstreaming/io/package-info.java | 0
.../structuredstreaming/metrics/BeamMetricSet.java | 0
.../metrics/MetricsAccumulator.java | 0
.../metrics/SparkBeamMetric.java | 0
.../metrics/SparkBeamMetricSource.java | 0
.../metrics/WithMetricsSupport.java | 0
.../structuredstreaming/metrics/package-info.java | 0
.../metrics/sink/CodahaleCsvSink.java | 0
.../metrics/sink/CodahaleGraphiteSink.java | 0
.../metrics/sink/package-info.java | 0
.../spark/structuredstreaming/package-info.java | 0
.../translation/EvaluationContext.java | 0
.../translation/PipelineTranslator.java | 0
.../translation/SparkSessionFactory.java | 0
.../translation/SparkTransformOverrides.java | 0
.../translation/TransformTranslator.java | 0
.../translation/batch/Aggregators.java | 0
.../batch/CombineGloballyTranslatorBatch.java | 0
.../batch/CombineGroupedValuesTranslatorBatch.java | 0
.../batch/CombinePerKeyTranslatorBatch.java | 0
.../batch/DoFnPartitionIteratorFactory.java | 0
.../translation/batch/DoFnRunnerFactory.java | 4 +-
.../translation/batch/DoFnRunnerWithMetrics.java | 14 +-
.../translation/batch/FlattenTranslatorBatch.java | 0
.../translation/batch/GroupByKeyHelpers.java | 0
.../batch/GroupByKeyTranslatorBatch.java | 0
.../translation/batch/ImpulseTranslatorBatch.java | 0
.../translation/batch/ParDoTranslatorBatch.java | 0
.../translation/batch/PipelineTranslatorBatch.java | 0
.../batch/ReadSourceTranslatorBatch.java | 0
.../batch/ReshuffleTranslatorBatch.java | 0
.../batch/WindowAssignTranslatorBatch.java | 0
.../batch/functions/CachedSideInputReader.java | 0
.../GroupAlsoByWindowViaOutputBufferFn.java | 0
.../batch/functions/NoOpStepContext.java | 0
.../batch/functions/SideInputValues.java | 0
.../batch/functions/SparkSideInputReader.java | 0
.../translation/batch/functions/package-info.java | 0
.../translation/batch/package-info.java | 0
.../translation/helpers/CoderHelpers.java | 0
.../translation/helpers/EncoderFactory.java | 0
.../translation/helpers/EncoderHelpers.java | 77 +-
.../translation/helpers/EncoderProvider.java | 0
.../translation/helpers/package-info.java | 0
.../translation/package-info.java | 0
.../translation/utils/ScalaInterop.java | 0
.../translation/utils/package-info.java | 0
.../spark/translation/AbstractInOutIterator.java | 3 +-
.../spark/translation/DoFnRunnerWithMetrics.java | 14 +-
.../spark/translation/EvaluationContext.java | 2 +-
.../spark/translation/GroupCombineFunctions.java | 2 +-
.../spark/translation/SparkContextFactory.java | 2 +-
.../spark/translation/SparkInputDataProcessor.java | 4 +-
.../spark/translation/TransformTranslator.java | 4 +
.../spark/translation/TranslationUtils.java | 10 +-
.../streaming/StatefulStreamingParDoEvaluator.java | 4 +
.../streaming/StreamingTransformTranslator.java | 5 +
.../apache/beam/runners/spark/util/ByteArray.java | 2 +-
.../runners/spark/util/CachedSideInputReader.java | 10 +-
.../runners/spark/util/SideInputBroadcast.java | 2 +-
.../beam/runners/spark/util/SideInputStorage.java | 2 +-
.../apache/beam/runners/spark/util/TimerUtils.java | 5 +-
.../runners/spark/SparkRunnerRegistrarTest.java | 4 +-
.../aggregators/metrics/sink/InMemoryMetrics.java | 26 +-
.../coders/SparkRunnerKryoRegistratorTest.java | 3 +-
.../spark/metrics/SparkMetricsPusherTest.java | 2 +-
.../structuredstreaming/SparkSessionRule.java | 0
...parkStructuredStreamingRunnerRegistrarTest.java | 6 +-
.../StructuredStreamingPipelineStateTest.java | 0
.../aggregators/metrics/sink/InMemoryMetrics.java | 26 +-
.../metrics/sink/InMemoryMetricsSinkRule.java | 0
.../metrics/sink/SparkMetricsSinkTest.java | 0
.../metrics/SparkBeamMetricTest.java | 0
.../translation/batch/AggregatorsTest.java | 0
.../translation/batch/CombineGloballyTest.java | 0
.../batch/CombineGroupedValuesTest.java | 0
.../translation/batch/CombinePerKeyTest.java | 0
.../translation/batch/ComplexSourceTest.java | 0
.../translation/batch/FlattenTest.java | 0
.../translation/batch/GroupByKeyTest.java | 0
.../translation/batch/ParDoTest.java | 0
.../translation/batch/SimpleSourceTest.java | 0
.../translation/batch/WindowAssignTest.java | 0
.../batch/functions/SideInputValuesTest.java | 0
.../translation/helpers/EncoderHelpersTest.java | 2 +-
.../translation/streaming/SimpleSourceTest.java | 0
.../translation/AbstractInOutIteratorTest.java | 9 +-
.../translation/SparkInputDataProcessorTest.java | 4 +-
.../spark/translation/TransformTranslatorTest.java | 64 +
.../ValueAndCoderLazySerializableTest.java | 9 +-
.../twister2/Twister2TranslationContext.java | 2 +-
scripts/beam-sql.sh | 2 +-
scripts/ci/issue-report/package-lock.json | 14 +-
scripts/ci/issue-report/package.json | 2 +-
scripts/ci/pr-bot/findPrsNeedingAttention.ts | 30 +-
scripts/ci/pr-bot/package-lock.json | 12 +-
scripts/ci/pr-bot/processNewPrs.ts | 41 +-
scripts/ci/pr-bot/processPrUpdate.ts | 52 +-
scripts/tools/bomupgrader.py | 1 -
sdks/go.mod | 96 +-
sdks/go.sum | 210 +-
sdks/go/container/build.gradle | 6 +-
.../minimal_wordcount/minimal_wordcount.go | 23 +
.../ordered_list_state/ordered_list_state.go | 93 +
sdks/go/examples/snippets/04transforms.go | 32 +
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/core/graph/fn.go | 4 +-
.../core/runtime/contextreg/contextreg_test.go | 4 +-
sdks/go/pkg/beam/core/runtime/exec/data.go | 6 +
.../pkg/beam/core/runtime/exec/sideinput_test.go | 15 +
sdks/go/pkg/beam/core/runtime/exec/translate.go | 2 +
sdks/go/pkg/beam/core/runtime/exec/userstate.go | 150 ++
.../pkg/beam/core/runtime/exec/userstate_test.go | 25 +-
sdks/go/pkg/beam/core/runtime/graphx/translate.go | 16 +-
sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 2 +-
sdks/go/pkg/beam/core/runtime/harness/statemgr.go | 82 +
sdks/go/pkg/beam/core/state/state.go | 133 ++
sdks/go/pkg/beam/core/state/state_test.go | 243 +-
sdks/go/pkg/beam/io/xlang/debeziumio/debezium.go | 48 +
sdks/go/pkg/beam/runners/dataflow/dataflow_test.go | 16 +-
.../pkg/beam/runners/dataflow/dataflowlib/job.go | 6 +-
.../prism/internal/engine/elementmanager.go | 4 +-
.../runners/prism/internal/engine/teststream.go | 2 +-
sdks/go/pkg/beam/transforms/stats/quantiles.go | 20 +-
sdks/go/run_with_go_version.sh | 19 +-
sdks/go/test/build.gradle | 26 +-
sdks/go/test/integration/integration.go | 8 +
.../test/integration/io/xlang/debezium/debezium.go | 3 +-
.../integration/io/xlang/debezium/debezium_test.go | 3 +-
sdks/go/test/integration/primitives/state.go | 38 +
sdks/go/test/integration/primitives/state_test.go | 5 +
sdks/go/test/run_validatesrunner_tests.sh | 27 +-
sdks/java/bom/gcp/build.gradle | 14 +-
.../resources/beam/checkstyle/suppressions.xml | 3 +-
sdks/java/container/boot.go | 15 +-
.../container/license_scripts/dep_urls_java.yaml | 10 +-
.../container/license_scripts/license_script.sh | 20 +-
sdks/java/core/build.gradle | 3 +-
.../apache/beam/sdk/jmh/util/VarIntBenchmark.java | 2 +-
.../java/org/apache/beam/sdk/PipelineResult.java | 4 +-
.../org/apache/beam/sdk/coders/AtomicCoder.java | 1 +
.../apache/beam/sdk/coders/BigDecimalCoder.java | 4 +-
.../sdk/coders/CannotProvideCoderException.java | 2 +-
.../java/org/apache/beam/sdk/coders/Coder.java | 5 +
.../org/apache/beam/sdk/coders/DelegateCoder.java | 3 +-
.../java/org/apache/beam/sdk/coders/RowCoder.java | 1 +
.../apache/beam/sdk/coders/RowCoderGenerator.java | 28 +-
.../apache/beam/sdk/coders/SerializableCoder.java | 1 +
.../beam/sdk/coders/StringDelegateCoder.java | 3 +-
.../apache/beam/sdk/coders/StructuredCoder.java | 1 +
.../java/org/apache/beam/sdk/coders/ZstdCoder.java | 2 +-
.../sdk/fn/channel/AddHarnessIdInterceptor.java | 4 +-
.../sdk/fn/data/BeamFnDataOutboundAggregator.java | 15 +-
.../org/apache/beam/sdk/fn/data/WeightedList.java | 11 +-
.../server/GrpcContextHeaderAccessorProvider.java | 5 +-
.../org/apache/beam/sdk/fn/test/TestStreams.java | 20 +-
.../org/apache/beam/sdk/io/CompressedSource.java | 20 +-
.../apache/beam/sdk/io/DefaultFilenamePolicy.java | 11 +-
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 24 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 68 +-
.../java/org/apache/beam/sdk/io/FileSystem.java | 4 +-
.../java/org/apache/beam/sdk/io/FileSystems.java | 5 +-
.../src/main/java/org/apache/beam/sdk/io/Read.java | 4 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 10 +-
.../main/java/org/apache/beam/sdk/io/TextIO.java | 16 +-
.../java/org/apache/beam/sdk/io/TextSource.java | 8 +-
.../org/apache/beam/sdk/io/UnboundedSource.java | 1 +
.../org/apache/beam/sdk/io/range/ByteKeyRange.java | 4 +-
.../org/apache/beam/sdk/io/range/OffsetRange.java | 2 +-
.../org/apache/beam/sdk/lineage/LineageBase.java | 46 +
.../apache/beam/sdk/lineage/LineageOptions.java | 31 +-
.../org/apache/beam/sdk/lineage/package-info.java} | 18 +-
.../BoundedTrieMetricsLineage.java} | 36 +-
.../apache/beam/sdk/metrics/BoundedTrieResult.java | 2 +-
.../java/org/apache/beam/sdk/metrics/Lineage.java | 120 +-
.../apache/beam/sdk/metrics/MetricNameFilter.java | 8 +-
.../beam/sdk/metrics/MetricsEnvironment.java | 11 +-
.../beam/sdk/metrics/StringSetMetricsLineage.java | 33 +-
.../apache/beam/sdk/metrics/StringSetResult.java | 2 +-
.../options/DefaultPipelineOptionsRegistrar.java | 2 +
.../beam/sdk/options/PipelineOptionsFactory.java | 48 +-
.../beam/sdk/options/PipelineOptionsValidator.java | 6 +-
.../beam/sdk/options/ProxyInvocationHandler.java | 64 +-
.../apache/beam/sdk/options/SdkHarnessOptions.java | 15 +-
.../org/apache/beam/sdk/options/ValueProvider.java | 17 +-
.../apache/beam/sdk/options/ValueProviders.java | 4 +-
.../GenerateSequenceSchemaTransformProvider.java | 12 +-
.../apache/beam/sdk/runners/AppliedPTransform.java | 2 +-
.../apache/beam/sdk/schemas/AutoValueSchema.java | 10 +-
.../apache/beam/sdk/schemas/CachingFactory.java | 2 +-
.../beam/sdk/schemas/FromRowUsingCreator.java | 2 +-
.../sdk/schemas/GetterBasedSchemaProvider.java | 3 +-
.../apache/beam/sdk/schemas/JavaBeanSchema.java | 16 +-
.../apache/beam/sdk/schemas/JavaFieldSchema.java | 10 +-
.../java/org/apache/beam/sdk/schemas/Schema.java | 19 +-
.../org/apache/beam/sdk/schemas/SchemaCoder.java | 2 +-
.../apache/beam/sdk/schemas/SchemaTranslation.java | 2 +
.../io/payloads/JsonPayloadSerializerProvider.java | 53 +-
.../apache/beam/sdk/schemas/logicaltypes/Date.java | 20 +-
.../sdk/schemas/logicaltypes/EnumerationType.java | 2 +-
.../beam/sdk/schemas/logicaltypes/OneOfType.java | 2 +-
.../schemas/logicaltypes/UnknownLogicalType.java | 2 +-
.../apache/beam/sdk/schemas/transforms/Group.java | 4 +-
.../apache/beam/sdk/schemas/transforms/Select.java | 7 +-
.../providers/LoggingTransformProvider.java | 6 +-
.../beam/sdk/schemas/utils/ByteBuddyUtils.java | 2 +-
.../apache/beam/sdk/schemas/utils/YamlUtils.java | 33 +-
.../java/org/apache/beam/sdk/testing/PAssert.java | 2 +-
.../org/apache/beam/sdk/testing/StaticWindows.java | 3 +-
.../beam/sdk/testing/TestOutputReceiver.java | 2 +
.../beam/sdk/transforms/ApproximateQuantiles.java | 9 +-
.../beam/sdk/transforms/ApproximateUnique.java | 4 +-
.../org/apache/beam/sdk/transforms/CombineFns.java | 2 +-
.../java/org/apache/beam/sdk/transforms/Count.java | 2 +-
.../java/org/apache/beam/sdk/transforms/DoFn.java | 13 +-
.../beam/sdk/transforms/DoFnOutputReceivers.java | 29 +-
.../org/apache/beam/sdk/transforms/DoFnTester.java | 52 +-
.../org/apache/beam/sdk/transforms/Flatten.java | 2 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 24 +-
.../org/apache/beam/sdk/transforms/JsonToRow.java | 6 +-
.../org/apache/beam/sdk/transforms/MapKeys.java | 1 -
.../beam/sdk/transforms/PeriodicSequence.java | 2 +-
.../apache/beam/sdk/transforms/Redistribute.java | 7 +-
.../java/org/apache/beam/sdk/transforms/Reify.java | 11 +-
.../java/org/apache/beam/sdk/transforms/Sum.java | 6 +-
.../beam/sdk/transforms/display/DisplayData.java | 2 +
.../beam/sdk/transforms/join/RawUnionValue.java | 2 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 156 +-
.../beam/sdk/transforms/reflect/DoFnInvoker.java | 15 +
.../beam/sdk/transforms/reflect/DoFnSignature.java | 31 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 14 +-
.../transforms/resourcehints/ResourceHints.java | 2 +-
.../beam/sdk/transforms/windowing/AfterAll.java | 5 +
.../beam/sdk/transforms/windowing/AfterEach.java | 7 +-
.../beam/sdk/transforms/windowing/AfterFirst.java | 5 +
.../beam/sdk/transforms/windowing/AfterPane.java | 5 +
.../transforms/windowing/AfterProcessingTime.java | 5 +
.../windowing/AfterSynchronizedProcessingTime.java | 5 +
.../sdk/transforms/windowing/AfterWatermark.java | 14 +-
.../sdk/transforms/windowing/DefaultTrigger.java | 5 +
.../beam/sdk/transforms/windowing/Never.java | 5 +
.../sdk/transforms/windowing/OrFinallyTrigger.java | 5 +
.../beam/sdk/transforms/windowing/PaneInfo.java | 6 +-
.../beam/sdk/transforms/windowing/Repeatedly.java | 5 +
.../sdk/transforms/windowing/ReshuffleTrigger.java | 5 +
.../beam/sdk/transforms/windowing/Trigger.java | 3 +
.../sdk/transforms/windowing/TriggerVisitor.java | 53 +-
.../beam/sdk/util/ByteStringOutputStream.java | 12 +-
.../org/apache/beam/sdk/util/FluentBackoff.java | 2 +-
.../org/apache/beam/sdk/util/HistogramData.java | 2 +-
.../org/apache/beam/sdk/util/InstanceBuilder.java | 24 +-
.../java/org/apache/beam/sdk/util/MoreFutures.java | 4 +
.../org/apache/beam/sdk/util/MutationDetector.java | 2 +-
.../beam/sdk/util/RowJsonValueExtractors.java | 5 +-
.../beam/sdk/util/RowStringInterpolator.java | 8 +-
.../java/org/apache/beam/sdk/util/StreamUtils.java | 47 +-
.../util/UnboundedScheduledExecutorService.java | 12 +-
.../apache/beam/sdk/util/UnownedInputStream.java | 4 +
.../main/java/org/apache/beam/sdk/util/VarInt.java | 12 +-
.../org/apache/beam/sdk/util/WindowTracing.java | 3 +-
.../sdk/util/construction/PTransformMatchers.java | 2 +-
.../util/construction/PTransformReplacements.java | 6 +-
.../util/construction/PTransformTranslation.java | 7 +-
.../sdk/util/construction/ParDoTranslation.java | 2 +-
.../construction/SplittableParDoNaiveBounded.java | 38 +-
.../apache/beam/sdk/util/construction/Timer.java | 32 +-
.../sdk/util/construction/TransformUpgrader.java | 8 +-
.../sdk/util/construction/TriggerTranslation.java | 165 +-
.../sdk/util/construction/UnknownCoderWrapper.java | 2 +-
.../construction/resources/PipelineResources.java | 9 +-
.../org/apache/beam/sdk/values/CausedByDrain.java | 7 +-
.../org/apache/beam/sdk/values/OutputBuilder.java | 2 +-
.../apache/beam/sdk/values/PCollectionViews.java | 1 +
.../java/org/apache/beam/sdk/values/RowUtils.java | 2 +-
.../beam/sdk/values/ValueInSingleWindow.java | 24 +-
.../org/apache/beam/sdk/values/WindowedValue.java | 2 +-
.../org/apache/beam/sdk/values/WindowedValues.java | 94 +-
.../org/apache/beam/sdk/RequiresStableInputIT.java | 7 +-
.../test/java/org/apache/beam/sdk/TestUtils.java | 10 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 1 -
.../apache/beam/sdk/coders/CoderRegistryTest.java | 4 +-
.../org/apache/beam/sdk/coders/RowCoderTest.java | 19 +
.../beam/sdk/coders/SerializableCoderTest.java | 7 +-
.../coders/TimestampPrefixingWindowCoderTest.java | 2 +-
.../fn/data/BeamFnDataOutboundAggregatorTest.java | 5 +-
.../java/org/apache/beam/sdk/io/FileIOTest.java | 139 ++
.../apache/beam/sdk/io/GenerateSequenceTest.java | 5 +-
.../org/apache/beam/sdk/io/TextIOReadTest.java | 8 +-
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 155 +-
.../apache/beam/sdk/lineage/LineagePluginTest.java | 272 +++
.../org/apache/beam/sdk/lineage/TestLineage.java | 77 +
.../sdk/options/PipelineOptionsFactoryTest.java | 5 +-
.../sdk/options/ProxyInvocationHandlerTest.java | 8 +-
.../beam/sdk/schemas/AutoValueSchemaTest.java | 4 +-
.../beam/sdk/schemas/JavaBeanSchemaTest.java | 14 +-
.../beam/sdk/schemas/JavaFieldSchemaTest.java | 14 +-
.../beam/sdk/schemas/JsonSchemaConversionTest.java | 15 +-
.../apache/beam/sdk/schemas/SchemaCoderTest.java | 10 +-
.../beam/sdk/schemas/SchemaTranslationTest.java | 3 +
.../sdk/schemas/logicaltypes/LogicalTypesTest.java | 4 +-
.../beam/sdk/schemas/transforms/CoGroupTest.java | 2 +-
.../beam/sdk/schemas/transforms/ConvertTest.java | 8 +-
.../beam/sdk/schemas/transforms/SelectTest.java | 8 +-
.../TypedSchemaTransformProviderTest.java | 4 +-
.../beam/sdk/schemas/utils/JsonUtilsTest.java | 8 +-
.../beam/sdk/schemas/utils/TestJavaBeans.java | 41 +-
.../apache/beam/sdk/schemas/utils/TestPOJOs.java | 53 +-
.../apache/beam/sdk/testing/ExpectedLogsTest.java | 22 +-
.../apache/beam/sdk/testing/TestPipelineTest.java | 9 +-
.../apache/beam/sdk/transforms/CombineFnsTest.java | 2 +-
.../org/apache/beam/sdk/transforms/CreateTest.java | 7 +-
.../apache/beam/sdk/transforms/DoFnTesterTest.java | 2 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 12 +-
.../sdk/transforms/MetadataPropagationTest.java | 83 +
.../beam/sdk/transforms/ParDoSchemaTest.java | 8 +-
.../org/apache/beam/sdk/transforms/ToJsonTest.java | 6 +-
.../org/apache/beam/sdk/transforms/ViewTest.java | 1 +
.../org/apache/beam/sdk/transforms/WaitTest.java | 2 +-
.../sdk/transforms/display/DisplayDataTest.java | 5 +-
.../errorhandling/BadRecordRouterTest.java | 29 +-
.../beam/sdk/transforms/join/CoGbkResultTest.java | 2 +-
.../sdk/transforms/reflect/DefaultPackageTest.java | 2 +-
.../sdk/transforms/reflect/DoFnInvokersTest.java | 56 +-
.../reflect/DoFnSignaturesSplittableDoFnTest.java | 12 +-
.../sdk/transforms/reflect/DoFnSignaturesTest.java | 34 +-
.../GrowableOffsetRangeTrackerTest.java | 16 +-
.../beam/sdk/transforms/windowing/StubTrigger.java | 5 +
.../beam/sdk/transforms/windowing/TriggerTest.java | 10 +
.../beam/sdk/transforms/windowing/WindowTest.java | 2 +-
.../beam/sdk/util/ByteStringOutputStreamTest.java | 8 +
.../org/apache/beam/sdk/util/CoderUtilsTest.java | 10 +
.../beam/sdk/util/SerializableUtilsTest.java | 3 +-
.../org/apache/beam/sdk/util/StreamUtilsTest.java | 95 +
.../UnboundedScheduledExecutorServiceTest.java | 152 +-
.../apache/beam/sdk/util/WindowedValueTest.java | 5 +-
.../org/apache/beam/sdk/util/YamlUtilsTest.java | 30 +
.../util/construction/CombineTranslationTest.java | 2 +-
.../construction/PTransformReplacementsTest.java | 2 +-
.../construction/PTransformTranslationTest.java | 2 +-
.../util/construction/ParDoTranslationTest.java | 21 +-
.../util/construction/PipelineTranslationTest.java | 2 +-
.../sdk/util/construction/ReadTranslationTest.java | 4 +-
.../sdk/util/construction/SplittableParDoTest.java | 12 +-
.../beam/sdk/util/construction/TimerTest.java | 33 +-
.../construction/WindowIntoTranslationTest.java | 2 +-
.../WindowingStrategyTranslationTest.java | 4 +-
.../construction/graph/FieldAccessVisitorTest.java | 7 +-
.../graph/ProjectionProducerVisitorTest.java | 2 +-
.../graph/ProjectionPushdownOptimizerTest.java | 9 +-
.../construction/graph/QueryablePipelineTest.java | 2 +-
.../java/org/apache/beam/sdk/values/RowTest.java | 30 +-
.../sdk/expansion/service/ExpansionService.java | 14 +-
.../ExpansionServiceSchemaTransformProvider.java | 2 +-
.../sdk/expansion/service/TransformProvider.java | 2 +-
.../expansion/service/ExpansionServiceTest.java | 2 +-
sdks/java/extensions/avro/build.gradle | 2 +
.../beam/sdk/extensions/avro/coders/AvroCoder.java | 4 +-
.../sdk/extensions/avro/io/AvroDatumFactory.java | 2 +-
.../schemas/utils/AvroJavaTimeConversions.java | 4 +-
.../extensions/avro/schemas/utils/AvroUtils.java | 68 +-
.../sdk/extensions/avro/coders/AvroCoderTest.java | 5 +-
.../avro/io/AvroGeneratedUserFactory.java | 2 +-
.../extensions/avro/schemas/SchemaCoderTest.java | 2 +-
.../extensions/avro/schemas/TestAvroFactory.java | 2 +-
.../avro/schemas/transforms/ConvertTest.java | 4 +-
.../avro/schemas/utils/AvroUtilsTest.java | 131 +-
sdks/java/extensions/euphoria/build.gradle | 2 +-
.../euphoria/core/annotation/operator/Basic.java | 4 +-
.../euphoria/core/annotation/operator/Derived.java | 4 +-
.../core/annotation/operator/Recommended.java | 8 +-
.../extensions/euphoria/core/client/lib/Split.java | 4 +-
.../core/client/operator/AssignEventTime.java | 9 +-
.../euphoria/core/client/operator/FullJoin.java | 3 +-
.../euphoria/core/client/operator/ReduceByKey.java | 6 +-
.../core/client/operator/ReduceWindow.java | 2 +-
.../euphoria/core/client/operator/RightJoin.java | 3 +-
.../euphoria/core/client/util/SumsTest.java | 2 +-
.../SingleJvmAccumulatorProviderTest.java | 4 +-
.../collector/SingleValueCollectorTest.java | 4 +-
.../google-cloud-platform-core/build.gradle | 3 +
.../sdk/extensions/gcp/options/GcpOptions.java | 19 +-
.../sdk/extensions/gcp/util/CustomHttpErrors.java | 2 +-
.../sdk/extensions/gcp/util/GceMetadataUtil.java | 17 +-
.../beam/sdk/extensions/gcp/util/GcsUtil.java | 297 ++-
.../beam/sdk/extensions/gcp/util/GcsUtilV1.java | 20 +-
.../beam/sdk/extensions/gcp/util/GcsUtilV2.java | 484 ++++
.../extensions/gcp/util/HttpCallCustomError.java | 2 +-
.../sdk/extensions/gcp/util/HttpCallMatcher.java | 2 +-
.../LatencyRecordingHttpRequestInitializer.java | 2 +-
.../gcp/util/RetryHttpRequestInitializer.java | 20 +-
.../sdk/extensions/gcp/util/gcsfs/GcsPath.java | 17 +-
.../sdk/extensions/gcp/GcpCoreApiSurfaceTest.java | 15 +-
.../beam/sdk/extensions/gcp/util/GcsUtilIT.java | 9 +-
.../gcp/util/GcsUtilParameterizedIT.java | 590 +++++
.../beam/sdk/extensions/gcp/util/GcsUtilTest.java | 207 +-
.../sdk/extensions/gcp/util/gcsfs/GcsPathTest.java | 22 +
.../extensions/jackson/JacksonTransformsTest.java | 2 +-
.../kafka/factories/FileAwareFactoryFn.java | 8 +-
sdks/java/extensions/kryo/build.gradle | 2 +-
.../apache/beam/sdk/extensions/kryo/KryoCoder.java | 2 +-
.../beam/sdk/extensions/kryo/KryoCoderTest.java | 4 +-
.../beam/sdk/extensions/ml/DLPDeidentifyText.java | 53 +-
.../beam/sdk/extensions/ml/DLPInspectText.java | 40 +-
.../beam/sdk/extensions/ml/DLPReidentifyText.java | 52 +-
.../ml/RecommendationAICreateCatalogItem.java | 18 +-
.../ml/RecommendationAIImportCatalogItems.java | 33 +-
.../ml/RecommendationAIImportUserEvents.java | 41 +-
.../sdk/extensions/ml/RecommendationAIPredict.java | 32 +-
.../ml/RecommendationAIWriteUserEvent.java | 24 +-
.../ordered/ContiguousSequenceRange.java | 6 +-
.../ordered/GlobalSequencesProcessorDoFn.java | 14 +-
.../ordered/OrderedEventProcessorResult.java | 14 +-
.../ordered/OrderedProcessingHandler.java | 4 +-
.../ordered/OrderedProcessingStatus.java | 30 +-
.../beam/sdk/extensions/ordered/ProcessorDoFn.java | 13 +-
.../ordered/SequencePerKeyProcessorDoFn.java | 4 +-
.../ordered/combiner/DefaultSequenceCombiner.java | 2 +-
.../ordered/OrderedEventProcessorTestBase.java | 2 +-
.../sdk/extensions/protobuf/DynamicProtoCoder.java | 1 +
.../beam/sdk/extensions/protobuf/ProtoCoder.java | 1 +
.../beam/sdk/extensions/python/PythonService.java | 6 +-
.../sdk/extensions/sbe/SbeLogicalTypesTest.java | 7 +-
.../extensions/sketching/ApproximateDistinct.java | 5 +-
.../sorter/BufferedExternalSorterTest.java | 4 +-
.../extensions/sql/expansion-service/build.gradle | 1 -
sdks/java/extensions/sql/iceberg/build.gradle | 2 +
.../provider/iceberg/IcebergAlterTableOps.java | 50 +
.../sql/meta/provider/iceberg/IcebergCatalog.java | 50 +-
.../meta/provider/iceberg/IcebergMetastore.java | 16 +-
.../iceberg/BeamSqlCliIcebergAlterTest.java | 231 ++
.../sdk/extensions/sql/jdbc/BeamSqlLineIT.java | 1 +
.../provider/bigquery/BigQueryIOPushDownIT.java | 2 +-
.../extensions/sql/src/main/codegen/config.fmpp | 7 +
.../sql/src/main/codegen/includes/parserImpls.ftl | 120 +-
.../sdk/extensions/sql/example/model/Customer.java | 2 +-
.../sdk/extensions/sql/example/model/Order.java | 2 +-
.../sdk/extensions/sql/impl/CatalogSchema.java | 6 +-
.../beam/sdk/extensions/sql/impl/nfa/NFA.java | 31 +-
.../sql/impl/parser/SqlAlterCatalog.java | 159 ++
.../extensions/sql/impl/parser/SqlAlterTable.java | 254 ++
.../sql/impl/parser/SqlCreateExternalTable.java | 4 +-
.../extensions/sql/impl/parser/SqlDdlNodes.java | 41 +
.../sql/impl/parser/SqlDropDatabase.java | 6 +-
.../extensions/sql/impl/parser/SqlDropTable.java | 6 +-
.../extensions/sql/impl/parser/SqlUseDatabase.java | 2 +-
.../sdk/extensions/sql/impl/rel/BeamCalcRel.java | 143 +-
.../sql/impl/rel/BeamEnumerableConverter.java | 4 +-
.../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 2 +-
.../sdk/extensions/sql/impl/rel/BeamWindowRel.java | 10 +-
.../sql/impl/rule/BeamSideInputLookupJoinRule.java | 7 +-
.../sql/impl/transform/agg/CovarianceFn.java | 2 +-
.../sql/impl/transform/agg/VarianceFn.java | 2 +-
.../extensions/sql/impl/utils/CalciteUtils.java | 33 +
.../sdk/extensions/sql/meta/catalog/Catalog.java | 3 +
.../sql/meta/catalog/CatalogManager.java | 4 +-
.../sql/meta/catalog/InMemoryCatalog.java | 8 +-
.../sql/meta/provider/AlterTableOps.java | 45 +
.../sql/meta/provider/TableProvider.java | 5 +
.../provider/bigquery/BeamSqlUnparseContext.java | 2 +-
.../sql/meta/provider/bigquery/BigQueryTable.java | 2 +-
.../sql/meta/provider/kafka/BeamKafkaTable.java | 2 +-
.../sql/meta/provider/test/AlterTestTableOps.java | 101 +
.../sql/meta/provider/test/TestTableProvider.java | 21 +
.../sql/meta/provider/text/TextTable.java | 2 +-
.../sql/meta/store/InMemoryMetaStore.java | 11 +
.../sdk/extensions/sql/BeamComplexTypeTest.java | 28 +-
.../sdk/extensions/sql/BeamSqlCliCatalogTest.java | 93 +
.../beam/sdk/extensions/sql/BeamSqlCliTest.java | 249 ++
.../extensions/sql/InferredJavaBeanSqlTest.java | 4 +-
.../apache/beam/sdk/extensions/sql/TestUtils.java | 2 +-
.../sql/impl/rel/BeamEnumerableConverterTest.java | 2 +-
.../sql/impl/schema/BeamSqlRowCoderTest.java | 7 +-
.../sql/impl/utils/CalciteUtilsTest.java | 20 +
.../meta/provider/mongodb/MongoDbReadWriteIT.java | 44 +-
.../beam/sdk/extensions/timeseries/FillGaps.java | 8 +-
.../zetasketch/ApproximateCountDistinct.java | 8 +-
.../BigQueryHllSketchCompatibilityIT.java | 7 +-
.../fn/harness/jmh/ProcessBundleBenchmark.java | 6 +-
.../java/org/apache/beam/fn/harness/Caches.java | 57 +-
.../org/apache/beam/fn/harness/CombineRunners.java | 4 +-
.../beam/fn/harness/ExternalWorkerService.java | 2 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 199 +-
.../java/org/apache/beam/fn/harness/FnHarness.java | 35 +-
.../beam/fn/harness/PrecombineGroupingTable.java | 4 +-
.../SplittablePairWithRestrictionDoFnRunner.java | 8 +
...littableSplitAndSizeRestrictionsDoFnRunner.java | 4 +-
...ittableTruncateSizedRestrictionsDoFnRunner.java | 2 +
.../fn/harness/control/ExecutionStateSampler.java | 37 +-
.../fn/harness/control/FinalizeBundleHandler.java | 182 +-
.../fn/harness/control/ProcessBundleHandler.java | 4 +-
.../harness/data/PTransformFunctionRegistry.java | 4 +-
.../apache/beam/fn/harness/debug/DataSampler.java | 3 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 5 +-
.../apache/beam/fn/harness/logging/QuotaEvent.java | 16 +-
.../apache/beam/fn/harness/state/BagUserState.java | 2 +-
.../beam/fn/harness/state/FnApiStateAccessor.java | 2 +-
.../fn/harness/state/FnApiTimerBundleTracker.java | 2 +
.../fn/harness/state/StateFetchingIterators.java | 195 +-
.../beam/fn/harness/status/BeamFnStatusClient.java | 1 +
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 4 +-
...plittablePairWithRestrictionDoFnRunnerTest.java | 4 +-
.../beam/fn/harness/WindowMergingFnRunnerTest.java | 8 +-
.../harness/control/FinalizeBundleHandlerTest.java | 49 +
.../harness/control/ProcessBundleHandlerTest.java | 15 +-
.../harness/logging/BeamFnLoggingClientTest.java | 2 +-
.../state/LazyCachingIteratorToIterableTest.java | 10 +-
.../harness/state/StateFetchingIteratorsTest.java | 39 +
.../sdk/io/aws2/common/AsyncBatchWriteHandler.java | 6 +-
.../apache/beam/sdk/io/aws2/common/ObjectPool.java | 2 +-
.../sdk/io/aws2/kinesis/EFOShardSubscriber.java | 2 +-
.../io/aws2/kinesis/EFOShardSubscribersPool.java | 23 +-
.../apache/beam/sdk/io/aws2/kinesis/KinesisIO.java | 4 +-
.../beam/sdk/io/aws2/kinesis/KinesisRecord.java | 8 +-
.../io/aws2/kinesis/RateLimitPolicyFactory.java | 6 +-
.../beam/sdk/io/aws2/kinesis/ShardCheckpoint.java | 2 +-
.../beam/sdk/io/aws2/kinesis/StartingPoint.java | 2 +-
.../sdk/io/aws2/schemas/AwsSchemaProvider.java | 2 +-
.../org/apache/beam/sdk/io/aws2/sns/SnsIO.java | 2 +-
.../beam/sdk/io/aws2/sqs/SqsCheckpointMark.java | 2 +-
.../org/apache/beam/sdk/io/aws2/sqs/SqsIO.java | 7 +-
.../io/aws2/common/AsyncBatchWriteHandlerTest.java | 1 +
.../sdk/io/aws2/dynamodb/DynamoDBIOWriteTest.java | 2 +-
.../sdk/io/aws2/dynamodb/testing/DynamoDBIOIT.java | 2 +-
.../aws2/kinesis/EFOShardSubscribersPoolTest.java | 2 +-
.../sdk/io/aws2/kinesis/KinesisIOWriteTest.java | 10 +-
.../sdk/io/aws2/options/SerializationTestUtil.java | 2 +-
sdks/java/io/amqp/build.gradle | 4 +-
.../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 2 +-
.../azure/blobstore/AzureBlobStoreFileSystem.java | 1 +
.../blobstore/AzureBlobStoreFileSystemTest.java | 3 +-
.../apache/beam/sdk/io/cassandra/CassandraIO.java | 23 +-
.../apache/beam/sdk/io/cassandra/RingRange.java | 4 +-
.../beam/sdk/io/cassandra/CassandraIOTest.java | 8 +-
.../org/apache/beam/sdk/io/cdap/ConfigWrapper.java | 2 +-
.../io/cdap/PluginConfigInstantiationUtils.java | 4 +-
.../java/org/apache/beam/sdk/io/cdap/CdapIOIT.java | 2 +-
.../apache/beam/sdk/io/cdap/TestRowDBWritable.java | 6 +-
sdks/java/io/clickhouse/build.gradle | 9 +-
.../beam/sdk/io/clickhouse/ClickHouseIO.java | 424 +++-
.../sdk/io/clickhouse/ClickHouseJdbcUrlParser.java | 276 +++
.../apache/beam/sdk/io/clickhouse/TableSchema.java | 8 +-
.../beam/sdk/io/clickhouse/AtomicInsertTest.java | 28 +-
.../beam/sdk/io/clickhouse/BaseClickHouseTest.java | 140 +-
.../{ClickHouseIOTest.java => ClickHouseIOIT.java} | 192 +-
.../ClickHouseIOJdbcBackwardCompatibilityTest.java | 98 +
.../ClickHouseIOPropertyMergingTest.java | 211 ++
.../beam/sdk/io/clickhouse/ClickHouseIOTest.java | 447 +---
.../io/clickhouse/ClickHouseJdbcUrlParserTest.java | 429 ++++
.../beam/sdk/io/common/DatabaseTestHelper.java | 11 +-
.../org/apache/beam/sdk/io/common/HashingFn.java | 2 +-
.../beam/sdk/io/common/NetworkTestHelper.java | 2 +-
.../beam/sdk/io/common/SchemaAwareJavaBeans.java | 36 +-
.../org/apache/beam/sdk/io/common/TestRow.java | 10 +-
sdks/java/io/components/build.gradle | 5 +
.../io/components/deadletterqueue/DLQRouter.java | 21 +-
.../components/deadletterqueue}/package-info.java | 4 +-
.../deadletterqueue/sinks/ThrowingSink.java | 3 +-
.../deadletterqueue/sinks}/package-info.java | 4 +-
.../components/ratelimiter/EnvoyRateLimiter.java | 32 +-
.../ratelimiter/EnvoyRateLimiterContext.java | 65 +
.../ratelimiter/EnvoyRateLimiterFactory.java | 239 ++
.../sdk/io/components/ratelimiter/RateLimiter.java | 41 +
.../ratelimiter/RateLimiterClientCache.java | 103 +
.../components/ratelimiter/RateLimiterContext.java | 11 +-
.../components/ratelimiter/RateLimiterFactory.java | 57 +
.../components/ratelimiter/RateLimiterOptions.java | 68 +
.../io/components/ratelimiter}/package-info.java | 4 +-
.../components/throttling/ThrottlingSignaler.java | 11 +-
.../io/components/throttling}/package-info.java | 4 +-
.../components/deadletterqueue/DLQRouterTest.java | 30 +-
.../ratelimiter/EnvoyRateLimiterTest.java | 168 ++
.../ratelimiter/RateLimiterClientCacheTest.java | 115 +
.../ratelimiter/RateLimiterOptionsTest.java | 81 +
.../io/contextualtextio/ContextualTextIOTest.java | 12 +-
.../beam/sdk/io/datadog/DatadogEventPublisher.java | 10 +-
.../org/apache/beam/sdk/io/datadog/DatadogIO.java | 2 -
sdks/java/io/debezium/build.gradle | 8 +-
.../io/debezium/expansion-service/build.gradle | 2 +-
sdks/java/io/debezium/src/README.md | 7 +-
.../org/apache/beam/io/debezium/DebeziumIO.java | 104 +-
.../DebeziumReadSchemaTransformProvider.java | 4 +-
.../io/debezium/DebeziumTransformRegistrar.java | 47 +
.../beam/io/debezium/FileSystemOffsetRetainer.java | 166 ++
.../apache/beam/io/debezium/KafkaConnectUtils.java | 27 +-
.../beam/io/debezium/KafkaSourceConsumerFn.java | 175 +-
.../apache/beam/io/debezium/OffsetRetainer.java | 69 +
.../io/debezium/DebeziumIOMySqlConnectorIT.java | 6 +-
.../debezium/DebeziumIOPostgresSqlConnectorIT.java | 4 +-
.../apache/beam/io/debezium/DebeziumIOTest.java | 153 ++
.../io/debezium/FileSystemOffsetRetainerTest.java | 131 ++
.../beam/io/debezium/KafkaConnectSchemaTest.java | 41 +-
.../io/debezium/KafkaSourceConsumerFnTest.java | 89 +-
.../apache/beam/io/debezium/OffsetTrackerTest.java | 25 +-
.../io/elasticsearch/ElasticsearchIOTestUtils.java | 3 +-
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 18 +-
sdks/java/io/expansion-service/build.gradle | 20 +-
.../beam/sdk/io/common/FileBasedIOITHelper.java | 3 +-
.../sdk/io/fileschematransform/XmlRowAdapter.java | 2 +-
.../sdk/io/fileschematransform/XmlRowValue.java | 2 +-
.../FileReadSchemaTransformFormatProviderTest.java | 2 +-
.../JsonReadSchemaTransformFormatProviderTest.java | 8 +-
.../io/fileschematransform/XmlRowValueTest.java | 14 +-
sdks/java/io/google-ads/build.gradle | 4 +-
.../apache/beam/sdk/io/googleads/GoogleAdsIO.java | 6 +-
.../{GoogleAdsV19.java => GoogleAdsV23.java} | 56 +-
.../sdk/io/googleads/DummyRateLimitPolicy.java | 4 +-
.../beam/sdk/io/googleads/GoogleAdsIOTest.java | 40 +-
.../io/googleads/MockGoogleAdsClientFactory.java | 10 +-
sdks/java/io/google-cloud-platform/build.gradle | 16 +-
.../beam/sdk/io/gcp/bigquery/AppendClientInfo.java | 26 +-
.../AvroGenericRecordToStorageApiProto.java | 6 +-
.../io/gcp/bigquery/BeamRowToStorageApiProto.java | 14 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 15 +-
.../bigquery/BigQueryCoderProviderRegistrar.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 103 +-
.../sdk/io/gcp/bigquery/BigQueryIOTranslation.java | 47 +-
.../sdk/io/gcp/bigquery/BigQueryInsertError.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 8 +
.../beam/sdk/io/gcp/bigquery/BigQueryServices.java | 4 +
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 44 +-
.../sdk/io/gcp/bigquery/BigQuerySinkMetrics.java | 19 +-
.../io/gcp/bigquery/BigQueryStorageSourceBase.java | 71 +-
.../gcp/bigquery/BigQueryStorageStreamSource.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 27 +-
.../sdk/io/gcp/bigquery/ConvertMessagesDoFn.java | 297 +++
.../gcp/bigquery/DynamicDestinationsHelpers.java | 2 +-
.../sdk/io/gcp/bigquery/MergeSchemaCombineFn.java | 76 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 3 +-
.../sdk/io/gcp/bigquery/PatchTableSchemaDoFn.java | 163 ++
.../beam/sdk/io/gcp/bigquery/RetryManager.java | 8 +
.../io/gcp/bigquery/RowMutationInformation.java | 16 +-
.../sdk/io/gcp/bigquery/SchemaUpdateHoldingFn.java | 259 ++
.../sdk/io/gcp/bigquery/SplittingIterable.java | 76 +-
.../io/gcp/bigquery/StorageApiConvertMessages.java | 299 ++-
.../bigquery/StorageApiDynamicDestinations.java | 15 +-
.../StorageApiDynamicDestinationsBeamRow.java | 16 +-
...StorageApiDynamicDestinationsGenericRecord.java | 17 +-
.../StorageApiDynamicDestinationsProto.java | 16 +-
.../StorageApiDynamicDestinationsTableRow.java | 139 +-
.../gcp/bigquery/StorageApiFinalizeWritesDoFn.java | 16 +-
.../bigquery/StorageApiFlushAndFinalizeDoFn.java | 13 +-
.../beam/sdk/io/gcp/bigquery/StorageApiLoads.java | 23 +-
.../io/gcp/bigquery/StorageApiWritePayload.java | 9 +
.../bigquery/StorageApiWriteUnshardedRecords.java | 146 +-
.../bigquery/StorageApiWritesShardedRecords.java | 453 ++--
.../io/gcp/bigquery/StreamingInsertsMetrics.java | 2 +-
.../io/gcp/bigquery/TableRowToStorageApiProto.java | 517 +++-
.../beam/sdk/io/gcp/bigquery/TableSchemaCache.java | 121 +-
.../bigquery/TwoLevelMessageConverterCache.java | 9 +-
.../io/gcp/bigquery/UpdateSchemaDestination.java | 11 +-
.../sdk/io/gcp/bigquery/UpgradeTableSchema.java | 308 +++
.../beam/sdk/io/gcp/bigquery/WritePartition.java | 6 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 8 +-
...ueryStorageWriteApiSchemaTransformProvider.java | 4 +-
.../io/gcp/bigtable/BigtableConfigTranslator.java | 2 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 48 +-
.../sdk/io/gcp/bigtable/BigtableReadOptions.java | 12 +
.../io/gcp/bigtable/BigtableServiceFactory.java | 30 +-
.../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 31 +-
.../BigtableWriteSchemaTransformProvider.java | 2 +-
.../action/DetectNewPartitionsAction.java | 8 +-
.../gcp/bigtable/changestreams/dao/DaoFactory.java | 1 +
.../changestreams/dao/MetadataTableAdminDao.java | 2 +-
.../changestreams/dao/MetadataTableDao.java | 12 +-
.../changestreams/dofn/InitializeDoFn.java | 6 +-
.../beam/sdk/io/gcp/datastore/EntityToRow.java | 2 +-
.../beam/sdk/io/gcp/datastore/RowToEntity.java | 2 +-
.../beam/sdk/io/gcp/firestore/FirestoreDoFn.java | 8 +-
.../sdk/io/gcp/firestore/FirestoreV1WriteFn.java | 2 +-
.../apache/beam/sdk/io/gcp/firestore/RpcQos.java | 10 +-
.../beam/sdk/io/gcp/firestore/RpcQosImpl.java | 5 +-
.../apache/beam/sdk/io/gcp/healthcare/FhirIO.java | 41 +-
.../io/gcp/healthcare/FhirIOPatientEverything.java | 7 +-
.../sdk/io/gcp/healthcare/FhirSearchParameter.java | 2 +-
.../apache/beam/sdk/io/gcp/healthcare/HL7v2IO.java | 34 +-
.../io/gcp/healthcare/HttpHealthcareApiClient.java | 14 +-
.../sdk/io/gcp/pubsub/AddTimestampAttribute.java | 7 +-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 8 +-
.../beam/sdk/io/gcp/pubsub/PubsubMessages.java | 52 +-
.../pubsub/PubsubReadSchemaTransformProvider.java | 2 +-
.../beam/sdk/io/gcp/pubsub/PubsubRowToMessage.java | 8 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSource.java | 2 +-
.../apache/beam/sdk/io/gcp/pubsub/TestPubsub.java | 1 +
.../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java | 1 +
.../beam/sdk/io/gcp/spanner/BatchSpannerRead.java | 4 +-
.../beam/sdk/io/gcp/spanner/MutationUtils.java | 4 +-
.../beam/sdk/io/gcp/spanner/NaiveSpannerRead.java | 4 +-
.../beam/sdk/io/gcp/spanner/OrderedCode.java | 2 +-
.../beam/sdk/io/gcp/spanner/ReadSpannerSchema.java | 2 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 19 +
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 25 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 176 +-
.../SpannerWriteSchemaTransformProvider.java | 4 +-
.../changestreams/ChangeStreamsConstants.java | 23 +
.../changestreams/action/ActionFactory.java | 14 +-
.../action/ChildPartitionsRecordAction.java | 8 +-
.../action/DetectNewPartitionsAction.java | 9 +-
.../action/HeartbeatRecordAction.java | 14 +-
.../action/PartitionStartRecordAction.java | 6 +-
.../action/QueryChangeStreamAction.java | 89 +-
.../spanner/changestreams/dao/ChangeStreamDao.java | 50 +-
.../changestreams/dao/ChangeStreamResultSet.java | 31 +
.../gcp/spanner/changestreams/dao/DaoFactory.java | 16 +
.../changestreams/dao/PartitionMetadataDao.java | 186 +-
.../spanner/changestreams/dofn/InitializeDoFn.java | 93 +-
.../dofn/ReadChangeStreamPartitionDoFn.java | 26 +-
.../mapper/ChangeStreamRecordMapper.java | 12 +-
.../mapper/PartitionMetadataMapper.java | 5 +-
.../changestreams/model/PartitionMetadata.java | 32 +-
.../beam/sdk/io/gcp/testing/BigqueryClient.java | 33 +-
.../beam/sdk/io/gcp/testing/BigqueryMatcher.java | 4 +-
.../sdk/io/gcp/testing/FakeBigQueryServices.java | 3 +-
.../sdk/io/gcp/testing/FakeDatasetService.java | 251 +-
.../cloud/spanner/FakeBatchTransactionId.java | 2 +-
.../AvroGenericRecordToStorageApiProtoTest.java | 61 +-
.../gcp/bigquery/BeamRowToStorageApiProtoTest.java | 82 +-
.../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 2 +-
.../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 110 +-
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 212 +-
.../StorageApiDataTriggeredSchemaUpdateIT.java | 348 +++
.../bigquery/TableRowToStorageApiProtoTest.java | 493 +++-
.../io/gcp/bigquery/UpgradeTableSchemaTest.java | 361 +++
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 107 +-
.../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 47 +-
.../BigtableReadSchemaTransformProviderIT.java | 7 +-
.../io/gcp/bigtable/BigtableServiceImplTest.java | 5 +-
.../io/gcp/bigtable/BigtableSharedClientTest.java | 8 +-
...gtableSimpleWriteSchemaTransformProviderIT.java | 6 +-
.../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 5 +-
.../BigtableWriteSchemaTransformProviderIT.java | 6 +-
.../ReadChangeStreamPartitionActionTest.java | 10 +-
.../changestreams/dao/MetadataTableDaoTest.java | 2 +-
.../dofn/ReadChangeStreamPartitionDoFnTest.java | 2 +-
.../estimator/NullSizeEstimatorTest.java | 6 +-
.../changestreams/it/BigtableChangeStreamIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 1 +
...artitionQueryResponseToRunQueryRequestTest.java | 2 +-
.../beam/sdk/io/gcp/pubsub/PubsubMessagesTest.java | 176 ++
.../sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java | 2 +-
.../sdk/io/gcp/spanner/ReadSpannerSchemaTest.java | 4 +-
.../gcp/spanner/SpannerIOReadChangeStreamTest.java | 61 +
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 26 +-
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 12 +-
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 12 +-
.../SpannerChangeStreamErrorTest.java | 10 +-
.../action/ChildPartitionsRecordActionTest.java | 6 +-
.../action/HeartbeatRecordActionTest.java | 149 +-
.../action/PartitionStartRecordActionTest.java | 39 +-
.../action/QueryChangeStreamActionTest.java | 289 ++-
.../changestreams/dao/ChangeStreamDaoTest.java | 65 +-
.../dao/ChangeStreamResultSetTest.java | 59 +
.../dao/PartitionMetadataDaoTest.java | 117 +-
.../changestreams/dofn/InitializeDoFnTest.java | 80 +-
.../dofn/ReadChangeStreamPartitionDoFnTest.java | 42 +-
.../changestreams/it/IntegrationTestEnv.java | 67 +-
...StreamOrderedByTimestampAndTransactionIdIT.java | 86 +-
...nnerChangeStreamOrderedWithinKeyGloballyIT.java | 16 +-
.../it/SpannerChangeStreamOrderedWithinKeyIT.java | 22 +-
.../it/SpannerChangeStreamPlacementTableIT.java | 53 +-
...annerChangeStreamPlacementTablePostgresIT.java} | 40 +-
.../it/SpannerChangeStreamPostgresIT.java | 3 +-
...SpannerChangeStreamTransactionBoundariesIT.java | 18 +-
.../mapper/ChangeStreamRecordMapperTest.java | 105 +
.../mapper/PartitionMetadataMapperTest.java | 11 +-
.../changestreams/model/ModelEncodingTest.java | 3 +-
.../changestreams/model/PartitionMetadataTest.java | 56 +-
.../restriction/TimestampRangeTrackerTest.java | 7 +-
.../spanner/changestreams/util/TestJsonMapper.java | 10 +-
.../beam/sdk/io/gcp/storage/GcsKmsKeyIT.java | 7 +-
.../apache/beam/sdk/io/gcp/storage/GcsMatchIT.java | 9 +-
.../beam/sdk/io/hadoop/format/HadoopFormatIO.java | 14 +-
.../apache/beam/sdk/io/hadoop/format/Employee.java | 2 +-
.../sdk/io/hadoop/format/TestRowDBWritable.java | 6 +-
sdks/java/io/hbase/build.gradle | 2 +-
.../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 14 +-
.../beam/sdk/io/hbase/HBaseRowMutationsCoder.java | 2 -
.../beam/sdk/io/hbase/HBaseSharedConnection.java | 2 +-
.../org/apache/beam/sdk/io/hbase/HBaseIOIT.java | 2 +-
.../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 2 +-
.../sdk/io/hcatalog/HiveDatabaseTestHelper.java | 17 +-
sdks/java/io/iceberg/build.gradle | 12 +-
sdks/java/io/iceberg/hive/build.gradle | 2 +-
.../org/apache/beam/sdk/io/iceberg/AddFiles.java | 901 +++++++
.../iceberg/AddFilesSchemaTransformProvider.java | 199 ++
.../beam/sdk/io/iceberg/AppendFilesToTables.java | 18 +-
.../apache/beam/sdk/io/iceberg/FilterUtils.java | 31 +-
.../beam/sdk/io/iceberg/IcebergCatalogConfig.java | 149 +-
.../beam/sdk/io/iceberg/IcebergScanConfig.java | 20 +-
.../sdk/io/iceberg/IcebergTableCreateConfig.java | 14 +
.../apache/beam/sdk/io/iceberg/IcebergUtils.java | 25 +-
.../IcebergWriteSchemaTransformProvider.java | 13 +
.../apache/beam/sdk/io/iceberg/PartitionUtils.java | 42 +-
.../io/iceberg/PortableIcebergDestinations.java | 4 +
.../org/apache/beam/sdk/io/iceberg/ReadUtils.java | 40 +-
.../apache/beam/sdk/io/iceberg/RecordWriter.java | 25 +-
.../beam/sdk/io/iceberg/RecordWriterManager.java | 60 +-
.../apache/beam/sdk/io/iceberg/ScanTaskReader.java | 5 +-
.../beam/sdk/io/iceberg/SerializableDataFile.java | 2 +-
.../apache/beam/sdk/io/iceberg/SortOrderUtils.java | 135 ++
.../sdk/io/iceberg/WriteDirectRowsToFiles.java | 2 +-
.../org/apache/beam/sdk/io/iceberg/AddFilesIT.java | 538 +++++
.../apache/beam/sdk/io/iceberg/AddFilesTest.java | 709 ++++++
.../beam/sdk/io/iceberg/FilterUtilsTest.java | 22 +-
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 30 +
.../sdk/io/iceberg/RecordWriterManagerTest.java | 178 +-
.../beam/sdk/io/iceberg/SortOrderUtilsTest.java | 125 +
.../io/iceberg/catalog/IcebergCatalogBaseIT.java | 72 +-
.../sdk/io/iceberg/catalog/RESTCatalogBLMSIT.java | 75 +
.../io/iceberg/catalog/hiveutils/ScriptRunner.java | 2 +-
.../apache/beam/sdk/io/influxdb/GenerateData.java | 10 +-
.../beam/sdk/io/influxdb/InfluxDbIOTest.java | 1 +
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 6 +-
.../io/jdbc/JdbcReadSchemaTransformProvider.java | 4 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcUtil.java | 17 +-
.../io/jdbc/JdbcWriteSchemaTransformProvider.java | 6 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 5 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 3 +-
.../beam/sdk/io/jdbc/JdbcSchemaIOProviderTest.java | 3 +-
.../apache/beam/sdk/io/jdbc/JdbcTestHelper.java | 2 +-
.../org/apache/beam/sdk/io/jdbc/JdbcUtilTest.java | 7 +-
sdks/java/io/jms/build.gradle | 2 +-
.../apache/beam/sdk/io/jms/JmsCheckpointMark.java | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 10 +-
.../jms/MockNonSerializableConnectionFactory.java | 21 +
.../beam/sdk/io/kafka/KafkaExactlyOnceSink.java | 28 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 182 +-
.../KafkaIOReadImplementationCompatibility.java | 8 +-
.../org/apache/beam/sdk/io/kafka/KafkaMetrics.java | 2 +-
.../io/kafka/KafkaReadSchemaTransformProvider.java | 31 +-
.../beam/sdk/io/kafka/KafkaSourceDescriptor.java | 4 +-
.../beam/sdk/io/kafka/KafkaUnboundedReader.java | 124 +-
.../beam/sdk/io/kafka/KafkaUnboundedSource.java | 8 +-
.../org/apache/beam/sdk/io/kafka/KafkaWriter.java | 2 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 6 +-
.../sdk/io/kafka/WatchForKafkaTopicPartitions.java | 6 +-
.../org/apache/beam/sdk/io/kafka/KafkaDlqTest.java | 14 +-
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 98 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 5 +-
...KafkaIOReadImplementationCompatibilityTest.java | 16 +-
.../KafkaUnboundedReaderIdlePartitionTest.java | 273 +++
.../KafkaWriteSchemaTransformProviderTest.java | 22 +-
.../sdk/io/kafka/upgrade/KafkaIOTranslation.java | 7 +
.../io/kafka/upgrade/KafkaIOTranslationTest.java | 2 +
.../java/org/apache/beam/sdk/io/kudu/KuduIO.java | 6 +-
.../org/apache/beam/sdk/io/kudu/KuduIOTest.java | 4 +
sdks/java/io/mongodb/build.gradle | 2 +-
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 2 +-
.../org/apache/beam/sdk/io/mongodb/MongoDbIO.java | 8 +-
.../beam/sdk/io/mongodb/MongoDBGridFSIOTest.java | 66 +-
.../apache/beam/sdk/io/mongodb/MongoDBIOIT.java | 3 +-
.../apache/beam/sdk/io/mongodb/MongoDbIOTest.java | 73 +-
.../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 8 +-
.../java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java | 19 +-
.../org/apache/beam/sdk/io/parquet/ParquetIO.java | 90 +-
.../apache/beam/sdk/io/parquet/ParquetIOTest.java | 92 +
.../apache/beam/sdk/io/pulsar/PulsarIOUtils.java | 3 -
.../org/apache/beam/sdk/io/pulsar/PulsarIOIT.java | 2 +-
.../apache/beam/sdk/io/pulsar/PulsarIOTest.java | 9 +-
.../beam/sdk/io/rabbitmq/RabbitMqMessage.java | 7 +-
.../beam/sdk/io/rabbitmq/ExchangeTestPlan.java | 2 +-
sdks/java/io/redis/build.gradle | 2 +-
.../org/apache/beam/sdk/io/redis/RedisCursor.java | 2 +-
.../org/apache/beam/sdk/io/redis/RedisIOTest.java | 60 +-
.../org/apache/beam/io/requestresponse/Cache.java | 4 +
.../org/apache/beam/io/requestresponse/Call.java | 55 +-
.../apache/beam/io/requestresponse/Monitoring.java | 170 +-
.../apache/beam/io/requestresponse/Repeater.java | 1 +
.../beam/io/requestresponse/RequestResponseIO.java | 6 +-
.../beam/io/requestresponse/SetupTeardown.java | 2 +-
.../apache/beam/io/requestresponse/CallTest.java | 4 +-
.../EchoGRPCCallerWithSetupTeardown.java | 1 +
.../io/requestresponse/RequestResponseIOTest.java | 63 +-
.../beam/sdk/io/singlestore/SingleStoreIO.java | 9 +-
.../apache/beam/sdk/io/singlestore/TestHelper.java | 2 +-
sdks/java/io/snowflake/build.gradle | 8 +-
.../apache/beam/sdk/io/snowflake/SnowflakeIO.java | 64 +-
.../sdk/io/snowflake/data/SnowflakeDataType.java | 51 +-
.../test/FakeSnowflakeBasicDataSource.java | 6 +-
.../test/FakeSnowflakeBatchServiceImpl.java | 2 +-
.../io/snowflake/test/FakeSnowflakeDatabase.java | 12 +-
.../test/unit/DataSourceConfigurationTest.java | 2 +-
.../test/unit/write/SchemaDispositionTest.java | 2 +-
.../test/unit/write/SnowflakeIOWriteTest.java | 2 +-
.../test/unit/write/StreamingWriteTest.java | 2 +-
.../beam/sdk/io/solace/broker/SessionService.java | 14 +-
.../org/apache/beam/sdk/io/solace/data/Solace.java | 4 +-
.../sdk/io/solace/read/SolaceCheckpointMark.java | 6 +-
...entialsBasicAuthJcsmpSessionServiceFactory.java | 2 +-
.../sdk/io/solace/it/SolaceContainerManager.java | 2 +-
.../java/org/apache/beam/sdk/io/solr/SolrIO.java | 1 +
.../beam/sdk/io/sparkreceiver/HasOffset.java | 8 +-
.../beam/sdk/io/sparkreceiver/ReceiverBuilder.java | 4 +-
.../sparkreceiver/RabbitMqReceiverWithOffset.java | 4 +-
.../sdk/io/sparkreceiver/SparkReceiverIOIT.java | 7 +-
.../beam/sdk/io/splunk/HttpEventPublisher.java | 8 +-
.../beam/sdk/io/synthetic/BundleSplitter.java | 2 +-
.../org/apache/beam/sdk/io/thrift/ThriftIO.java | 6 +-
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 14 +-
.../testing/TestSchemaTransformProvider.java | 6 +-
.../src/main/resources/archetype-resources/pom.xml | 7 +
.../src/main/resources/archetype-resources/pom.xml | 7 +
.../src/main/java/StarterPipeline.java | 19 +-
sdks/java/ml/inference/openai/build.gradle | 9 +-
.../ml/inference/openai/OpenAIModelHandler.java | 85 +-
.../sdk/ml/inference/openai/OpenAIModelInput.java | 7 +-
.../ml/inference/openai/OpenAIModelParameters.java | 23 +-
.../ml/inference/openai/OpenAIModelResponse.java | 6 +-
.../sdk/ml/inference/openai}/package-info.java | 4 +-
.../ml/inference/openai/OpenAIModelHandlerIT.java | 597 ++---
.../inference/openai/OpenAIModelHandlerTest.java | 163 +-
sdks/java/ml/inference/remote/build.gradle | 6 +-
.../beam/sdk/ml/inference/remote/BaseInput.java | 12 +-
.../sdk/ml/inference/remote/BaseModelHandler.java | 55 +-
.../ml/inference/remote/BaseModelParameters.java | 26 +-
.../beam/sdk/ml/inference/remote/BaseResponse.java | 16 +-
.../sdk/ml/inference/remote/PredictionResult.java | 12 +-
.../sdk/ml/inference/remote/RemoteInference.java | 115 +-
.../beam/sdk/ml/inference/remote/RetryHandler.java | 46 +-
.../sdk/ml/inference/remote}/package-info.java | 4 +-
.../ml/inference/remote/RemoteInferenceTest.java | 448 ++--
.../java/org/apache/beam/sdk/jpmstests/JpmsIT.java | 6 +-
.../beam/sdk/testing/TestPipelineExtension.java | 1 -
.../org/apache/beam/sdk/loadtests/LoadTest.java | 2 +-
.../beam/sdk/nexmark/NexmarkConfiguration.java | 5 +-
.../apache/beam/sdk/nexmark/NexmarkQueryName.java | 4 +-
.../org/apache/beam/sdk/nexmark/NexmarkUtils.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Auction.java | 2 +-
.../apache/beam/sdk/nexmark/model/AuctionBid.java | 2 +-
.../beam/sdk/nexmark/model/AuctionCount.java | 2 +-
.../beam/sdk/nexmark/model/AuctionPrice.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Bid.java | 2 +-
.../beam/sdk/nexmark/model/BidsPerSession.java | 2 +-
.../beam/sdk/nexmark/model/CategoryPrice.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Done.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Event.java | 2 +-
.../beam/sdk/nexmark/model/IdNameReserve.java | 2 +-
.../beam/sdk/nexmark/model/NameCityStateId.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Person.java | 2 +-
.../apache/beam/sdk/nexmark/model/SellerPrice.java | 2 +-
.../apache/beam/sdk/nexmark/queries/Query10.java | 25 +-
.../beam/sdk/nexmark/queries/Query4Model.java | 2 +-
.../apache/beam/sdk/nexmark/queries/Query5.java | 2 +-
.../beam/sdk/nexmark/queries/Query6Model.java | 2 +-
.../beam/sdk/nexmark/queries/WinningBids.java | 4 +-
.../sdk/nexmark/sources/generator/Generator.java | 2 +-
.../nexmark/sources/generator/GeneratorConfig.java | 2 +-
.../sdk/testutils/metrics/MetricsReaderTest.java | 2 +-
.../apache/beam/sdk/tpcds/BeamSqlEnvRunner.java | 1 +
.../apache/beam/sdk/tpcds/SqlTransformRunner.java | 1 +
.../beam/sdk/tpcds/TpcdsParametersReader.java | 2 -
.../org/apache/beam/sdk/tpcds/TpcdsRunResult.java | 2 +
.../sdk/testing/watermarks/WatermarkLatency.java | 2 +-
.../launcher/TransformServiceLauncher.java | 12 +-
sdks/python/apache_beam/__init__.py | 2 +-
sdks/python/apache_beam/coders/coder_impl.pxd | 4 +
sdks/python/apache_beam/coders/coder_impl.py | 26 +-
sdks/python/apache_beam/coders/coders.py | 54 +-
.../apache_beam/coders/coders_test_common.py | 219 +-
sdks/python/apache_beam/coders/row_coder.py | 5 +-
sdks/python/apache_beam/coders/row_coder_test.py | 4 +-
sdks/python/apache_beam/coders/typecoders.py | 42 +-
.../apache_beam/examples/inference/README.md | 74 +-
.../inference/pytorch_language_modeling.py | 3 +-
.../examples/inference/rate_limiter_vertex_ai.py | 2 +
.../inference/sklearn_examples_requirements.txt | 3 +-
.../examples/inference/table_row_batch_example.py | 194 ++
.../examples/inference/table_row_inference.py | 369 +++
.../examples/inference/table_row_inference_test.py | 196 ++
.../inference/table_row_inference_utils.py | 310 +++
.../examples/inference/vllm_gemma_batch.py | 1 -
.../examples/inference/vllm_text_completion.py | 53 +-
.../apache_beam/examples/rate_limiter_simple.py | 4 +
.../apache_beam/examples/snippets/snippets.py | 124 +
.../transforms/aggregation/groupby_attr.py | 13 +-
.../transforms/aggregation/groupby_attr_expr.py | 10 +-
.../transforms/aggregation/groupby_expr.py | 5 +-
.../aggregation/groupby_expr_aggregate.py | 10 +-
.../aggregation/groupby_global_aggregate.py | 6 +-
.../aggregation/groupby_simple_aggregate.py | 5 +-
.../transforms/aggregation/groupby_test.py | 21 +-
.../transforms/aggregation/groupby_two_exprs.py | 9 +-
.../internal/cloudpickle/cloudpickle.py | 23 +-
.../apache_beam/internal/cloudpickle_pickler.py | 13 +-
sdks/python/apache_beam/io/aws/s3io.py | 28 +-
sdks/python/apache_beam/io/aws/s3io_test.py | 36 +-
sdks/python/apache_beam/io/azure/blobstorageio.py | 26 +-
sdks/python/apache_beam/io/debezium.py | 44 +-
.../io/external/xlang_bigqueryio_it_test.py | 2 +-
.../io/external/xlang_debeziumio_it_test.py | 2 +-
.../io/external/xlang_jdbcio_it_test.py | 13 +-
.../apache_beam/io/gcp/bigquery_change_history.py | 1349 +++++++++++
.../io/gcp/bigquery_change_history_it_test.py | 638 +++++
.../io/gcp/bigquery_change_history_test.py | 209 ++
.../apache_beam/io/gcp/bigquery_file_loads.py | 4 +-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 15 +-
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 10 +-
sdks/python/apache_beam/io/gcp/gcsio.py | 26 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 5 +-
sdks/python/apache_beam/io/jdbc.py | 6 +
sdks/python/apache_beam/io/kafka.py | 25 +-
sdks/python/apache_beam/io/parquetio_test.py | 2 +-
sdks/python/apache_beam/io/tfrecordio.py | 5 -
sdks/python/apache_beam/metrics/metric.py | 30 +-
.../apache_beam/ml/gcp/recommendations_ai.py | 20 +-
.../apache_beam/ml/gcp/recommendations_ai_test.py | 8 +-
.../ml/gcp/recommendations_ai_test_it.py | 7 +-
.../ml/inference/agent_development_kit.py | 302 +++
.../ml/inference/agent_development_kit_test.py | 334 +++
.../ml/inference/anthropic_inference.py | 296 +++
.../ml/inference/anthropic_inference_it_test.py | 215 ++
.../ml/inference/anthropic_inference_test.py | 351 +++
.../inference/anthropic_tests_requirements.txt} | 4 +-
sdks/python/apache_beam/ml/inference/base.py | 208 +-
sdks/python/apache_beam/ml/inference/base_test.py | 334 ++-
.../apache_beam/ml/inference/gemini_inference.py | 57 +-
.../ml/inference/gemini_inference_test.py | 25 +
.../ml/inference/huggingface_inference.py | 120 +-
.../apache_beam/ml/inference/model_manager.py | 820 +++++++
.../ml/inference/model_manager_it_test.py | 191 ++
.../apache_beam/ml/inference/model_manager_test.py | 694 ++++++
.../apache_beam/ml/inference/onnx_inference.py | 41 +-
.../apache_beam/ml/inference/pytorch_inference.py | 96 +-
.../apache_beam/ml/inference/sklearn_inference.py | 80 +-
...ts.txt => table_row_inference_requirements.txt} | 10 +-
.../ml/inference/tensorflow_inference.py | 82 +-
.../ml/inference/tensorflow_tests_requirements.txt | 3 +
.../apache_beam/ml/inference/tensorrt_inference.py | 41 +-
.../ml/inference/vertex_ai_inference.py | 22 +-
.../ml/inference/vertex_ai_inference_it_test.py | 5 +-
.../apache_beam/ml/inference/vllm_inference.py | 70 +-
.../ml/inference/vllm_tests_requirements.txt | 3 +-
.../apache_beam/ml/inference/xgboost_inference.py | 34 +-
sdks/python/apache_beam/ml/rag/embeddings/base.py | 66 +-
.../apache_beam/ml/rag/embeddings/base_test.py | 47 +-
.../apache_beam/ml/rag/embeddings/huggingface.py | 151 +-
.../ml/rag/embeddings/huggingface_test.py | 135 ++
.../apache_beam/ml/rag/embeddings/vertex_ai.py | 133 +-
.../ml/rag/embeddings/vertex_ai_test.py | 132 +-
.../ml/rag/enrichment/bigquery_vector_search.py | 127 +-
.../apache_beam/ml/rag/enrichment/milvus_search.py | 71 +-
.../ml/rag/enrichment/milvus_search_it_test.py | 9 +
.../ml/rag/enrichment/milvus_search_test.py | 4 +-
.../python/apache_beam/ml/rag/ingestion/alloydb.py | 2 +-
sdks/python/apache_beam/ml/rag/ingestion/base.py | 34 +-
.../apache_beam/ml/rag/ingestion/bigquery.py | 121 +-
.../ml/rag/ingestion/bigquery_it_test.py | 2 +-
.../apache_beam/ml/rag/ingestion/cloudsql.py | 2 +-
.../apache_beam/ml/rag/ingestion/milvus_search.py | 32 +-
.../ml/rag/ingestion/milvus_search_it_test.py | 33 +-
sdks/python/apache_beam/ml/rag/ingestion/mysql.py | 15 +-
.../apache_beam/ml/rag/ingestion/mysql_common.py | 133 +-
.../apache_beam/ml/rag/ingestion/postgres.py | 14 +-
.../ml/rag/ingestion/postgres_common.py | 155 +-
.../python/apache_beam/ml/rag/ingestion/spanner.py | 92 +-
.../ml/rag/ingestion/spanner_it_test.py | 2 +-
sdks/python/apache_beam/ml/rag/test_utils.py | 58 +-
sdks/python/apache_beam/ml/rag/types.py | 140 +-
sdks/python/apache_beam/ml/rag/utils.py | 4 +-
sdks/python/apache_beam/ml/transforms/base_test.py | 4 +-
.../ml/transforms/embeddings/huggingface_test.py | 2 +-
.../transforms/embeddings/tensorflow_hub_test.py | 2 +-
.../ml/transforms/embeddings/vertex_ai.py | 10 +-
.../{vertex_ai_test.py => vertex_ai_it_test.py} | 14 +-
.../python/apache_beam/options/pipeline_options.py | 19 +
.../options/pipeline_options_context.py | 65 +
.../options/pipeline_options_context_test.py | 335 +++
.../apache_beam/options/pipeline_options_test.py | 66 +
sdks/python/apache_beam/pipeline.py | 22 +-
sdks/python/apache_beam/pipeline_test.py | 2 +
sdks/python/apache_beam/portability/common_urns.py | 1 +
sdks/python/apache_beam/pvalue.py | 12 +-
.../runners/dataflow/dataflow_runner.py | 6 +-
.../runners/dataflow/dataflow_runner_test.py | 42 +
.../runners/dataflow/internal/apiclient.py | 4 +-
.../runners/dataflow/internal/apiclient_test.py | 2 +-
.../clients/dataflow/dataflow_v1b3_client.py | 515 ++--
.../clients/dataflow/dataflow_v1b3_messages.py | 879 ++++---
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
.../apache_beam/runners/interactive/README.md | 15 +-
.../runners/interactive/cache_manager.py | 2 +-
.../apache-beam-jupyterlab-sidepanel/README.md | 52 +-
.../apache-beam-jupyterlab-sidepanel/install.json | 5 +
.../apache-beam-jupyterlab-sidepanel/package.json | 3 +-
.../pyproject.toml | 2 +
.../style/index.js | 13 +
.../apache-beam-jupyterlab-sidepanel/tsconfig.json | 1 +
.../runners/interactive/interactive_runner.py | 6 +
.../runners/interactive/interactive_runner_test.py | 20 +
.../runners/interactive/sql/utils_test.py | 3 +-
.../runners/portability/flink_runner_test.py | 29 +-
.../runners/portability/portable_runner.py | 9 +-
.../runners/portability/portable_runner_test.py | 98 +
.../runners/portability/sdk_container_builder.py | 28 +-
sdks/python/apache_beam/runners/runner.py | 2 +
.../apache_beam/runners/worker/operations.py | 5 +
.../apache_beam/runners/worker/sdk_worker_main.py | 27 +-
.../testing/analyzers/perf_analysis_utils.py | 6 -
.../testing/benchmarks/cloudml/requirements.txt | 12 +-
.../testing/benchmarks/inference/README.md | 15 +
.../inference/table_row_inference_benchmark.py | 109 +
.../testing/benchmarks/sort_and_batch_benchmark.py | 655 ++++++
.../testing/load_tests/dataflow_cost_benchmark.py | 156 +-
.../apache_beam/testing/load_tests/load_test.py | 10 +-
sdks/python/apache_beam/testing/test_pipeline.py | 14 +
.../apache_beam/transforms/combiners_test.py | 73 +-
sdks/python/apache_beam/transforms/core.py | 169 +-
sdks/python/apache_beam/transforms/core_test.py | 212 ++
.../transforms/enrichment_handlers/bigquery.py | 35 +-
.../enrichment_handlers/bigquery_test.py | 82 +
.../transforms/enrichment_handlers/cloudsql.py | 15 +-
.../enrichment_handlers/cloudsql_test.py | 53 +
sdks/python/apache_beam/transforms/external.py | 7 +-
.../external_transform_provider_it_test.py | 12 +-
.../transforms/managed_iceberg_it_test.py | 23 +-
sdks/python/apache_beam/transforms/ptransform.py | 34 +-
sdks/python/apache_beam/transforms/sideinputs.py | 3 +-
.../apache_beam/transforms/sideinputs_test.py | 35 +
sdks/python/apache_beam/transforms/sql_test.py | 26 +
sdks/python/apache_beam/transforms/util.py | 394 +++-
sdks/python/apache_beam/transforms/util_test.py | 646 ++++-
sdks/python/apache_beam/typehints/decorators.py | 182 +-
.../apache_beam/typehints/decorators_test.py | 125 +
.../typehints/native_type_compatibility.py | 54 +-
.../typehints/native_type_compatibility_test.py | 63 +
sdks/python/apache_beam/typehints/row_type.py | 50 +-
sdks/python/apache_beam/typehints/row_type_test.py | 177 ++
.../apache_beam/typehints/schema_registry.py | 11 +-
sdks/python/apache_beam/typehints/schemas.py | 124 +-
sdks/python/apache_beam/typehints/schemas_test.py | 70 +-
.../typehints/tagged_output_typehints_test.py | 440 ++++
sdks/python/apache_beam/typehints/typehints.py | 7 +-
.../python/apache_beam/typehints/typehints_test.py | 29 +-
.../apache_beam/utils/multi_process_shared.py | 239 +-
.../apache_beam/utils/multi_process_shared_test.py | 227 +-
sdks/python/apache_beam/utils/windowed_value.py | 2 +-
sdks/python/apache_beam/version.py | 2 +-
.../yaml/examples/testing/examples_test.py | 18 +-
.../yaml/examples/testing/input_data.py | 5 +
.../examples/transforms/io/bigquery_write.yaml | 59 +
.../transforms/jinja/inheritance/README.md | 77 +
.../jinja/inheritance/base/base_pipeline.yaml | 55 +
.../jinja/inheritance/wordCountInheritance.yaml} | 24 +-
.../databases/bigquery_dynamic_destinations.yaml | 97 +
.../yaml/extended_tests/messaging/kafka.yaml | 54 +-
sdks/python/apache_beam/yaml/integration_tests.py | 24 +-
sdks/python/apache_beam/yaml/json_utils.py | 30 +-
sdks/python/apache_beam/yaml/json_utils_test.py | 124 +
sdks/python/apache_beam/yaml/main.py | 6 +-
sdks/python/apache_beam/yaml/readme_test.py | 2 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 47 +-
sdks/python/apache_beam/yaml/tests/create.yaml | 18 +
.../apache_beam/yaml/tests/iceberg_add_files.yaml | 115 +
.../yaml/tests/iceberg_add_files_batch.yaml | 88 +
sdks/python/apache_beam/yaml/yaml_ml.py | 7 +-
sdks/python/apache_beam/yaml/yaml_provider.py | 43 +-
.../apache_beam/yaml/yaml_provider_unit_test.py | 13 +
sdks/python/apache_beam/yaml/yaml_transform.py | 88 +-
.../python/apache_beam/yaml/yaml_transform_test.py | 20 +
sdks/python/build.gradle | 5 +-
sdks/python/conftest.py | 2 +-
sdks/python/container/Dockerfile | 6 +-
.../container/base_image_requirements_manual.txt | 4 +
sdks/python/container/boot.go | 72 +-
sdks/python/container/build.gradle | 3 +-
sdks/python/container/common.gradle | 25 +-
.../container/distroless/py314/build.gradle} | 16 +-
.../container/license_scripts/dep_urls_py.yaml | 4 +-
.../container/license_scripts/pull_licenses_py.py | 2 +-
.../container/ml/py310/base_image_requirements.txt | 164 +-
.../container/ml/py310/gpu_image_requirements.txt | 167 +-
.../container/ml/py311/base_image_requirements.txt | 171 +-
.../container/ml/py311/gpu_image_requirements.txt | 174 +-
.../container/ml/py312/base_image_requirements.txt | 171 +-
.../container/ml/py312/gpu_image_requirements.txt | 170 +-
.../container/ml/py313/base_image_requirements.txt | 166 +-
.../container/ml/py314/build.gradle} | 16 +-
sdks/python/container/piputil.go | 53 +-
.../container/py310/base_image_requirements.txt | 139 +-
.../container/py311/base_image_requirements.txt | 144 +-
.../container/py312/base_image_requirements.txt | 144 +-
.../container/py313/base_image_requirements.txt | 139 +-
.../{py313 => py314}/base_image_requirements.txt | 141 +-
.../container/py314/build.gradle} | 16 +-
sdks/python/container/run_generate_requirements.sh | 4 +-
sdks/python/gen_xlang_wrappers.py | 6 +-
sdks/python/pyproject.toml | 3 +
sdks/python/pyrefly.toml | 60 +
sdks/python/pytest.ini | 1 +
sdks/python/ruff.toml | 96 +
sdks/python/scripts/run_pylint.sh | 7 +-
sdks/python/setup.py | 134 +-
sdks/python/test-suites/dataflow/common.gradle | 28 +
.../test-suites/dataflow/py314/build.gradle} | 14 +-
.../test-suites/direct/py314/build.gradle} | 14 +-
sdks/python/test-suites/gradle.properties | 28 +-
sdks/python/test-suites/portable/common.gradle | 101 +-
.../test-suites/portable/py314/build.gradle} | 14 +-
sdks/python/test-suites/tox/py310/build.gradle | 50 +-
.../test-suites/tox/py314/build.gradle} | 18 +-
sdks/python/test-suites/tox/pycommon/build.gradle | 3 -
sdks/python/tox.ini | 80 +-
sdks/typescript/package-lock.json | 2477 ++++++++++++--------
sdks/typescript/package.json | 14 +-
.../typescript/src/apache_beam/worker/operators.ts | 44 +-
sdks/typescript/test/primitives_test.ts | 12 +-
settings.gradle.kts | 13 +-
start-build-env.sh | 4 +-
website/Dockerfile | 19 +-
website/build.gradle | 55 +
website/www/site/assets/js/language-switch-v2.js | 2 +
.../www/site/assets/scss/_syntax-highlighting.scss | 3 +
website/www/site/config.toml | 2 +-
website/www/site/content/en/blog/beam-2.71.0.md | 8 +-
website/www/site/content/en/blog/beam-2.72.0.md | 63 +
.../en/documentation/io/built-in/iceberg.md | 433 ++++
.../site/content/en/documentation/io/connectors.md | 302 ++-
.../site/content/en/documentation/io/managed-io.md | 796 +++----
.../content/en/documentation/patterns/overview.md | 3 +
.../en/documentation/patterns/rate-limiting.md | 97 +
.../content/en/documentation/programming-guide.md | 4 +
.../site/content/en/documentation/runners/flink.md | 10 +
.../sdks/python-pipeline-dependencies.md | 14 +
.../content/en/documentation/sdks/yaml-schema.md | 48 +
.../www/site/content/en/documentation/sdks/yaml.md | 6 +-
.../www/site/content/en/get-started/downloads.md | 16 +-
website/www/site/content/en/performance/_index.md | 2 +
.../en/performance/tablerowinference/_index.md | 45 +
.../tablerowinferencestreaming/_index.md | 43 +
website/www/site/data/performance.yaml | 33 +
.../partials/section-menu/en/documentation.html | 18 +-
.../www/site/layouts/shortcodes/button-colab.html | 2 +-
.../site/layouts/shortcodes/language-switcher.html | 3 +
.../site/layouts/shortcodes/performance_looks.html | 21 +-
.../shortcodes/{button-colab.html => section.html} | 14 +-
.../shortcodes/{button-colab.html => tab.html} | 26 +-
website/www/site/static/images/colab/icon32.png | Bin 0 -> 1418 bytes
website/www/yarn.lock | 19 +-
2116 files changed, 59419 insertions(+), 16609 deletions(-)
create mode 100644 .agent/skills/README.md
create mode 100644 .agent/skills/adding-new-metadata/SKILL.md
create mode 100644 .agent/skills/beam-concepts/SKILL.md
create mode 100644 .agent/skills/beam-dofn-modernizer/SKILL.md
create mode 100644 .agent/skills/ci-cd/SKILL.md
create mode 100644 .agent/skills/contributing/SKILL.md
create mode 100644 .agent/skills/gradle-build/SKILL.md
create mode 100644 .agent/skills/io-connectors/SKILL.md
create mode 100644 .agent/skills/java-development/SKILL.md
create mode 100644 .agent/skills/license-compliance/SKILL.md
create mode 100644 .agent/skills/python-development/SKILL.md
create mode 100644 .agent/skills/runners/SKILL.md
copy .github/trigger_files/{beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.json => beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_Engine.json} (79%)
copy .github/trigger_files/{beam_PostCommit_Java_ValidatesRunner_Dataflow.json => beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.json} (80%)
copy .github/trigger_files/{beam_PostCommit_Java_Examples_Dataflow_ARM.json => beam_PostCommit_PortableJar_Flink.json} (100%)
create mode 100644 .github/trigger_files/beam_PostCommit_Python_Examples_Flink.json
copy .github/workflows/{beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml => beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_Engine.yml} (82%)
copy .github/workflows/{beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml => beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.yml} (79%)
copy .github/workflows/{beam_PreCommit_SQL.yml => beam_PreCommit_Java_Dataflow.yml} (57%)
copy .github/workflows/load-tests-pipeline-options/{beam_Inference_Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt => beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Batch.txt} (59%)
copy .github/workflows/load-tests-pipeline-options/{beam_Inference_Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt => beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Stream.txt} (60%)
create mode 100644 examples/java/iceberg/src/main/java/org/apache/beam/examples/iceberg/snippets/IcebergBeamSchemaAndRow.java
create mode 100644 examples/java/iceberg/src/main/java/org/apache/beam/examples/iceberg/snippets/Quickstart.java
create mode 100644 examples/java/src/main/java/org/apache/beam/examples/PicoWordCount.java
create mode 100644 examples/java/src/main/java/org/apache/beam/examples/RateLimitedPubSubReader.java
create mode 100644 examples/java/src/main/java/org/apache/beam/examples/RateLimiterSimple.java
create mode 100755 examples/terraform/envoy-ratelimiter/deploy.sh
mode change 100755 => 100644 gradlew.bat
create mode 100644 it/datadog/build.gradle
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogClientFactory.java
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogLogEntry.java
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogResourceManager.java
copy runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetricsSinkRule.java => it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogResourceManagerException.java (67%)
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/DatadogResourceManagerUtils.java
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/conditions/DatadogLogEntriesCheck.java
copy {runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io => it/datadog/src/main/java/org/apache/beam/it/datadog/conditions}/package-info.java (87%)
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/matchers/DatadogAsserts.java
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/matchers/package-info.java
create mode 100644 it/datadog/src/main/java/org/apache/beam/it/datadog/package-info.java
create mode 100644 it/datadog/src/test/java/org/apache/beam/it/datadog/DatadogResourceManagerIT.java
create mode 100644 it/datadog/src/test/java/org/apache/beam/it/datadog/DatadogResourceManagerTest.java
create mode 100644 it/datadog/src/test/java/org/apache/beam/it/datadog/DatadogResourceManagerUtilsTest.java
create mode 100644 runners/core-java/src/main/java/org/apache/beam/runners/core/serialization/SentinelBitSetCoder.java
create mode 100644 runners/core-java/src/test/java/org/apache/beam/runners/core/serialization/SentinelBitSetCoderTest.java
copy runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{windmill/client/grpc/stubs/WindmillStubFactoryFactory.java => WindmillTimerType.java} (80%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/stubs/FailoverChannel.java
copy sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullSizeEstimatorTest.java => runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillTimerData.java (59%)
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcDispatcherClientTest.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/stubs/FailoverChannelTest.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/stubs/NoopClientCall.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/StreamingCommitFinalizerTest.java
copy runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineOptions.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineResult.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/examples/WordCount.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/BoundedDatasetFactory.java (100%)
copy runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/package-info.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/BeamMetricSet.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsAccumulator.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java (100%)
copy runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetricSource.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/package-info.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleCsvSink.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleGraphiteSink.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/package-info.java (100%)
copy runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkSessionFactory.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/Aggregators.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGloballyTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGroupedValuesTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnPartitionIteratorFactory.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerFactory.java (98%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerWithMetrics.java (91%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyHelpers.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ImpulseTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/CachedSideInputReader.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/GroupAlsoByWindowViaOutputBufferFn.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/NoOpStepContext.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValues.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SparkSideInputReader.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/package-info.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/CoderHelpers.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java (93%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderProvider.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/package-info.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/ScalaInterop.java (100%)
rename runners/spark/{3 => }/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/package-info.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkSessionRule.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java (91%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/StructuredStreamingPipelineStateTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetrics.java (73%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetricsSinkRule.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/SparkMetricsSinkTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetricTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/AggregatorsTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGloballyTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGroupedValuesTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ComplexSourceTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/SimpleSourceTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SideInputValuesTest.java (100%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java (99%)
rename runners/spark/{3 => }/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/SimpleSourceTest.java (100%)
create mode 100644 sdks/go/examples/ordered_list_state/ordered_list_state.go
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/lineage/LineageBase.java
copy runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineOptions.java => sdks/java/core/src/main/java/org/apache/beam/sdk/lineage/LineageOptions.java (51%)
copy sdks/java/{extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/HttpCallMatcher.java => core/src/main/java/org/apache/beam/sdk/lineage/package-info.java} (62%)
copy sdks/java/core/src/main/java/org/apache/beam/sdk/{options/DefaultPipelineOptionsRegistrar.java => metrics/BoundedTrieMetricsLineage.java} (51%)
rename runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineOptions.java => sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/StringSetMetricsLineage.java (52%)
rename runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillNamespacePrefix.java => sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/TriggerVisitor.java (52%)
copy runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io/package-info.java => sdks/java/core/src/main/java/org/apache/beam/sdk/values/CausedByDrain.java (88%)
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/lineage/LineagePluginTest.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/lineage/TestLineage.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MetadataPropagationTest.java
create mode 100644 sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilV2.java
create mode 100644 sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilParameterizedIT.java
create mode 100644 sdks/java/extensions/sql/iceberg/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergAlterTableOps.java
create mode 100644 sdks/java/extensions/sql/iceberg/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/BeamSqlCliIcebergAlterTest.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlAlterCatalog.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/parser/SqlAlterTable.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/AlterTableOps.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/test/AlterTestTableOps.java
create mode 100644 sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseJdbcUrlParser.java
copy sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/{ClickHouseIOTest.java => ClickHouseIOIT.java} (70%)
create mode 100644 sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIOJdbcBackwardCompatibilityTest.java
create mode 100644 sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIOPropertyMergingTest.java
create mode 100644 sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseJdbcUrlParserTest.java
copy {runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming => sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/deadletterqueue}/package-info.java (86%)
copy {runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming => sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/deadletterqueue/sinks}/package-info.java (86%)
rename runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetricSource.java => sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/EnvoyRateLimiter.java (53%)
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/EnvoyRateLimiterContext.java
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/EnvoyRateLimiterFactory.java
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiter.java
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterClientCache.java
rename runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java => sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterContext.java (74%)
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterFactory.java
create mode 100644 sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterOptions.java
copy {runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io => sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/ratelimiter}/package-info.java (86%)
copy {runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io => sdks/java/io/components/src/main/java/org/apache/beam/sdk/io/components/throttling}/package-info.java (87%)
create mode 100644 sdks/java/io/components/src/test/java/org/apache/beam/sdk/io/components/ratelimiter/EnvoyRateLimiterTest.java
create mode 100644 sdks/java/io/components/src/test/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterClientCacheTest.java
create mode 100644 sdks/java/io/components/src/test/java/org/apache/beam/sdk/io/components/ratelimiter/RateLimiterOptionsTest.java
create mode 100644 sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/FileSystemOffsetRetainer.java
create mode 100644 sdks/java/io/debezium/src/main/java/org/apache/beam/io/debezium/OffsetRetainer.java
create mode 100644 sdks/java/io/debezium/src/test/java/org/apache/beam/io/debezium/FileSystemOffsetRetainerTest.java
rename sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/{GoogleAdsV19.java => GoogleAdsV23.java} (93%)
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ConvertMessagesDoFn.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/MergeSchemaCombineFn.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PatchTableSchemaDoFn.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SchemaUpdateHoldingFn.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/UpgradeTableSchema.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiDataTriggeredSchemaUpdateIT.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/UpgradeTableSchemaTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessagesTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dao/ChangeStreamResultSetTest.java
copy sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/it/{SpannerChangeStreamPostgresIT.java => SpannerChangeStreamPlacementTablePostgresIT.java} (91%)
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AddFiles.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AddFilesSchemaTransformProvider.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SortOrderUtils.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/AddFilesIT.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/AddFilesTest.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/SortOrderUtilsTest.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/catalog/RESTCatalogBLMSIT.java
create mode 100644 sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaUnboundedReaderIdlePartitionTest.java
copy {runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io => sdks/java/ml/inference/openai/src/main/java/org/apache/beam/sdk/ml/inference/openai}/package-info.java (88%)
rename {runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/io => sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote}/package-info.java (88%)
create mode 100644 sdks/python/apache_beam/examples/inference/table_row_batch_example.py
create mode 100644 sdks/python/apache_beam/examples/inference/table_row_inference.py
create mode 100644 sdks/python/apache_beam/examples/inference/table_row_inference_test.py
create mode 100644 sdks/python/apache_beam/examples/inference/table_row_inference_utils.py
create mode 100644 sdks/python/apache_beam/io/gcp/bigquery_change_history.py
create mode 100644 sdks/python/apache_beam/io/gcp/bigquery_change_history_it_test.py
create mode 100644 sdks/python/apache_beam/io/gcp/bigquery_change_history_test.py
create mode 100644 sdks/python/apache_beam/ml/inference/agent_development_kit.py
create mode 100644 sdks/python/apache_beam/ml/inference/agent_development_kit_test.py
create mode 100644 sdks/python/apache_beam/ml/inference/anthropic_inference.py
create mode 100644 sdks/python/apache_beam/ml/inference/anthropic_inference_it_test.py
create mode 100644 sdks/python/apache_beam/ml/inference/anthropic_inference_test.py
copy sdks/python/apache_beam/{version.py => ml/inference/anthropic_tests_requirements.txt} (90%)
create mode 100644 sdks/python/apache_beam/ml/inference/model_manager.py
create mode 100644 sdks/python/apache_beam/ml/inference/model_manager_it_test.py
create mode 100644 sdks/python/apache_beam/ml/inference/model_manager_test.py
copy sdks/python/apache_beam/ml/inference/{vllm_tests_requirements.txt => table_row_inference_requirements.txt} (90%)
rename sdks/python/apache_beam/ml/transforms/embeddings/{vertex_ai_test.py => vertex_ai_it_test.py} (98%)
create mode 100644 sdks/python/apache_beam/options/pipeline_options_context.py
create mode 100644 sdks/python/apache_beam/options/pipeline_options_context_test.py
create mode 100644 sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/install.json
create mode 100644 sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/style/index.js
create mode 100644 sdks/python/apache_beam/testing/benchmarks/inference/table_row_inference_benchmark.py
create mode 100644 sdks/python/apache_beam/testing/benchmarks/sort_and_batch_benchmark.py
create mode 100644 sdks/python/apache_beam/typehints/row_type_test.py
create mode 100644 sdks/python/apache_beam/typehints/tagged_output_typehints_test.py
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/io/bigquery_write.yaml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/jinja/inheritance/README.md
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/jinja/inheritance/base/base_pipeline.yaml
copy sdks/python/apache_beam/{version.py => yaml/examples/transforms/jinja/inheritance/wordCountInheritance.yaml} (55%)
create mode 100644 sdks/python/apache_beam/yaml/extended_tests/databases/bigquery_dynamic_destinations.yaml
create mode 100644 sdks/python/apache_beam/yaml/tests/iceberg_add_files.yaml
create mode 100644 sdks/python/apache_beam/yaml/tests/iceberg_add_files_batch.yaml
copy sdks/{java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseInput.java => python/container/distroless/py314/build.gradle} (73%)
copy sdks/{java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseInput.java => python/container/ml/py314/build.gradle} (73%)
copy sdks/python/container/{py313 => py314}/base_image_requirements.txt (67%)
copy sdks/{java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseInput.java => python/container/py314/build.gradle} (73%)
create mode 100644 sdks/python/pyrefly.toml
create mode 100644 sdks/python/ruff.toml
copy sdks/{java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseInput.java => python/test-suites/dataflow/py314/build.gradle} (73%)
copy sdks/{java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseInput.java => python/test-suites/direct/py314/build.gradle} (73%)
copy sdks/{java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseInput.java => python/test-suites/portable/py314/build.gradle} (73%)
copy sdks/{java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/BaseInput.java => python/test-suites/tox/py314/build.gradle} (74%)
create mode 100644 website/www/site/content/en/blog/beam-2.72.0.md
create mode 100644 website/www/site/content/en/documentation/io/built-in/iceberg.md
create mode 100644 website/www/site/content/en/documentation/patterns/rate-limiting.md
create mode 100644 website/www/site/content/en/performance/tablerowinference/_index.md
create mode 100644 website/www/site/content/en/performance/tablerowinferencestreaming/_index.md
copy website/www/site/layouts/shortcodes/{button-colab.html => section.html} (64%)
copy website/www/site/layouts/shortcodes/{button-colab.html => tab.html} (57%)
create mode 100644 website/www/site/static/images/colab/icon32.png