You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2025/10/08 18:49:09 UTC
(beam) branch dependabot/gradle/com.gradleup.shadow-shadow-gradle-plugin-9.2.2 updated (b64327bc337 -> 637e1670f37)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch dependabot/gradle/com.gradleup.shadow-shadow-gradle-plugin-9.2.2
in repository https://gitbox.apache.org/repos/asf/beam.git
discard b64327bc337 Bump com.gradleup.shadow:shadow-gradle-plugin from 8.3.8 to 9.2.2
add 18d1d9fbb22 increase timeout (#36317)
add 50e14ace7f6 feat(bigquery): add BigLake configuration support for Storage Write API (#36225)
add c84f28f84aa Add option to pickle relative filepaths in cloudpickle. (#36300)
add 8cdd2f595ff Fix PerformanceTests jobs (#36315)
add abdec1bcecd Bump github.com/nats-io/nats.go from 1.45.0 to 1.46.0 in /sdks (#36312)
add 34aa17de6cd Allow users to pass service name for profiler for Java And Go SDK (#35903)
add 970f6b26c5d [GSoC 25][Blog Post]: Beam ML Vector DB/Feature Store Project (#36301)
add 292484f96a2 fix license script (#36328)
add bb340c2f66a further increase timeout (#36329)
add 4d87e040ac2 Add deterministic redistribute sharding for KafkaIO read. (#36112)
add 449ab577d32 Update python SDK container image (#36341)
add de8a6375242 Add Python 3.13
add 7a815d458f4 Add Python 3.13 to supported versions
add 881de9ba98a Update tensorflow version for Python 3.13
add 5f216696371 Merge pull request #36319 from apache/fix-update-dependencies
add da00474b4ea Add temp_location to all BigQuery Write examples and integration tests
add 75866588752 Merge pull request #36336 from tarun-google/fix_temp_location
add 09b9c471c8d Add @SchemaFieldDescription annotations and description to Bigtable config (#36344)
add 15b8560975a Fix Managed I/O link in the blog (#36268)
add 2f9a910e1f7 Implement a hierarchy for Beam SQL Calcite Schemas (#35787)
add 69b70263e72 Add ability for coders to set version tags for update compat checks. (#36316)
add e4c891fffe7 Update coders.py
add 7de704e9e9a Add dill test workflow (#36348)
add 340d420c2d3 Remove workflow_run. (#36351)
add a0831e0d4b1 Add AftersynchronizedProcessing Time as continuation trigger (#36285)
add 97b35eb5d7f Bump docker/login-action from 2 to 3 (#36232)
add d78d004d0a5 Skip SqlTransformExample (#36354)
add 6b9d534137c Set user agent when downloading jars with local_jar (#36342)
add e61b4f39b09 Add Java25 container (#36349)
add 666c75adc07 Add google-cloud-secret-managager to beam python container deps (#36353)
add a3fd151f8f9 Fix sql example in playground (#36358)
add b2954eef3b5 Fix playground example dataset format value
add 3f6d8210845 Merge pull request #36360 from apache/fix-dataset-path
add 3903623850d Exit gradle process after build complete (#36361)
add 7c88161b18b Consolidate beam Publish Bem SDK Snapshot matrix (#36363)
add 3534960b9c8 [Prism] Support AfterProcessingTime triggers - part 2 (#36333)
add df3384f96a2 Also check the existence of _fields in addition to__annotations__ for NamedTuple. (#36337)
add 1c8bcf9bae1 Fix race condition and nil pointer dereferencing (#36370)
add 45c36901859 Bump google.golang.org/protobuf from 1.36.8 to 1.36.10 in /sdks (#36371)
add 24d3a7bac85 Fix errorprone move serialization to private method (#36352)
add a03e96ad981 Update GitHub Actions to run Python 3.13 Tests (#35056)
add bf39e48ebdc Update Python SDK container (#36374)
add bd5cfdacc07 Make ml tests portable (#36377)
add 586cb119224 [Prism] Support AfterSynchronizedProcessingTime and enable java processing-time trigger tests (#36379)
add c0774c9b636 Trigger python coverage workflow on prism changes (#36378)
add 659cc4dfbd6 Add pipeline option to enforce gbek (#36321)
add 9f8e00e4393 Switch streaming engine worker harness based on job settings (#35901)
add b9c277244f2 Add Jpms tests for Java 25 (#36380)
add 800e2d7da5f Implement histogram user metric for python SDK (#36335)
add 82952c8c29a Update python metrics example in programming guide (#36334)
add 5af82f5f78c feat(subprocess_server): add fallback to Google Maven mirror (#36365)
add 1e973630e67 fix(sdk:python): Avoid AttributeError for transforms without hints (#36251)
add 5485467f230 Handle pending adjustment for processing time bundle correctly. (#36384)
add 312509fd019 Add back missing licenses (#36386)
add c1fc3698a38 Java GroupByEncryptedKey (#36217)
add bd77a1eff91 ci: update Python version from 3.12 to 3.13 in Flink pre-commit workflow (#36389)
add 7174991d487 fix(python-sdk): restrict transformers version for python 3.9 compat (#36385)
add 60436605e19 Add schema provider support for Kafka redistribute options (#36332)
add 517bef3290f fix: handle empty requests in milvus search to avoid connection attempts (#36388)
add 0b9fe991555 [Python] cloudsql_it_test fixture runs before setupclass (#36406)
add 25f65521f5b input is reshuffled so we can't guarantee that same input row will get specific job id based on which we have assert. (#36402)
add 10a049ecf81 Fix python 3.13 version for different workflows (#36399)
add 7542aeabbcf enrichment_test.EnrichmentTest flaky fix (#36407)
add 1282e6cb0bd [Dataflow Streaming] Fix race in GetDataStream that could leave get data requests orphaned on stream errors after half-closing. (#36401)
add 0b337b7df9d Add ibrahim user access (#36410)
add 7cedc0d2380 Update Dev Image tag to latest (#36411)
add 09aa10c52f1 Python examples to use IObase IOs sinks in streaming (and batch) (#35811)
add 1a6ec3a0826 Per element schema parsing in ConvertToBeamRows (#36393)
add 9076b1a1e0f Add labels to unlabeled transforms (#36381)
add 6d5b9843198 Fix Python 3.13 workflows (#36416)
add 060b1b7b858 [Dataflow Streaming] Remove one wait for GetData (#36417)
add 7b03da94989 Bump actions/setup-python from 4 to 6 (#36277)
add c52cb21ba05 chore: update milvus container image version to v2.6.2 (#36413)
add 6fe2c280df9 Bump github.com/docker/docker in /sdks (#36398)
add 3d5eaf3969f Add ThrottlingSignaler class to the Java SDK (#36119)
add 492d270efd8 Bump nodemailer from 6.9.9 to 7.0.7 in /scripts/ci/issue-report (#36419)
add 9944acf243e Add permissions for storage (#36423)
add ec35441d7f3 CombinePerKey with gbek (Python) (#36382)
add 4181f6f2d43 Pickle the relative path for code objects when cloudpickle is used to encode special deterministic types. (#36345)
add e8b41d7664a Fix execute gradle task for examples (#36421)
add d82f4fb25b2 Add Java 25 tests (#36409)
add 661c730d788 Revert "Bump actions/setup-python from 4 to 6 (#36277)" (#36427)
add 77973ca0aea GCP User IAM workflow modified to post Terraform plan as a comment on PRs (#36375)
add f9feffcaec7 GHA fix: GCP User Roles (#36429)
add d19b534ba0b Fix split thresholds for BQ sink (#36422)
add 27ad139bb18 Allow passing label through GBK (#36426)
add c8df4da229d Add pipeline option to force GBEK (Java) (#36346)
add 8566b2dc4ba Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#36433)
add 9cd906b98fd Bump golang.org/x/net from 0.43.0 to 0.45.0 in /sdks (#36435)
add ebe8de603ec Bump docker/setup-buildx-action from 1 to 3 (#36260)
add 3896ca7839c Modify GCP User Roles workflow to run when a PR is merged (#36430)
add 0d121ec273f Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#36434)
add 580db936824 [Prism] LP SOME coder and its data elements for TestStream (#36424)
add 8d4b096441c Fix cloudpickle import path. (#36439)
add 303c4a2ce68 Fix changes.md (#36442)
add 637e1670f37 Bump com.gradleup.shadow:shadow-gradle-plugin from 8.3.8 to 9.2.2
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b64327bc337)
\
N -- N -- N refs/heads/dependabot/gradle/com.gradleup.shadow-shadow-gradle-plugin-9.2.2 (637e1670f37)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.../gradle-command-self-hosted-action/action.yml | 2 +-
.../test-properties.json | 8 +-
.github/trigger_files/beam_PostCommit_Java.json | 2 +-
.../beam_PostCommit_Java_DataflowV1.json | 2 +-
.../beam_PostCommit_Java_DataflowV2.json | 6 +-
.github/trigger_files/beam_PostCommit_Python.json | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.json | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.json | 2 +-
...PostCommit_XVR_PythonUsingJavaSQL_Dataflow.json | 2 +-
.github/workflows/README.md | 19 +-
.../beam_Infrastructure_UsersPermissions.yml | 43 +-
.github/workflows/beam_PostCommit_Go.yml | 2 +-
.../workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow_ARM.yml | 4 +-
...beam_PostCommit_Java_Examples_Dataflow_Java.yml | 2 +-
...m_PostCommit_Java_Examples_Dataflow_V2_Java.yml | 2 +-
....yml => beam_PostCommit_Java_Jpms_Dataflow.yml} | 16 +-
...eam_PostCommit_Java_Jpms_Dataflow_Versions.yml} | 23 +-
...11.yml => beam_PostCommit_Java_Jpms_Direct.yml} | 17 +-
.../beam_PostCommit_Java_Jpms_Direct_Java21.yml | 101 ----
... beam_PostCommit_Java_Jpms_Direct_Versions.yml} | 23 +-
..._Java_ValidatesRunner_Dataflow_JavaVersions.yml | 2 +-
...it_Java_ValidatesRunner_Direct_JavaVersions.yml | 2 +-
.github/workflows/beam_PostCommit_Python.yml | 2 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 4 +-
.../beam_PostCommit_Python_Dependency.yml | 2 +-
.../beam_PostCommit_Python_Examples_Dataflow.yml | 6 +-
.../beam_PostCommit_Python_Examples_Direct.yml | 2 +-
.../beam_PostCommit_Python_Examples_Flink.yml | 2 +-
.../beam_PostCommit_Python_Examples_Spark.yml | 2 +-
.../beam_PostCommit_Python_MongoDBIO_IT.yml | 8 +-
...stCommit_Python_ValidatesContainer_Dataflow.yml | 2 +-
..._Python_ValidatesContainer_Dataflow_With_RC.yml | 2 +-
..._PostCommit_Python_ValidatesRunner_Dataflow.yml | 2 +-
...eam_PostCommit_Python_ValidatesRunner_Flink.yml | 2 +-
...eam_PostCommit_Python_ValidatesRunner_Samza.yml | 2 +-
...eam_PostCommit_Python_ValidatesRunner_Spark.yml | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.yml | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.yml | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.yml | 2 +-
.../beam_PostCommit_TransformService_Direct.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Direct.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Flink.yml | 2 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 2 +-
...eam_PostCommit_XVR_JavaUsingPython_Dataflow.yml | 2 +-
..._PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml | 6 +-
...eam_PostCommit_XVR_PythonUsingJava_Dataflow.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Samza.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Spark3.yml | 2 +-
.../workflows/beam_PreCommit_CommunityMetrics.yml | 2 +-
.github/workflows/beam_PreCommit_Prism_Python.yml | 2 +-
.github/workflows/beam_PreCommit_Python.yml | 2 +-
.github/workflows/beam_PreCommit_PythonDocker.yml | 4 +-
.../workflows/beam_PreCommit_Python_Coverage.yml | 4 +-
.../workflows/beam_PreCommit_Python_Dataframes.yml | 2 +-
...ython_ML.yml => beam_PreCommit_Python_Dill.yml} | 50 +-
.../workflows/beam_PreCommit_Python_Examples.yml | 2 +-
.../beam_PreCommit_Python_Integration.yml | 2 +-
.github/workflows/beam_PreCommit_Python_ML.yml | 6 +-
.../workflows/beam_PreCommit_Python_PVR_Flink.yml | 6 +-
.../workflows/beam_PreCommit_Python_Runners.yml | 2 +-
.../workflows/beam_PreCommit_Python_Transforms.yml | 2 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 35 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 4 +-
.github/workflows/build_release_candidate.yml | 4 +-
.github/workflows/build_runner_image.yml | 2 +-
.github/workflows/build_wheels.yml | 6 +-
.github/workflows/finalize_release.yml | 2 +-
.github/workflows/python_dependency_tests.yml | 1 +
.github/workflows/python_tests.yml | 3 +-
.../republish_released_docker_containers.yml | 2 +-
.github/workflows/update_python_dependencies.yml | 3 +-
.../postgres/postgres-service-for-local-dev.yml | 15 +-
CHANGES.md | 3 +
build.gradle.kts | 3 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 44 +-
examples/java/build.gradle | 28 --
examples/java/common.gradle | 31 ++
examples/java/iceberg/build.gradle | 28 --
examples/java/sql/build.gradle | 26 +-
infra/iam/users.yml | 27 +-
local-env-setup.sh | 4 +-
playground/backend/containers/java/Dockerfile | 14 +-
playground/infrastructure/ci_cd.py | 2 +
playground/infrastructure/models.py | 2 +-
runners/google-cloud-dataflow-java/build.gradle | 4 +-
.../examples/build.gradle | 100 ++--
.../dataflow/internal/DataflowGroupByKey.java | 52 ++-
.../options/DataflowStreamingPipelineOptions.java | 3 +
.../dataflow/worker/StreamingDataflowWorker.java | 512 +++++++++++++++------
.../harness/SingleSourceWorkerHarness.java | 13 +-
.../windmill/client/grpc/GrpcGetDataStream.java | 22 +-
.../client/grpc/GrpcGetDataStreamRequests.java | 9 +-
.../dataflow/worker/FakeWindmillServer.java | 47 ++
.../worker/StreamingDataflowWorkerTest.java | 148 ++++++
.../client/grpc/GrpcGetDataStreamTest.java | 24 +-
.../worker/windmill/src/main/proto/windmill.proto | 9 +
runners/prism/java/build.gradle | 28 +-
scripts/ci/issue-report/package-lock.json | 15 +-
scripts/ci/issue-report/package.json | 2 +-
sdks/go.mod | 38 +-
sdks/go.sum | 76 +--
sdks/go/container/boot.go | 42 +-
sdks/go/container/boot_test.go | 127 +++--
sdks/go/pkg/beam/core/runtime/graphx/coder.go | 10 +-
sdks/go/pkg/beam/runners/prism/internal/coders.go | 25 +
.../prism/internal/engine/elementmanager.go | 188 +++++++-
.../beam/runners/prism/internal/engine/strategy.go | 81 +++-
sdks/go/pkg/beam/runners/prism/internal/execute.go | 27 +-
.../prism/internal/jobservices/management.go | 15 +-
.../runners/prism/internal/unimplemented_test.go | 8 +-
sdks/go/test/integration/integration.go | 1 +
sdks/go/test/integration/primitives/windowinto.go | 24 +-
.../test/integration/primitives/windowinto_test.go | 6 +
.../resources/beam/checkstyle/suppressions.xml | 4 +
sdks/java/container/boot.go | 88 +++-
sdks/java/container/boot_test.go | 45 ++
sdks/java/container/build.gradle | 1 +
sdks/java/container/distroless/build.gradle | 7 +-
.../java/container/{java21 => java25}/build.gradle | 4 +-
.../java25-security.properties} | 0
.../container/{java21 => java25}/option-jamm.json | 0
.../option-java25-security.json} | 2 +-
.../license_scripts/pull_licenses_java.py | 3 +-
sdks/java/core/build.gradle | 5 +
.../apache/beam/sdk/options/PipelineOptions.java | 35 ++
.../beam/sdk/transforms/GroupByEncryptedKey.java | 282 ++++++++++++
.../org/apache/beam/sdk/transforms/GroupByKey.java | 34 ++
.../apache/beam/sdk/transforms/Redistribute.java | 2 +-
.../java/org/apache/beam/sdk/util/GcpSecret.java | 67 +++
.../main/java/org/apache/beam/sdk/util/Secret.java | 85 ++++
.../beam/sdk/util/construction/Environments.java | 5 +-
.../util/construction/GroupByKeyTranslation.java | 14 +
.../util/construction/PTransformTranslation.java | 2 +
.../apache/beam/sdk/transforms/CombineTest.java | 2 -
.../sdk/transforms/GroupByEncryptedKeyTest.java | 189 ++++++++
.../apache/beam/sdk/transforms/GroupByKeyIT.java | 192 ++++++++
.../apache/beam/sdk/transforms/GroupByKeyTest.java | 106 +++++
.../java/org/apache/beam/sdk/util/SecretTest.java | 67 +++
.../sdk/util/construction/EnvironmentsTest.java | 10 +-
.../extensions/protobuf/ProtoBeamConverter.java | 9 +-
.../beam/sdk/extensions/protobuf/ProtobufUtil.java | 18 -
.../sql/meta/provider/iceberg/IcebergCatalog.java | 30 +-
.../meta/provider/iceberg/IcebergMetastore.java | 154 +++++++
.../sql/meta/provider/iceberg/IcebergTable.java | 5 +-
.../provider/iceberg/IcebergTableProvider.java | 96 ----
.../provider/iceberg/BeamSqlCliIcebergTest.java | 116 ++++-
.../provider/iceberg/IcebergMetastoreTest.java | 97 ++++
.../meta/provider/iceberg/IcebergReadWriteIT.java | 46 +-
.../provider/iceberg/IcebergTableProviderTest.java | 83 ----
.../meta/provider/iceberg/PubsubToIcebergIT.java | 18 +-
.../sql/src/main/codegen/includes/parserImpls.ftl | 30 +-
.../beam/sdk/extensions/sql/SqlTransform.java | 5 +-
.../apache/beam/sdk/extensions/sql/TableUtils.java | 4 +
.../sdk/extensions/sql/impl/BeamCalciteSchema.java | 58 +--
.../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 18 +-
.../extensions/sql/impl/CatalogManagerSchema.java | 278 +++++++++++
.../sdk/extensions/sql/impl/CatalogSchema.java | 229 +++++++++
.../sdk/extensions/sql/impl/JdbcConnection.java | 6 +-
.../beam/sdk/extensions/sql/impl/TableName.java | 30 ++
.../sql/impl/parser/SqlCreateCatalog.java | 43 +-
.../sql/impl/parser/SqlCreateDatabase.java | 74 ++-
.../sql/impl/parser/SqlCreateExternalTable.java | 46 +-
.../extensions/sql/impl/parser/SqlDdlNodes.java | 29 +-
.../extensions/sql/impl/parser/SqlDropCatalog.java | 46 +-
.../sql/impl/parser/SqlDropDatabase.java | 59 +--
.../extensions/sql/impl/parser/SqlDropTable.java | 45 ++
.../sql/impl/parser/SqlSetOptionBeam.java | 35 +-
.../extensions/sql/impl/parser/SqlUseCatalog.java | 38 +-
.../extensions/sql/impl/parser/SqlUseDatabase.java | 61 ++-
.../apache/beam/sdk/extensions/sql/meta/Table.java | 3 +-
.../sdk/extensions/sql/meta/catalog/Catalog.java | 25 +-
.../sql/meta/catalog/CatalogManager.java | 13 +-
.../sql/meta/catalog/EmptyCatalogManager.java | 14 +-
.../sql/meta/catalog/InMemoryCatalog.java | 46 +-
.../sql/meta/catalog/InMemoryCatalogManager.java | 29 +-
.../sql/meta/store/InMemoryMetaStore.java | 60 ++-
.../sdk/extensions/sql/meta/store/MetaStore.java | 6 +
.../sdk/extensions/sql/BeamSqlCliCatalogTest.java | 333 ++++++++++++++
.../sdk/extensions/sql/BeamSqlCliDatabaseTest.java | 133 +++++-
.../beam/sdk/extensions/sql/BeamSqlCliTest.java | 182 +-------
.../sdk/extensions/sql/impl/JdbcDriverTest.java | 22 +-
.../sql/impl/parser/BeamDDLNestedTypesTest.java | 2 +-
.../extensions/sql/impl/parser/BeamDDLTest.java | 28 +-
.../sdk/extensions/sql/impl/rel/BaseRelTest.java | 2 +
.../sql/impl/rule/JoinReorderingTest.java | 6 +-
.../sql/meta/store/InMemoryMetaStoreTest.java | 9 +-
.../components/throttling/ThrottlingSignaler.java} | 27 +-
.../DebeziumReadSchemaTransformProvider.java | 1 -
.../sdk/io/gcp/bigquery/SplittingIterable.java | 15 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 17 +-
...ueryStorageWriteApiSchemaTransformProvider.java | 3 +
.../providers/BigQueryWriteConfiguration.java | 11 +
.../BigtableReadSchemaTransformProvider.java | 19 +
.../BigtableWriteSchemaTransformProvider.java | 18 +
.../beam/sdk/io/iceberg/IcebergCatalogConfig.java | 40 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 53 ++-
.../KafkaIOReadImplementationCompatibility.java | 6 +
.../beam/sdk/io/kafka/KafkaReadRedistribute.java | 124 +++++
.../KafkaReadSchemaTransformConfiguration.java | 30 ++
.../io/kafka/KafkaReadSchemaTransformProvider.java | 29 ++
...KafkaIOReadImplementationCompatibilityTest.java | 6 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 87 +++-
.../sdk/io/kafka/KafkaReadRedistributeTest.java | 231 ++++++++++
.../KafkaReadSchemaTransformProviderTest.java | 7 +-
.../sdk/io/kafka/upgrade/KafkaIOTranslation.java | 10 +
.../io/kafka/upgrade/KafkaIOTranslationTest.java | 1 +
sdks/java/testing/test-utils/build.gradle | 2 +-
.../testutils/jvmverification/JvmVerification.java | 16 +-
.../apache/beam/sdk/tpcds/BeamSqlEnvRunner.java | 5 +-
sdks/python/apache_beam/coders/coder_impl.pxd | 1 +
sdks/python/apache_beam/coders/coder_impl.py | 14 +-
sdks/python/apache_beam/coders/coders.py | 48 +-
.../apache_beam/coders/coders_test_common.py | 57 ++-
.../anomaly_detection_pipeline/setup.py | 2 +-
.../inference/sklearn_examples_requirements.txt | 2 +-
sdks/python/apache_beam/examples/sinks/README.md | 59 +++
.../python/apache_beam/examples/sinks}/__init__.py | 0
.../apache_beam/examples/sinks/generate_event.py | 144 ++++++
.../examples/sinks/test_periodicimpulse.py | 68 +++
.../examples/sinks/test_write_bounded.py | 98 ++++
.../examples/sinks/test_write_unbounded.py | 166 +++++++
.../transforms/elementwise/enrichment_test.py | 3 +-
.../internal/cloudpickle/cloudpickle.py | 116 ++++-
sdks/python/apache_beam/internal/metrics/cells.py | 21 +-
.../apache_beam/internal/metrics/metric_test.py | 39 ++
.../io/external/xlang_bigqueryio_it_test.py | 96 ++++
sdks/python/apache_beam/io/gcp/bigquery.py | 11 +-
.../apache_beam/io/gcp/bigquery_biglake_test.py | 117 +++++
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 6 +-
sdks/python/apache_beam/metrics/execution.py | 12 +-
sdks/python/apache_beam/metrics/metric.py | 1 +
.../python/apache_beam/metrics/monitoring_infos.py | 39 +-
.../apache_beam/metrics/monitoring_infos_test.py | 35 ++
sdks/python/apache_beam/ml/inference/base_test.py | 4 +-
.../apache_beam/ml/rag/enrichment/milvus_search.py | 3 +
.../ml/rag/enrichment/milvus_search_it_test.py | 2 +-
.../python/apache_beam/options/pipeline_options.py | 21 +
sdks/python/apache_beam/pipeline.py | 35 +-
sdks/python/apache_beam/pipeline_test.py | 33 ++
.../iqr_test.py => programming_guide_test.py} | 44 +-
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
.../apache_beam/runners/direct/direct_metrics.py | 33 ++
.../testing/integration/tests/screen_diff_test.py | 2 -
.../portability/flink_uber_jar_job_server.py | 4 +-
.../apache_beam/runners/portability/job_server.py | 5 +-
.../portability/spark_uber_jar_job_server.py | 4 +-
sdks/python/apache_beam/transforms/core.py | 82 +++-
sdks/python/apache_beam/transforms/core_it_test.py | 109 +++++
.../enrichment_handlers/cloudsql_it_test.py | 3 +-
sdks/python/apache_beam/transforms/external.py | 43 +-
.../python/apache_beam/transforms/external_java.py | 7 +-
.../python/apache_beam/transforms/external_test.py | 2 +-
.../apache_beam/transforms/ptransform_test.py | 19 +
sdks/python/apache_beam/transforms/trigger.py | 119 ++++-
sdks/python/apache_beam/transforms/trigger_test.py | 50 ++
sdks/python/apache_beam/transforms/util.py | 94 +++-
sdks/python/apache_beam/transforms/util_test.py | 77 +++-
.../typehints/native_type_compatibility.py | 2 +-
sdks/python/apache_beam/utils/histogram.py | 72 ++-
sdks/python/apache_beam/utils/subprocess_server.py | 83 +++-
.../apache_beam/utils/subprocess_server_test.py | 57 ++-
.../utils/transform_service_launcher.py | 7 +-
.../transforms/blueprint/gcs_text_to_bigquery.yaml | 3 +
.../transforms/blueprint/jdbc_to_bigquery.yaml | 3 +-
.../transforms/blueprint/mysql_to_bigquery.yaml | 3 +
.../transforms/blueprint/oracle_to_bigquery.yaml | 3 +
.../transforms/blueprint/postgres_to_bigquery.yaml | 3 +
.../transforms/blueprint/spanner_to_bigquery.yaml | 3 +
.../blueprint/sqlserver_to_bigquery.yaml | 3 +
.../ml/log_analysis/ml_preprocessing.yaml | 1 +
.../yaml/extended_tests/databases/bigquery.yaml | 4 +-
sdks/python/build.gradle | 2 +-
.../ml_image_requirements.txt} | 128 +++---
.../container/py310/base_image_requirements.txt | 92 ++--
.../container/py311/base_image_requirements.txt | 94 ++--
.../container/py312/base_image_requirements.txt | 94 ++--
.../container/py313/base_image_requirements.txt | 103 +++--
.../container/py39/base_image_requirements.txt | 82 ++--
sdks/python/pyproject.toml | 2 +-
sdks/python/setup.py | 86 ++--
sdks/python/test-suites/gradle.properties | 28 +-
sdks/python/test-suites/tox/py313/build.gradle | 2 +
sdks/python/tox.ini | 12 +
settings.gradle.kts | 1 +
.../site/content/en/blog/gsoc-25-ml-connectors.md | 254 ++++++++++
.../en/blog/gsoc-25-yaml-user-accessibility.md | 2 +-
.../content/en/documentation/programming-guide.md | 16 +-
.../en/documentation/runtime/environments.md | 1 +
website/www/site/data/authors.yml | 4 +
291 files changed, 8040 insertions(+), 2277 deletions(-)
rename .github/workflows/{beam_PostCommit_Java_Jpms_Dataflow_Java11.yml => beam_PostCommit_Java_Jpms_Dataflow.yml} (88%)
rename .github/workflows/{beam_PostCommit_Java_Jpms_Dataflow_Java17.yml => beam_PostCommit_Java_Jpms_Dataflow_Versions.yml} (83%)
rename .github/workflows/{beam_PostCommit_Java_Jpms_Direct_Java11.yml => beam_PostCommit_Java_Jpms_Direct.yml} (87%)
delete mode 100644 .github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml
rename .github/workflows/{beam_PostCommit_Java_Jpms_Direct_Java17.yml => beam_PostCommit_Java_Jpms_Direct_Versions.yml} (83%)
copy .github/workflows/{beam_PreCommit_Python_ML.yml => beam_PreCommit_Python_Dill.yml} (73%)
copy sdks/java/container/{java21 => java25}/build.gradle (97%)
copy sdks/java/container/{java21/java21-security.properties => java25/java25-security.properties} (100%)
copy sdks/java/container/{java21 => java25}/option-jamm.json (100%)
copy sdks/java/container/{java11/option-java11-security.json => java25/option-java25-security.json} (86%)
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByEncryptedKey.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcpSecret.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/Secret.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByEncryptedKeyTest.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyIT.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/util/SecretTest.java
create mode 100644 sdks/java/extensions/sql/iceberg/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergMetastore.java
delete mode 100644 sdks/java/extensions/sql/iceberg/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergTableProvider.java
create mode 100644 sdks/java/extensions/sql/iceberg/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergMetastoreTest.java
delete mode 100644 sdks/java/extensions/sql/iceberg/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/iceberg/IcebergTableProviderTest.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/CatalogManagerSchema.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/CatalogSchema.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlCliCatalogTest.java
copy sdks/java/io/{google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/firestore/CounterFactory.java => components/src/main/java/org/apache/beam/sdk/io/components/throttling/ThrottlingSignaler.java} (51%)
create mode 100644 sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaReadRedistribute.java
create mode 100644 sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaReadRedistributeTest.java
create mode 100644 sdks/python/apache_beam/examples/sinks/README.md
copy {examples/notebooks/notebook_test_scripts => sdks/python/apache_beam/examples/sinks}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/examples/sinks/generate_event.py
create mode 100644 sdks/python/apache_beam/examples/sinks/test_periodicimpulse.py
create mode 100644 sdks/python/apache_beam/examples/sinks/test_write_bounded.py
create mode 100644 sdks/python/apache_beam/examples/sinks/test_write_unbounded.py
create mode 100644 sdks/python/apache_beam/io/gcp/bigquery_biglake_test.py
copy sdks/python/apache_beam/{ml/anomaly/detectors/iqr_test.py => programming_guide_test.py} (50%)
create mode 100644 sdks/python/apache_beam/transforms/core_it_test.py
copy sdks/python/container/ml/{py312/base_image_requirements.txt => py313/ml_image_requirements.txt} (75%)
create mode 100644 website/www/site/content/en/blog/gsoc-25-ml-connectors.md