You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2025/03/18 14:47:07 UTC
(beam) branch dependabot/go_modules/sdks/github.com/fsouza/fake-gcs-server-1.52.2 updated (e4ba164aaab -> 534fcfd9f7f)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch dependabot/go_modules/sdks/github.com/fsouza/fake-gcs-server-1.52.2
in repository https://gitbox.apache.org/repos/asf/beam.git
discard e4ba164aaab Bump github.com/fsouza/fake-gcs-server from 1.52.1 to 1.52.2 in /sdks
add bc33a33763c Allow users to specify custom audit entries in pipeline options. (#34062)
add 25977b8b05c Bump github.com/docker/docker in /sdks (#34095)
add e14bba64129 Bump github.com/go-sql-driver/mysql from 1.8.1 to 1.9.0 in /sdks (#34033)
add 94d62d695d0 Speed up release by pushing docker containers in different jobs (#34099)
add 285414d0a5c Update kafka.py (#34079)
add 9d92dd7f41c Add alloydb language connector to schemaio expansion service. (#34106)
add 6fbdaa23b3b (fix #34038) Use CivilTimeEncoder to encode Time values in AvroGenericRecordToStorageApiProto (#34059)
add c8172fbaadb Bump github.com/aws/aws-sdk-go-v2 from 1.36.1 to 1.36.3 in /sdks (#34114)
add d0fbfcdabdc Fix mobilegaming-java-dataflow.groovy script (#34119)
add 10a9f227f2a Bump to Mokito 4 (#34054)
add 93aa342709b Update routing of types into convert_to_beam_type in normalize() (#34103)
add a8c62f39aed Coherce values in assert_that to lists from iterables (#34121)
add 633043282b3 Update Dataflow containers (#34122)
add 492051a2c41 Fix indent for code-change-guide.md (#34026)
add 047c78887ca fix flakes (#34111)
add 228f028c942 Add log4j2 deps for spark validatesRunner tasks. (#34123)
add 468001b3a09 Fix race condition (#34058)
add ac1c7886598 Update 2024.md (#34137)
add fb061f27e26 Automatically refresh Performance Metrics Graphs using Looker (#34097)
add 0b261e8b9f8 Call out race condition fix in CHANGES (#34147)
add 9c8a6969ff5 Make pydoc docstring reflecting deprecated (#34136)
add 2f703aef0ee Update republish workflow to split docker pushes (#34086)
add d91d656b99b Add resource hint capabilities to YAML.
add c1bff62e83f Support for composite-level resource hints.
add 70af19f764d Add Baeam YAML resource hints to docs.
add 65c6204ec5b Apply suggestions from code review
add 810469bc396 Merge pull request #34087 Add resource hint capabilities to YAML.
add fd23344744f Use bigdataoss 3.x-compatible API in BigQueryIO's BatchLoads (#34105)
add 5ec0407ed59 Spark Runner : Replace queueStream with custom DStream in Spark streaming Flatten transform (#34080)
add a994291b15b Add explicit schema support to JdbcIO read and xlang transform. (#34128)
add 468aa4e49eb Add docs about withQueryFn, logic to detect other functions, and new … (#34127)
add 7f749a9939d Fix hadoop version tests. (#34155)
add a140b5ed789 updated Go to 1.24.0 (#34163)
add bd2541662ee [Java] Allow users to specify GCS custom audit entries in pipeline options (#34134)
add 103eb8182e3 Add sleep to give enough time for server to be up (#34133)
add 7bff2833a63 enable kafka metrics by default for streaming dataflow jobs (#34153)
add ed8b1f77811 [Java] Fix UnboundedReaderAsSdfFn to avoid using unstarted unbounded reader. (#34146)
add 0faa7b8e3be Run Python Postcommit on High mem 22. Increase max nodes and replicas to 8. (#34170)
add 14b1fa5098f fix for adding unexpected Empty Records in Nested Arrays in BigQueryIO (#34102)
add c15bd4c380e Add Sequences support to Breaking Changes (#34169)
add c6dee85231c Updates Managed Javadocs and pydocs to refer to runner specific features
add b76e45acf4c Merge pull request #34072: Updates Managed Javadocs and pydocs to refer to runner specific features
add 451ee3d5256 [Spark] Skip unused outputs of ParDo in SparkRunner (#33771) (#33772)
add 2fb383b7a6d Install docker compose to run xlang_jdbc tests in xlang direct workflow. (#34182)
add 88031b46f6c Add histogram to metrics container (#33043)
add 82713351f8b Fix cleanPython (#34185)
add 9e05b789af3 Add AlloyDB embeddings colab. (#34184)
add 9da70060e8f Correctly parse labels if they are passed as a single string instead of a list (#34183)
add 0733fa5e82d Alloy language connector (#34156)
add c7edbb3dda1 Catch union-of-iterables case in get_yielded_type() (#34186)
add 0c9561db30d Add test ensuring pipeline options observes kwargs.
add c9b55a389cc Merge pull request #34177 Add test ensuring pipeline options observes kwargs.
add 5bb7cf54a1d Remove retry count assertion from testInvalidRecordReceived (#34192)
add 8870dd81759 Add more time for grpc cleanup (#34180)
add 9303e3ead19 Fix failing test: Increase timeout (#34181)
add 57c644a9ae3 Fix test (#34162)
add c1d0fa4f850 Clarify BigQuery InsertRetryPolicy behavior for non-200 responses (#34118)
add a2d9e213b82 Bump dataflow java container version to beam-master-20250307 (#34211)
add 1db2a5ae995 cloudpickle support is no longer experimental. (#34200)
add 0415bd6a771 Use individual windows rather than window sets in the combining table. (#34193)
add dfc45f0d1fd Fix PostCommit Java DataflowV2 (#34209)
add 2507861de9f Updates the information regarding Managed I/O
add 8297f112c20 Merge pull request #34152: Updates the information regarding Managed I/O
add 3096c31e3ee Bump jinja2 from 3.1.4 to 3.1.6 in /.test-infra/jenkins/metrics_report (#34189)
add d89ccc96ca3 Implement ClickHouse Integration Testing Resource Manager (#34230)
add 9015b2b180b [AnomalyDetection] Add detectors and some necessary univariate trackers. (#34232)
add 9429df9b13b Clean up ExternalTransformRegistrar (#34237)
add caf80e844b9 Pin python-snappy==0.6.1 (#34233)
add 53baae84bd8 Kafka add counters v1 uw2 (#33503)
add c23dea40e4f Update go_tests.yml (#34241)
add 060d6f7f6dc call out the snappy version in the warning message (#34225)
add 1ffea2a3d5f Fix Docker dev environment set up (#34142)
add 611eb6a7e4a Remove obsolete release script. (#34239)
add 5d38b8efedc Factor out data file discovery. (#34238)
add 62bdf56e210 Issue warning on long running DoFn.Setup on legacy Dataflow runner (#34215)
add a08b1234333 Fix typo on resource hints page. (#34216)
add df09b1d624f Don't attempt to cache the bulk multimap lookup call.
add cad3dc0174e Merge pull request #34150 Don't attempt to cache the bulk multimap lookup call.
add a1b6a2aa8cf [AnomalyDetection] Add main and auxiliary transforms. (#34234)
add 747141117d6 Refactor code and add some docstrings. (#34235)
add 2f804c2db18 Bump dataflow java container version - 20250312 (#34256)
add 12bc35874f5 Use same format for next and current release versions (#34262)
add a69094861cb Putting the default retry settings which was overridden in Dataflow templates (#34161)
add 86f43f29a0a Update Python container (#34255)
add 6dceef51658 Add yaml examples generation to the release scripts. (#34217)
add 404f7e17d03 [Java] Added Metrics Configuration Support to Iceberg Data Writers (#34140)
add 3708ad6563c Fix PostCommit Java PVR Spark3 Streaming job (#34253)
add d7ef7b1e9ff Update to latest bom (#34231)
add 01092b043fd Update pinned kafka version to 3.9.0 to support OAuth in python pipelines on KafkaIO. (#34197)
add ee62d7def54 Fix PostCommit Python ValidatesContainer Dataflow With RC job (#34246)
add 5892df81b24 Update code-change-guide.md (#34266)
add 89c6402690e Clean up per worker histogram reporting path (#34244)
add bd02aa62fda Remove 2.1 and 2.2 and add 2.8 for Kafka compatibility test (#34265)
add 347d1f15f03 Fix Stress Tests Java Bigquery (#34078)
add fd30941a67c Fix spotless (#34279)
add f831f91268b Eliminate springframework dependency in KafkaIO (#34278)
add bdb26939350 Spark Runner : Fix invalid translateImpulse (#34272)
add 5950a17fdf8 Update join beam doc (#34284)
add 8f7513aee3c Update Lineage to use BoundedTrie
add 8b420bd5412 Update IOs to work with Lineage based on BoundedTrie
add 5a7311c5fcd Escape last segment path as whole
add df98cbc433e Fix comments
add a00f1840935 Merge pull request #33891 Use BoundedTrie metric to track lineage in IO.
add 00861d8276a Updates 2.64.0 release notes to highlight recent additions to the Managed API
add 56409b2e052 Merge pull request #34291: Updates 2.64.0 release notes to highlight recent additions to the Managed API
add 085140af5b4 Use Avro 1.8-compatible Schema constructors in Storage Write API translator (#34281)
add 4d8710d43d5 [AnomalyDetection] Support functions and classes as init arguments in specifiable. (#34273)
add d42bbec28c6 add default port for HostAndPort instances used in Windmill (#34061)
add c51d6d31009 docs: update CHANGES.md for Spark Runner (#34295)
add d4c2028e084 docs: fix typo in snippets_test.py (#34297)
add d6ce383b054 Update REVIEWERS.yml (#34299)
add d3b1312e882 Fix pre/postprocess type hints (#34298)
add 0aca8f1dfdc refactor main.py so that it can be called by runner.run_async().
add 8a418f513d9 Merge pull request #34290 Refactor main.py so that it can be called by runner.run_async().
add c7d67a8958a Better error messages for unavailable providers.
add fbdd8b32107 Merge pull request #34304 Better error messages for unavailable providers.
add bef5238b76b Add a STRING format to PubSub reading that interpretes the payload as utf-8 encoded. (#34301)
add e75577c8079 Add copyright year and Apache Beam (#34306)
add fa064b260b1 Honor JAVA_HOME when it is set (#34313)
add 76b0df92051 Consider workflow flaky if the 5 last runs were failed (#34321)
add 82a9be5709d added test_accessing_valueprovider_info_after_run to test accessing_valueprovider_info_after_run (#34315)
add 41cac5956e0 Fix Precommit SQL (#34320)
add b8505511d9b Fix PostCommit Python ValidatesContainer with RC workflow (#34332)
add 1bf14ad0745 Bump golang.org/x/net from 0.35.0 to 0.37.0 in /sdks (#34328)
add 534fcfd9f7f Bump github.com/fsouza/fake-gcs-server from 1.52.1 to 1.52.2 in /sdks
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (e4ba164aaab)
\
N -- N -- N refs/heads/dependabot/go_modules/sdks/github.com/fsouza/fake-gcs-server-1.52.2 (534fcfd9f7f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/REVIEWERS.yml | 4 -
.../actions/setup-environment-action/action.yml | 2 +-
.../arc/environments/beam.env | 4 +-
.../arc/images/Dockerfile | 4 +-
.../IO_Iceberg_Integration_Tests.json | 2 +-
.../beam_PostCommit_Java_Hadoop_Versions.json | 2 +-
.../beam_PostCommit_Java_PVR_Spark3_Streaming.json | 2 +-
...beam_PostCommit_Java_ValidatesRunner_Spark.json | 5 +-
...a_ValidatesRunner_SparkStructuredStreaming.json | 5 +-
...stCommit_Java_ValidatesRunner_Spark_Java11.json | 4 +-
.github/trigger_files/beam_PostCommit_Python.json | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.json | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.json | 2 +-
.github/workflows/beam_PostCommit_Python.yml | 2 +-
..._Python_ValidatesContainer_Dataflow_With_RC.yml | 4 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.yml | 6 +-
.github/workflows/beam_PreCommit_Spotless.yml | 12 +-
.github/workflows/build_release_candidate.yml | 20 +-
.github/workflows/cut_release_branch.yml | 8 +-
.github/workflows/go_tests.yml | 2 +-
.github/workflows/python_dependency_tests.yml | 2 +
...st_detection.yml => refresh_looker_metrics.yml} | 25 +-
.../republish_released_docker_containers.yml | 19 +-
.../jenkins/metrics_report/requirements.txt | 2 +-
.../github/github_runs_prefetcher/code/main.py | 8 +-
.test-infra/tools/refresh_looker_metrics.py | 119 ++
CHANGES.md | 8 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 25 +-
contributor-docs/code-change-guide.md | 19 +-
contributor-docs/discussion-docs/2024.md | 1 +
dev-support/docker/Dockerfile | 7 +-
dev-support/docker/pkglist | 3 +-
.../alloydb_product_catalog_embeddings.ipynb | 2172 ++++++++++++++++++++
it/{elasticsearch => clickhouse}/build.gradle | 35 +-
.../it/clickhouse/ClickHouseResourceManager.java | 353 ++++
.../ClickHouseResourceManagerException.java | 14 +-
.../apache/beam/it/clickhouse/ClickHouseUtils.java | 58 +
.../apache/beam/it/clickhouse}/package-info.java | 4 +-
.../it/clickhouse/ClickHouseResourceManagerIT.java | 85 +
.../clickhouse/ClickHouseResourceManagerTest.java | 186 ++
.../beam/it/clickhouse/ClickHouseUtilsTest.java | 96 +
.../apache/beam/it/gcp/bigquery/BigQueryIOST.java | 8 +-
.../apache/beam/model/pipeline/v1/metrics.proto | 17 +
.../cloudbuild/playground_ci_examples.sh | 8 +-
.../main/groovy/mobilegaming-java-dataflow.groovy | 40 +-
.../src/main/scripts/build_release_candidate.sh | 439 ----
.../runners/core/metrics/DefaultMetricResults.java | 10 +-
.../beam/runners/core/metrics/HistogramCell.java | 6 +
.../beam/runners/core/metrics/MetricUpdates.java | 18 +-
.../runners/core/metrics/MetricsContainerImpl.java | 112 +-
.../core/metrics/MetricsContainerStepMap.java | 7 +
.../core/metrics/MonitoringInfoConstants.java | 16 +
.../core/metrics/MonitoringInfoMetricName.java | 1 +
.../core/metrics/SimpleMonitoringInfoBuilder.java | 11 +
.../runners/core/metrics/HistogramCellTest.java | 9 +
.../core/metrics/MetricsContainerImplTest.java | 38 +
.../core/metrics/MonitoringInfoConstantsTest.java | 18 +
.../apache/beam/runners/direct/DirectMetrics.java | 4 +-
.../beam/runners/direct/DirectMetricsTest.java | 15 +-
.../metrics/CustomMetricQueryResults.java | 6 +
.../extensions/metrics/MetricsHttpSinkTest.java | 4 +-
runners/flink/flink_runner.gradle | 3 +-
.../flink/metrics/FlinkMetricContainerTest.java | 6 +-
.../unbounded/FlinkUnboundedSourceReaderTest.java | 4 +-
runners/google-cloud-dataflow-java/build.gradle | 4 +-
.../beam/runners/dataflow/DataflowMetrics.java | 6 +-
.../runners/dataflow/DataflowRunnerStaticTest.java | 196 ++
.../beam/runners/dataflow/DataflowRunnerTest.java | 112 +-
.../runners/dataflow/util/PackageUtilTest.java | 42 +-
.../google-cloud-dataflow-java/worker/build.gradle | 1 -
.../dataflow/worker/DataflowMetricsContainer.java | 11 +-
.../dataflow/worker/DoFnInstanceManagers.java | 14 +
.../worker/StreamingStepMetricsContainer.java | 64 +-
.../worker/windmill/WindmillEndpoints.java | 4 +-
.../client/grpc/stubs/WindmillChannelFactory.java | 13 +-
.../runners/dataflow/worker/HotKeyLoggerTest.java | 7 +-
.../worker/StreamingStepMetricsContainerTest.java | 88 +-
.../dataflow/worker/streaming/StageInfoTest.java | 2 +-
.../FanOutStreamingEngineWorkerHarnessTest.java | 7 +-
.../windmill/state/WindmillStateInternalsTest.java | 26 +-
.../control/BundleFinalizationHandlersTest.java | 6 +-
.../runners/jet/FailedRunningPipelineResults.java | 6 +
.../beam/runners/jet/metrics/JetMetricResults.java | 9 +
.../runners/jet/metrics/JetMetricsContainer.java | 7 +
.../beam/runners/portability/PortableMetrics.java | 4 +-
.../samza/runtime/ClassicBundleManagerTest.java | 16 +-
.../translation/batch/ParDoTranslatorBatch.java | 6 +-
runners/spark/spark_runner.gradle | 16 +
.../runners/spark/DependentTransformsVisitor.java | 50 +
.../org/apache/beam/runners/spark/SparkRunner.java | 10 +-
.../runners/spark/TestSparkPipelineOptions.java | 6 +
.../apache/beam/runners/spark/TestSparkRunner.java | 6 +-
.../spark/translation/EvaluationContext.java | 41 +
.../spark/translation/SingleEmitInputDStream.java | 63 +
.../SparkStreamingPortablePipelineTranslator.java | 39 +-
.../spark/translation/TransformTranslator.java | 45 +-
.../streaming/StreamingTransformTranslator.java | 19 +-
.../spark/DependentTransformsVisitorTest.java | 143 ++
.../runners/spark/translation/RDDTreeParser.java | 6 +-
.../translation/SingleEmitInputDStreamTest.java | 163 ++
.../SparkExecutableStageFunctionTest.java | 4 +-
.../spark/translation/TransformTranslatorTest.java | 45 +-
.../StreamingTransformTranslatorTest.java | 241 +++
sdks/go.mod | 21 +-
sdks/go.sum | 36 +-
sdks/go/run_with_go_version.sh | 2 +-
.../main/resources/beam/checkstyle/checkstyle.xml | 8 -
.../container/license_scripts/dep_urls_java.yaml | 2 +-
.../src/main/java/org/apache/beam/sdk/io/Read.java | 73 +-
.../apache/beam/sdk/metrics/DelegatingGauge.java | 25 +-
.../beam/sdk/metrics/DelegatingHistogram.java | 25 +-
.../org/apache/beam/sdk/metrics/Histogram.java | 5 +
.../beam/sdk/metrics/LabeledMetricNameUtils.java | 16 +-
.../java/org/apache/beam/sdk/metrics/Lineage.java | 197 +-
.../org/apache/beam/sdk/metrics/MetricName.java | 21 +-
.../beam/sdk/metrics/MetricQueryResults.java | 10 +-
.../java/org/apache/beam/sdk/metrics/Metrics.java | 8 +
.../apache/beam/sdk/metrics/MetricsContainer.java | 17 -
.../org/apache/beam/sdk/metrics/NoOpGauge.java | 42 -
.../org/apache/beam/sdk/util/HistogramData.java | 11 +
.../test/java/org/apache/beam/sdk/io/ReadTest.java | 6 +
.../sdk/metrics/LabeledMetricNameUtilsTest.java | 27 +-
.../org/apache/beam/sdk/metrics/LineageTest.java | 64 +-
sdks/java/expansion-service/build.gradle | 4 -
.../sdk/extensions/gcp/options/GcsOptions.java | 64 +
.../sdk/extensions/gcp/storage/GcsFileSystem.java | 2 +-
.../beam/sdk/extensions/gcp/util/Transport.java | 14 +-
.../sdk/extensions/gcp/GcpCoreApiSurfaceTest.java | 2 +
.../sdk/extensions/gcp/options/GcsOptionsTest.java | 78 +
.../extensions/gcp/storage/GcsFileSystemTest.java | 2 +-
.../sdk/extensions/gcp/util/TransportTest.java | 21 +-
.../schemaio-expansion-service/build.gradle | 2 +
.../ExternalSchemaIOTransformRegistrar.java | 41 +-
.../beam/fn/harness/state/MultimapSideInput.java | 4 +-
.../apache/beam/sdk/io/aws2/s3/S3FileSystem.java | 2 +-
.../beam/sdk/io/aws2/s3/S3FileSystemTest.java | 2 +-
.../azure/blobstore/AzureBlobStoreFileSystem.java | 5 +-
.../blobstore/AzureBlobStoreFileSystemTest.java | 2 +-
sdks/java/io/expansion-service/build.gradle | 8 +-
sdks/java/io/google-cloud-platform/build.gradle | 3 +-
.../AvroGenericRecordToStorageApiProto.java | 22 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 3 +-
.../sdk/io/gcp/bigquery/BigQuerySinkMetrics.java | 4 +-
.../sdk/io/gcp/bigquery/InsertRetryPolicy.java | 17 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 4 +-
.../io/gcp/bigquery/TableRowToStorageApiProto.java | 15 +-
.../beam/sdk/io/gcp/firestore/RpcQosImpl.java | 7 +
.../sdk/io/gcp/pubsub/PreparePubsubWriteDoFn.java | 6 +-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 5 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 13 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 23 +-
.../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 1 +
.../AvroGenericRecordToStorageApiProtoTest.java | 20 +-
.../io/gcp/bigquery/BigQuerySinkMetricsTest.java | 36 +-
.../gcp/bigquery/StorageApiSinkSchemaUpdateIT.java | 15 +-
.../gcp/bigquery/StreamingInsertsMetricsTest.java | 6 +-
.../bigquery/TableRowToStorageApiProtoTest.java | 82 +
.../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +-
.../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 4 +-
.../gcp/firestore/BaseFirestoreV1WriteFnTest.java | 2 +-
.../sdk/io/gcp/pubsub/PubsubIOExternalTest.java | 11 +-
.../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java | 50 +-
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 77 +-
.../SpannerChangeStreamErrorTest.java | 5 -
.../action/ChildPartitionsRecordActionTest.java | 4 +-
.../dao/PartitionMetadataDaoTest.java | 17 +-
.../sdk/io/gcp/testing/BigqueryClientTest.java | 18 +-
.../sdk/io/gcp/testing/BigqueryMatcherTest.java | 21 +-
.../io/hadoop/format/HadoopFormatIOReadTest.java | 3 +-
.../apache/beam/sdk/io/iceberg/RecordWriter.java | 6 +
.../sdk/io/iceberg/RecordWriterManagerTest.java | 184 +-
sdks/java/io/influxdb/build.gradle | 10 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 55 +-
.../beam/sdk/io/jdbc/JdbcSchemaIOProvider.java | 13 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 82 +-
.../beam/sdk/io/jdbc/JdbcSchemaIOProviderTest.java | 87 +
sdks/java/io/kafka/build.gradle | 25 +-
.../io/kafka/{kafka-211 => kafka-282}/build.gradle | 4 +-
.../io/kafka/{kafka-222 => kafka-390}/build.gradle | 4 +-
.../org/apache/beam/sdk/io/kafka/ConsumerSpEL.java | 31 -
.../beam/sdk/io/kafka/KafkaIOInitializer.java | 5 +-
.../org/apache/beam/sdk/io/kafka/KafkaMetrics.java | 40 +-
.../apache/beam/sdk/io/kafka/KafkaSinkMetrics.java | 26 +-
.../beam/sdk/io/kafka/KafkaUnboundedReader.java | 6 +-
.../org/apache/beam/sdk/io/kafka/KafkaWriter.java | 6 +
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 17 +-
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 7 +-
.../apache/beam/sdk/io/kafka/KafkaMetricsTest.java | 64 +-
.../beam/sdk/io/kafka/KafkaSinkMetricsTest.java | 23 +-
.../beam/sdk/io/kafka/ProducerRecordCoderTest.java | 22 +-
.../org/apache/beam/sdk/io/mongodb/MongoDbIO.java | 15 +-
.../apache/beam/sdk/io/mongodb/FindQueryTest.java} | 18 +-
.../apache/beam/sdk/io/mongodb/MongoDbIOTest.java | 17 +
.../org/apache/beam/sdk/io/mqtt/MqttIOTest.java | 2 +
.../java/org/apache/beam/sdk/managed/Managed.java | 7 +
.../apache_beam/examples/snippets/snippets.py | 2 +-
.../apache_beam/examples/snippets/snippets_test.py | 14 +-
.../io/external/xlang_jdbcio_it_test.py | 399 +++-
sdks/python/apache_beam/io/gcp/gcsio.py | 15 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 12 +-
sdks/python/apache_beam/io/jdbc.py | 23 +-
sdks/python/apache_beam/io/kafka.py | 7 +
sdks/python/apache_beam/io/tfrecordio.py | 2 +-
sdks/python/apache_beam/ml/anomaly/aggregations.py | 23 +-
sdks/python/apache_beam/ml/anomaly/base.py | 15 +-
.../{internal => ml/anomaly/detectors}/__init__.py | 4 +-
.../python/apache_beam/ml/anomaly/detectors/iqr.py | 129 ++
.../anomaly/detectors/iqr_test.py} | 35 +-
.../ml/anomaly/detectors/robust_zscore.py | 116 ++
.../anomaly/detectors/robust_zscore_test.py} | 34 +-
.../apache_beam/ml/anomaly/detectors/zscore.py | 126 ++
.../ml/anomaly/detectors/zscore_test.py | 83 +
sdks/python/apache_beam/ml/anomaly/specifiable.py | 97 +-
.../apache_beam/ml/anomaly/specifiable_test.py | 111 +-
sdks/python/apache_beam/ml/anomaly/thresholds.py | 159 --
.../apache_beam/ml/anomaly/thresholds_test.py | 157 --
sdks/python/apache_beam/ml/anomaly/transforms.py | 529 +++++
.../apache_beam/ml/anomaly/transforms_test.py | 413 ++++
.../apache_beam/ml/anomaly/univariate/base.py | 2 +
.../apache_beam/ml/anomaly/univariate/mad.py | 87 +
.../anomaly/univariate/mad_test.py} | 36 +-
.../apache_beam/ml/anomaly/univariate/median.py | 67 +
.../ml/anomaly/univariate/median_test.py | 72 +
.../apache_beam/ml/anomaly/univariate/quantile.py | 58 +-
sdks/python/apache_beam/ml/inference/base.py | 4 +-
.../python/apache_beam/ml/rag/ingestion/alloydb.py | 129 +-
.../ml/rag/ingestion/alloydb_it_test.py | 88 +
.../python/apache_beam/options/pipeline_options.py | 75 +
.../apache_beam/options/pipeline_options_test.py | 72 +
.../runners/dataflow/internal/apiclient.py | 4 +
.../runners/dataflow/internal/apiclient_test.py | 40 +
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
.../portability/fn_api_runner/fn_runner_test.py | 21 +
.../apache_beam/runners/worker/operations.pxd | 1 +
.../apache_beam/runners/worker/operations.py | 63 +-
sdks/python/apache_beam/testing/util.py | 2 +-
sdks/python/apache_beam/transforms/managed.py | 7 +
sdks/python/apache_beam/typehints/typehints.py | 13 +-
.../python/apache_beam/typehints/typehints_test.py | 20 +
sdks/python/apache_beam/utils/annotations.py | 24 +
sdks/python/apache_beam/utils/annotations_test.py | 16 +
sdks/python/apache_beam/utils/subprocess_server.py | 23 +-
...yaml => map_to_fields_with_resource_hints.yaml} | 11 +-
sdks/python/apache_beam/yaml/main.py | 65 +-
sdks/python/apache_beam/yaml/pipeline.schema.yaml | 9 +
sdks/python/apache_beam/yaml/yaml_io.py | 11 +-
sdks/python/apache_beam/yaml/yaml_io_test.py | 22 +
sdks/python/apache_beam/yaml/yaml_provider.py | 39 +-
sdks/python/apache_beam/yaml/yaml_transform.py | 60 +-
.../python/apache_beam/yaml/yaml_transform_test.py | 64 +-
sdks/python/apache_beam/yaml/yaml_utils.py | 5 +
sdks/python/build.gradle | 3 +-
.../container/base_image_requirements_manual.txt | 3 +-
.../container/py310/base_image_requirements.txt | 2 +-
.../container/py311/base_image_requirements.txt | 2 +-
.../container/py312/base_image_requirements.txt | 2 +-
.../container/py39/base_image_requirements.txt | 2 +-
sdks/python/container/run_validatescontainer.sh | 11 +-
sdks/python/scripts/generate_pydoc.sh | 1 +
sdks/python/test-suites/portable/common.gradle | 3 +-
settings.gradle.kts | 12 +-
.../www/site/content/en/contribute/attributes.md | 2 +-
.../site/content/en/documentation/io/connectors.md | 9 +-
.../en/documentation/runtime/resource-hints.md | 34 +-
.../sdks/python-pipeline-dependencies.md | 2 +-
website/www/site/data/en/join_beam.yaml | 2 +-
268 files changed, 9484 insertions(+), 2156 deletions(-)
copy .github/workflows/{flaky_test_detection.yml => refresh_looker_metrics.yml} (62%)
create mode 100644 .test-infra/tools/refresh_looker_metrics.py
create mode 100644 examples/notebooks/beam-ml/alloydb_product_catalog_embeddings.ipynb
copy it/{elasticsearch => clickhouse}/build.gradle (58%)
create mode 100644 it/clickhouse/src/main/java/org/apache/beam/it/clickhouse/ClickHouseResourceManager.java
copy learning/katas/java/IO/Built-in IOs/Built-in IOs/test/org/apache/beam/learning/katas/io/builtinios/TaskTest.java => it/clickhouse/src/main/java/org/apache/beam/it/clickhouse/ClickHouseResourceManagerException.java (73%)
create mode 100644 it/clickhouse/src/main/java/org/apache/beam/it/clickhouse/ClickHouseUtils.java
copy it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datastream => clickhouse/src/main/java/org/apache/beam/it/clickhouse}/package-info.java (88%)
create mode 100644 it/clickhouse/src/test/java/org/apache/beam/it/clickhouse/ClickHouseResourceManagerIT.java
create mode 100644 it/clickhouse/src/test/java/org/apache/beam/it/clickhouse/ClickHouseResourceManagerTest.java
create mode 100644 it/clickhouse/src/test/java/org/apache/beam/it/clickhouse/ClickHouseUtilsTest.java
delete mode 100755 release/src/main/scripts/build_release_candidate.sh
create mode 100644 runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerStaticTest.java
create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/DependentTransformsVisitor.java
create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SingleEmitInputDStream.java
create mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/DependentTransformsVisitorTest.java
create mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SingleEmitInputDStreamTest.java
create mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslatorTest.java
delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/NoOpGauge.java
create mode 100644 sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcsOptionsTest.java
rename sdks/java/io/kafka/{kafka-211 => kafka-282}/build.gradle (90%)
rename sdks/java/io/kafka/{kafka-222 => kafka-390}/build.gradle (90%)
copy sdks/java/io/mongodb/src/{main/java/org/apache/beam/sdk/io/mongodb/FindQuery.java => test/java/org/apache/beam/sdk/io/mongodb/FindQueryTest.java} (87%)
copy sdks/python/apache_beam/{internal => ml/anomaly/detectors}/__init__.py (81%)
create mode 100644 sdks/python/apache_beam/ml/anomaly/detectors/iqr.py
copy sdks/python/apache_beam/{io/flink/flink_streaming_impulse_source_test.py => ml/anomaly/detectors/iqr_test.py} (62%)
create mode 100644 sdks/python/apache_beam/ml/anomaly/detectors/robust_zscore.py
copy sdks/python/apache_beam/{io/flink/flink_streaming_impulse_source_test.py => ml/anomaly/detectors/robust_zscore_test.py} (60%)
create mode 100644 sdks/python/apache_beam/ml/anomaly/detectors/zscore.py
create mode 100644 sdks/python/apache_beam/ml/anomaly/detectors/zscore_test.py
create mode 100644 sdks/python/apache_beam/ml/anomaly/transforms.py
create mode 100644 sdks/python/apache_beam/ml/anomaly/transforms_test.py
create mode 100644 sdks/python/apache_beam/ml/anomaly/univariate/mad.py
copy sdks/python/apache_beam/{io/flink/flink_streaming_impulse_source_test.py => ml/anomaly/univariate/mad_test.py} (55%)
create mode 100644 sdks/python/apache_beam/ml/anomaly/univariate/median.py
create mode 100644 sdks/python/apache_beam/ml/anomaly/univariate/median_test.py
copy sdks/python/apache_beam/yaml/examples/transforms/elementwise/{map_to_fields_callable.yaml => map_to_fields_with_resource_hints.yaml} (86%)