You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2025/05/12 15:23:13 UTC
(beam) branch users/damccorm/prismByDefault updated (cbaa083f415 -> 7e13214d068)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a change to branch users/damccorm/prismByDefault
in repository https://gitbox.apache.org/repos/asf/beam.git
from cbaa083f415 Merge branch 'master' of https://github.com/apache/beam into users/damccorm/prismByDefault
add 9331662c934 consume flow control settings updates from job settings (#34539)
add 8de3d3d74f2 [KafkaIO] BUG FIX : fix-ReadFromKafkaViaSDF-bug-shall-set-coder (#34505)
add 8fb42c7a730 Update CHANGES.md (#34639)
add 4ead94006b5 Inject SDK-side flattens while handling input/output coder mismatch in flattens. (#34641)
add a142d78d4d1 Fix FanOutStreamingEngineWorkerHarnessTest - testOnNewWorkerMetadata_redistributesBudget (#34531)
add 56f1aa1bbda Skip rewriting internal pcollection coders in a stage (#34655)
add a847f066916 Restore previous handling of Callables, fix bare collections types (#34645)
add 58e1616d775 Get rid of unnecessary cogbk when running offline detector. (#34656)
add 38192def9fa make validation for BQ export source configurable (#34651)
add 698148e2c9e Avoid use of deprecated datetime.utcnow(). (#34572)
add b2db7a41bd4 Remove unnecessary checkNotNull (#34664)
add 90b77e726e7 Regenerate Python requirement (#34666)
add 795e45d5215 Ignore test_flatten_and_gbk for spark and samza runners. (#34663)
add 1773bd66109 Update link in yaml-providers.md (#34570)
add 7b27e8ac775 Added run_rc_validation_python_yaml to validate the Beam YAML RC (#34670)
add 10775fefc28 Provide kafka unbounded reader to checkpoint mark when offset based deduplication is supported. (#34669)
add 2be5a611971 fix the doc (#34684)
add de7f1bd85ae Minor fixes to yaml testing docs. (#34618)
add 32620186a00 move Java snippet to Java, remove redundant line (#33242)
add d0def26d3ec Update release-guide.md (#34681)
add 6e1bd6f4fff Update REVIEWERS.yml - remove lostluck from fallback (#34696)
add 201d207de4f update py container (#34691)
add 8dd07803bfd Clickhouse IT RM JDBC fix + ClickHouseRowsCheck (#34407)
add c600069a71a Add adapters to address output conversion for OfflineDetector. (#34662)
add 73739725633 fixed jdbc io (#34704)
add f9101840e13 Properly propagate schemas of Beam YAML Partition transform.
add 1fe7ee9b696 Merge pull request #34595 Properly propagate schemas of Beam YAML Partition transform.
add 83261806299 add contributing dev doc for yaml integration tests (#34636)
add b7bd1274039 Add new resource hints: max active bundles per worker (#34529)
add 9abd394d3fb Add some more cloudpickle tests. (#34683)
add bf417db65d1 support MicrosInstant type (#34707)
add b3c317957f2 Add core and missing transformers for yaml integration testing - phase 2sh (#34654)
add 238233ded62 Fix input validation for non-chain YAML-provider transforms. (#34701)
add c8e5118d6d1 Updated the container doc (#34682)
add 248380af25b Created using Colab (#34713)
add 0cc82ec55c3 Add schema definition for yaml test specs.
add 22f8e4bba2f Merge pull request #34710 Add schema definition for yaml test specs.
add 5c4b42f81c6 Phase 1sh - more yaml integration tests (#34692)
add 27edce710bb Bump golang.org/x/net from 0.36.0 to 0.38.0 in /learning/katas/go (#34650)
add fac2b1cb75f Bump golang.org/x/crypto from 0.21.0 to 0.35.0 in /.test-infra/mock-apis (#34632)
add 8b354f3f27b Bump cloud.google.com/go/spanner from 1.78.0 to 1.79.0 in /sdks (#34609)
add c6d42f13206 Update run_rc_validation_python_yaml.yml (#34675)
add 72a04378515 Bump github.com/aws/aws-sdk-go-v2/credentials in /sdks (#34634)
add 8ba84370053 Bump github.com/testcontainers/testcontainers-go in /sdks (#34642)
add 7ea4f06c874 Bump github.com/avast/retry-go/v4 from 4.6.0 to 4.6.1 in /sdks (#34373)
add 5645a032499 Add a md5sum check before overwriting a cached prism file (#34715)
add 8509a87e185 Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#34610)
add edd00b24580 Add support and unit test for PyOD models (#34709)
add 49c8f266c38 Update beam_PreCommit_Java.yml (#34685)
add 9de36d5c72d Bump @octokit/request, @actions/github and @octokit/rest (#34545)
add 582a2464cb3 [extensions-proto] Return correct TypeDescriptor for ProtoCoder (#34561)
add 8d400da0034 Merge pull request #34562: [extensions-proto] Return correct TypeDescriptor for ProtoCoder (#34561)
add 0a511a751cd Revert "Bump @octokit/request, @actions/github and @octokit/rest (#34545)" (#34725)
add c594706fa8f Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#34727)
add 563ab826623 Bump cloud.google.com/go/bigtable from 1.36.0 to 1.37.0 in /sdks (#34730)
add 15d2ff0e504 Ensure watermark updates when position advances
add f81486ec864 Ensure expectedOffset advances immediately after a claim
add 46b2021295c Merge pull request #34202: [KafkaIO] Update tracker and watermark for non-visible progress
add 89d6bc323cc Fix Iceberg Integration tests (#34686)
add fb5962385f6 Update code-change-guide.md (#34697)
add c841d8924ab Bump google.golang.org/grpc from 1.71.1 to 1.72.0 in /sdks (#34729)
add 3b5a2b67bb6 Bump github.com/nats-io/nats.go from 1.41.0 to 1.41.2 in /sdks (#34726)
add e69a7baf0d9 Update RunInference type hints to PEP 585 standards (#34719)
add 8cd0a2970d9 Bump github.com/go-jose/go-jose/v4 from 4.0.4 to 4.0.5 in /sdks (#34735)
add 971ad8d1b42 Skip this failed test until further investigation is done. (#34737)
add 5832c61f769 Update contributor doc reflecting #33954 (#34640)
add 1540776dc30 Fix IndexOutOfBoundsException invoking expansion service (#34428)
add 404170a351e Remove myself from java reviewers and focus on python and go. (#34741)
add 908d43e65ca Setting num_bytes to DEFAULT_READ_BUFFER_SIZE if it is None originally (#34475)
add f2819ca987b Bump cloud.google.com/go/spanner from 1.79.0 to 1.80.0 in /sdks (#34745)
add a71dd28aec7 Deep copy PipelineOptions and keep the input intact. (#34723)
add d3b526ad260 Bump cloud.google.com/go/storage from 1.51.0 to 1.52.0 in /sdks (#34742)
add ae7bf20db96 Add option to pickler dumps() for best-effort determinism (#34698)
add a2fcbd95287 Only declare state backed iterables on code channel.
add 82540a889aa Merge pull request #34740 Only declare state backed iterables on code channel.
add 930d14abc99 Fix sigint handler for StopOnExitJobServer (#34748)
add ca64fcf8b67 Fix handling of collections.Counter types (#34706)
add cefc9a20144 Add option to create metedata filter with function. (#34754)
add fb415f7b55b Enable cloudpickle by default (#34695)
add 0dd5dd327d7 Update schedule cron time (#34765)
add b9fa49a9827 Use Java 17 for Debezium IO #34747 (#34751)
add 5f2403c747b Created using Colab (#34753)
add 95965ffd5c6 pin pip==25.0.1 (#34774)
add 8901e69ec04 Add reshuffle before triggering load jobs. (#34657)
add 29b8b31740c Bump github.com/nats-io/nats-server/v2 from 2.11.1 to 2.11.2 in /sdks (#34757)
add face5d372f2 Fixing Typo on Blog Post (#34778)
add 4d797cab6fb [ManagedIO] Single source of truth for ManagedIO configs (#34720)
add 735ad8b8803 Skip deep copying runner instance in pipeline options. (#34775)
add 49a46c9c571 Improve validation error message for chain transform (#34708)
add 681cc3c0ad5 Modernize Anomaly Detection type hints (#34779)
add b46a6361d2b Improve the apache beam type system to handle TypedDict. (#34780)
add e038ad3a696 [IcebergIO] Fetch source schema directly from the table (#34773)
add fe2bc29a077 Support writing to Pubsub with ordering key; Add PubsubMessage SchemaCoder (#31608)
add 363bcc0d328 [ManagedIO] Fail expansion when encountering extra or unknown configuration (#34525)
add 9628c1ef9ba [KafkaIO] Improve caching in backlog estimation and processing (#34331)
add 74c514fc392 Spark Runner : Support for Streaming side-inputs for Spark Runner (#34560)
add 966d1fff903 [Dataflow Streaming] BoundedQueueExecutor: Add an experiment to use fair monitor (#34787)
add f3e0f6d4962 Updates YAML SDK to replace Kafka read/write transforms with equivalent managed transforms (#34755)
add cfae8675690 re-introduce the fail fast mechanic with fixes for documented bugs. (#34659)
add 3a27dfbeb8c Adjusting the YAML Kafka Managed I/O compat version (#34790)
add 5b4178fd6f4 Moving to 2.66.0-SNAPSHOT on master branch.
add edcd5cebccd Update CHANGES.md (#34791)
add 9e8e3c32d32 Update CHANGES.md after 2.64 release cut (#34793)
add 86b7067d925 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#34785)
add c2e26f4c94c Revert "[ManagedIO] Fail expansion when encountering extra or unknown configu…" (#34802)
add 43f3f44151a Fix issue link in trivial_inference.py (#34803)
add b607ce44f23 Add name to BQ File Loads lambda to ensure changes are update compatible (#34807)
add 82ef9d81066 add changes note about TFRecord support in beam yaml and rename integration test (#34635)
add 9fd7d092490 Fix test retry on throttling (#34786)
add 1763e7b9048 Fix loopback (#34678)
add 4e6cbfc8bf5 Prism windowed value coder (#34830)
add c483ab8fe00 Bump google.golang.org/api from 0.229.0 to 0.231.0 in /sdks (#34784)
add 3816dfbc1f8 Bump github.com/go-sql-driver/mysql from 1.9.1 to 1.9.2 in /sdks (#34759)
add c8c9ba85a1c Bump github.com/golang-cz/devslog from 0.0.11 to 0.0.13 in /sdks (#34758)
add 2832abe7a5a updated command for the wordcount powershell (#34837)
add c9a0e908154 Add OrderedList and Set state (#34836)
add ff9a22af4e1 Encode paneinfo with PaneInfoCoder. (#34824)
add 6f2f960b359 use batch delete for GCS IO (#34835)
add 4c99387ecc5 Update code-change-guide.md
add 5e29867ba53 Merge pull request #34776: Update code-change-guide.md
add ed7624155f8 Fix Nexmark Dataflow V2 use runner v2
add 891e8a44704 Merge pull request #34823: Fix Nexmark Dataflow V2 use runner v2
add 5288c548e9e Modify the assert to accept any instance of AvroCoder, including subclasses. (#34843)
add 0a2d8cbf535 Bump github.com/nats-io/nats-server/v2 from 2.11.2 to 2.11.3 in /sdks (#34838)
add 7f1e14bb909 Use checkerframework Nullable/NonNull (#34851)
add 1398655862d Revert "Bump @octokit/request and @octokit/rest in /scripts/ci/issue-report (…" (#34857)
add 1e010b3f3b9 Fix Tour of Beam Go Integration tests (#34854)
add 4c213e5a796 Update breaking changes with PaneInfoCoder change. (#34865)
add 91b09c8c763 Add label when PR gets reassigned (#34866)
add db3b8332770 Fix checkstyle rule for guava within vendored grpc
add a23cc4dce18 Improve clarity of PTransformRunnerFactory in Java SDK harness
add e1804030702 Merge pull request #34846: Improve clarity of PTransformRunnerFactory in Java SDK harness
add b3636fab625 fix: correct spanner column schema type parser (#34867)
add f935ec7962a Add yapf version upgrade doc to discussion-docs for 2025 (#34871)
add e543c6b33e7 Fix github client variable (#34877)
add 6261411d199 reorder opt-out review comment (#34872)
add 399ae726a54 [IcebergIO] Support column pruning (#34856)
add 35c1e32e3a1 Replace deprecated model version. (#34875)
add 93ec4e09663 Bump google.golang.org/api from 0.231.0 to 0.232.0 in /sdks (#34876)
add cbf57c4c608 [AnomalyDetection] Add a notebook for using iforest for anomaly detection (#34845)
add c72b351275d [AnomalyDetection] Add a notebook for anomaly detection with Z-Score (#34459)
add deeb4c45049 Streamline non-cached state backed iterable.
add 6b8ef134636 Address reviewer comments.
add 877f1112d2c Remove tests wrongly asserting that reiteration is still cached.
add 5b862dd510b Merge pull request #34746 Streamline non-cached state backed iterable.
add 1caa1c23120 [AnomalyDetection] Update notebook. (#34884)
add d799c958af1 Reference managed-io page in our docs (#34882)
add 4a349769854 yapf 0.43.0 migration (#34801)
add d7fdefe20af Fix typecoders_test.py formatting (#34889)
add cccf870ad24 Add empty outputs to notebooks which can't run without help (#34895)
add b7d1e068d0c Remove type suppressions from MetricsPusher and MetricsSink
add ee22eacf669 Merge pull request #33918: Remove type suppressions from MetricsPusher and MetricsSink
add 4f5b78f5ccb Update Iceberg table field documentation (#34847)
add dc874bd5b9e Add PyTorch DistilBERT Sentiment Analysis Batch/Streaming pipelines for ML Benchmarks (#34577)
add b6038e6ce41 Correcting regex for _PROJECT_PATTERN for bigquery table name (resolves #32611) (#34898)
add 3a967752fdd Enable certain Beam module compile with newer Java version (#34858)
add d15a1639fbe Revert "Unpin Dataflow legacy worker container for Nexmark test (#33224)" (#34804)
add 4c59b6a736d Add cloudpickle default to changes.md. (#34904)
add d427d3731e4 Revert "Correcting regex for _PROJECT_PATTERN for bigquery table name (resolv…" (#34901)
add 1f31eb0a04d use blob.exists to check the GCS file (#34818)
add c2c9275c680 [Python] Added the validate support for ReadAllFromBigQuery (#34910)
add e7cdee37bb3 [Dataflow Streaming] Remove unused quota throttling metrics from windmill streams (#34891)
add 63242c9413a [Java SDK] Added cache for AvroCoder DatumReader/DatumWriter to reduce memory footprint (#34873)
add 23dc4470592 [Dataflow] Fix flaky BoundedQueueExecutor test (#34878)
add 28df7cb27f6 [AnomalyDetection] Support unkeyed input for the transform. (#34897)
add 7e13214d068 Merge branch 'master' of https://github.com/apache/beam into users/damccorm/prismByDefault
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 +
.github/REVIEWERS.yml | 3 +-
.../IO_Iceberg_Integration_Tests_Dataflow.json | 2 +-
.../beam_PostCommit_Java_Nexmark_Dataflow.json | 2 +-
...beam_PostCommit_Java_ValidatesRunner_Spark.json | 3 +-
...stCommit_Java_ValidatesRunner_Spark_Java11.json | 3 +-
.github/trigger_files/beam_PostCommit_Python.json | 2 +-
...tCommit_Python_ValidatesContainer_Dataflow.json | 2 +-
...am_PostCommit_Python_ValidatesRunner_Flink.json | 3 +-
...am_PostCommit_Python_ValidatesRunner_Samza.json | 3 +
...am_PostCommit_Python_ValidatesRunner_Spark.json | 3 +
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.json | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 2 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.json | 2 +-
.../beam_Inference_Python_Benchmarks_Dataflow.yml | 24 +
.../beam_PostCommit_Java_Nexmark_Dataflow_V2.yml | 7 +
...PostCommit_Java_ValidatesRunner_Flink_Java8.yml | 2 +-
.github/workflows/beam_PreCommit_Java.yml | 5 +-
.../beam_PreCommit_Java_Debezium_IO_Direct.yml | 10 +-
.../beam_PreCommit_Xlang_Generated_Transforms.yml | 2 +
.../workflows/beam_PreCommit_Yaml_Xlang_Direct.yml | 2 +-
...ch_Sentiment_Batch_DistilBert_Base_Uncased.txt} | 15 +-
...entiment_Streaming_DistilBert_Base_Uncased.txt} | 15 +-
.../workflows/run_rc_validation_python_yaml.yml | 286 ++++
.../workflows/tour_of_beam_backend_integration.yml | 2 +
.pre-commit-config.yaml | 4 +-
.test-infra/mock-apis/go.mod | 9 +-
.test-infra/mock-apis/go.sum | 16 +-
.test-infra/tools/refresh_looker_metrics.py | 2 +
CHANGES.md | 69 +-
build.gradle.kts | 5 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 146 +-
contributor-docs/code-change-guide.md | 28 +-
contributor-docs/discussion-docs/2025.md | 2 +
contributor-docs/release-guide.md | 5 +
.../alloydb_product_catalog_embeddings.ipynb | 781 ++++++----
.../anomaly_detection_iforest.ipynb | 650 ++++++++
.../anomaly_detection_zscore.ipynb | 779 +++++++++
.../beam-ml/bigquery_enrichment_transform.ipynb | 42 +-
.../bigquery_vector_ingestion_and_search.ipynb | 1648 ++++++++++++++++++++
.../gemma_2_sentiment_and_summarization.ipynb | 16 +-
.../notebooks/beam-ml/run_inference_vllm.ipynb | 30 +-
gradle.properties | 4 +-
it/clickhouse/build.gradle | 1 +
.../it/clickhouse/ClickHouseResourceManager.java | 8 +-
.../conditions/ClickHouseRowsCheck.java} | 36 +-
.../clickhouse/{ => conditions}/package-info.java | 2 +-
.../clickhouse/ClickHouseResourceManagerTest.java | 18 +
learning/katas/go/go.mod | 2 +-
learning/katas/go/go.sum | 18 +-
.../emulators/firebase/Dockerfile | 2 +-
.../map/group-by-key/description.md | 29 +-
.../beam/model/pipeline/v1/beam_runner_api.proto | 4 +
.../beam/runners/core/metrics/MetricsPusher.java | 5 +-
.../extensions/metrics/MetricsGraphiteSink.java | 7 +-
.../extensions/metrics/MetricsHttpSink.java | 27 +-
.../beam/runners/dataflow/DataflowRunner.java | 10 +
.../dataflow/worker/DataflowSystemMetrics.java | 1 -
.../dataflow/worker/StreamingDataflowWorker.java | 127 +-
.../config/StreamingGlobalConfigHandle.java | 3 +-
.../FanOutStreamingEngineWorkerHarness.java | 21 +-
.../harness/SingleSourceWorkerHarness.java | 13 +-
.../streaming/harness/StreamingCounters.java | 4 -
.../streaming/harness/StreamingWorkerHarness.java | 3 +-
.../harness/StreamingWorkerStatusPages.java | 11 +-
.../harness/StreamingWorkerStatusReporter.java | 10 -
.../streaming/harness/WindmillStreamSender.java | 36 +-
.../dataflow/worker/util/BoundedQueueExecutor.java | 6 +-
.../worker/windmill/WindmillConnection.java | 16 +-
.../worker/windmill/WindmillServerBase.java | 6 -
.../worker/windmill/WindmillServerStub.java | 3 -
.../windmill/client/AbstractWindmillStream.java | 12 -
.../windmill/client/grpc/GrpcCommitWorkStream.java | 13 -
.../client/grpc/GrpcDirectGetWorkStream.java | 12 -
.../windmill/client/grpc/GrpcDispatcherClient.java | 21 +-
.../windmill/client/grpc/GrpcGetDataStream.java | 12 -
.../windmill/client/grpc/GrpcGetWorkStream.java | 12 -
.../client/grpc/GrpcGetWorkerMetadataStream.java | 11 -
.../windmill/client/grpc/GrpcWindmillServer.java | 21 +-
.../client/grpc/GrpcWindmillStreamFactory.java | 29 +-
.../windmill/client/grpc/stubs/ChannelCache.java | 66 +-
.../client/grpc/stubs/IsolationChannel.java | 34 +-
.../client/grpc/stubs/WindmillChannelFactory.java | 119 +-
...llChannelFactory.java => WindmillChannels.java} | 100 +-
.../grpc/stubs/WindmillStubFactoryFactoryImpl.java | 22 +-
.../throttling/StreamingEngineThrottleTimers.java | 41 -
.../windmill/client/throttling/ThrottleTimer.java | 69 -
.../client/throttling/ThrottledTimeTracker.java | 32 -
.../dataflow/worker/FakeWindmillServer.java | 5 -
.../worker/StreamingDataflowWorkerTest.java | 17 +-
.../FanOutStreamingEngineWorkerHarnessTest.java | 32 +-
.../harness/SingleSourceWorkerHarnessTest.java | 2 -
.../harness/StreamingWorkerStatusReporterTest.java | 2 -
.../harness/WindmillStreamSenderTest.java | 34 +-
.../worker/util/BoundedQueueExecutorTest.java | 21 +-
.../client/AbstractWindmillStreamTest.java | 3 -
.../client/grpc/GrpcCommitWorkStreamTest.java | 5 +-
.../client/grpc/GrpcDirectGetWorkStreamTest.java | 41 +-
.../client/grpc/GrpcGetDataStreamTest.java | 5 +-
.../grpc/GrpcGetWorkerMetadataStreamTest.java | 2 -
.../client/grpc/GrpcWindmillServerTest.java | 120 +-
.../client/grpc/stubs/ChannelCacheTest.java | 103 +-
.../windmill/testing/FakeWindmillStubFactory.java | 4 +-
.../failures/WorkFailureProcessorTest.java | 3 +-
.../work/refresh/ActiveWorkRefresherTest.java | 3 +-
runners/prism/java/build.gradle | 12 -
runners/spark/spark_runner.gradle | 1 -
.../beam/runners/spark/SparkPipelineRunner.java | 4 +-
.../runners/spark/SparkTransformOverrides.java | 17 +
.../beam/runners/spark/coders/CoderHelpers.java | 18 +
.../spark/translation/EvaluationContext.java | 28 +
.../spark/translation/MultiDoFnFunction.java | 10 +-
.../spark/translation/SideInputMetadata.java | 70 +
.../spark/translation/SparkPCollectionView.java | 37 +-
.../spark/translation/TransformTranslator.java | 4 +-
.../streaming/CreateStreamingSparkView.java} | 43 +-
.../translation/streaming/ParDoStateUpdateFn.java | 11 +-
.../streaming/StatefulStreamingParDoEvaluator.java | 3 +-
.../streaming/StreamingTransformTranslator.java | 79 +-
.../runners/spark/util/SideInputBroadcast.java | 15 +-
.../runners/spark/util/SideInputReaderFactory.java | 52 +
.../runners/spark/util/SparkSideInputReader.java | 57 +-
.../StreamingTransformTranslatorTest.java | 200 ++-
scripts/ci/issue-report/package-lock.json | 465 +++---
scripts/ci/issue-report/package.json | 2 +-
scripts/ci/pr-bot/findPrsNeedingAttention.ts | 6 +
scripts/ci/pr-bot/shared/commentStrings.ts | 5 +-
sdks/go.mod | 61 +-
sdks/go.sum | 128 +-
sdks/go/pkg/beam/core/core.go | 2 +-
.../beam/model/pipeline_v1/beam_runner_api.pb.go | 1543 +++++++++---------
.../model/pipeline_v1/beam_runner_api_grpc.pb.go | 150 --
.../model/pipeline_v1/external_transforms.pb.go | 361 +++--
sdks/go/pkg/beam/model/pipeline_v1/metrics.pb.go | 1276 +++++++++++----
sdks/go/pkg/beam/options/resource/hint.go | 49 +
sdks/go/pkg/beam/options/resource/hint_test.go | 43 +-
sdks/go/pkg/beam/runners/prism/internal/coders.go | 40 +
.../beam/runners/prism/internal/handlerunner.go | 67 +-
.../pkg/beam/runners/prism/internal/preprocess.go | 3 +
sdks/go/pkg/beam/runners/prism/internal/stage.go | 7 +-
.../main/resources/beam/checkstyle/checkstyle.xml | 2 +-
sdks/java/container/agent/build.gradle | 47 +-
.../apache/beam/sdk/schemas/utils/YamlUtils.java | 5 +
.../sdk/transforms/resourcehints/ResourceHint.java | 5 +-
.../transforms/resourcehints/ResourceHints.java | 43 +-
.../java/org/apache/beam/sdk/util/RowFilter.java | 22 +
.../beam/sdk/util/construction/Environments.java | 19 +-
.../main/java/org/apache/beam/sdk/values/Row.java | 2 +-
.../resourcehints/ResourceHintsTest.java | 23 +-
.../sdk/expansion/service/ExpansionService.java | 19 +-
.../beam/sdk/extensions/avro/coders/AvroCoder.java | 172 +-
.../extensions/avro/coders/AvroGenericCoder.java | 12 +-
.../sdk/extensions/avro/coders/AvroCoderTest.java | 92 ++
.../beam/sdk/extensions/protobuf/ProtoCoder.java | 5 +
.../sdk/extensions/protobuf/ProtoCoderTest.java | 6 +
sdks/java/extensions/python/build.gradle | 1 +
.../python/PythonExternalTransformTest.java | 20 +
.../sql/impl/LazyAggregateCombineFn.java | 2 +-
.../beam/fn/harness/BeamFnDataReadRunner.java | 13 +-
.../beam/fn/harness/BeamFnDataWriteRunner.java | 10 +-
.../org/apache/beam/fn/harness/CombineRunners.java | 18 +-
.../org/apache/beam/fn/harness/FlattenRunner.java | 18 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 7 +-
.../org/apache/beam/fn/harness/MapFnRunners.java | 17 +-
.../beam/fn/harness/PTransformRunnerFactory.java | 11 +-
.../fn/harness/control/ProcessBundleHandler.java | 336 ++--
.../beam/fn/harness/state/StateBackedIterable.java | 4 +-
.../fn/harness/state/StateFetchingIterators.java | 92 ++
.../beam/fn/harness/AssignWindowsRunnerTest.java | 2 +-
.../beam/fn/harness/BeamFnDataReadRunnerTest.java | 6 +-
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 2 +-
.../apache/beam/fn/harness/CombineRunnersTest.java | 10 +-
.../apache/beam/fn/harness/FlattenRunnerTest.java | 4 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 46 +-
.../apache/beam/fn/harness/MapFnRunnersTest.java | 6 +-
.../PTransformRunnerFactoryTestContext.java | 5 +
.../harness/control/ProcessBundleHandlerTest.java | 114 +-
.../fn/harness/state/StateBackedIterableTest.java | 70 -
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 2 +
.../sdk/io/gcp/pubsub/PreparePubsubWriteDoFn.java | 34 +-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 95 +-
.../beam/sdk/io/gcp/pubsub/PubsubJsonClient.java | 6 +-
.../beam/sdk/io/gcp/pubsub/PubsubMessage.java | 10 +
.../io/gcp/pubsub/PubsubMessageSchemaCoder.java | 78 +
...hAttributesAndMessageIdAndOrderingKeyCoder.java | 7 +-
...bsubMessageWithAttributesAndMessageIdCoder.java | 7 +-
.../pubsub/PubsubMessageWithAttributesCoder.java | 7 +-
.../pubsub/PubsubMessageWithMessageIdCoder.java | 3 +
.../io/gcp/pubsub/PubsubMessageWithTopicCoder.java | 7 +-
.../beam/sdk/io/gcp/pubsub/PubsubTestClient.java | 29 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 96 +-
.../beam/sdk/io/gcp/spanner/SpannerSchema.java | 6 +-
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 18 +
.../BigQuerySchemaTransformTranslationTest.java | 4 +-
.../beam/sdk/io/gcp/firestore/RpcQosTest.java | 2 +-
.../gcp/firestore/it/FirestoreTestingHelper.java | 2 +-
.../io/gcp/pubsub/PreparePubsubWriteDoFnTest.java | 39 +
.../pubsub/PubSubWritePayloadTranslationTest.java | 3 +
...Test.java => PubsubMessageSchemaCoderTest.java} | 23 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java | 172 +-
.../beam/sdk/io/gcp/pubsub/PubsubWriteIT.java | 87 +-
.../beam/sdk/io/gcp/spanner/SpannerSchemaTest.java | 33 +-
sdks/java/io/iceberg/build.gradle | 2 +-
.../beam/sdk/io/iceberg/CreateReadTasksDoFn.java | 6 +-
.../IcebergCdcReadSchemaTransformProvider.java | 16 +-
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 132 +-
.../IcebergReadSchemaTransformProvider.java | 16 +-
.../beam/sdk/io/iceberg/IcebergScanConfig.java | 55 +
.../IcebergWriteSchemaTransformProvider.java | 4 +-
.../beam/sdk/io/iceberg/IncrementalScanSource.java | 2 +-
.../apache/beam/sdk/io/iceberg/ReadFromTasks.java | 7 +-
.../org/apache/beam/sdk/io/iceberg/ReadUtils.java | 7 +-
.../org/apache/beam/sdk/io/iceberg/ScanSource.java | 10 +-
.../apache/beam/sdk/io/iceberg/ScanTaskReader.java | 9 +-
.../apache/beam/sdk/io/iceberg/ScanTaskSource.java | 4 +-
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 188 ++-
.../apache/beam/sdk/io/iceberg/ReadUtilsTest.java | 3 +-
.../catalog/BigQueryMetastoreCatalogIT.java | 7 +-
.../beam/sdk/io/iceberg/catalog/HiveCatalogIT.java | 4 +-
.../io/iceberg/catalog/IcebergCatalogBaseIT.java | 54 +-
.../beam/sdk/io/kafka/KafkaCheckpointMark.java | 3 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 94 +-
.../KafkaIOReadImplementationCompatibility.java | 1 +
.../org/apache/beam/sdk/io/kafka/KafkaIOUtils.java | 15 +
.../io/kafka/KafkaReadSchemaTransformProvider.java | 2 +-
.../beam/sdk/io/kafka/KafkaUnboundedReader.java | 4 +-
.../beam/sdk/io/kafka/KafkaUnboundedSource.java | 65 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 757 +++++----
.../sdk/io/kafka/WatchForKafkaTopicPartitions.java | 10 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 377 ++++-
.../KafkaReadSchemaTransformProviderTest.java | 23 +-
.../KafkaWriteSchemaTransformProviderTest.java | 24 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java | 8 +-
.../sdk/io/kafka/upgrade/KafkaIOTranslation.java | 4 +
.../io/kafka/upgrade/KafkaIOTranslationTest.java | 1 +
.../java/org/apache/beam/sdk/managed/Managed.java | 8 +-
.../managed/ManagedSchemaTransformProvider.java | 27 +-
.../sdk/managed/ManagedTransformConstants.java | 76 -
.../src/main/resources/available_configs.yaml} | 36 +-
.../src/main/resources/config_aliases.yaml} | 17 +-
sdks/java/testing/nexmark/build.gradle | 4 +
sdks/python/apache_beam/coders/coder_impl.py | 10 +-
sdks/python/apache_beam/coders/coders.py | 27 +-
.../coders/coders_property_based_test.py | 4 +-
.../apache_beam/coders/coders_test_common.py | 24 +-
sdks/python/apache_beam/coders/row_coder_test.py | 3 +-
.../apache_beam/coders/standard_coders_test.py | 91 +-
sdks/python/apache_beam/coders/typecoders.py | 2 +
sdks/python/apache_beam/coders/typecoders_test.py | 18 +
sdks/python/apache_beam/dataframe/convert.py | 10 +-
sdks/python/apache_beam/dataframe/doctests.py | 36 +-
sdks/python/apache_beam/dataframe/frames.py | 181 +--
sdks/python/apache_beam/dataframe/frames_test.py | 105 +-
sdks/python/apache_beam/dataframe/io.py | 20 +-
.../apache_beam/dataframe/pandas_doctests_test.py | 143 +-
.../dataframe/pandas_top_level_functions.py | 11 +-
sdks/python/apache_beam/dataframe/transforms.py | 14 +-
.../apache_beam/dataframe/transforms_test.py | 6 +-
sdks/python/apache_beam/examples/avro_nyc_trips.py | 10 +-
.../examples/cookbook/bigtableio_it_test.py | 5 +-
.../inference/pytorch_language_modeling.py | 27 +-
.../examples/inference/pytorch_sentiment.py | 290 ++++
.../inference/xgboost_iris_classification.py | 4 +-
.../ml_transform/vocab_tfidf_processing.py | 3 +-
.../apache_beam/examples/online_clustering.py | 3 +-
.../apache_beam/examples/snippets/snippets_test.py | 17 +-
.../aggregation/approximatequantiles_test.py | 3 +-
.../aggregation/approximateunique_test.py | 3 +-
.../combineperkey_side_inputs_singleton.py | 3 +-
.../combinevalues_side_inputs_singleton.py | 3 +-
.../snippets/transforms/aggregation/top_of.py | 7 +-
.../snippets/transforms/aggregation/top_per_key.py | 7 +-
.../elementwise/filter_side_inputs_dict.py | 3 +-
.../elementwise/filter_side_inputs_iter.py | 3 +-
.../elementwise/filter_side_inputs_singleton.py | 3 +-
.../elementwise/flatmap_side_inputs_singleton.py | 3 +-
.../snippets/transforms/elementwise/map_context.py | 5 +-
.../transforms/elementwise/map_side_inputs_iter.py | 3 +-
.../elementwise/map_side_inputs_singleton.py | 3 +-
.../elementwise/withtimestamps_event_time.py | 23 +-
.../examples/snippets/transforms/other/window.py | 48 +-
sdks/python/apache_beam/examples/sql_taxi.py | 15 +-
.../internal/cloudpickle/cloudpickle.py | 12 +-
.../apache_beam/internal/cloudpickle_pickler.py | 10 +-
.../internal/cloudpickle_pickler_test.py | 81 +
sdks/python/apache_beam/internal/dill_pickler.py | 20 +-
sdks/python/apache_beam/internal/metrics/metric.py | 4 +-
sdks/python/apache_beam/internal/module_test.py | 32 +
sdks/python/apache_beam/internal/pickler.py | 15 +-
sdks/python/apache_beam/internal/pickler_test.py | 151 +-
sdks/python/apache_beam/internal/set_pickler.py | 164 ++
.../apache_beam/internal/set_pickler_test.py | 278 ++++
sdks/python/apache_beam/internal/util.py | 7 +-
sdks/python/apache_beam/io/avroio.py | 9 +-
sdks/python/apache_beam/io/avroio_test.py | 4 +-
.../io/external/xlang_debeziumio_it_test.py | 4 +-
.../io/external/xlang_jdbcio_it_test.py | 8 +-
sdks/python/apache_beam/io/filebasedsink.py | 8 +-
sdks/python/apache_beam/io/fileio.py | 13 +-
sdks/python/apache_beam/io/fileio_test.py | 8 +-
sdks/python/apache_beam/io/filesystem.py | 2 +-
sdks/python/apache_beam/io/filesystem_test.py | 22 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 49 +-
.../apache_beam/io/gcp/bigquery_file_loads.py | 27 +-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 72 +-
.../apache_beam/io/gcp/bigquery_read_internal.py | 7 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 69 +-
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 4 +-
.../apache_beam/io/gcp/bigquery_tools_test.py | 10 +-
.../apache_beam/io/gcp/bigquery_write_it_test.py | 6 +-
.../apache_beam/io/gcp/bigtableio_it_test.py | 2 +-
.../apache_beam/io/gcp/experimental/spannerio.py | 35 +-
.../apache_beam/io/gcp/gcsfilesystem_test.py | 6 +-
sdks/python/apache_beam/io/gcp/gcsio.py | 30 +-
sdks/python/apache_beam/io/gcp/gcsio_retry.py | 33 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 39 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 6 +-
.../apache_beam/io/gcp/tests/bigquery_matcher.py | 8 +-
sdks/python/apache_beam/io/gcp/tests/utils_test.py | 9 +-
sdks/python/apache_beam/io/localfilesystem_test.py | 4 +-
sdks/python/apache_beam/io/mongodbio.py | 12 +-
sdks/python/apache_beam/io/mongodbio_test.py | 7 +-
sdks/python/apache_beam/io/range_trackers.py | 6 +-
sdks/python/apache_beam/io/source_test_utils.py | 8 +-
sdks/python/apache_beam/metrics/cells.py | 6 +-
sdks/python/apache_beam/metrics/cells_test.py | 3 +-
sdks/python/apache_beam/metrics/execution.py | 22 +-
.../python/apache_beam/metrics/monitoring_infos.py | 3 +-
sdks/python/apache_beam/ml/anomaly/aggregations.py | 4 +-
sdks/python/apache_beam/ml/anomaly/base.py | 7 +-
.../apache_beam/ml/anomaly/detectors/offline.py | 67 +-
.../ml/anomaly/detectors/pyod_adapter.py | 109 ++
.../ml/anomaly/detectors/pyod_adapter_test.py | 160 ++
sdks/python/apache_beam/ml/anomaly/specifiable.py | 22 +-
.../apache_beam/ml/anomaly/specifiable_test.py | 6 +-
sdks/python/apache_beam/ml/anomaly/transforms.py | 235 +--
.../apache_beam/ml/anomaly/transforms_test.py | 336 ++--
sdks/python/apache_beam/ml/inference/base.py | 142 +-
sdks/python/apache_beam/ml/inference/base_test.py | 21 +-
.../ml/inference/huggingface_inference.py | 70 +-
.../ml/inference/huggingface_inference_test.py | 13 +-
.../apache_beam/ml/inference/onnx_inference.py | 15 +-
.../ml/inference/onnx_inference_test.py | 69 +-
.../apache_beam/ml/inference/pytorch_inference.py | 51 +-
.../ml/inference/pytorch_inference_it_test.py | 4 +-
.../ml/inference/pytorch_inference_test.py | 80 +-
.../apache_beam/ml/inference/sklearn_inference.py | 19 +-
.../ml/inference/sklearn_inference_test.py | 19 +-
.../ml/inference/tensorflow_inference.py | 25 +-
.../ml/inference/tensorflow_inference_test.py | 48 +-
.../apache_beam/ml/inference/tensorrt_inference.py | 16 +-
.../ml/inference/tensorrt_inference_test.py | 24 +-
sdks/python/apache_beam/ml/inference/utils.py | 9 +-
.../apache_beam/ml/inference/vllm_inference.py | 26 +-
.../apache_beam/ml/inference/xgboost_inference.py | 21 +-
.../ml/inference/xgboost_inference_test.py | 3 +-
.../ml/rag/embeddings/vertex_ai_test.py | 2 +-
.../ml/rag/enrichment/bigquery_vector_search.py | 33 +-
.../enrichment/bigquery_vector_search_it_test.py | 308 +++-
.../ml/rag/ingestion/alloydb_it_test.py | 32 +-
.../apache_beam/ml/rag/ingestion/bigquery.py | 11 +-
.../ml/rag/ingestion/bigquery_it_test.py | 3 +-
sdks/python/apache_beam/ml/transforms/base.py | 4 +-
sdks/python/apache_beam/ml/transforms/base_test.py | 15 +-
.../ml/transforms/embeddings/huggingface.py | 4 +-
.../ml/transforms/embeddings/vertex_ai.py | 2 +-
.../ml/transforms/embeddings/vertex_ai_test.py | 8 +-
sdks/python/apache_beam/ml/transforms/tft_test.py | 18 +-
.../python/apache_beam/options/pipeline_options.py | 11 +-
.../options/pipeline_options_validator.py | 4 +-
sdks/python/apache_beam/pipeline.py | 57 +-
sdks/python/apache_beam/pipeline_test.py | 52 +-
sdks/python/apache_beam/pvalue.py | 13 +-
sdks/python/apache_beam/runners/common.py | 211 +--
sdks/python/apache_beam/runners/common_test.py | 5 +-
.../apache_beam/runners/dask/dask_runner_test.py | 9 +-
.../dataflow/dataflow_exercise_metrics_pipeline.py | 10 +-
.../runners/dataflow/dataflow_job_service.py | 3 +-
.../runners/dataflow/dataflow_metrics.py | 4 +-
.../runners/dataflow/dataflow_metrics_test.py | 10 +-
.../runners/dataflow/dataflow_runner.py | 7 +-
.../runners/dataflow/dataflow_runner_test.py | 4 +-
.../runners/dataflow/internal/apiclient.py | 21 +-
.../clients/cloudbuild/cloudbuild_v1_client.py | 168 +-
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
.../consumer_tracking_pipeline_visitor_test.py | 6 +-
.../apache_beam/runners/direct/direct_metrics.py | 20 +-
.../runners/direct/direct_runner_test.py | 24 +-
.../runners/direct/evaluation_context.py | 4 +-
.../runners/direct/transform_evaluator.py | 3 +-
.../interactive/background_caching_job_test.py | 24 +-
.../runners/interactive/cache_manager.py | 11 +-
.../runners/interactive/caching/read_cache.py | 4 +-
.../runners/interactive/caching/write_cache.py | 17 +-
.../runners/interactive/interactive_beam_test.py | 16 +-
.../runners/interactive/interactive_environment.py | 3 +-
.../runners/interactive/pipeline_instrument.py | 14 +-
.../runners/interactive/recording_manager.py | 11 +-
.../interactive/sql/beam_sql_magics_test.py | 24 +-
.../runners/interactive/sql/sql_chain.py | 3 +-
.../apache_beam/runners/interactive/sql/utils.py | 4 +-
.../python/apache_beam/runners/pipeline_context.py | 1 +
sdks/python/apache_beam/runners/pipeline_utils.py | 11 +-
.../runners/portability/abstract_job_service.py | 12 +-
.../runners/portability/artifact_service.py | 3 +-
.../runners/portability/expansion_service.py | 15 +-
.../runners/portability/flink_runner_test.py | 13 +-
.../runners/portability/fn_api_runner/execution.py | 7 +-
.../runners/portability/fn_api_runner/fn_runner.py | 28 +-
.../portability/fn_api_runner/fn_runner_test.py | 74 +-
.../portability/fn_api_runner/translations.py | 56 +-
.../portability/fn_api_runner/translations_test.py | 4 +-
.../fn_api_runner/trigger_manager_test.py | 12 +-
.../portability/fn_api_runner/worker_handlers.py | 155 +-
.../apache_beam/runners/portability/job_server.py | 10 +-
.../runners/portability/local_job_service.py | 12 +-
.../runners/portability/portable_runner.py | 3 +-
.../runners/portability/prism_runner.py | 54 +-
.../runners/portability/prism_runner_test.py | 45 +-
.../runners/portability/samza_runner_test.py | 10 +
.../runners/portability/spark_runner_test.py | 9 +-
.../apache_beam/runners/portability/stager.py | 4 +-
sdks/python/apache_beam/runners/render.py | 5 +-
sdks/python/apache_beam/runners/runner.py | 4 +
sdks/python/apache_beam/runners/trivial_runner.py | 8 +-
.../apache_beam/runners/worker/bundle_processor.py | 28 +-
.../apache_beam/runners/worker/data_plane.py | 5 +-
.../apache_beam/runners/worker/data_sampler.py | 8 +-
.../apache_beam/runners/worker/log_handler.py | 4 +-
.../apache_beam/runners/worker/log_handler_test.py | 3 +-
.../apache_beam/runners/worker/opcounters.py | 15 +-
.../apache_beam/runners/worker/operation_specs.py | 23 +-
.../apache_beam/runners/worker/operations.py | 112 +-
.../apache_beam/runners/worker/sdk_worker.py | 16 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 3 +-
.../runners/worker/sdk_worker_main_test.py | 8 +-
.../apache_beam/runners/worker/sdk_worker_test.py | 55 +-
.../apache_beam/runners/worker/statecache_test.py | 31 +-
.../apache_beam/runners/worker/worker_status.py | 4 +-
.../apache_beam/testing/analyzers/perf_analysis.py | 4 +-
.../benchmarks/cloudml/pipelines/workflow.py | 3 +-
...nchmarks.py => pytorch_sentiment_benchmarks.py} | 14 +-
.../testing/benchmarks/nexmark/queries/query1.py | 5 +-
.../testing/benchmarks/nexmark/queries/query3.py | 6 +-
.../apache_beam/testing/datatype_inference.py | 13 +-
.../apache_beam/testing/datatype_inference_test.py | 4 +-
.../testing/load_tests/load_test_metrics_utils.py | 7 +-
.../apache_beam/testing/synthetic_pipeline.py | 3 +-
.../apache_beam/transforms/batch_dofn_test.py | 3 +-
sdks/python/apache_beam/transforms/combiners.py | 21 +-
sdks/python/apache_beam/transforms/core.py | 58 +-
sdks/python/apache_beam/transforms/display.py | 3 +-
.../enrichment_handlers/bigtable_it_test.py | 6 +-
sdks/python/apache_beam/transforms/environments.py | 81 +-
sdks/python/apache_beam/transforms/external.py | 105 +-
.../python/apache_beam/transforms/external_test.py | 66 +-
.../external_transform_provider_it_test.py | 65 +-
.../fully_qualified_named_transform_test.py | 13 +-
sdks/python/apache_beam/transforms/managed.py | 43 +-
sdks/python/apache_beam/transforms/ptransform.py | 45 +-
.../apache_beam/transforms/ptransform_test.py | 45 +-
sdks/python/apache_beam/transforms/resources.py | 29 +
.../apache_beam/transforms/resources_test.py | 20 +
.../apache_beam/transforms/sideinputs_test.py | 32 +-
sdks/python/apache_beam/transforms/sql_test.py | 4 +-
sdks/python/apache_beam/transforms/trigger.py | 7 +-
sdks/python/apache_beam/transforms/trigger_test.py | 98 +-
sdks/python/apache_beam/transforms/util.py | 8 +-
sdks/python/apache_beam/transforms/util_test.py | 203 ++-
sdks/python/apache_beam/transforms/window.py | 5 +-
.../typehints/arrow_type_compatibility.py | 19 +-
.../typehints/arrow_type_compatibility_test.py | 39 +-
sdks/python/apache_beam/typehints/batch.py | 3 +-
sdks/python/apache_beam/typehints/decorators.py | 13 +-
.../typehints/native_type_compatibility.py | 20 +-
.../typehints/native_type_compatibility_test.py | 64 +
.../typehints/pandas_type_compatibility.py | 9 +-
.../typehints/pandas_type_compatibility_test.py | 26 +-
.../typehints/pytorch_type_compatibility.py | 3 +-
sdks/python/apache_beam/typehints/row_type.py | 3 +-
sdks/python/apache_beam/typehints/schemas_test.py | 22 +-
.../apache_beam/typehints/trivial_inference.py | 8 +-
.../typehints/trivial_inference_test.py | 89 +-
sdks/python/apache_beam/typehints/typehints.py | 4 +-
.../python/apache_beam/typehints/typehints_test.py | 20 +
sdks/python/apache_beam/utils/urns.py | 6 +-
sdks/python/apache_beam/version.py | 2 +-
.../yaml/examples/testing/examples_test.py | 3 +-
sdks/python/apache_beam/yaml/generate_yaml_docs.py | 3 +-
sdks/python/apache_beam/yaml/json_utils.py | 6 +-
sdks/python/apache_beam/yaml/pipeline.schema.yaml | 45 +
sdks/python/apache_beam/yaml/readme_test.py | 8 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 10 +-
.../apache_beam/yaml/tests/assign_timestamps.yaml | 84 +
.../tests/{tfrecord_write.yaml => create.yaml} | 64 +-
sdks/python/apache_beam/yaml/tests/explode.yaml | 79 +
.../yaml/tests/extract_windowing_Info.yaml | 131 ++
sdks/python/apache_beam/yaml/tests/filter.yaml | 147 ++
sdks/python/apache_beam/yaml/tests/flatten.yaml | 125 ++
.../apache_beam/yaml/tests/ml_transform.yaml | 92 ++
sdks/python/apache_beam/yaml/tests/partition.yaml | 268 ++++
.../python/apache_beam/yaml/tests/pytransform.yaml | 159 ++
.../tests/{tfrecord_write.yaml => tfrecord.yaml} | 0
.../yaml/tests/validate_with_schema.yaml | 95 ++
sdks/python/apache_beam/yaml/yaml_io.py | 8 +-
sdks/python/apache_beam/yaml/yaml_io_test.py | 7 +-
sdks/python/apache_beam/yaml/yaml_join_test.py | 4 +-
sdks/python/apache_beam/yaml/yaml_mapping.py | 19 +-
sdks/python/apache_beam/yaml/yaml_mapping_test.py | 1 +
sdks/python/apache_beam/yaml/yaml_provider.py | 65 +-
.../apache_beam/yaml/yaml_provider_unit_test.py | 32 +-
sdks/python/apache_beam/yaml/yaml_testing.py | 3 +-
sdks/python/apache_beam/yaml/yaml_transform.py | 17 +-
.../python/apache_beam/yaml/yaml_transform_test.py | 13 +-
.../apache_beam/yaml/yaml_transform_unit_test.py | 23 +
sdks/python/apache_beam/yaml/yaml_udf_test.py | 8 +-
sdks/python/apache_beam/yaml/yaml_utils.py | 4 +-
sdks/python/build.gradle | 16 +
.../container/base_image_requirements_manual.txt | 1 -
.../container/py310/base_image_requirements.txt | 40 +-
.../container/py311/base_image_requirements.txt | 40 +-
.../container/py312/base_image_requirements.txt | 40 +-
.../container/py39/base_image_requirements.txt | 38 +-
sdks/python/gen_managed_doc.py | 359 +++++
sdks/python/gen_xlang_wrappers.py | 23 +-
sdks/python/pyproject.toml | 2 +-
sdks/python/setup.cfg | 2 +
sdks/python/setup.py | 3 +
sdks/python/tox.ini | 4 +-
sdks/standard_external_transforms.yaml | 122 +-
sdks/typescript/package.json | 2 +-
.../blog/apache-beam-flink-and-kubernetes-part2.md | 2 +-
.../en/blog/apache-beam-flink-and-kubernetes.md | 2 +-
.../site/content/en/documentation/io/managed-io.md | 823 ++++++++++
.../content/en/documentation/programming-guide.md | 77 +
.../en/documentation/runtime/environments.md | 2 +-
.../en/documentation/sdks/yaml-providers.md | 16 +-
.../content/en/documentation/sdks/yaml-testing.md | 10 +-
.../site/content/en/get-started/quickstart-java.md | 30 +
website/www/site/content/en/performance/_index.md | 9 +-
.../en/performance/pytorchbertbase/_index.md | 6 +
.../en/performance/pytorchbertlarge/_index.md | 6 +
.../_index.md | 16 +-
.../_index.md | 16 +-
.../en/performance/pytorchresnet101/_index.md | 6 +
.../en/performance/pytorchresnet152/_index.md | 6 +
.../en/performance/pytorchresnet152tesla/_index.md | 6 +
.../en/performance/tensorflowmnist/_index.md | 9 +-
website/www/site/data/performance.yaml | 32 +
.../partials/section-menu/en/documentation.html | 1 +
550 files changed, 18398 insertions(+), 7610 deletions(-)
create mode 100644 .github/trigger_files/beam_PostCommit_Python_ValidatesRunner_Samza.json
create mode 100644 .github/trigger_files/beam_PostCommit_Python_ValidatesRunner_Spark.json
copy .github/workflows/load-tests-pipeline-options/{beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt => beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Sentiment_Batch_DistilBert_Base_Uncased.txt} (75%)
copy .github/workflows/load-tests-pipeline-options/{beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt => beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Sentiment_Streaming_DistilBert_Base_Uncased.txt} (76%)
create mode 100644 .github/workflows/run_rc_validation_python_yaml.yml
create mode 100644 examples/notebooks/beam-ml/anomaly_detection/anomaly_detection_iforest.ipynb
create mode 100644 examples/notebooks/beam-ml/anomaly_detection/anomaly_detection_zscore.ipynb
create mode 100644 examples/notebooks/beam-ml/bigquery_vector_ingestion_and_search.ipynb
copy it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/BigQueryRowsCheck.java => clickhouse/src/main/java/org/apache/beam/it/clickhouse/conditions/ClickHouseRowsCheck.java} (66%)
copy it/clickhouse/src/main/java/org/apache/beam/it/clickhouse/{ => conditions}/package-info.java (94%)
copy runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/stubs/{WindmillChannelFactory.java => WindmillChannels.java} (55%)
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/throttling/StreamingEngineThrottleTimers.java
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/throttling/ThrottleTimer.java
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/throttling/ThrottledTimeTracker.java
create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SideInputMetadata.java
copy runners/{flink/src/main/java/org/apache/beam/runners/flink/CreateStreamingFlinkView.java => spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/CreateStreamingSparkView.java} (79%)
create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/util/SideInputReaderFactory.java
delete mode 100644 sdks/go/pkg/beam/model/pipeline_v1/beam_runner_api_grpc.pb.go
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubMessageSchemaCoder.java
copy sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/{PubsubMessageWithAttributesCoderTest.java => PubsubMessageSchemaCoderTest.java} (75%)
delete mode 100644 sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/ManagedTransformConstants.java
copy sdks/{python/apache_beam/coders/avro_record.py => java/managed/src/main/resources/available_configs.yaml} (50%)
copy sdks/{python/apache_beam/examples/wordcount_dataframe.py => java/managed/src/main/resources/config_aliases.yaml} (72%)
create mode 100644 sdks/python/apache_beam/examples/inference/pytorch_sentiment.py
create mode 100644 sdks/python/apache_beam/internal/set_pickler.py
create mode 100644 sdks/python/apache_beam/internal/set_pickler_test.py
create mode 100644 sdks/python/apache_beam/ml/anomaly/detectors/pyod_adapter.py
create mode 100644 sdks/python/apache_beam/ml/anomaly/detectors/pyod_adapter_test.py
copy sdks/python/apache_beam/testing/benchmarks/inference/{pytorch_language_modeling_benchmarks.py => pytorch_sentiment_benchmarks.py} (75%)
create mode 100644 sdks/python/apache_beam/yaml/tests/assign_timestamps.yaml
copy sdks/python/apache_beam/yaml/tests/{tfrecord_write.yaml => create.yaml} (52%)
create mode 100644 sdks/python/apache_beam/yaml/tests/explode.yaml
create mode 100644 sdks/python/apache_beam/yaml/tests/extract_windowing_Info.yaml
create mode 100644 sdks/python/apache_beam/yaml/tests/filter.yaml
create mode 100644 sdks/python/apache_beam/yaml/tests/flatten.yaml
create mode 100644 sdks/python/apache_beam/yaml/tests/ml_transform.yaml
create mode 100644 sdks/python/apache_beam/yaml/tests/partition.yaml
create mode 100644 sdks/python/apache_beam/yaml/tests/pytransform.yaml
rename sdks/python/apache_beam/yaml/tests/{tfrecord_write.yaml => tfrecord.yaml} (100%)
create mode 100644 sdks/python/apache_beam/yaml/tests/validate_with_schema.yaml
create mode 100644 sdks/python/gen_managed_doc.py
create mode 100644 website/www/site/content/en/documentation/io/managed-io.md
copy website/www/site/content/en/performance/{pytorchbertbase => pytorchbertsentimentbatch}/_index.md (52%)
copy website/www/site/content/en/performance/{pytorchbertbase => pytorchbertsentimentstreaming}/_index.md (51%)