You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2024/08/06 23:11:43 UTC
(beam) branch dependabot/go_modules/sdks/google.golang.org/grpc-1.65.0 updated (44523a0c9ab -> 13d40ce5c45)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch dependabot/go_modules/sdks/google.golang.org/grpc-1.65.0
in repository https://gitbox.apache.org/repos/asf/beam.git
discard 44523a0c9ab Bump google.golang.org/grpc from 1.64.0 to 1.65.0 in /sdks
add 4df89c704b0 Allow Firestore project to be configurable (#31808)
add 080c80a9573 Moving to 2.59.0-SNAPSHOT on master branch.
add 43b237e5040 Modernize python type hints for apache_beam
add cd495e9cf8a Modernize python type hints for apache_beam/coders
add 6143cd0e0a8 Modernize python type hints for apache_beam/dataframe
add d75916b6342 Modernize python type hints for apache_beam/examples/cookbook
add c842252b4b5 Modernize python type hints for apache_beam/internal
add 33bde4de9c5 Modernize python type hints for apache_beam/internal/metrics
add d73982af054 Modernize python type hints for apache_beam/io
add 8f6f24dc1ff Modernize python type hints for apache_beam/io/azure
add b8029e9aa1e Modernize python type hints for apache_beam/io/flink
add f49a29a4685 Modernize python type hints for apache_beam/io/gcp
add 5d5a09b6b3e Modernize python type hints for apache_beam/metrics
add fbafe8d780b Modernize python type hints for apache_beam/ml/gcp
add 0eab29802a3 Modernize python type hints for apache_beam/options
add 842b8ecab01 Modernize python type hints for apache_beam/runners
add 0763d7e50f3 Modernize python type hints for apache_beam/runners/direct
add 3cf0c5512a3 Modernize python type hints for apache_beam/runners/interactive
add b41698210df Modernize python type hints for apache_beam/runners/job
add 8b540eb5075 Modernize python type hints for apache_beam/runners/portability
add 8fdbe88dc6e Modernize python type hints for apache_beam/runners/worker
add 77d81895109 Modernize python type hints for apache_beam/testing/benchmarks
add acfd72c7c06 Modernize python type hints for apache_beam/testing/load_tests
add 4402f2d44e9 Modernize python type hints for apache_beam/transforms
add 79d4ffd20ba Modernize python type hints for apache_beam/typehints
add f2ffa5ec49e Modernize python type hints for apache_beam/utils
add abdb1b742a9 Fix circular references, mypy complaints.
add d4de077a2a9 Fix bad type declarations.
add 14c52d66ec6 Fix bad typing in PubSub tests.
add 64e6194b948 Preserve existing linter comments.
add a0ba8dea7d8 isort
add 36961405769 Merge pull request #31755 Modernize type hints.
add dd0912460c4 add doc warning against using icebergio directly (#31833)
add 7c0cf39001a Merge pull request #31823 Add lineage information for BigQuery sinks.
add 8d5c3b5ee2c Locate and download Prism binary (#31796)
add 00bf1c6d036 Change orphan file log to warning (#31835)
add 041ccdbe5d0 playground python image update openjdk to 17 (#31843)
add 9ee961fc0c2 Use fileNameTemplate attribute for file prefix (#31844)
add b34c014888b Dedup SerializableSupplier (#31829)
add e646c28d2ac [CsvIO] Implemented CsvIOParseHelpers:parseCell (#31802)
add 50a3403a474 Export string sets in monitoring infos. (#31838)
add 441840afb5b Fix Python test auth (#31850)
add eb59788cf4b [CsvIO] Changed CsvIOParseConfiguration to include DLQ (#31852)
add b573c8ffc8b Add a little more info on snapshot containers (#31861)
add 4561fd10634 [CsvIO] Implement CsvIOParseHelpers::validate(CSVFormat) (#31853)
add 4d429dde47b Add options to control number of Storage API connections when using multiplexing (#31721)
add ff2731bc7e9 Switch to use self-hosted runner for build_wheels action (#31866)
add ba27c36c073 [CsvIO] implement CsvIOParseHelpers::validate(CSVFormat, Schema) (#31869)
add 7c07f7ff453 Create CsvIOParse scaffold (#31878)
add 86ea7806eca Remove auth@1 in tests running on self hosted runner (#31881)
add 36f16314052 [CsvIO] Create scaffold CsvIOParseKV class (#31880)
add faff55c2882 Fix auth for clean up actions (#31888)
add 0b61035f36f Increase retry backoff for Storage API batch to survive AppendRows quota refill (#31837)
add 408f67cfc83 Create CsvIOParseHelpers::mapFieldPositions method (#31889)
add 3e7614a50e5 Bump braces from 3.0.2 to 3.0.3 in /scripts/ci/pr-bot (#31886)
add ea761115998 [CsvIO] Create CsvIOStringToCsvRecord Class (#31857)
add d74a2f53e47 Fix ReadAllFromBigQuery leak temp dataset (#31895)
add 5ea57c3f8bd Update CHANGES.md with 2.59.0 Section (#31831)
add 2547e0b482e Bump github.com/nats-io/nats-server/v2 from 2.10.12 to 2.10.16 in /sdks (#31611)
add 53409cca5bb Set setFailIfPoolExhausted in SessionPoolOptions for SpannerAccessor (#31663)
add 6b6d9aa1f98 Remove remaining use of setup-cloud@v0 action (#31907)
add 2d6d55b98ce Fix generateYamlDocs gradle task (#31909)
add f3e6c66c0a5 Improve performance of BigQueryIO connector when withPropagateSuccessfulStorageApiWrites(true) is used (#31840)
add ff15999dfc4 [CsvIO] Change method signature of CsvIOStringToCsvRecord class (#31920)
add a767d41daaf [CsvIO] update input Pcollection type arguments due to change in implementation details. (#31891)
add 24f22f2b2cf Better assertion error messages for PAssert.thatSingleton (#31761)
add 8f2c72e8d22 Disable Dataflow run for java_test and python_test (#31934)
add 521586e04a7 Update Python artifact name in release process (#31933)
add 8760a677e81 Update Dataflow containers (#31936)
add be6216357bd Clean up integration test manual run parameters (#31938)
add c0f6ac980ba PubsubMessageWithTopicCoder.of() is returning wrong coder (#31619)
add 1284986ed05 Modify casing in workflow (#31911)
add 12ad2afd2d6 Use default auth for Iceberg integration tests (#31940)
add bdd5fff78c8 [Prism] Implement PrismPipelineResult (#31937)
add 147d3e495bd Callout beam-pyio (#31860)
add 04ed95d6037 JCSMP properties providers for new SolaceIO write connector (#31906)
add 53a804e25b5 Update Dataflow containers (#31944)
add 04dd443bda7 Update bug.yml (#31947)
add edf5e547695 Add infra option for remaining templates + autolabel
add 229477976fe Merge pull request #31952 from apache/users/damccorm/issue-templates
add aa3cfe59d57 Move dataset cleanup to finishBundle (#31955)
add 9b6c805e1cd Use correct device ordinal when GPU is detected (#31951)
add 35cfad98818 remove listing topics when processing each element (#31897)
add eadb81fd01f Expand test coverage (#31957)
add bfffff7f254 Auth with project id in our notebook (#31966)
add 92c5d57d529 Dedup Lineage and getTableToExtract call in BigQuerySourceBase (#31960)
add e2b8acb8bbf Adds ordered list user state support to fnapi accessor cache. (#31923)
add 799405c4c84 Make the SolaceIO.Read constructor private. (#31962)
add 74f1f7097cf Generic throttling metrics namespace and BigtableIO write throttling counter (#31924)
add dc1e1347de7 correct beam pyio details (#31949)
add 6a1d917a5ee [yaml] Fix yaml provider schema validation and merging (#31974)
add 8709f1126f6 Note SpannerIO.read new validation in 2.58.0 breaking change section (#31973)
add 496e7124272 Use buffered loggers that periodically flush. (#31977)
add b9a0c2b72ac Requirements_cache shouldn't create a cache folder when skipped. (#31961)
add 1e8c091a14d [CsvIO]: add Coder and FromRowFn to CsvIOParseConfiguration class. (#31989)
add 8eb09bfd6d7 Explicitly close the FlowReceiver (#31982)
add 44792814ff3 [CsvIO]: update class for parsing date time. (#31996)
add 7930a1fc88b Revert "Avoid publishing string set metrics on the Dataflow legacy runner." (#32002)
add ffae5b5bf0b Update Build Wheels to only build once on RCs (#32009)
add 3bdf702e6ab [CsvIO]: add CsvIORecordToObjects Class (#32006)
add 493e0ba0be6 This is a followup PR to #31906, and part of the issue #31905 (#31953)
add 0a65b4b065b Regenerate Dataflow Python client (#31997)
add 9e431b49bc7 Correctly chose earliest or latest in pane. (#31979)
add 5d13894975b Revert "Update Build Wheels to only build once on RCs" (#32014)
add 25a4ffe8e09 Bump setuptools from 68.2.2 to 70.0.0 in /.test-infra/mock-apis (#31893)
add ee3d57fb13c Bump com.gradle.develocity from 3.17.5 to 3.17.6 (#31948)
add 17ef888a783 Add StringSet metrics to Python SDK (#31969)
add 89d5e2f2961 Validate commits in StreamingDataflowWorker (#31822)
add 98f8b869327 beam-sql-udf-doc-mistake (#32019)
add 570f2f89647 Fix/Refactor GetData interfaces for Direct Path integration (#31784)
add c624e02b852 [CsvIO] update error and result handling. (#32023)
add 121ac713fa0 [#31991][prism] Allow Empty Composites (#32024)
add 88a01027f4c [#32003][prism] Support empty transform input sets, such as for flattens. (#32029)
add e7847c9448c [#32004] Ensure all pcollection coders are length prefixed if necessary. (#32012)
add 835630b12e2 Replace Class with TypeDescriptor in SchemaProvider implementations (#31785)
add b61ef7591fe update document in AwsOptions (#32036)
add 2824944530f Remove `--impersonate_service_account` whenever PipelineOptions are serialized (#32031)
add 2aefd5b9c0c Improve IcebergIO utils (#31958)
add 25e839b1f47 Fix Dataflow v2 test failed tag container if project contains colon (#32011)
add eec20689f21 Fix StringSet tests on portable runners (#31999)
add 346011b0a21 Fix row ranges issue in Bigtable Read. (#31990)
add 6d3b547adcf [#31992][prism] Send side inputs for all SDF phases. (#32042)
add f1c72c5e0c4 Support metrics at Source.split for Direct Runner (#32022)
add 56aa17b59ec Change the gradle task definition in quickstart. (#32043)
add 21009e68eab StateWatcher watches and reports changed Pipeline State (#32040)
add 6aac47c8354 [CsvIO] Create CsvIOParse Class (#32028)
add b795a61f094 Adds null checks when accessing OperationalLimits in config since they might not have been set yet. (#32053)
add 202fa56be77 Enable ExternalWorkerService during Prism Runner lifecycle (#32057)
add 0b4b8ea9423 Handle rc container in _update_container_image_for_dataflow (#32049)
add d96fa7d4009 Add some large model troubleshooting steps (#31862)
add bf42a8153af [#32064] Keep elements heap in sequence order. (#32065)
add 7e750873152 Update top_wikipedia_sessions to be more idiomatic with beam.Map. (#32041)
add ca744ae9f65 Add WorkProvider interfaces and implementations (#31883)
add bfc64d5c14a Fix error when ActiveWorkRefresher processed empty heartbeat map. (#32078)
add 80ae93217c5 Minor optimization for the common case of merging empty string sets. (#31803)
add 5b2bfe96f83 [Prism] Enable an artifact resolver for the Prism runner (#32058)
add fb49e9644a4 Fix load test dataproc cluster name exceeded allowed length (#32062)
add c60623524ae Beam Website Updates for 2.58.0 Release (#31925)
add d09c3237c8a Added support for the TOKENLIST type in Spanner (#32038)
add e9b5dc69532 Enforce java.nio.charset.StandardCharsets against guava Charsets (#32083)
add 99a23830037 Enable artifact staging during Prism Runner lifecycle (#32084)
add 741facf0099 Bump github.com/docker/docker in /sdks (#32046)
add 5ab908b984d Add Lineage metrics for BigtableIO (#32068)
add 17283bb8294 Add Lineage metrics to PubsubIO (#32037)
add e3e44544577 [#32085][prism] Fix session windowing. (#32086)
add 0a42afa9f5c [prism] Use non-deprecated docker types in environment. (#32092)
add 13d40ce5c45 Bump google.golang.org/grpc from 1.64.0 to 1.65.0 in /sdks
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (44523a0c9ab)
\
N -- N -- N refs/heads/dependabot/go_modules/sdks/google.golang.org/grpc-1.65.0 (13d40ce5c45)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 +
.github/ISSUE_TEMPLATE/bug.yml | 2 +
.github/ISSUE_TEMPLATE/failing_test.yml | 2 +
.github/ISSUE_TEMPLATE/feature.yml | 2 +
.github/ISSUE_TEMPLATE/task.yml | 2 +
.github/issue-rules.yml | 2 +
.../IO_Iceberg_Integration_Tests.json | 2 +-
...eam_PostCommit_Java_ValidatesRunner_Direct.json | 2 +-
.github/workflows/IO_Iceberg_Integration_Tests.yml | 6 -
.github/workflows/IO_Iceberg_Performance_Tests.yml | 6 -
.github/workflows/IO_Iceberg_Unit_Tests.yml | 6 -
.github/workflows/beam_CancelStaleDataflowJobs.yml | 6 -
.github/workflows/beam_CleanUpGCPResources.yml | 9 +-
.../workflows/beam_CleanUpPrebuiltSDKImages.yml | 6 -
.../beam_LoadTests_Python_CoGBK_Flink_Batch.yml | 2 +-
.../beam_LoadTests_Python_ParDo_Flink_Batch.yml | 2 +-
...beam_LoadTests_Python_ParDo_Flink_Streaming.yml | 2 +-
.../beam_PerformanceTests_AvroIOIT_HDFS.yml | 6 -
.github/workflows/beam_PerformanceTests_Cdap.yml | 6 -
...m_PerformanceTests_Compressed_TextIOIT_HDFS.yml | 6 -
.../beam_PerformanceTests_HadoopFormat.yml | 6 -
.github/workflows/beam_PerformanceTests_JDBC.yml | 6 -
.../workflows/beam_PerformanceTests_Kafka_IO.yml | 6 -
...am_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml | 6 -
.../beam_PerformanceTests_MongoDBIO_IT.yml | 6 -
.../beam_PerformanceTests_ParquetIOIT_HDFS.yml | 6 -
.../beam_PerformanceTests_SingleStoreIO.yml | 6 -
.../beam_PerformanceTests_SparkReceiver_IO.yml | 6 -
.../beam_PerformanceTests_TFRecordIOIT_HDFS.yml | 6 -
.../beam_PerformanceTests_XmlIOIT_HDFS.yml | 6 -
.../beam_PerformanceTests_xlang_KafkaIO_Python.yml | 6 -
.../beam_PostCommit_Java_InfluxDbIO_IT.yml | 6 -
.../beam_PostCommit_Java_SingleStoreIO_IT.yml | 10 +-
.../workflows/beam_StressTests_Java_KafkaIO.yml | 6 -
.github/workflows/build_runner_image.yml | 8 -
.github/workflows/build_wheels.yml | 53 +-
.github/workflows/java_tests.yml | 11 +-
.github/workflows/playground_backend_precommit.yml | 2 +-
.github/workflows/python_tests.yml | 9 +-
.github/workflows/run_perf_alert_tool.yml | 6 -
.github/workflows/typescript_tests.yml | 6 +-
.test-infra/mock-apis/poetry.lock | 13 +-
.test-infra/tools/stale_bq_datasets_cleaner.sh | 4 +-
CHANGES.md | 40 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +
contributor-docs/code-change-guide.md | 30 +-
contributor-docs/release-guide.md | 12 +-
.../kafkatopubsub/transforms/FormatTransform.java | 5 +-
.../beam-ml/bigtable_enrichment_transform.ipynb | 28 +-
gradle.properties | 4 +-
.../apache/beam/model/pipeline/v1/metrics.proto | 11 +
playground/backend/containers/python/Dockerfile | 3 +-
release/src/main/scripts/run_rc_validation.sh | 48 +-
.../runners/core/metrics/MetricsContainerImpl.java | 53 +-
.../core/metrics/MonitoringInfoConstants.java | 2 +
.../core/metrics/SimpleMonitoringInfoBuilder.java | 11 +
.../beam/runners/core/metrics/StringSetData.java | 17 +-
.../core/metrics/MetricsContainerImplTest.java | 32 +
.../direct/ExecutorServiceParallelExecutor.java | 13 +-
runners/flink/flink_runner.gradle | 2 +
runners/flink/job-server/flink_job_server.gradle | 1 -
.../streaming/io/StreamingImpulseSource.java | 5 +-
.../runners/flink/FlinkJobServerDriverTest.java | 12 +-
.../FlinkPipelineExecutionEnvironmentTest.java | 4 +-
.../beam/runners/flink/FlinkSubmissionTest.java | 4 +-
.../streaming/ExecutableStageDoFnOperatorTest.java | 9 +-
.../wrappers/streaming/FlinkKeyUtilsTest.java | 4 +-
.../google-cloud-dataflow-java/arm/build.gradle | 12 +-
runners/google-cloud-dataflow-java/build.gradle | 57 +-
.../beam/runners/dataflow/DataflowMetrics.java | 4 +-
.../dataflow/DataflowPipelineTranslator.java | 7 +-
.../dataflow/worker/BatchModeExecutionContext.java | 45 +-
.../dataflow/worker/DataflowSystemMetrics.java | 5 +-
.../worker/MetricTrackingWindmillServerStub.java | 355 -----
.../runners/dataflow/worker/OperationalLimits.java | 64 +
...Exception.java => OutputTooLargeException.java} | 19 +-
.../dataflow/worker/StreamingDataflowWorker.java | 455 +++---
.../worker/StreamingModeExecutionContext.java | 18 +
.../worker/StreamingStepMetricsContainer.java | 7 +-
.../beam/runners/dataflow/worker/WindmillSink.java | 25 +
.../worker/WorkItemCancelledException.java | 4 +
.../beam/runners/dataflow/worker/graph/Nodes.java | 6 +-
.../dataflow/worker/streaming/ActiveWorkState.java | 61 +-
.../worker/streaming/ComputationState.java | 13 +-
.../worker/streaming/ComputationWorkExecutor.java | 7 +-
.../dataflow/worker/streaming/ExecutableWork.java | 2 +-
.../RefreshableWork.java} | 29 +-
.../dataflow/worker/streaming/StageInfo.java | 14 +-
.../runners/dataflow/worker/streaming/Work.java | 72 +-
.../runners/dataflow/worker/streaming/WorkId.java | 4 +-
.../StreamingEngineComputationConfigFetcher.java | 18 +-
.../config/StreamingEnginePipelineConfig.java | 10 +
.../FanOutStreamingEngineWorkerHarness.java} | 91 +-
.../harness/SingleSourceWorkerHarness.java | 284 ++++
.../harness}/StreamingEngineConnectionState.java | 2 +-
.../streaming/harness/StreamingWorkerHarness.java | 15 +-
.../harness}/WindmillStreamSender.java | 48 +-
.../streaming/sideinput/SideInputStateFetcher.java | 114 +-
.../sideinput/SideInputStateFetcherFactory.java | 46 +
.../worker/windmill/ApplianceWindmillClient.java | 22 +-
.../windmill/StreamingEngineWindmillClient.java | 54 +
.../worker/windmill/WindmillConnection.java | 13 +-
.../worker/windmill/WindmillServerBase.java | 5 -
.../worker/windmill/WindmillServerStub.java | 58 +-
.../windmill/client/AbstractWindmillStream.java | 56 +-
.../worker/windmill/client/WindmillStream.java | 27 +-
.../worker/windmill/client/WindmillStreamPool.java | 4 +-
.../commits/StreamingEngineWorkCommitter.java | 113 +-
.../client/getdata/ApplianceGetDataClient.java | 220 +++
.../windmill/client/getdata/GetDataClient.java | 57 +
.../client/getdata/StreamGetDataClient.java | 101 ++
.../client/getdata/StreamPoolGetDataClient.java | 80 ++
.../getdata/ThrottlingGetDataMetricTracker.java | 108 ++
.../windmill/client/grpc/ChannelzServlet.java | 27 +-
.../client/grpc/GetWorkResponseChunkAssembler.java | 139 ++
.../windmill/client/grpc/GrpcCommitWorkStream.java | 7 +-
.../client/grpc/GrpcDirectGetWorkStream.java | 178 +--
.../windmill/client/grpc/GrpcDispatcherClient.java | 4 +-
.../windmill/client/grpc/GrpcGetDataStream.java | 22 +-
.../windmill/client/grpc/GrpcGetWorkStream.java | 142 +-
.../client/grpc/GrpcGetWorkerMetadataStream.java | 4 +-
.../windmill/client/grpc/GrpcWindmillServer.java | 5 -
.../client/grpc/GrpcWindmillStreamFactory.java | 18 +-
.../StreamObserverCancelledException.java | 14 +-
.../worker/windmill/work/WorkItemScheduler.java | 4 -
.../work/budget/EvenGetWorkBudgetDistributor.java | 27 +-
.../work/budget/GetWorkBudgetDistributor.java | 5 +-
.../windmill/work/budget/GetWorkBudgetSpender.java | 20 +-
.../work/processing/StreamingWorkScheduler.java | 38 +-
.../windmill/work/refresh/ActiveWorkRefresher.java | 104 +-
.../work/refresh/ActiveWorkRefreshers.java | 50 -
.../work/refresh/ApplianceHeartbeatSender.java | 62 +
.../refresh/DispatchedActiveWorkRefresher.java | 68 -
.../work/refresh/FixedStreamHeartbeatSender.java | 93 ++
.../windmill/work/refresh/HeartbeatSender.java | 18 +-
.../worker/windmill/work/refresh/Heartbeats.java | 70 +
.../work/refresh/StreamPoolHeartbeatSender.java | 48 +
.../worker/BatchModeExecutionContextTest.java | 3 +-
.../dataflow/worker/FakeWindmillServer.java | 84 +-
.../worker/StreamingDataflowWorkerTest.java | 120 +-
.../worker/StreamingModeExecutionContextTest.java | 14 +-
.../worker/StreamingStepMetricsContainerTest.java | 2 -
.../dataflow/worker/WorkerCustomSourcesTest.java | 13 +-
.../worker/streaming/ActiveWorkStateTest.java | 77 +-
.../streaming/ComputationStateCacheTest.java | 7 +-
.../FanOutStreamingEngineWorkerHarnessTest.java} | 61 +-
.../harness}/WindmillStreamSenderTest.java | 56 +-
.../sideinput/SideInputStateFetcherTest.java | 93 +-
.../worker/util/BoundedQueueExecutorTest.java | 26 +-
.../worker/GroupingShuffleEntryIteratorTest.java | 7 +-
.../windmill/client/WindmillStreamPoolTest.java | 14 +-
.../StreamingApplianceWorkCommitterTest.java | 13 +-
.../commits/StreamingEngineWorkCommitterTest.java | 54 +-
.../client/getdata/FakeGetDataClient.java} | 35 +-
.../ThrottlingGetDataMetricTrackerTest.java | 277 ++++
.../windmill/client/grpc/ChannelzServletTest.java | 6 +-
.../grpc/GrpcGetWorkerMetadataStreamTest.java | 2 +-
.../client/grpc/GrpcWindmillServerTest.java | 39 +-
.../windmill/state/WindmillStateInternalsTest.java | 11 +-
.../windmill/state/WindmillStateReaderTest.java | 12 +-
.../budget/EvenGetWorkBudgetDistributorTest.java | 123 +-
.../failures/WorkFailureProcessorTest.java | 8 +-
...esherTest.java => ActiveWorkRefresherTest.java} | 95 +-
.../artifact/ArtifactStagingService.java | 4 +-
.../fnexecution/state/StateRequestHandlers.java | 5 +-
.../artifact/ArtifactRetrievalServiceTest.java | 4 +-
.../beam/runners/portability/PortableMetrics.java | 1 +
.../portability/testing/TestUniversalRunner.java | 5 +-
runners/prism/java/build.gradle | 7 +
.../beam/runners/prism/PrismArtifactResolver.java | 110 ++
.../beam/runners/prism/PrismArtifactStager.java | 173 +++
.../apache/beam/runners/prism/PrismExecutor.java | 8 +
.../apache/beam/runners/prism/PrismLocator.java | 221 +++
.../beam/runners/prism/PrismPipelineResult.java | 109 ++
.../apache/beam/runners/prism/StateListener.java | 17 +-
.../apache/beam/runners/prism/StateWatcher.java | 146 ++
.../apache/beam/runners/prism/WorkerService.java | 116 ++
.../runners/prism/PrismArtifactResolverTest.java | 45 +
.../runners/prism/PrismArtifactStagerTest.java | 143 ++
.../beam/runners/prism/PrismLocatorTest.java | 125 ++
.../runners/prism/PrismPipelineResultTest.java | 130 ++
.../beam/runners/prism/StateWatcherTest.java | 136 ++
.../beam/runners/prism/WorkerServiceTest.java | 85 ++
runners/samza/job-server/build.gradle | 1 -
runners/spark/job-server/spark_job_server.gradle | 2 -
runners/spark/spark_runner.gradle | 6 +
scripts/ci/pr-bot/package-lock.json | 28 +-
sdks/go.mod | 28 +-
sdks/go.sum | 86 +-
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/core/runtime/exec/hash.go | 7 +
.../prism/internal/engine/elementmanager.go | 52 +-
.../beam/runners/prism/internal/engine/timers.go | 1 +
.../beam/runners/prism/internal/environments.go | 8 +-
.../pkg/beam/runners/prism/internal/handlepardo.go | 14 +-
.../beam/runners/prism/internal/handlerunner.go | 56 +-
.../prism/internal/jobservices/management.go | 15 +-
.../pkg/beam/runners/prism/internal/preprocess.go | 16 +-
sdks/go/pkg/beam/runners/prism/internal/stage.go | 55 +-
.../beam/runners/prism/internal/worker/worker.go | 2 +-
.../main/resources/beam/checkstyle/checkstyle.xml | 8 +
.../java/org/apache/beam/sdk/metrics/Lineage.java | 43 +-
.../java/org/apache/beam/sdk/metrics/Metrics.java | 17 +-
.../beam/sdk/metrics/MetricsEnvironment.java | 11 +-
.../apache/beam/sdk/schemas/AutoValueSchema.java | 35 +-
.../apache/beam/sdk/schemas/CachingFactory.java | 11 +-
.../java/org/apache/beam/sdk/schemas/Factory.java | 3 +-
.../beam/sdk/schemas/FromRowUsingCreator.java | 38 +-
.../sdk/schemas/GetterBasedSchemaProvider.java | 68 +-
.../sdk/schemas/GetterBasedSchemaProviderV2.java | 56 +
.../apache/beam/sdk/schemas/JavaBeanSchema.java | 59 +-
.../apache/beam/sdk/schemas/JavaFieldSchema.java | 44 +-
.../sdk/schemas/SetterBasedCreatorFactory.java | 7 +-
.../sdk/schemas/annotations/DefaultSchema.java | 2 +-
.../schemas/transforms/providers/JavaRowUdf.java | 2 +-
.../providers/LoggingTransformProvider.java | 4 +-
.../beam/sdk/schemas/utils/AutoValueUtils.java | 30 +-
.../sdk/schemas/utils/FieldValueTypeSupplier.java | 7 +-
.../beam/sdk/schemas/utils/JavaBeanUtils.java | 57 +-
.../apache/beam/sdk/schemas/utils/POJOUtils.java | 72 +-
.../beam/sdk/schemas/utils/ReflectUtils.java | 13 +
.../sdk/schemas/utils/StaticSchemaInference.java | 28 +-
.../java/org/apache/beam/sdk/testing/PAssert.java | 15 +-
.../beam/sdk/testing/SerializableMatchers.java | 14 +-
.../sdk/transforms/errorhandling/BadRecord.java | 6 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 6 +-
.../transforms/resourcehints/ResourceHints.java | 8 +-
.../java/org/apache/beam/sdk/util/StringUtils.java | 35 +
.../org/apache/beam/sdk/values/RowWithGetters.java | 2 +-
.../beam/sdk/coders/StructuralByteArrayTest.java | 10 +-
.../org/apache/beam/sdk/io/FileBasedSinkTest.java | 2 +-
.../java/org/apache/beam/sdk/io/FileIOTest.java | 6 +-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 7 +-
.../org/apache/beam/sdk/io/TextIOReadTest.java | 8 +-
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 6 +-
.../beam/sdk/io/TextRowCountEstimatorTest.java | 10 +-
.../org/apache/beam/sdk/io/WriteFilesTest.java | 5 +-
.../beam/sdk/metrics/MetricResultsMatchers.java | 18 +-
.../org/apache/beam/sdk/metrics/MetricsTest.java | 187 ++-
.../sdk/options/PipelineOptionsFactoryTest.java | 24 +-
.../beam/sdk/schemas/SchemaTranslationTest.java | 5 +-
.../beam/sdk/schemas/utils/JavaBeanUtilsTest.java | 33 +-
.../beam/sdk/schemas/utils/POJOUtilsTest.java | 40 +-
.../org/apache/beam/sdk/testing/PAssertTest.java | 31 +
.../BufferedElementCountingOutputStreamTest.java | 6 +-
.../sdk/util/ExposedByteArrayInputStreamTest.java | 6 +-
.../sdk/util/ExposedByteArrayOutputStreamTest.java | 4 +-
.../beam/sdk/util/SerializableUtilsTest.java | 4 +-
.../org/apache/beam/sdk/util/StringUtilsTest.java | 23 +
.../expansion/service/ExpansionServiceTest.java | 4 +-
.../beam/sdk/extensions/arrow/ArrowConversion.java | 3 +-
.../avro/AvroGenericCoderTranslator.java | 6 +-
.../extensions/avro/schemas/AvroRecordSchema.java | 18 +-
.../extensions/avro/schemas/utils/AvroUtils.java | 45 +-
.../beam/sdk/extensions/avro/io/AvroIOTest.java | 6 +-
.../gcp/util/RetryHttpRequestInitializer.java | 2 +-
.../extensions/protobuf/ProtoByteBuddyUtils.java | 5 +-
.../extensions/protobuf/ProtoMessageSchema.java | 27 +-
.../extensions/python/PythonExternalTransform.java | 5 +-
.../beam/sdk/extensions/python/PythonService.java | 5 +-
sdks/java/extensions/sql/jdbc/build.gradle | 2 +-
.../beam/sdk/extensions/sql/jdbc/BeamSqlLine.java | 6 +-
.../meta/provider/text/TextTableProviderTest.java | 20 +-
.../beam/fn/harness/state/FnApiStateAccessor.java | 8 +-
.../beam/sdk/io/aws2/options/AwsOptions.java | 8 +-
.../sdk/io/aws2/schemas/AwsSchemaProvider.java | 40 +-
.../apache/beam/sdk/io/aws2/schemas/AwsTypes.java | 4 +-
.../beam/sdk/io/clickhouse/ClickHouseWriter.java | 4 +-
.../beam/sdk/io/common/SchemaAwareJavaBeans.java | 8 +-
.../io/contextualtextio/ContextualTextIOTest.java | 16 +-
.../java/org/apache/beam/sdk/io/csv/CsvIO.java | 228 +++
.../org/apache/beam/sdk/io/csv/CsvIOParse.java | 84 ++
.../beam/sdk/io/csv/CsvIOParseConfiguration.java | 37 +-
.../apache/beam/sdk/io/csv/CsvIOParseError.java | 24 +
.../apache/beam/sdk/io/csv/CsvIOParseHelpers.java | 129 +-
.../org/apache/beam/sdk/io/csv/CsvIOParseKV.java | 46 +
.../apache/beam/sdk/io/csv/CsvIOParseResult.java | 100 ++
.../org/apache/beam/sdk/io/csv/CsvIOReadFiles.java | 4 +-
.../beam/sdk/io/csv/CsvIORecordToObjects.java | 129 ++
.../beam/sdk/io/csv/CsvIOStringToCsvRecord.java | 114 ++
.../beam/sdk/io/csv/CsvIOParseHelpersTest.java | 608 ++++++++
.../apache/beam/sdk/io/csv/CsvIOParseKVTest.java} | 16 +-
.../org/apache/beam/sdk/io/csv/CsvIOParseTest.java | 197 +++
.../beam/sdk/io/csv/CsvIORecordToObjectsTest.java | 422 ++++++
.../sdk/io/csv/CsvIOStringToCsvRecordTest.java | 565 ++++++++
.../java/org/apache/beam/sdk/io/csv/CsvIOTest.java | 300 ++++
sdks/java/io/google-cloud-platform/build.gradle | 4 +
.../beam/sdk/io/gcp/bigquery/AppendClientInfo.java | 8 +-
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 21 +-
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 27 +
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 13 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 4 +-
.../io/gcp/bigquery/BigQueryStorageSourceBase.java | 4 +-
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 8 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 55 +-
.../bigquery/StorageApiWritesShardedRecords.java | 6 +
.../io/gcp/bigquery/TableRowToStorageApiProto.java | 2 +-
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 6 +
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 6 +
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 120 +-
.../beam/sdk/io/gcp/bigtable/BigtableService.java | 6 +
.../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 39 +-
.../sdk/io/gcp/bigtable/BigtableWriteOptions.java | 5 +
.../dofn/ReadChangeStreamPartitionDoFn.java | 1 +
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 2 +-
.../sdk/io/gcp/datastore/RampupThrottlingFn.java | 3 +-
.../sdk/io/gcp/firestore/FirestoreOptions.java | 11 +
.../sdk/io/gcp/firestore/FirestoreV1ReadFn.java | 9 +-
.../sdk/io/gcp/firestore/FirestoreV1WriteFn.java | 9 +-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 28 +
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 11 +-
.../io/gcp/pubsub/PubsubMessageWithTopicCoder.java | 4 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 13 +
.../sdk/io/gcp/pubsub/PubsubUnboundedSource.java | 14 +
.../internal/SubscriptionPartitionLoader.java | 2 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 5 +-
.../beam/sdk/io/gcp/spanner/SpannerSchema.java | 3 +
.../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 23 +-
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 23 +-
.../io/gcp/bigquery/BigQuerySinkMetricsTest.java | 4 +-
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 15 +
.../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 21 +-
.../io/gcp/bigtable/BigtableServiceImplTest.java | 62 +
.../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 18 +-
.../dofn/ReadChangeStreamPartitionDoFnTest.java | 1 +
.../gcp/firestore/it/FirestoreTestingHelper.java | 5 +-
.../beam/sdk/io/gcp/pubsub/PubsubClientTest.java | 2 +
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 9 +-
.../PubsubReadSchemaTransformProviderTest.java | 6 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java | 2 +-
.../gcp/pubsublite/internal/FakeSerializable.java | 2 +-
.../internal/SubscriptionPartitionLoaderTest.java | 2 +-
.../sdk/io/gcp/spanner/SpannerAccessorTest.java | 2 +
.../beam/sdk/io/gcp/spanner/SpannerSchemaTest.java | 4 +-
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 10 +-
...emaAndRowConversions.java => IcebergUtils.java} | 217 ++-
.../apache/beam/sdk/io/iceberg/RecordWriter.java | 4 +-
.../org/apache/beam/sdk/io/iceberg/ScanSource.java | 2 +-
.../apache/beam/sdk/io/iceberg/ScanTaskReader.java | 4 +-
.../apache/beam/sdk/io/iceberg/IcebergIOIT.java | 8 +-
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 8 +-
.../beam/sdk/io/iceberg/IcebergIOWriteTest.java | 12 +-
.../IcebergReadSchemaTransformProviderTest.java | 8 +-
.../beam/sdk/io/iceberg/IcebergUtilsTest.java | 676 +++++++++
.../IcebergWriteSchemaTransformProviderTest.java | 5 +-
.../apache/beam/sdk/io/iceberg/ScanSourceTest.java | 6 +-
.../io/iceberg/SchemaAndRowConversionsTest.java | 268 ----
.../apache/beam/sdk/io/iceberg/TestFixtures.java | 2 +-
.../java/org/apache/beam/sdk/io/jms/CommonJms.java | 5 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 34 +-
.../beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java | 8 +-
.../org/apache/beam/io/requestresponse/Call.java | 1 +
.../DefaultSerializableBackoffSupplier.java | 1 +
.../beam/io/requestresponse/RequestResponseIO.java | 1 +
.../requestresponse/WindowedCallShouldBackoff.java | 1 +
.../io/requestresponse/RequestResponseIOTest.java | 1 +
.../WindowedCallShouldBackoffTest.java | 1 +
.../apache/beam/sdk/io/snowflake/SnowflakeIO.java | 6 +-
sdks/java/io/solace/build.gradle | 3 +
.../org/apache/beam/sdk/io/solace/SolaceIO.java | 441 +++++-
.../broker/BasicAuthJcsmpSessionService.java | 38 +-
.../BasicAuthJcsmpSessionServiceFactory.java | 3 +-
.../broker/GCPSecretSessionServiceFactory.java | 169 +++
.../beam/sdk/io/solace/broker/MessageReceiver.java | 3 +
.../beam/sdk/io/solace/broker/SessionService.java | 196 ++-
.../io/solace/broker/SessionServiceFactory.java | 3 +-
.../io/solace/broker/SolaceMessageReceiver.java | 7 +
.../org/apache/beam/sdk/io/solace/data/Solace.java | 99 ++
.../beam/sdk/io/solace/write/SolaceOutput.java | 104 ++
.../beam/sdk/io/solace/write/package-info.java} | 11 +-
.../sdk/io/solace/MockEmptySessionService.java | 8 +-
.../beam/sdk/io/solace/MockSessionService.java | 31 +-
.../solace/broker/BasicAuthWriterSessionTest.java | 106 ++
.../broker/OverrideWriterPropertiesTest.java | 56 +
.../beam/sdk/io/synthetic/SyntheticStep.java | 2 +-
.../apache/beam/sdk/io/thrift/ThriftSchema.java | 23 +-
.../org/apache/beam/sdk/tpcds/QueryReader.java | 4 +-
.../apache/beam/sdk/tpcds/SqlTransformRunner.java | 4 +-
.../beam/sdk/tpcds/TableSchemaJSONLoader.java | 4 +-
.../launcher/TransformServiceLauncherTest.java | 10 +-
sdks/python/apache_beam/coders/observable_test.py | 2 +-
sdks/python/apache_beam/coders/row_coder.py | 3 +-
sdks/python/apache_beam/coders/slow_stream.py | 32 +-
.../apache_beam/coders/standard_coders_test.py | 2 +-
sdks/python/apache_beam/coders/typecoders.py | 20 +-
sdks/python/apache_beam/dataframe/convert.py | 38 +-
sdks/python/apache_beam/dataframe/frame_base.py | 20 +-
sdks/python/apache_beam/dataframe/partitionings.py | 9 +-
sdks/python/apache_beam/dataframe/schemas.py | 9 +-
sdks/python/apache_beam/dataframe/schemas_test.py | 85 +-
.../examples/complete/top_wikipedia_sessions.py | 34 +-
.../complete/top_wikipedia_sessions_test.py | 2 +
.../examples/cookbook/bigtableio_it_test.py | 2 +-
.../examples/cookbook/datastore_wordcount.py | 4 +-
.../apache_beam/internal/cloudpickle_pickler.py | 4 +-
sdks/python/apache_beam/internal/dill_pickler.py | 7 +-
sdks/python/apache_beam/internal/metrics/cells.py | 27 +-
sdks/python/apache_beam/internal/metrics/metric.py | 66 +-
sdks/python/apache_beam/internal/module_test.py | 4 +-
sdks/python/apache_beam/internal/pickler.py | 3 +-
sdks/python/apache_beam/internal/util.py | 11 +-
.../apache_beam/io/azure/blobstoragefilesystem.py | 4 -
sdks/python/apache_beam/io/filebasedsource.py | 24 +-
sdks/python/apache_beam/io/fileio.py | 14 +-
sdks/python/apache_beam/io/filesystem.py | 77 +-
.../io/flink/flink_streaming_impulse_source.py | 2 +-
.../apache_beam/io/gcp/bigquery_avro_tools.py | 26 +-
.../apache_beam/io/gcp/bigquery_read_internal.py | 49 +-
.../apache_beam/io/gcp/bigquery_read_it_test.py | 4 +-
.../apache_beam/io/gcp/bigquery_schema_tools.py | 5 +-
.../apache_beam/io/gcp/datastore/v1new/helper.py | 4 +-
.../apache_beam/io/gcp/datastore/v1new/types.py | 19 +-
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 8 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 58 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 10 +-
sdks/python/apache_beam/io/hadoopfilesystem.py | 12 +-
sdks/python/apache_beam/io/iobase.py | 79 +-
sdks/python/apache_beam/io/jdbc.py | 18 +-
sdks/python/apache_beam/io/localfilesystem.py | 8 +-
sdks/python/apache_beam/io/restriction_trackers.py | 9 +-
sdks/python/apache_beam/io/textio.py | 66 +-
sdks/python/apache_beam/metrics/cells.pxd | 6 +
sdks/python/apache_beam/metrics/cells.py | 75 +
sdks/python/apache_beam/metrics/cells_test.py | 24 +
sdks/python/apache_beam/metrics/execution.py | 20 +-
sdks/python/apache_beam/metrics/execution_test.py | 9 +
sdks/python/apache_beam/metrics/metric.py | 125 +-
sdks/python/apache_beam/metrics/metricbase.py | 28 +-
.../python/apache_beam/metrics/monitoring_infos.py | 53 +-
.../apache_beam/metrics/monitoring_infos_test.py | 25 +
.../python/apache_beam/ml/gcp/naturallanguageml.py | 36 +-
.../ml/inference/huggingface_inference.py | 2 +-
.../python/apache_beam/options/pipeline_options.py | 14 +
sdks/python/apache_beam/options/value_provider.py | 2 +-
sdks/python/apache_beam/pvalue.py | 125 +-
.../runners/dataflow/dataflow_metrics.py | 15 +-
.../runners/dataflow/internal/apiclient.py | 6 +
.../runners/dataflow/internal/apiclient_test.py | 37 +
.../clients/dataflow/dataflow_v1b3_client.py | 81 +-
.../clients/dataflow/dataflow_v1b3_messages.py | 1497 ++++++++++++++++----
.../apache_beam/runners/direct/bundle_factory.py | 34 +-
.../direct/consumer_tracking_pipeline_visitor.py | 15 +-
.../apache_beam/runners/direct/direct_metrics.py | 15 +-
.../apache_beam/runners/direct/direct_runner.py | 5 +-
.../runners/direct/direct_runner_test.py | 9 +
.../runners/direct/evaluation_context.py | 123 +-
sdks/python/apache_beam/runners/direct/executor.py | 101 +-
.../runners/direct/sdf_direct_runner.py | 17 +-
.../apache_beam/runners/direct/test_stream_impl.py | 5 +-
.../runners/direct/transform_evaluator.py | 62 +-
.../runners/direct/watermark_manager.py | 84 +-
.../runners/interactive/background_caching_job.py | 4 +-
.../runners/interactive/cache_manager.py | 4 +-
.../runners/interactive/cache_manager_test.py | 2 +-
.../runners/interactive/display/pipeline_graph.py | 17 +-
.../interactive/display/pipeline_graph_renderer.py | 30 +-
.../runners/interactive/options/capture_control.py | 10 +-
.../interactive/options/capture_limiters.py | 17 +-
.../runners/interactive/recording_manager.py | 143 +-
.../testing/integration/notebook_executor.py | 3 +-
.../interactive/testing/integration/screen_diff.py | 7 +-
sdks/python/apache_beam/runners/job/utils.py | 6 +-
.../python/apache_beam/runners/pipeline_context.py | 140 +-
.../runners/portability/abstract_job_service.py | 149 +-
.../runners/portability/artifact_service.py | 18 +-
.../runners/portability/fn_api_runner/execution.py | 307 ++--
.../runners/portability/fn_api_runner/fn_runner.py | 336 ++---
.../portability/fn_api_runner/fn_runner_test.py | 11 +-
.../portability/fn_api_runner/watermark_manager.py | 11 +-
.../apache_beam/runners/portability/job_server.py | 6 +-
.../runners/portability/local_job_service.py | 41 +-
.../runners/portability/portable_metrics.py | 13 +-
.../runners/portability/portable_runner.py | 81 +-
.../runners/portability/prism_runner_test.py | 22 +
.../apache_beam/runners/portability/stager.py | 3 +-
.../apache_beam/runners/portability/stager_test.py | 2 +-
sdks/python/apache_beam/runners/runner.py | 42 +-
sdks/python/apache_beam/runners/sdf_utils.py | 26 +-
.../apache_beam/runners/worker/log_handler.py | 35 +-
sdks/python/apache_beam/runners/worker/logger.py | 29 +-
.../apache_beam/runners/worker/statecache.py | 76 +-
.../apache_beam/runners/worker/statesampler.py | 50 +-
.../runners/worker/statesampler_slow.py | 51 +-
.../runners/worker/worker_id_interceptor.py | 3 +-
.../apache_beam/runners/worker/worker_pool_main.py | 37 +-
.../apache_beam/runners/worker/worker_status.py | 4 +-
.../testing/benchmarks/nexmark/monitor.py | 3 +-
.../testing/benchmarks/nexmark/nexmark_launcher.py | 3 +-
.../testing/benchmarks/nexmark/nexmark_perf.py | 4 +-
.../testing/benchmarks/nexmark/nexmark_util.py | 18 +-
.../testing/load_tests/load_test_metrics_utils.py | 32 +-
.../transforms/combinefn_lifecycle_pipeline.py | 2 +-
sdks/python/apache_beam/transforms/combiners.py | 2 +-
.../python/apache_beam/transforms/external_java.py | 5 +-
sdks/python/apache_beam/transforms/resources.py | 39 +-
sdks/python/apache_beam/transforms/sideinputs.py | 28 +-
sdks/python/apache_beam/transforms/trigger.py | 7 +-
sdks/python/apache_beam/transforms/userstate.py | 149 +-
.../apache_beam/transforms/userstate_test.py | 2 +-
sdks/python/apache_beam/transforms/window.py | 164 +--
sdks/python/apache_beam/typehints/decorators.py | 80 +-
.../typehints/native_type_compatibility.py | 2 +-
sdks/python/apache_beam/typehints/typehints.py | 2 +-
sdks/python/apache_beam/utils/profiler.py | 24 +-
sdks/python/apache_beam/utils/proto_utils.py | 34 +-
sdks/python/apache_beam/utils/python_callable.py | 6 +-
sdks/python/apache_beam/utils/sharded_key.py | 5 +-
sdks/python/apache_beam/utils/shared.py | 28 +-
sdks/python/apache_beam/utils/timestamp.py | 160 +--
sdks/python/apache_beam/utils/urns.py | 60 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/apache_beam/yaml/pipeline.schema.yaml | 6 +-
sdks/python/apache_beam/yaml/yaml_transform.py | 4 +-
sdks/python/build.gradle | 2 +-
sdks/python/container/piputil.go | 13 +-
sdks/python/test-suites/dataflow/common.gradle | 8 +-
sdks/python/test-suites/direct/common.gradle | 2 +-
sdks/python/test-suites/direct/xlang/build.gradle | 2 +-
sdks/typescript/package.json | 2 +-
settings.gradle.kts | 2 +-
website/www/site/config.toml | 2 +-
website/www/site/content/en/blog/beam-2.49.0.md | 1 +
website/www/site/content/en/blog/beam-2.58.0.md | 130 ++
.../dsls/sql/extensions/user-defined-functions.md | 2 +-
.../site/content/en/documentation/io/connectors.md | 16 +
.../en/documentation/ml/large-language-modeling.md | 50 +-
.../www/site/content/en/get-started/downloads.md | 13 +-
.../site/content/en/get-started/quickstart-java.md | 2 +-
528 files changed, 15730 insertions(+), 6148 deletions(-)
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/MetricTrackingWindmillServerStub.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/OperationalLimits.java
copy runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{WorkItemCancelledException.java => OutputTooLargeException.java} (63%)
copy runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{windmill/work/budget/GetWorkBudgetDistributor.java => streaming/RefreshableWork.java} (58%)
rename runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{windmill/client/grpc/StreamingEngineClient.java => streaming/harness/FanOutStreamingEngineWorkerHarness.java} (86%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/harness/SingleSourceWorkerHarness.java
rename runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{windmill/client/grpc => streaming/harness}/StreamingEngineConnectionState.java (97%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/SerializableSupplier.java => runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/harness/StreamingWorkerHarness.java (71%)
rename runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{windmill/client/grpc => streaming/harness}/WindmillStreamSender.java (81%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/sideinput/SideInputStateFetcherFactory.java
copy sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/Factory.java => runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/ApplianceWindmillClient.java (51%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/StreamingEngineWindmillClient.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/ApplianceGetDataClient.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/GetDataClient.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/StreamGetDataClient.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/StreamPoolGetDataClient.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/ThrottlingGetDataMetricTracker.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GetWorkResponseChunkAssembler.java
copy sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/Factory.java => runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/observers/StreamObserverCancelledException.java (71%)
copy sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/SerializableSupplier.java => runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetSpender.java (64%)
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/ActiveWorkRefreshers.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/ApplianceHeartbeatSender.java
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/DispatchedActiveWorkRefresher.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/FixedStreamHeartbeatSender.java
copy sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/SerializableSupplier.java => runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/HeartbeatSender.java (66%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/Heartbeats.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/StreamPoolHeartbeatSender.java
rename runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/{windmill/client/grpc/StreamingEngineClientTest.java => streaming/harness/FanOutStreamingEngineWorkerHarnessTest.java} (90%)
rename runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/{windmill/client/grpc => streaming/harness}/WindmillStreamSenderTest.java (81%)
copy runners/google-cloud-dataflow-java/worker/src/{main/java/org/apache/beam/runners/dataflow/worker/streaming/ExecutableWork.java => test/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/FakeGetDataClient.java} (55%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/getdata/ThrottlingGetDataMetricTrackerTest.java
rename runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/refresh/{DispatchedActiveWorkRefresherTest.java => ActiveWorkRefresherTest.java} (76%)
create mode 100644 runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismArtifactResolver.java
create mode 100644 runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismArtifactStager.java
create mode 100644 runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java
create mode 100644 runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineResult.java
rename sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/SerializableSupplier.java => runners/prism/java/src/main/java/org/apache/beam/runners/prism/StateListener.java (69%)
create mode 100644 runners/prism/java/src/main/java/org/apache/beam/runners/prism/StateWatcher.java
create mode 100644 runners/prism/java/src/main/java/org/apache/beam/runners/prism/WorkerService.java
create mode 100644 runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismArtifactResolverTest.java
create mode 100644 runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismArtifactStagerTest.java
create mode 100644 runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java
create mode 100644 runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismPipelineResultTest.java
create mode 100644 runners/prism/java/src/test/java/org/apache/beam/runners/prism/StateWatcherTest.java
create mode 100644 runners/prism/java/src/test/java/org/apache/beam/runners/prism/WorkerServiceTest.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/GetterBasedSchemaProviderV2.java
create mode 100644 sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIOParse.java
create mode 100644 sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIOParseKV.java
create mode 100644 sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIOParseResult.java
create mode 100644 sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIORecordToObjects.java
create mode 100644 sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIOStringToCsvRecord.java
create mode 100644 sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOParseHelpersTest.java
copy sdks/java/{core/src/main/java/org/apache/beam/sdk/schemas/Factory.java => io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOParseKVTest.java} (73%)
create mode 100644 sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOParseTest.java
create mode 100644 sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIORecordToObjectsTest.java
create mode 100644 sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOStringToCsvRecordTest.java
create mode 100644 sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOTest.java
rename sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/{SchemaAndRowConversions.java => IcebergUtils.java} (50%)
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java
delete mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/SchemaAndRowConversionsTest.java
create mode 100644 sdks/java/io/solace/src/main/java/org/apache/beam/sdk/io/solace/broker/GCPSecretSessionServiceFactory.java
create mode 100644 sdks/java/io/solace/src/main/java/org/apache/beam/sdk/io/solace/write/SolaceOutput.java
copy sdks/java/{core/src/main/java/org/apache/beam/sdk/schemas/Factory.java => io/solace/src/main/java/org/apache/beam/sdk/io/solace/write/package-info.java} (72%)
create mode 100644 sdks/java/io/solace/src/test/java/org/apache/beam/sdk/io/solace/broker/BasicAuthWriterSessionTest.java
create mode 100644 sdks/java/io/solace/src/test/java/org/apache/beam/sdk/io/solace/broker/OverrideWriterPropertiesTest.java
create mode 100644 website/www/site/content/en/blog/beam-2.58.0.md