You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2024/06/01 04:15:16 UTC
(beam) branch nightly-refs/heads/master updated (b50ad0fe8fc -> f93a67a6ae8)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from b50ad0fe8fc [ManagedIO] pass underlying transform URN as an annotation (#31398)
add 6911a1af720 Updating Commons Compress to 1.26.2
add f410b71c076 Merge pull request #31464: Updating Commons Compress to 1.26.2
add daf81143388 Merge pull request #31449 Pass through docs (and configs) for SqlProviders.
add 19d57d01266 Change configurations for stress tests. Fix BigQueryIOST (#31439)
add 7b6f9415c10 Updates Expansion Service Container to support upgrading using the schema-transform ID (#31451)
add f93a67a6ae8 remove processing/scheduling logic from StreamingDataflowWorker (#31317)
No new revisions were added by this update.
Summary of changes:
...owV1.json => IO_Iceberg_Integration_Tests.json} | 0
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 12 +-
.../apache/beam/it/gcp/bigquery/BigQueryIOST.java | 7 +-
.../apache/beam/it/gcp/bigtable/BigTableIOST.java | 2 +-
.../apache/beam/it/gcp/spanner/SpannerIOST.java | 2 +-
.../beam/runners/dataflow/worker/PubsubReader.java | 2 +-
.../beam/runners/dataflow/worker/ReaderCache.java | 4 +-
.../dataflow/worker/StreamingDataflowWorker.java | 614 +++------------------
.../worker/StreamingModeExecutionContext.java | 176 +++---
.../dataflow/worker/UngroupedWindmillReader.java | 2 +-
.../dataflow/worker/WindmillTimerInternals.java | 39 +-
.../dataflow/worker/WindowingWindmillReader.java | 2 +-
.../dataflow/worker/WorkerCustomSources.java | 2 +-
.../dataflow/worker/streaming/ActiveWorkState.java | 77 +--
.../worker/streaming/ComputationState.java | 52 +-
.../worker/streaming/ComputationStateCache.java | 1 +
.../worker/streaming/ComputationWorkExecutor.java | 118 ++++
.../{ShardedKey.java => ExecutableWork.java} | 26 +-
.../dataflow/worker/streaming/ExecutionState.java | 54 --
.../dataflow/worker/streaming/Watermarks.java | 69 +++
.../runners/dataflow/worker/streaming/Work.java | 289 ++++++----
.../runners/dataflow/worker/streaming/WorkId.java | 8 +
.../streaming/sideinput/SideInputStateFetcher.java | 2 +
.../worker/util/common/worker/MapTaskExecutor.java | 2 +-
.../client/grpc/GrpcDirectGetWorkStream.java | 53 +-
.../windmill/client/grpc/GrpcDispatcherClient.java | 32 +-
.../client/grpc/GrpcWindmillStreamFactory.java | 11 +-
.../client/grpc/StreamingEngineClient.java | 161 ++++--
.../windmill/client/grpc/WindmillStreamSender.java | 37 +-
.../worker/windmill/state/WindmillStateReader.java | 32 +-
.../windmill/work/ProcessWorkItemClient.java | 52 --
...rkItemProcessor.java => WorkItemScheduler.java} | 29 +-
.../processing/ComputationWorkExecutorFactory.java | 291 ++++++++++
.../work/processing/StreamingCommitFinalizer.java | 85 +++
.../work/processing/StreamingWorkScheduler.java | 428 ++++++++++++++
.../processing/failures/WorkFailureProcessor.java | 16 +-
.../runners/dataflow/worker/PubsubReaderTest.java | 2 +-
.../worker/StreamingDataflowWorkerTest.java | 172 +++---
.../worker/StreamingModeExecutionContextTest.java | 40 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 83 +--
.../worker/streaming/ActiveWorkStateTest.java | 213 ++++---
.../streaming/ComputationStateCacheTest.java | 45 +-
.../StreamingApplianceWorkCommitterTest.java | 19 +-
.../commits/StreamingEngineWorkCommitterTest.java | 28 +-
.../client/grpc/StreamingEngineClientTest.java | 37 +-
.../client/grpc/WindmillStreamSenderTest.java | 42 +-
.../budget/EvenGetWorkBudgetDistributorTest.java | 14 +-
.../failures/WorkFailureProcessorTest.java | 75 ++-
.../refresh/DispatchedActiveWorkRefresherTest.java | 54 +-
.../construction/ExternalTranslationOptions.java | 4 +-
.../util/construction/PTransformTranslation.java | 2 +-
.../sdk/util/construction/TransformUpgrader.java | 56 +-
.../util/construction/TransformUpgraderTest.java | 139 ++++-
.../container/expansion_service_config.yml | 3 +
.../sdk/expansion/service/ExpansionService.java | 67 ++-
sdks/java/io/expansion-service/build.gradle | 5 +
sdks/java/io/iceberg/build.gradle | 4 +-
.../managed/ManagedSchemaTransformProvider.java | 63 ++-
sdks/python/apache_beam/yaml/yaml_join.py | 45 +-
sdks/python/apache_beam/yaml/yaml_provider.py | 27 +-
60 files changed, 2519 insertions(+), 1509 deletions(-)
copy .github/trigger_files/{beam_PostCommit_Java_DataflowV1.json => IO_Iceberg_Integration_Tests.json} (100%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationWorkExecutor.java
copy runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/{ShardedKey.java => ExecutableWork.java} (58%)
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ExecutionState.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Watermarks.java
delete mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/ProcessWorkItemClient.java
rename runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/{WorkItemProcessor.java => WorkItemScheduler.java} (61%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/ComputationWorkExecutorFactory.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/StreamingCommitFinalizer.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/StreamingWorkScheduler.java