You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2024/03/16 03:49:28 UTC
(beam) branch nightly-refs/heads/master updated (49c7864f0ad -> 3aa78d21f70)

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git


    from 49c7864f0ad add flag for direct path that reads from system properties (#30588)
     add c298da550f8 Refactor commit logic out of StreamingDataflowWorker (#30312)
     add 3aa78d21f70 Better error message for large elements. (#30639)

No new revisions were added by this update.

Summary of changes:
 .../dataflow/worker/StreamingDataflowWorker.java   | 237 +++-------------
 .../dataflow/worker/WindmillComputationKey.java    |   5 +
 .../worker/windmill/client/CloseableStream.java    |  28 +-
 .../worker/windmill/client/WindmillStreamPool.java |   7 +
 .../client/commits}/Commit.java                    |  10 +-
 .../windmill/client/commits/CompleteCommit.java    |  67 +++++
 .../commits/StreamingApplianceWorkCommitter.java   | 167 +++++++++++
 .../commits/StreamingEngineWorkCommitter.java      | 233 ++++++++++++++++
 .../windmill/client/commits/WorkCommitter.java     |  54 ++++
 .../worker/windmill/state/WindmillStateCache.java  |   5 +
 .../dataflow/worker/FakeWindmillServer.java        |  32 ++-
 .../worker/StreamingDataflowWorkerTest.java        |   2 +-
 .../StreamingApplianceWorkCommitterTest.java       | 140 ++++++++++
 .../commits/StreamingEngineWorkCommitterTest.java  | 308 +++++++++++++++++++++
 .../apache_beam/runners/worker/data_plane.py       |   9 +
 15 files changed, 1086 insertions(+), 218 deletions(-)
 copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/estimator/NullSizeEstimator.java => runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/CloseableStream.java (56%)
 rename runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/{streaming => windmill/client/commits}/Commit.java (81%)
 create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/CompleteCommit.java
 create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/StreamingApplianceWorkCommitter.java
 create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/StreamingEngineWorkCommitter.java
 create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/WorkCommitter.java
 create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/StreamingApplianceWorkCommitterTest.java
 create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/commits/StreamingEngineWorkCommitterTest.java