You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2023/12/14 03:54:21 UTC
(beam) branch nightly-refs/heads/master updated (276aa023903 -> def026bfe11)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 276aa023903 Add suffix for vocab files in tft transforms (#29720)
add 8be85d29f53 fix: optimize segment reader (#29694)
add b2073c6c986 Fix typo in notebook (#29728)
add c8deda71743 Upgrade go version to resolve vulnerabilities (#29743)
add 5d42cb654c0 revert kafka host change (#29725)
add 86720287c21 Deep-copy the process bundle descriptor when creating a new bundle processor. (#29742)
add 06c3340e338 Add logs to confirm JvmInitializer completed running and Spanner successfully connected. (#29727)
add 8f90fdd62d3 Upgrade Debian Bullseye to Bookworm (#29721)
add 951b3b1a81b Per DoFn latency instrumentation (#29592)
add fa47640888c Remove comment body from action (#29750)
add ad2d8dc6e30 Update Bigtable python client version (#29753)
add e37fa77c3b6 Pass getPort method through JDBC Resource Managers (#29732)
add 8d41b62e828 Moving to 2.54.0-SNAPSHOT on master branch.
add c49e9ec7dc7 test revert _beam_schema_proto (#29754)
add db0bb446aa0 Add outputWindowedValue capability to Java SDK
add ef0ee769e8f Merge pull request #29616: Add outputWindowedValue capability to Java SDK
add 19858e990ad [#29760] Only respond to sampling request while data sampling is enabled (#29761)
add a04a4ad4cd2 [yaml] Normalize drop on MapToFields
add c124e473fa4 Merge pull request #29755 from Polber/jkinard/mapping-drop
add a9f5ab14d06 Add support to use side inputs with Combine.PerKeyWithHotKeyFanout (#28867)
add def026bfe11 Bump google.golang.org/api from 0.153.0 to 0.154.0 in /sdks (#29734)
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 +
.github/actions/setup-action/action.yml | 6 +-
CHANGES.md | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
.../org/apache/beam/examples/KafkaStreaming.java | 2 +-
.../notebooks/beam-ml/run_custom_inference.ipynb | 2 +-
gradle.properties | 4 +-
.../beam/it/jdbc/AbstractJDBCResourceManager.java | 6 +-
release/go-licenses/Dockerfile | 2 +-
release/src/main/Dockerfile | 2 +-
.../runners/core/construction/SplittableParDo.java | 10 +
.../construction/SplittableParDoNaiveBounded.java | 38 +++
...TimeBoundedSplittableProcessElementInvoker.java | 25 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 79 +++++-
.../core/metrics/ExecutionStateSampler.java | 10 +-
.../core/metrics/ExecutionStateTracker.java | 9 +-
.../dataflow/worker/ActiveMessageMetadata.java | 14 +-
.../dataflow/worker/DataflowExecutionContext.java | 60 ++++-
.../worker/DataflowExecutionStateSampler.java | 135 ++++++++++
.../dataflow/worker/StreamingDataflowWorker.java | 25 +-
.../dataflow/worker/streaming/ActiveWorkState.java | 13 +-
.../worker/streaming/ComputationState.java | 10 +-
.../runners/dataflow/worker/streaming/Work.java | 74 +++++-
.../worker/DataflowExecutionContextTest.java | 76 ++++++
.../worker/DataflowExecutionStateSamplerTest.java | 148 +++++++++++
.../dataflow/worker/FakeWindmillServer.java | 10 +-
.../worker/StreamingDataflowWorkerTest.java | 89 ++++++-
.../worker/streaming/ActiveWorkStateTest.java | 4 +-
.../worker/windmill/src/main/proto/windmill.proto | 16 ++
sdks/go.mod | 17 +-
sdks/go.sum | 37 ++-
sdks/go/container/Dockerfile | 2 +-
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/core/runtime/harness/harness.go | 22 +-
.../org/apache/beam/sdk/transforms/Combine.java | 24 +-
.../org/apache/beam/sdk/transforms/Create.java | 191 +++++++++++++
.../java/org/apache/beam/sdk/transforms/DoFn.java | 58 ++++
.../beam/sdk/transforms/DoFnOutputReceivers.java | 27 ++
.../org/apache/beam/sdk/transforms/DoFnTester.java | 22 ++
.../sdk/transforms/windowing/GlobalWindow.java | 5 +
.../apache/beam/sdk/transforms/CombineTest.java | 24 ++
.../org/apache/beam/sdk/transforms/CreateTest.java | 81 ++++++
.../sdk/transforms/reflect/DoFnInvokersTest.java | 22 ++
.../extensions/sql/zetasql/BeamZetaSqlCalcRel.java | 12 +
.../org/apache/beam/sdk/fn/JvmInitializers.java | 2 +
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 295 ++++++++++++++++++++-
.../bigquery/StorageApiWriteUnshardedRecords.java | 21 ++
.../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 9 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 1 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 12 +
.../io/gcp/bigtable/BigtableServiceImplTest.java | 63 +++++
.../beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java | 11 +
.../beam/sdk/io/pulsar/ReadFromPulsarDoFnTest.java | 13 +
.../ReadFromSparkReceiverWithOffsetDoFnTest.java | 13 +
.../apache_beam/runners/worker/sdk_worker.py | 11 +-
sdks/python/apache_beam/typehints/schemas.py | 1 -
sdks/python/apache_beam/version.py | 2 +-
sdks/python/apache_beam/yaml/yaml_mapping.py | 13 +-
sdks/python/apache_beam/yaml/yaml_transform.py | 2 +
sdks/python/container/Dockerfile | 2 +-
.../container/py310/base_image_requirements.txt | 2 +-
.../container/py311/base_image_requirements.txt | 2 +-
.../container/py38/base_image_requirements.txt | 2 +-
.../container/py39/base_image_requirements.txt | 2 +-
sdks/python/expansion-service-container/Dockerfile | 2 +-
sdks/python/test-suites/direct/common.gradle | 2 +-
sdks/typescript/package.json | 2 +-
.../en/documentation/runtime/environments.md | 10 +-
68 files changed, 1790 insertions(+), 125 deletions(-)
copy sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcWriteResult.java => runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ActiveMessageMetadata.java (72%)
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateSampler.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateSamplerTest.java