You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2023/12/12 03:54:53 UTC
(beam) branch nightly-refs/heads/master updated (a96fa74f0cc -> bdaec7a13ed)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from a96fa74f0cc Pass Java SchemaTransform descriptions to Python SDK (#29606)
add 1061fcf0969 Bump github.com/nats-io/nats-server/v2 from 2.10.6 to 2.10.7 in /sdks (#29698)
add 75ba21ff5c3 [runners-flink] #29558 emit watermark before source shutdown
add a3d011549d7 Merge pull request #29563: [runners-flink] #29558 emit watermark before source shutdown
add 5c7ec723a44 Bump github.com/aws/aws-sdk-go-v2/config from 1.26.0 to 1.26.1 in /sdks (#29702)
add 218af9de868 Add per test timeout in recently changed dataflow legacy worker tests (#29696)
add aacf1ee5910 Support Embeddings in mltransform (#29564)
add b91a5174d14 [Python] Log dependencies in runtime environment (#29705)
add f93423009fd Save Job IDs of dataflow load tests (#29693)
add 75bce947ceb Add a runner capability for multimap keys values side input reading.
add ccd93a95a04 Plumb runner capabilities to Python workers.
add 5c879cc5c04 Toggle use of bulk multimap state read on runner capabilities.
add bdaec7a13ed Merge pull request #29695 Enable keys values multimap protocol based on runner capabilities.
No new revisions were added by this update.
Summary of changes:
CHANGES.md | 1 +
.../beam/model/pipeline/v1/beam_runner_api.proto | 5 +
.../streaming/io/source/FlinkSourceReaderBase.java | 80 +---
.../io/source/FlinkSourceSplitEnumerator.java | 1 -
.../source/bounded/FlinkBoundedSourceReader.java | 92 ++++-
.../unbounded/FlinkUnboundedSourceReader.java | 51 +++
.../io/source/FlinkSourceReaderTestBase.java | 8 +-
.../bounded/FlinkBoundedSourceReaderTest.java | 18 +-
.../dataflow/worker/BatchDataflowWorkerTest.java | 2 +
.../worker/DataflowWorkUnitClientTest.java | 3 +
.../worker/StreamingDataflowWorkerTest.java | 1 -
.../worker/StreamingModeExecutionContextTest.java | 4 +-
.../worker/StreamingSideInputDoFnRunnerTest.java | 4 +-
.../worker/StreamingSideInputFetcherTest.java | 4 +-
.../worker/StreamingStepMetricsContainerTest.java | 4 +-
.../DataflowWorkerLoggingInitializerTest.java | 2 +
.../worker/streaming/ActiveWorkStateTest.java | 4 +-
.../worker/streaming/WeightBoundedQueueTest.java | 3 +
.../sideinput/SideInputStateFetcherTest.java | 3 +
.../windmill/client/WindmillStreamPoolTest.java | 3 +
.../grpc/GrpcGetWorkerMetadataStreamTest.java | 2 +
.../client/grpc/GrpcWindmillServerTest.java | 3 +-
.../client/grpc/StreamingEngineClientTest.java | 2 +
.../client/grpc/WindmillStreamSenderTest.java | 2 +
.../windmill/state/WindmillStateCacheTest.java | 4 +-
.../windmill/state/WindmillStateInternalsTest.java | 4 +-
.../windmill/state/WindmillStateReaderTest.java | 3 +
.../budget/EvenGetWorkBudgetDistributorTest.java | 2 +
.../work/budget/GetWorkBudgetRefresherTest.java | 3 +
.../windmill/work/budget/GetWorkBudgetTest.java | 3 +
sdks/go.mod | 16 +-
sdks/go.sum | 32 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 4 +
.../beam/fn/harness/state/FnApiStateAccessor.java | 12 +-
.../beam/fn/harness/state/MultimapSideInput.java | 11 -
sdks/python/apache_beam/ml/transforms/base.py | 446 +++++++++++++++++++--
sdks/python/apache_beam/ml/transforms/base_test.py | 339 +++++++++++++++-
.../transforms/embeddings}/__init__.py | 6 +-
.../ml/transforms/embeddings/huggingface.py | 131 ++++++
.../ml/transforms/embeddings/huggingface_test.py | 278 +++++++++++++
.../ml/transforms/embeddings/vertex_ai.py | 154 +++++++
.../ml/transforms/embeddings/vertex_ai_test.py | 249 ++++++++++++
sdks/python/apache_beam/ml/transforms/handlers.py | 6 +-
.../apache_beam/ml/transforms/handlers_test.py | 12 +-
sdks/python/apache_beam/ml/transforms/tft.py | 15 +
sdks/python/apache_beam/ml/transforms/tft_test.py | 7 +-
sdks/python/apache_beam/ml/transforms/utils.py | 16 +-
sdks/python/apache_beam/portability/common_urns.py | 2 +
.../portability/fn_api_runner/worker_handlers.py | 9 +-
.../apache_beam/runners/worker/bundle_processor.py | 17 +-
.../runners/worker/bundle_processor_test.py | 6 +-
.../apache_beam/runners/worker/log_handler_test.py | 2 +-
.../apache_beam/runners/worker/sdk_worker.py | 6 +
.../apache_beam/runners/worker/sdk_worker_main.py | 10 +-
.../apache_beam/runners/worker/sdk_worker_test.py | 8 +-
.../testing/load_tests/load_test_metrics_utils.py | 28 ++
sdks/python/apache_beam/typehints/typehints.py | 8 +-
sdks/python/container/boot.go | 28 ++
.../container/py310/base_image_requirements.txt | 1 +
.../container/py311/base_image_requirements.txt | 1 +
.../container/py38/base_image_requirements.txt | 1 +
.../container/py39/base_image_requirements.txt | 1 +
sdks/python/gen_protos.py | 1 -
sdks/python/scripts/generate_pydoc.sh | 4 +-
sdks/python/setup.py | 1 +
sdks/python/test-suites/tox/py38/build.gradle | 4 +
sdks/python/tox.ini | 12 +
67 files changed, 1978 insertions(+), 227 deletions(-)
copy sdks/python/apache_beam/{testing/analyzers => ml/transforms/embeddings}/__init__.py (83%)
create mode 100644 sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py
create mode 100644 sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py
create mode 100644 sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai.py
create mode 100644 sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py