You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2021/10/16 04:07:19 UTC
[beam] tag nightly-master updated (0542886 -> 0cddc44)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 0542886 (commit)
to 0cddc44 (commit)
from 0542886 [BEAM-12907] Run DataFrame API tests with multiple pandas versions (#15587)
add 2e4ddad Remove unnecessary ERROR logging.
add 08b8d5f Merge pull request #15730 from vachan-shetty/fix-logging
add fdf0636 Minor: Add more links to DataFrame API documentation (#15661)
add 9e22bb1 Corrected Join Example
add 52a178f Merge pull request #15666: Corrected Join Example
add 83bd5d7 [BEAM-11480] Use snippets for DataFrame examples (#15600)
add 802e174 Allow multiple Python worker processe to share the same VM. (#15642)
add 72b861d [BEAM-12564] Implement Series.hasnans (#15729)
add 672a08b Minor: Fix frames_test.py equality check for non-frame outputs (#15734)
add 024d96c [BEAM-12769] Adds integration tests for Java Class Lookup based cross-language expansion (#15690)
add 0a26344 Address BEAM-4028 cleanup TODOs
add c93cf97 Bump Dataflow container to beam-master-20211015
add 25770e3 [BEAM-4028] Address cleanup TODOs (#15652)
add 0cddc44 [BEAM-13052] Restructure pubsublite folder to move non-user interface classes to internal folder (#15724)
No new revisions were added by this update.
Summary of changes:
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 3 +
.../src/main/proto/beam_provision_api.proto | 5 +
.../pipeline/src/main/proto/beam_runner_api.proto | 5 +
.../model/fnexecution_v1/beam_provision_api.pb.go | 65 +++++----
sdks/go/pkg/beam/util/execx/exec.go | 12 ++
.../sdk/io/gcp/pubsublite/PublisherOptions.java | 28 +---
.../beam/sdk/io/gcp/pubsublite/PubsubLiteIO.java | 4 +
.../pubsublite/SerializableSubscriberFactory.java | 33 -----
.../sdk/io/gcp/pubsublite/SubscriberOptions.java | 145 +--------------------
.../gcp/pubsublite/UuidDeduplicationOptions.java | 5 +-
.../{ => internal}/AddUuidsTransform.java | 5 +-
.../gcp/pubsublite/{ => internal}/DlqProvider.java | 4 +-
.../{ => internal}/InitialOffsetReader.java | 2 +-
.../{ => internal}/InitialOffsetReaderImpl.java | 2 +-
.../{ => internal}/LimitingTopicBacklogReader.java | 2 +-
.../ManagedBacklogReaderFactory.java | 2 +-
.../ManagedBacklogReaderFactoryImpl.java | 2 +-
.../{ => internal}/OffsetByteProgress.java | 2 +-
.../pubsublite/{ => internal}/OffsetByteRange.java | 2 +-
.../{ => internal}/OffsetByteRangeCoder.java | 2 +-
.../{ => internal}/OffsetByteRangeTracker.java | 2 +-
.../{ => internal}/PerServerPublisherCache.java | 2 +-
.../PerSubscriptionPartitionSdf.java | 2 +-
.../PublisherAssembler.java} | 31 ++---
.../pubsublite/{ => internal}/PublisherCache.java | 8 +-
.../{ => internal}/PublisherOrError.java | 2 +-
.../pubsublite/{ => internal}/PubsubLiteSink.java | 18 +--
.../{ => internal}/SubscribeTransform.java | 18 +--
.../pubsublite/internal/SubscriberAssembler.java | 121 +++++++++++++++++
.../{ => internal}/SubscriptionPartition.java | 2 +-
.../{ => internal}/SubscriptionPartitionCoder.java | 2 +-
.../SubscriptionPartitionLoader.java | 2 +-
.../SubscriptionPartitionProcessor.java | 2 +-
.../SubscriptionPartitionProcessorFactory.java | 2 +-
.../SubscriptionPartitionProcessorImpl.java | 2 +-
.../{ => internal}/TopicBacklogReader.java | 2 +-
.../{ => internal}/TopicBacklogReaderImpl.java | 2 +-
.../{ => internal}/TopicBacklogReaderSettings.java | 2 +-
.../{ => internal}/TrackerWithProgress.java | 2 +-
.../sdk/io/gcp/pubsublite/{ => internal}/Uuid.java | 2 +-
.../gcp/pubsublite/{ => internal}/UuidCoder.java | 2 +-
.../{ => internal}/UuidDeduplicationTransform.java | 7 +-
.../package-info.java} | 13 +-
.../{ => internal}/AddUuidsTransformTest.java | 2 +-
.../{ => internal}/FakeSerializable.java | 3 +-
.../{ => internal}/OffsetByteRangeTrackerTest.java | 2 +-
.../PerSubscriptionPartitionSdfTest.java | 2 +-
.../{ => internal}/PubsubLiteSinkTest.java | 3 +-
.../SubscriptionPartitionLoaderTest.java | 2 +-
.../SubscriptionPartitionProcessorImplTest.java | 2 +-
.../{ => internal}/TopicBacklogReaderImplTest.java | 2 +-
.../UuidDeduplicationTransformTest.java | 3 +-
.../test_expansion_service_allowlist.yaml | 26 ++++
sdks/python/apache_beam/dataframe/frames.py | 17 +++
sdks/python/apache_beam/dataframe/frames_test.py | 11 +-
.../apache_beam/examples/dataframe/taxiride.py | 4 +
.../apache_beam/examples/dataframe/wordcount.py | 4 +
.../io/external/generate_sequence_test.py | 46 ++++++-
sdks/python/apache_beam/io/gcp/bigquery.py | 5 +-
sdks/python/apache_beam/runners/common.py | 35 -----
.../runners/dataflow/dataflow_runner_test.py | 4 +-
.../apache_beam/runners/dataflow/internal/names.py | 4 +-
.../apache_beam/runners/worker/bundle_processor.py | 4 +-
.../apache_beam/runners/worker/operation_specs.py | 71 ----------
.../apache_beam/runners/worker/operations.py | 17 +--
sdks/python/apache_beam/transforms/environments.py | 6 +-
.../apache_beam/transforms/environments_test.py | 3 +-
sdks/python/container/boot.go | 31 +++--
sdks/python/scripts/run_expansion_services.sh | 7 +-
.../en/documentation/dsls/dataframes/overview.md | 50 +++----
.../en/documentation/dsls/sql/walkthrough.md | 5 +-
.../layouts/partials/section-menu/en/sdks.html | 5 +-
72 files changed, 457 insertions(+), 497 deletions(-)
delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SerializableSubscriberFactory.java
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/AddUuidsTransform.java (92%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/DlqProvider.java (94%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/InitialOffsetReader.java (94%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/InitialOffsetReaderImpl.java (97%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/LimitingTopicBacklogReader.java (98%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/ManagedBacklogReaderFactory.java (95%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/ManagedBacklogReaderFactoryImpl.java (97%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/OffsetByteProgress.java (95%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/OffsetByteRange.java (95%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/OffsetByteRangeCoder.java (97%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/OffsetByteRangeTracker.java (99%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/PerServerPublisherCache.java (95%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/PerSubscriptionPartitionSdf.java (98%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{Publishers.java => internal/PublisherAssembler.java} (77%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/PublisherCache.java (91%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/PublisherOrError.java (96%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/PubsubLiteSink.java (91%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/SubscribeTransform.java (88%)
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/SubscriberAssembler.java
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/SubscriptionPartition.java (96%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/SubscriptionPartitionCoder.java (97%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/SubscriptionPartitionLoader.java (98%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/SubscriptionPartitionProcessor.java (95%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/SubscriptionPartitionProcessorFactory.java (96%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/SubscriptionPartitionProcessorImpl.java (99%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/TopicBacklogReader.java (96%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/TopicBacklogReaderImpl.java (97%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/TopicBacklogReaderSettings.java (98%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/TrackerWithProgress.java (95%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/Uuid.java (97%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/UuidCoder.java (97%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/UuidDeduplicationTransform.java (89%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{SerializableSupplier.java => internal/package-info.java} (72%)
rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/AddUuidsTransformTest.java (98%)
rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/FakeSerializable.java (93%)
rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/OffsetByteRangeTrackerTest.java (99%)
rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/PerSubscriptionPartitionSdfTest.java (99%)
rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/PubsubLiteSinkTest.java (98%)
rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/SubscriptionPartitionLoaderTest.java (98%)
rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/SubscriptionPartitionProcessorImplTest.java (99%)
rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/TopicBacklogReaderImplTest.java (98%)
rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/{ => internal}/UuidDeduplicationTransformTest.java (97%)
create mode 100644 sdks/java/testing/expansion-service/src/test/resources/test_expansion_service_allowlist.yaml