You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2021/04/01 08:27:21 UTC
[beam] tag nightly-master updated (fd3075f -> 194fe17)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from fd3075f (commit)
to 194fe17 (commit)
from fd3075f Merge pull request #13995 from [BEAM-11322] Apache Beam Example to tokenize sensitive data
add 6eb3c24 Update dependencies
add eb8ceeb Update dependencies
add 460e093 Merge pull request #14379: Update python dependencies
add 3d8798e Moving from vendored gRPC 1.26 to 1.36
add 76c49ac Merge pull request #14295: [BEAM-11227] Moving from vendored gRPC 1.26 to 1.36
add ab509b3 [BEAM-9185] Publish pre-release python artifacts (RCs) to PyPI (#14325)
add 93a6927 Update httplib2 to version 1.19.1 on Python container
add 288d379 Merge pull request #14386 from iemejia/update-python-deps
add 50031c6 Update website build dependencies
add ec9e413 Merge pull request #14387: Update website build dependencies
add dcd8778 Merge pull request #14113 from [BEAM-11277] Respect schemaUpdateOptions during BigQuery load with temporary tables
add 1499d16 Reenable type checking for iobase (#14281)
add 12a2d18 Better dataframe support for beam notebooks.
add a582fd6 Merge pull request #14356 Better dataframe support for beam notebooks.
add e1571c3 Documenting temp_dataset parameter in ReadFromBigQuery
add 196d55f Merge pull request #14376 from apache/pabloem-patch-2
add 194fe17 [BEAM-12062] Python-Katas Doc Fixes and Lesson Enhancements (#14353)
No new revisions were added by this update.
Summary of changes:
.github/workflows/build_wheels.yml | 93 ++++++
.../jenkins/metrics_report/requirements.txt | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 10 +-
.../apache/beam/gradle/GrpcVendoring_1_26_0.groovy | 222 -------------
examples/java/build.gradle | 2 +-
.../transforms/DataProtectors.java | 6 +-
.../kafkatopubsub/kafka/consumer/Utils.java | 4 +-
.../Common Transforms/Aggregation/Largest/task.md | 2 +-
.../Common Transforms/Aggregation/Largest/task.py | 2 +-
.../Common Transforms/Aggregation/Largest/tests.py | 4 +-
.../Common Transforms/WithKeys/WithKeys/task.md | 2 +-
.../Windowing/Adding Timestamp/ParDo/task.md | 2 +-
release/src/main/scripts/choose_rc_commit.sh | 0
.../main/scripts/deploy_release_candidate_pypi.sh | 172 ++++++++++
.../scripts/download_github_actions_artifacts.py | 30 +-
runners/core-construction-java/build.gradle | 2 +-
.../beam/runners/core/construction/BeamUrns.java | 2 +-
.../core/construction/CoderTranslation.java | 2 +-
.../core/construction/CoderTranslators.java | 2 +-
.../core/construction/CombineTranslation.java | 2 +-
.../CreatePCollectionViewTranslation.java | 2 +-
.../DefaultExpansionServiceClientFactory.java | 2 +-
.../core/construction/DisplayDataTranslation.java | 2 +-
.../runners/core/construction/Environments.java | 4 +-
.../beam/runners/core/construction/External.java | 6 +-
.../runners/core/construction/ModelCoders.java | 2 +-
.../construction/PCollectionViewTranslation.java | 2 +-
.../core/construction/ParDoTranslation.java | 4 +-
.../construction/PipelineOptionsTranslation.java | 6 +-
.../runners/core/construction/ReadTranslation.java | 4 +-
.../core/construction/TestStreamTranslation.java | 2 +-
.../core/construction/WindowIntoTranslation.java | 2 +-
.../construction/WindowingStrategyTranslation.java | 8 +-
.../core/construction/WriteFilesTranslation.java | 2 +-
.../graph/GreedyPCollectionFusers.java | 2 +-
.../core/construction/graph/QueryablePipeline.java | 2 +-
.../runners/core/construction/CommonCoderTest.java | 2 +-
.../PipelineOptionsTranslationTest.java | 6 +-
.../core/construction/ValidateRunnerXlangTest.java | 8 +-
.../construction/WindowIntoTranslationTest.java | 2 +-
.../construction/graph/ProtoOverridesTest.java | 2 +-
runners/core-java/build.gradle | 2 +-
.../core/metrics/MetricsContainerStepMap.java | 4 +-
.../core/metrics/MonitoringInfoEncodings.java | 2 +-
.../core/metrics/MonitoringInfoEncodingsTest.java | 2 +-
runners/direct-java/build.gradle | 4 +-
runners/flink/flink_runner.gradle | 2 +-
.../FlinkBatchPortablePipelineTranslator.java | 2 +-
.../apache/beam/runners/flink/FlinkJobInvoker.java | 2 +-
.../beam/runners/flink/FlinkPipelineRunner.java | 2 +-
.../FlinkStreamingPortablePipelineTranslator.java | 2 +-
.../utils/FlinkPortableRunnerUtils.java | 2 +-
.../streaming/ExecutableStageDoFnOperator.java | 4 +-
.../wrappers/streaming/FlinkKeyUtils.java | 2 +-
.../FlinkExecutableStageFunctionTest.java | 2 +-
.../streaming/ExecutableStageDoFnOperatorTest.java | 4 +-
.../wrappers/streaming/FlinkKeyUtilsTest.java | 2 +-
runners/google-cloud-dataflow-java/build.gradle | 2 +-
.../dataflow/DataflowPipelineTranslator.java | 2 +-
.../beam/runners/dataflow/DataflowRunner.java | 4 +-
.../beam/runners/dataflow/DataflowRunnerTest.java | 2 +-
.../google-cloud-dataflow-java/worker/build.gradle | 2 +-
.../worker/legacy-worker/build.gradle | 4 +-
.../runners/dataflow/worker/ByteStringCoder.java | 2 +-
.../dataflow/worker/DataflowRunnerHarness.java | 2 +-
.../worker/DataflowWorkerHarnessHelper.java | 2 +-
.../worker/GroupAlsoByWindowParDoFnFactory.java | 2 +-
.../beam/runners/dataflow/worker/PubsubSink.java | 2 +-
.../beam/runners/dataflow/worker/StateFetcher.java | 2 +-
.../dataflow/worker/StreamingDataflowWorker.java | 4 +-
.../worker/StreamingModeExecutionContext.java | 2 +-
.../dataflow/worker/StreamingSideInputFetcher.java | 4 +-
.../dataflow/worker/WindmillComputationKey.java | 4 +-
.../dataflow/worker/WindmillNamespacePrefix.java | 2 +-
.../beam/runners/dataflow/worker/WindmillSink.java | 2 +-
.../dataflow/worker/WindmillStateCache.java | 2 +-
.../dataflow/worker/WindmillStateInternals.java | 2 +-
.../dataflow/worker/WindmillStateReader.java | 2 +-
.../dataflow/worker/WindmillTimerInternals.java | 2 +-
.../dataflow/worker/WorkerCustomSources.java | 2 +-
.../dataflow/worker/fn/BeamFnControlService.java | 6 +-
.../control/RegisterAndProcessBundleOperation.java | 4 +-
.../worker/fn/data/BeamFnDataGrpcService.java | 4 +-
.../worker/fn/logging/BeamFnLoggingService.java | 4 +-
.../fn/stream/ServerStreamObserverFactory.java | 6 +-
.../graph/CreateExecutableStageNodeFunction.java | 4 +-
...nsertFetchAndFilterStreamingSideInputNodes.java | 2 +-
.../worker/graph/RegisterNodeFunction.java | 4 +-
.../worker/windmill/DirectStreamObserver.java | 4 +-
.../windmill/ForwardingClientResponseObserver.java | 6 +-
.../worker/windmill/GrpcWindmillServer.java | 30 +-
.../worker/windmill/StreamObserverFactory.java | 4 +-
.../runners/dataflow/harness/test/TestStreams.java | 4 +-
.../runners/dataflow/worker/DataflowMatchers.java | 2 +-
.../worker/DataflowWorkerHarnessHelperTest.java | 2 +-
.../runners/dataflow/worker/PubsubReaderTest.java | 2 +-
.../runners/dataflow/worker/PubsubSinkTest.java | 2 +-
.../runners/dataflow/worker/ReaderCacheTest.java | 2 +-
.../runners/dataflow/worker/StateFetcherTest.java | 2 +-
.../worker/StreamingDataflowWorkerTest.java | 6 +-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 2 +-
...reamingGroupAlsoByWindowsReshuffleDoFnTest.java | 2 +-
.../worker/StreamingModeExecutionContextTest.java | 2 +-
.../worker/StreamingSideInputDoFnRunnerTest.java | 2 +-
.../worker/StreamingSideInputFetcherTest.java | 2 +-
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 2 +-
.../worker/WindmillReaderIteratorBaseTest.java | 2 +-
.../dataflow/worker/WindmillStateCacheTest.java | 2 +-
.../worker/WindmillStateInternalsTest.java | 2 +-
.../dataflow/worker/WindmillStateReaderTest.java | 4 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 2 +-
.../worker/fn/BeamFnControlServiceTest.java | 6 +-
.../RegisterAndProcessBundleOperationTest.java | 2 +-
.../worker/fn/data/BeamFnDataGrpcServiceTest.java | 32 +-
.../fn/logging/BeamFnLoggingServiceTest.java | 12 +-
.../fn/stream/ServerStreamObserverFactoryTest.java | 4 +-
...tFetchAndFilterStreamingSideInputNodesTest.java | 2 +-
.../logging/DataflowWorkerLoggingHandlerTest.java | 2 +-
.../worker/windmill/GrpcWindmillServerTest.java | 14 +-
runners/java-fn-execution/build.gradle | 2 +-
.../apache/beam/runners/fnexecution/FnService.java | 6 +-
.../GrpcContextHeaderAccessorProvider.java | 16 +-
.../beam/runners/fnexecution/GrpcFnServer.java | 2 +-
.../fnexecution/InProcessServerFactory.java | 8 +-
.../beam/runners/fnexecution/ServerFactory.java | 20 +-
.../artifact/ArtifactRetrievalService.java | 8 +-
.../artifact/ArtifactStagingService.java | 10 +-
.../control/BundleCheckpointHandlers.java | 2 +-
.../fnexecution/control/FnApiControlClient.java | 6 +-
.../control/FnApiControlClientPoolService.java | 6 +-
.../control/ProcessBundleDescriptors.java | 2 +-
.../runners/fnexecution/data/GrpcDataService.java | 2 +-
.../environment/ExternalEnvironmentFactory.java | 2 +-
.../fnexecution/logging/GrpcLoggingService.java | 2 +-
.../runners/fnexecution/provisioning/JobInfo.java | 2 +-
.../provisioning/StaticGrpcProvisionService.java | 2 +-
.../fnexecution/state/GrpcStateService.java | 4 +-
.../fnexecution/state/StateRequestHandlers.java | 2 +-
.../status/BeamWorkerStatusGrpcService.java | 2 +-
.../fnexecution/status/WorkerStatusClient.java | 2 +-
.../translation/PipelineTranslatorUtils.java | 2 +-
.../runners/fnexecution/wire/ByteStringCoder.java | 2 +-
.../GrpcContextHeaderAccessorProviderTest.java | 20 +-
.../runners/fnexecution/ServerFactoryTest.java | 12 +-
.../artifact/ArtifactRetrievalServiceTest.java | 10 +-
.../artifact/ArtifactStagingServiceTest.java | 12 +-
.../control/DefaultJobBundleFactoryTest.java | 4 +-
.../control/FnApiControlClientPoolServiceTest.java | 8 +-
.../control/FnApiControlClientTest.java | 2 +-
.../fnexecution/control/RemoteExecutionTest.java | 2 +-
.../fnexecution/control/SdkHarnessClientTest.java | 2 +-
.../fnexecution/data/GrpcDataServiceTest.java | 8 +-
.../logging/GrpcLoggingServiceTest.java | 6 +-
.../StaticGrpcProvisionServiceTest.java | 10 +-
.../fnexecution/state/GrpcStateServiceTest.java | 4 +-
.../state/StateRequestHandlersTest.java | 2 +-
.../status/BeamWorkerStatusGrpcServiceTest.java | 8 +-
.../fnexecution/status/WorkerStatusClientTest.java | 2 +-
.../fnexecution/wire/ByteStringCoderTest.java | 2 +-
runners/java-job-service/build.gradle | 2 +-
.../runners/jobsubmission/InMemoryJobService.java | 10 +-
.../beam/runners/jobsubmission/JobInvocation.java | 2 +-
.../beam/runners/jobsubmission/JobInvoker.java | 2 +-
.../beam/runners/jobsubmission/JobPreparation.java | 2 +-
.../jobsubmission/PortablePipelineJarCreator.java | 4 +-
.../jobsubmission/PortablePipelineJarUtils.java | 6 +-
.../jobsubmission/InMemoryJobServiceTest.java | 6 +-
.../runners/jobsubmission/JobInvocationTest.java | 2 +-
.../PortablePipelineJarCreatorTest.java | 2 +-
runners/portability/java/build.gradle | 2 +-
.../runners/portability/ExternalWorkerService.java | 2 +-
.../portability/JobServicePipelineResult.java | 2 +-
.../beam/runners/portability/PortableRunner.java | 4 +-
.../portability/testing/TestJobService.java | 2 +-
.../runners/portability/PortableRunnerTest.java | 8 +-
runners/samza/build.gradle | 2 +-
.../beam/runners/samza/SamzaJobServerDriver.java | 2 +-
.../samza/translation/WindowAssignTranslator.java | 2 +-
.../samza/util/SamzaPipelineTranslatorUtils.java | 2 +-
runners/spark/spark_runner.gradle | 2 +-
.../apache/beam/runners/spark/SparkJobInvoker.java | 2 +-
.../beam/runners/spark/SparkPipelineRunner.java | 2 +-
runners/twister2/build.gradle | 2 +-
.../functions/AssignWindowsFunction.java | 2 +-
.../translators/functions/DoFnFunction.java | 2 +-
.../functions/GroupByWindowFunction.java | 2 +-
sdks/java/core/build.gradle | 2 +-
.../java/org/apache/beam/sdk/io/TextSource.java | 2 +-
.../java/org/apache/beam/sdk/io/range/ByteKey.java | 4 +-
.../apache/beam/sdk/schemas/SchemaTranslation.java | 2 +-
sdks/java/expansion-service/build.gradle | 2 +-
.../sdk/expansion/service/ExpansionServer.java | 4 +-
.../sdk/expansion/service/ExpansionService.java | 6 +-
.../expansion/service/ExpansionServiceTest.java | 2 +-
.../beam/sdk/expansion/service/ExternalTest.java | 4 +-
.../schemaio-expansion-service/build.gradle | 2 +-
.../ExternalSchemaIOTransformRegistrar.java | 2 +-
sdks/java/fn-execution/build.gradle | 2 +-
.../beam/sdk/fn/channel/ManagedChannelFactory.java | 18 +-
.../beam/sdk/fn/channel/SocketAddressFactory.java | 2 +-
.../data/BeamFnDataBufferingOutboundObserver.java | 2 +-
.../sdk/fn/data/BeamFnDataGrpcMultiplexer.java | 6 +-
.../sdk/fn/data/BeamFnDataInboundObserver.java | 2 +-
...amFnDataSizeBasedBufferingOutboundObserver.java | 4 +-
...amFnDataTimeBasedBufferingOutboundObserver.java | 2 +-
.../beam/sdk/fn/data/DecodingFnDataReceiver.java | 2 +-
.../beam/sdk/fn/data/RemoteGrpcPortRead.java | 2 +-
.../beam/sdk/fn/data/RemoteGrpcPortWrite.java | 2 +-
.../sdk/fn/stream/BufferingStreamObserver.java | 4 +-
.../org/apache/beam/sdk/fn/stream/DataStreams.java | 2 +-
.../beam/sdk/fn/stream/DirectStreamObserver.java | 4 +-
.../stream/ForwardingClientResponseObserver.java | 6 +-
.../sdk/fn/stream/OutboundObserverFactory.java | 4 +-
.../sdk/fn/stream/SynchronizedStreamObserver.java | 2 +-
.../fn/test/InProcessManagedChannelFactory.java | 4 +-
.../org/apache/beam/sdk/fn/test/TestStreams.java | 4 +-
.../sdk/fn/windowing/EncodedBoundedWindow.java | 2 +-
.../sdk/fn/channel/ManagedChannelFactoryTest.java | 6 +-
.../sdk/fn/channel/SocketAddressFactoryTest.java | 2 +-
.../sdk/fn/data/BeamFnDataGrpcMultiplexerTest.java | 2 +-
...DataSizeBasedBufferingOutboundObserverTest.java | 2 +-
...DataTimeBasedBufferingOutboundObserverTest.java | 2 +-
.../beam/sdk/fn/data/RemoteGrpcPortReadTest.java | 2 +-
.../beam/sdk/fn/data/RemoteGrpcPortWriteTest.java | 2 +-
.../apache/beam/sdk/fn/stream/DataStreamsTest.java | 2 +-
.../ForwardingClientResponseObserverTest.java | 6 +-
.../sdk/fn/stream/OutboundObserverFactoryTest.java | 4 +-
.../sdk/fn/windowing/EncodedBoundedWindowTest.java | 2 +-
sdks/java/harness/build.gradle | 2 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 16 +-
.../java/org/apache/beam/fn/harness/FnHarness.java | 4 +-
.../harness/control/AddHarnessIdInterceptor.java | 8 +-
.../fn/harness/control/BeamFnControlClient.java | 4 +-
.../fn/harness/control/ProcessBundleHandler.java | 4 +-
.../beam/fn/harness/data/BeamFnDataClient.java | 2 +-
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 4 +-
.../fn/harness/data/QueueingBeamFnDataClient.java | 2 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 12 +-
.../apache/beam/fn/harness/state/BagUserState.java | 2 +-
.../harness/state/BeamFnStateGrpcClientCache.java | 4 +-
.../beam/fn/harness/state/FnApiStateAccessor.java | 2 +-
.../beam/fn/harness/state/IterableSideInput.java | 2 +-
.../beam/fn/harness/state/MultimapSideInput.java | 2 +-
.../beam/fn/harness/state/StateBackedIterable.java | 2 +-
.../fn/harness/state/StateFetchingIterators.java | 2 +-
.../stream/HarnessStreamObserverFactories.java | 2 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 38 +--
.../org/apache/beam/fn/harness/FnHarnessTest.java | 8 +-
.../harness/control/BeamFnControlClientTest.java | 8 +-
.../harness/control/ProcessBundleHandlerTest.java | 4 +-
.../fn/harness/data/BeamFnDataGrpcClientTest.java | 14 +-
.../data/BeamFnDataInboundObserverTest.java | 2 +-
.../harness/data/QueueingBeamFnDataClientTest.java | 14 +-
.../harness/logging/BeamFnLoggingClientTest.java | 16 +-
.../beam/fn/harness/state/BagUserStateTest.java | 2 +-
.../state/BeamFnStateGrpcClientCacheTest.java | 16 +-
.../fn/harness/state/FakeBeamFnStateClient.java | 2 +-
.../fn/harness/state/MultimapSideInputTest.java | 2 +-
.../fn/harness/state/StateBackedIterableTest.java | 2 +-
.../harness/state/StateFetchingIteratorsTest.java | 2 +-
.../stream/HarnessStreamObserverFactoriesTest.java | 4 +-
sdks/java/io/contextualtextio/build.gradle | 2 +-
.../contextualtextio/ContextualTextIOSource.java | 2 +-
sdks/java/io/debezium/build.gradle | 2 +-
.../apache/beam/io/debezium/SourceRecordJson.java | 4 +-
sdks/java/io/google-cloud-platform/build.gradle | 2 +-
.../io/gcp/spanner/SpannerTransformRegistrar.java | 2 +-
.../sdk/io/gcp/pubsub/PubsubIOExternalTest.java | 4 +-
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 4 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 4 +
.../apache_beam/io/gcp/bigquery_file_loads.py | 180 ++++++++++-
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 54 +++-
.../apache_beam/io/gcp/bigquery_tools_test.py | 29 ++
.../apache_beam/io/gcp/bigquery_write_it_test.py | 45 +++
sdks/python/apache_beam/io/gcp/pubsub.py | 9 +-
sdks/python/apache_beam/io/iobase.py | 89 ++++--
.../runners/interactive/interactive_beam.py | 10 +-
.../runners/interactive/interactive_runner_test.py | 24 ++
.../runners/interactive/recording_manager.py | 8 +
.../apache_beam/runners/interactive/utils.py | 13 +-
sdks/python/container/base_image_requirements.txt | 4 +-
sdks/python/mypy.ini | 3 +
sdks/python/setup.py | 2 +-
settings.gradle.kts | 1 -
vendor/grpc-1_26_0/build.gradle | 37 ---
.../site/content/en/contribute/release-guide.md | 31 +-
website/www/yarn.lock | 355 ++++++++++-----------
287 files changed, 1449 insertions(+), 1056 deletions(-)
delete mode 100644 buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_26_0.groovy
mode change 100644 => 100755 release/src/main/scripts/choose_rc_commit.sh
create mode 100755 release/src/main/scripts/deploy_release_candidate_pypi.sh
delete mode 100644 vendor/grpc-1_26_0/build.gradle