You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/10/09 02:43:31 UTC
[beam] tag nightly-master updated (4602fa4 -> 571338b)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 4602fa4 (commit)
to 571338b (commit)
from 4602fa4 [BEAM-11019] Fixing Go SDF progress reporting with multiple windows. (#13013)
add 75aff81 Update BigQueryAvroUtils.java
add 5b34cbe allowing BOOL and BOOLEAN
add 10f1f58 Merge pull request #12855 from Update BigQueryAvroUtils.java to properly support BOOL type of Big Query
add f8f405b Use more specific version of Python in Dockerfiles (#13047)
add e58d4e1 [BEAM-8017] Plumb errors and remove panics from package graphx (#13028)
add 19ff045 Add support for gracefully aborting workers. On complete_work_status received (by the work progress updater thread), abort the corresponding map task executor thread. This relies on Thread.interrupt() to unblock the worker thread if blocked and raise an exception -- the exception is actually ignored by the backend since the backend already decided to abort this thread. Must also check the current thread for interrupts in the valuesiterator when it consumes a stream of [...]
add 42a4957 .
add 0980283 .
add c14d176 spotlessApply
add c543663 .
add ce47d21 Added async abort signaling up to the GroupingShuffleReader's (values) iterator
add c1b61ac Update abort message and add async abort signaling up to the GroupingShuffleReader's (values) iterator
add cba6498 When aborting the work item, log the status received from the backend
add 1958eae Merge pull request #12994 from tudorm/worker-abort
add cfca7f1 [BEAM-10845] Add Union support to AvroUtils (#12746)
add 81bf0f1 Minor programming guide improvements (#12953)
add 710cbd7 [BEAM-11039] Resolve conflicts between TFMA and Facets imports
add 61fc430 Merge pull request #13045 from [BEAM-11039] Resolve conflicts between TFMA and Facets imports
add ea3b282 Fail task when local job service fails to start
add 166cb08 Create parent directory tree for local job service pid, port, stdout, stderr files
add c37ba52 Merge pull request #13037: [BEAM-11031] Fix flaking in ULR ValidatesRunner suite
add 44eb961 [BEAM-10670] Make Spark by default execute Read.Bounded using SplittableDoFn.
add e9838b7 Merge pull request #13021: [BEAM-10670] Make Spark by default execute Read.Bounded using SplittableDoFn.
add 4f067f6 [BEAM-11022] Support bundle finalization for portable Flink in streaming.
add 7f69e2b Merge pull request #12952 from [BEAM-11022] Support bundle finalization for portable Flink in streaming.
add 278f74b Enable SDF testPairWithIndexWindowedTimestampedBounded in Spark Structured Streaming runner This passes now after some recent changes in core
add 571338b Merge pull request #13050: Enable SDF testPairWithIndexWindowedTimestampedBounded in Spark Structured Streaming runner
No new revisions were added by this update.
Summary of changes:
..._PortableValidatesRunner_Flink_Streaming.groovy | 5 +
.test-infra/metrics/influxdb/Dockerfile | 2 +-
.test-infra/metrics/influxdb/gsutil/Dockerfile | 2 +-
.test-infra/metrics/sync/github/Dockerfile | 2 +-
.test-infra/metrics/sync/jenkins/Dockerfile | 2 +-
.test-infra/metrics/sync/jira/Dockerfile | 2 +-
CHANGES.md | 1 +
runners/flink/job-server/flink_job_server.gradle | 73 +++--
.../wrappers/streaming/DoFnOperator.java | 2 +-
.../streaming/ExecutableStageDoFnOperator.java | 23 +-
.../FlinkExecutableStageFunctionTest.java | 10 +-
.../streaming/ExecutableStageDoFnOperatorTest.java | 27 +-
.../worker/DataflowWorkProgressUpdater.java | 14 +
.../dataflow/worker/GroupingShuffleReader.java | 48 ++-
.../worker/util/common/worker/MapTaskExecutor.java | 21 +-
.../worker/util/common/worker/NativeReader.java | 9 +
.../worker/util/common/worker/ReadOperation.java | 18 ++
.../worker/DataflowWorkProgressUpdaterTest.java | 23 ++
.../util/common/worker/MapTaskExecutorTest.java | 23 ++
.../control/DefaultJobBundleFactory.java | 11 +-
.../fnexecution/control/SdkHarnessClient.java | 21 +-
.../SingleEnvironmentInstanceJobBundleFactory.java | 22 +-
.../fnexecution/control/StageBundleFactory.java | 15 +-
.../fnexecution/control/RemoteExecutionTest.java | 3 +-
runners/portability/java/build.gradle | 5 +-
runners/spark/build.gradle | 1 -
.../org/apache/beam/runners/spark/SparkRunner.java | 12 +-
.../beam/runners/spark/SparkRunnerDebugger.java | 16 +-
.../SparkStructuredStreamingRunner.java | 12 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 5 +-
.../SparkExecutableStageFunctionTest.java | 10 +-
.../ResumeFromCheckpointStreamingTest.java | 3 +
sdks/go/pkg/beam/core/runtime/graphx/coder.go | 99 ++++---
sdks/go/pkg/beam/core/runtime/graphx/coder_test.go | 6 +-
sdks/go/pkg/beam/core/runtime/graphx/cogbk.go | 31 +-
sdks/go/pkg/beam/core/runtime/graphx/dataflow.go | 6 +-
sdks/go/pkg/beam/core/runtime/graphx/serialize.go | 47 +--
sdks/go/pkg/beam/core/runtime/graphx/translate.go | 327 +++++++++++++++------
sdks/go/pkg/beam/core/runtime/graphx/xlang.go | 12 +-
sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go | 25 +-
sdks/go/pkg/beam/core/runtime/xlangx/resolve.go | 6 +-
sdks/go/pkg/beam/core/runtime/xlangx/translate.go | 36 ++-
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 7 +-
sdks/go/pkg/beam/runners/universal/universal.go | 7 +-
.../apache/beam/sdk/schemas/utils/AvroUtils.java | 44 ++-
.../beam/sdk/schemas/utils/AvroUtilsTest.java | 122 +++-----
.../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 1 +
.../apache_beam/examples/snippets/snippets_test.py | 2 -
.../interactive/display/pcoll_visualization.py | 63 ++--
.../src/inspector/InteractiveInspectorWidget.tsx | 5 -
.../runners/interactive/interactive_environment.py | 4 -
.../runners/portability/flink_runner_test.py | 18 ++
.../runners/portability/local_job_service_main.py | 5 +
.../content/en/documentation/programming-guide.md | 13 +-
.../static/images/unwindowed-pipeline-bounded.svg | 31 +-
.../static/images/windowing-pipeline-bounded.svg | 37 +--
.../static/images/windowing-pipeline-unbounded.svg | 64 ++--
57 files changed, 1018 insertions(+), 443 deletions(-)