You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ie...@apache.org on 2020/06/26 22:13:59 UTC
[beam] branch master updated (6ae620b -> 9838377)
This is an automated email from the ASF dual-hosted git repository.
iemejia pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 6ae620b [BEAM-10023] Don't add interactive extra for dependency check in python 2 (#12105)
add 1e0eef0 adding twister2 runner with support for batch piplines
add 9838377 Merge pull request #10888: [BEAM-7304] Twister2 Beam runner
No new revisions were added by this update.
Summary of changes:
.github/PULL_REQUEST_TEMPLATE.md | 12 +-
.github/autolabeler.yml | 1 +
...PostCommit_Java_ValidatesRunner_Twister2.groovy | 44 +++
CHANGES.md | 1 +
README.md | 1 +
release/src/main/scripts/mass_comment.py | 1 +
runners/twister2/build.gradle | 86 +++++
.../runners/twister2/BeamBatchTSetEnvironment.java | 39 +++
.../beam/runners/twister2/BeamBatchWorker.java | 162 ++++++++++
.../twister2/Twister2BatchTranslationContext.java | 44 +++
.../Twister2PipelineExecutionEnvironment.java | 114 +++++++
.../runners/twister2/Twister2PipelineOptions.java | 82 +++++
.../runners/twister2/Twister2PipelineResult.java | 58 ++++
.../beam/runners/twister2/Twister2Runner.java | 341 ++++++++++++++++++++
.../runners/twister2/Twister2RunnerRegistrar.java | 53 +++
.../twister2/Twister2StreamTranslationContext.java | 33 ++
.../beam/runners/twister2/Twister2TestRunner.java | 60 ++++
.../twister2/Twister2TranslationContext.java | 122 +++++++
.../apache/beam/runners/twister2/package-info.java | 20 ++
.../wrappers/Twister2BoundedSource.java | 274 ++++++++++++++++
.../translation/wrappers/Twister2EmptySource.java | 38 +++
.../translation/wrappers/package-info.java | 20 ++
.../translators/BatchTransformTranslator.java | 26 ++
.../translators/StreamTransformTranslator.java | 26 ++
.../Twister2BatchPipelineTranslator.java | 89 +++++
.../translators/Twister2PipelineTranslator.java | 33 ++
.../Twister2StreamPipelineTranslator.java | 31 ++
.../batch/AssignWindowTranslatorBatch.java | 47 +++
.../translators/batch/FlattenTranslatorBatch.java | 66 ++++
.../batch/GroupByKeyTranslatorBatch.java | 76 +++++
.../batch/PCollectionViewTranslatorBatch.java | 77 +++++
.../batch/ParDoMultiOutputTranslatorBatch.java | 128 ++++++++
.../batch/ReadSourceTranslatorBatch.java | 47 +++
.../twister2/translators/batch/package-info.java | 20 ++
.../functions/AssignWindowsFunction.java | 119 +++++++
.../functions/ByteToWindowFunction.java | 106 ++++++
.../functions/ByteToWindowFunctionPrimitive.java | 110 +++++++
.../translators/functions/DoFnFunction.java | 357 +++++++++++++++++++++
.../functions/GroupByWindowFunction.java | 234 ++++++++++++++
.../translators/functions/MapToTupleFunction.java | 106 ++++++
.../translators/functions/OutputTagFilter.java | 55 ++++
.../functions/Twister2SinkFunction.java | 47 +++
.../internal/SystemReduceFnBuffering.java | 116 +++++++
.../functions/internal/package-info.java | 20 ++
.../translators/functions/package-info.java | 20 ++
.../runners/twister2/translators/package-info.java | 20 ++
.../streaming/ReadSourceTranslatorStream.java | 32 ++
.../translators/streaming/package-info.java | 20 ++
.../runners/twister2/utils/NoOpStepContext.java | 36 +++
.../runners/twister2/utils/TranslationUtils.java | 67 ++++
.../twister2/utils/Twister2AssignContext.java | 51 +++
.../twister2/utils/Twister2SideInputReader.java | 112 +++++++
.../beam/runners/twister2/utils/package-info.java | 20 ++
settings.gradle | 1 +
54 files changed, 3915 insertions(+), 6 deletions(-)
create mode 100644 .test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Twister2.groovy
create mode 100644 runners/twister2/build.gradle
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/BeamBatchTSetEnvironment.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/BeamBatchWorker.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2BatchTranslationContext.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2PipelineExecutionEnvironment.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2PipelineOptions.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2PipelineResult.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2Runner.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2RunnerRegistrar.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2StreamTranslationContext.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2TestRunner.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/Twister2TranslationContext.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/package-info.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translation/wrappers/Twister2BoundedSource.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translation/wrappers/Twister2EmptySource.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translation/wrappers/package-info.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/BatchTransformTranslator.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/StreamTransformTranslator.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/Twister2BatchPipelineTranslator.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/Twister2PipelineTranslator.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/Twister2StreamPipelineTranslator.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/AssignWindowTranslatorBatch.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/FlattenTranslatorBatch.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/GroupByKeyTranslatorBatch.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/PCollectionViewTranslatorBatch.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/ParDoMultiOutputTranslatorBatch.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/ReadSourceTranslatorBatch.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/batch/package-info.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/AssignWindowsFunction.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/ByteToWindowFunction.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/ByteToWindowFunctionPrimitive.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/DoFnFunction.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/GroupByWindowFunction.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/MapToTupleFunction.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/OutputTagFilter.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/Twister2SinkFunction.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/internal/SystemReduceFnBuffering.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/internal/package-info.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/functions/package-info.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/package-info.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/streaming/ReadSourceTranslatorStream.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/translators/streaming/package-info.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/NoOpStepContext.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/TranslationUtils.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2AssignContext.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/Twister2SideInputReader.java
create mode 100644 runners/twister2/src/main/java/org/apache/beam/runners/twister2/utils/package-info.java