You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by jo...@apache.org on 2018/11/30 09:14:19 UTC
[incubator-nemo] 03/03: Merge branch 'master' into
upgrade-to-0.2-SNAPSHOT
This is an automated email from the ASF dual-hosted git repository.
jooykim pushed a commit to branch upgrade-to-0.2-SNAPSHOT
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
commit f3f8f3b449c5736a15a615c261c027b8fa2cc47d
Merge: 06c1e11 4e40550
Author: Joo Yeon Kim <jo...@apache.org>
AuthorDate: Fri Nov 30 17:32:04 2018 +0900
Merge branch 'master' into upgrade-to-0.2-SNAPSHOT
DISCLAIMER | 6 +
.../org/apache/nemo/common/dag/DAGBuilder.java | 9 --
.../apache/nemo/common/punctuation/Watermark.java | 2 +-
.../ExampleTestArgs.java} | 32 ++---
.../frontend/beam/InMemorySideInputReader.java | 108 ++++++++++++++
.../frontend/beam/PipelineTranslationContext.java | 109 ++++++++------
.../compiler/frontend/beam/PipelineTranslator.java | 69 ++++++---
.../compiler/frontend/beam/SideInputElement.java | 36 +++--
.../frontend/beam/coder/SideInputCoder.java | 74 ++++++++++
.../beam/source/BeamBoundedSourceVertex.java | 12 +-
.../beam/source/BeamUnboundedSourceVertex.java | 12 +-
.../beam/transform/AbstractDoFnTransform.java | 99 +++++++++----
.../BroadcastVariableSideInputReader.java | 63 --------
.../beam/transform/CreateViewTransform.java | 28 ++--
.../frontend/beam/transform/DoFnTransform.java | 26 +---
.../GroupByKeyAndWindowDoFnTransform.java | 41 ++----
.../beam/transform/GroupByKeyTransform.java | 8 --
.../beam/transform/PushBackDoFnTransform.java | 160 +++++++++++++++++++++
.../beam/transform/SideInputTransform.java | 72 ++++++++++
.../frontend/beam/transform/WindowFnTransform.java | 7 +-
.../org/apache/nemo/compiler/CompilerTestUtil.java | 10 +-
.../frontend/beam/BeamFrontendALSTest.java | 2 +-
.../frontend/beam/BeamFrontendMLRTest.java | 4 +-
.../beam/transform/CreateViewTransformTest.java | 2 +
.../frontend/beam/transform/DoFnTransformTest.java | 121 ++++++++++------
.../GroupByKeyAndWindowDoFnTransformTest.java | 19 ++-
.../reshaping/LoopInvariantCodeMotionPassTest.java | 2 +-
.../nemo/examples/beam/AlternatingLeastSquare.java | 2 -
.../beam/AlternatingLeastSquareInefficient.java | 2 -
.../nemo/examples/beam}/FloatArrayCoder.java | 2 +-
.../apache/nemo/examples/beam}/IntArrayCoder.java | 2 +-
.../nemo/examples/beam/WindowedBroadcast.java | 90 ++++++++++++
.../nemo/examples/beam/WindowedWordCount.java | 2 +-
.../beam/AlternatingLeastSquareITCase.java | 23 ++-
.../nemo/examples/beam/BeamSimpleSumSQLITCase.java | 15 +-
.../apache/nemo/examples/beam/BroadcastITCase.java | 21 ++-
.../beam/MultinomialLogisticRegressionITCase.java | 12 +-
.../examples/beam/NetworkTraceAnalysisITCase.java | 27 ++--
.../beam/PartitionWordsByLengthITCase.java | 27 ++--
.../nemo/examples/beam/PerKeyMedianITCase.java | 19 ++-
.../examples/beam/PerPercentileAverageITCase.java | 19 ++-
...SQLITCase.java => WindowedBroadcastITCase.java} | 48 +++----
.../examples/beam/WindowedWordCountITCase.java | 53 ++++---
.../apache/nemo/examples/beam/WordCountITCase.java | 33 +++--
.../beam_test_executor_resources.json | 0
.../beam_test_one_executor_resources.json | 0
.../beam_test_poisoned_executor_resources.json | 0
.../spark_test_executor_resources.json | 0
examples/resources/{ => inputs}/test_input_als | 0
.../{ => inputs}/test_input_employees.json | 0
examples/resources/{ => inputs}/test_input_median | 0
examples/resources/{ => inputs}/test_input_mlr | 0
.../resources/{ => inputs}/test_input_network0 | 0
.../resources/{ => inputs}/test_input_network1 | 0
.../resources/{ => inputs}/test_input_partition | 0
.../resources/{ => inputs}/test_input_people.json | 0
.../resources/{ => inputs}/test_input_people.txt | 0
examples/resources/{ => inputs}/test_input_tag | 0
.../{ => inputs}/test_input_windowed_wordcount | 0
.../resources/{ => inputs}/test_input_wordcount | 0
.../{ => inputs}/test_input_wordcount_spark | 0
.../resources/{ => outputs}/expected_output_als | 0
.../{ => outputs}/expected_output_broadcast | 0
.../resources/{ => outputs}/expected_output_median | 0
.../{ => outputs}/expected_output_network | 0
.../{ => outputs}/expected_output_partition_0 | 0
.../{ => outputs}/expected_output_partition_1 | 0
.../{ => outputs}/expected_output_partition_2 | 0
.../{ => outputs}/expected_output_partition_3 | 0
.../{ => outputs}/expected_output_partition_4 | 0
.../{ => outputs}/expected_output_partition_5 | 0
.../{ => outputs}/expected_output_partition_6 | 0
.../{ => outputs}/expected_output_partition_7 | 0
.../{ => outputs}/expected_output_partition_8 | 0
.../{ => outputs}/expected_output_partition_9 | 0
.../expected_output_reversed_wordcount_spark | 0
.../{ => outputs}/expected_output_simplesql | 0
.../expected_output_sliding_windowed_wordcount | 0
.../{ => outputs}/expected_output_tag_long | 0
.../{ => outputs}/expected_output_tag_short | 0
.../{ => outputs}/expected_output_tag_very_long | 0
.../expected_output_tag_very_very_long | 0
.../expected_output_windowed_wordcount | 0
.../expected_output_word_and_line_count | 0
.../{ => outputs}/expected_output_wordcount | 0
.../{ => outputs}/expected_output_wordcount_spark | 0
.../org/apache/nemo/examples/spark/MRJava.java | 33 +++--
.../org/apache/nemo/examples/spark/SparkJava.java | 29 ++--
.../org/apache/nemo/examples/spark/SparkScala.java | 49 ++++---
pom.xml | 32 +++--
.../org/apache/nemo/runtime/executor/Executor.java | 2 +-
.../executor/data/BroadcastManagerWorker.java | 68 +++------
.../DedicatedKeyPerElementPartitioner.java | 1 +
.../datatransfer/MultiInputWatermarkManager.java | 1 -
.../OperatorVertexOutputCollector.java | 3 -
.../executor/datatransfer/PipeOutputWriter.java | 15 +-
.../datatransfer/SingleInputWatermarkManager.java | 3 +
.../nemo/runtime/executor/task/DataFetcher.java | 8 ++
.../task/MultiThreadParentTaskDataFetcher.java | 6 +-
.../executor/task/ParentTaskDataFetcher.java | 3 +-
.../nemo/runtime/executor/task/TaskExecutor.java | 143 +++++++++---------
.../executor/task/ParentTaskDataFetcherTest.java | 7 +-
.../runtime/executor/task/TaskExecutorTest.java | 25 +---
103 files changed, 1201 insertions(+), 757 deletions(-)