You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by jo...@apache.org on 2018/11/30 09:14:16 UTC
[incubator-nemo] branch upgrade-to-0.2-SNAPSHOT created (now
f3f8f3b)
This is an automated email from the ASF dual-hosted git repository.
jooykim pushed a change to branch upgrade-to-0.2-SNAPSHOT
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git.
at f3f8f3b Merge branch 'master' into upgrade-to-0.2-SNAPSHOT
This branch includes the following new commits:
new 5a52c3e Add a script to change version
new 06c1e11 Prepare for next development iteration, master is now at ${NEXT_VERSION_IN_BASE_BRANCH}
new f3f8f3b Merge branch 'master' into upgrade-to-0.2-SNAPSHOT
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-nemo] 03/03: Merge branch 'master' into
upgrade-to-0.2-SNAPSHOT
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jooykim pushed a commit to branch upgrade-to-0.2-SNAPSHOT
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
commit f3f8f3b449c5736a15a615c261c027b8fa2cc47d
Merge: 06c1e11 4e40550
Author: Joo Yeon Kim <jo...@apache.org>
AuthorDate: Fri Nov 30 17:32:04 2018 +0900
Merge branch 'master' into upgrade-to-0.2-SNAPSHOT
DISCLAIMER | 6 +
.../org/apache/nemo/common/dag/DAGBuilder.java | 9 --
.../apache/nemo/common/punctuation/Watermark.java | 2 +-
.../ExampleTestArgs.java} | 32 ++---
.../frontend/beam/InMemorySideInputReader.java | 108 ++++++++++++++
.../frontend/beam/PipelineTranslationContext.java | 109 ++++++++------
.../compiler/frontend/beam/PipelineTranslator.java | 69 ++++++---
.../compiler/frontend/beam/SideInputElement.java | 36 +++--
.../frontend/beam/coder/SideInputCoder.java | 74 ++++++++++
.../beam/source/BeamBoundedSourceVertex.java | 12 +-
.../beam/source/BeamUnboundedSourceVertex.java | 12 +-
.../beam/transform/AbstractDoFnTransform.java | 99 +++++++++----
.../BroadcastVariableSideInputReader.java | 63 --------
.../beam/transform/CreateViewTransform.java | 28 ++--
.../frontend/beam/transform/DoFnTransform.java | 26 +---
.../GroupByKeyAndWindowDoFnTransform.java | 41 ++----
.../beam/transform/GroupByKeyTransform.java | 8 --
.../beam/transform/PushBackDoFnTransform.java | 160 +++++++++++++++++++++
.../beam/transform/SideInputTransform.java | 72 ++++++++++
.../frontend/beam/transform/WindowFnTransform.java | 7 +-
.../org/apache/nemo/compiler/CompilerTestUtil.java | 10 +-
.../frontend/beam/BeamFrontendALSTest.java | 2 +-
.../frontend/beam/BeamFrontendMLRTest.java | 4 +-
.../beam/transform/CreateViewTransformTest.java | 2 +
.../frontend/beam/transform/DoFnTransformTest.java | 121 ++++++++++------
.../GroupByKeyAndWindowDoFnTransformTest.java | 19 ++-
.../reshaping/LoopInvariantCodeMotionPassTest.java | 2 +-
.../nemo/examples/beam/AlternatingLeastSquare.java | 2 -
.../beam/AlternatingLeastSquareInefficient.java | 2 -
.../nemo/examples/beam}/FloatArrayCoder.java | 2 +-
.../apache/nemo/examples/beam}/IntArrayCoder.java | 2 +-
.../nemo/examples/beam/WindowedBroadcast.java | 90 ++++++++++++
.../nemo/examples/beam/WindowedWordCount.java | 2 +-
.../beam/AlternatingLeastSquareITCase.java | 23 ++-
.../nemo/examples/beam/BeamSimpleSumSQLITCase.java | 15 +-
.../apache/nemo/examples/beam/BroadcastITCase.java | 21 ++-
.../beam/MultinomialLogisticRegressionITCase.java | 12 +-
.../examples/beam/NetworkTraceAnalysisITCase.java | 27 ++--
.../beam/PartitionWordsByLengthITCase.java | 27 ++--
.../nemo/examples/beam/PerKeyMedianITCase.java | 19 ++-
.../examples/beam/PerPercentileAverageITCase.java | 19 ++-
...SQLITCase.java => WindowedBroadcastITCase.java} | 48 +++----
.../examples/beam/WindowedWordCountITCase.java | 53 ++++---
.../apache/nemo/examples/beam/WordCountITCase.java | 33 +++--
.../beam_test_executor_resources.json | 0
.../beam_test_one_executor_resources.json | 0
.../beam_test_poisoned_executor_resources.json | 0
.../spark_test_executor_resources.json | 0
examples/resources/{ => inputs}/test_input_als | 0
.../{ => inputs}/test_input_employees.json | 0
examples/resources/{ => inputs}/test_input_median | 0
examples/resources/{ => inputs}/test_input_mlr | 0
.../resources/{ => inputs}/test_input_network0 | 0
.../resources/{ => inputs}/test_input_network1 | 0
.../resources/{ => inputs}/test_input_partition | 0
.../resources/{ => inputs}/test_input_people.json | 0
.../resources/{ => inputs}/test_input_people.txt | 0
examples/resources/{ => inputs}/test_input_tag | 0
.../{ => inputs}/test_input_windowed_wordcount | 0
.../resources/{ => inputs}/test_input_wordcount | 0
.../{ => inputs}/test_input_wordcount_spark | 0
.../resources/{ => outputs}/expected_output_als | 0
.../{ => outputs}/expected_output_broadcast | 0
.../resources/{ => outputs}/expected_output_median | 0
.../{ => outputs}/expected_output_network | 0
.../{ => outputs}/expected_output_partition_0 | 0
.../{ => outputs}/expected_output_partition_1 | 0
.../{ => outputs}/expected_output_partition_2 | 0
.../{ => outputs}/expected_output_partition_3 | 0
.../{ => outputs}/expected_output_partition_4 | 0
.../{ => outputs}/expected_output_partition_5 | 0
.../{ => outputs}/expected_output_partition_6 | 0
.../{ => outputs}/expected_output_partition_7 | 0
.../{ => outputs}/expected_output_partition_8 | 0
.../{ => outputs}/expected_output_partition_9 | 0
.../expected_output_reversed_wordcount_spark | 0
.../{ => outputs}/expected_output_simplesql | 0
.../expected_output_sliding_windowed_wordcount | 0
.../{ => outputs}/expected_output_tag_long | 0
.../{ => outputs}/expected_output_tag_short | 0
.../{ => outputs}/expected_output_tag_very_long | 0
.../expected_output_tag_very_very_long | 0
.../expected_output_windowed_wordcount | 0
.../expected_output_word_and_line_count | 0
.../{ => outputs}/expected_output_wordcount | 0
.../{ => outputs}/expected_output_wordcount_spark | 0
.../org/apache/nemo/examples/spark/MRJava.java | 33 +++--
.../org/apache/nemo/examples/spark/SparkJava.java | 29 ++--
.../org/apache/nemo/examples/spark/SparkScala.java | 49 ++++---
pom.xml | 32 +++--
.../org/apache/nemo/runtime/executor/Executor.java | 2 +-
.../executor/data/BroadcastManagerWorker.java | 68 +++------
.../DedicatedKeyPerElementPartitioner.java | 1 +
.../datatransfer/MultiInputWatermarkManager.java | 1 -
.../OperatorVertexOutputCollector.java | 3 -
.../executor/datatransfer/PipeOutputWriter.java | 15 +-
.../datatransfer/SingleInputWatermarkManager.java | 3 +
.../nemo/runtime/executor/task/DataFetcher.java | 8 ++
.../task/MultiThreadParentTaskDataFetcher.java | 6 +-
.../executor/task/ParentTaskDataFetcher.java | 3 +-
.../nemo/runtime/executor/task/TaskExecutor.java | 143 +++++++++---------
.../executor/task/ParentTaskDataFetcherTest.java | 7 +-
.../runtime/executor/task/TaskExecutorTest.java | 25 +---
103 files changed, 1201 insertions(+), 757 deletions(-)
[incubator-nemo] 01/03: Add a script to change version
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jooykim pushed a commit to branch upgrade-to-0.2-SNAPSHOT
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
commit 5a52c3e43d145d7111c0ddcde8cf14f339fe8980
Author: Joo Yeon Kim <jo...@apache.org>
AuthorDate: Tue Nov 20 15:14:16 2018 +0900
Add a script to change version
---
update_branch_version.sh | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/update_branch_version.sh b/update_branch_version.sh
new file mode 100644
index 0000000..a9c49ff
--- /dev/null
+++ b/update_branch_version.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Update the development version in existing pom files
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' client/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' common/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' compiler/backend/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' compiler/frontend/beam/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' compiler/frontend/spark/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' compiler/optimizer/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' compiler/test/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' compiler/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' conf/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' examples/beam/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' examples/spark/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' examples/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' runtime/common/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' runtime/driver/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' runtime/executor/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' runtime/master/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' runtime/test/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' runtime/pom.xml
+sed -i -e 's/${RELEASE}/${NEXT_VERSION_IN_BASE_BRANCH}/g' pom.xml
[incubator-nemo] 02/03: Prepare for next development iteration,
master is now at ${NEXT_VERSION_IN_BASE_BRANCH}
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jooykim pushed a commit to branch upgrade-to-0.2-SNAPSHOT
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
commit 06c1e1169753491cdf6d656eb0cbb787bb6110bd
Author: Joo Yeon Kim <jo...@apache.org>
AuthorDate: Tue Nov 20 16:05:25 2018 +0900
Prepare for next development iteration, master is now at ${NEXT_VERSION_IN_BASE_BRANCH}
---
client/pom.xml | 2 +-
common/pom.xml | 2 +-
compiler/backend/pom.xml | 2 +-
compiler/frontend/beam/pom.xml | 2 +-
compiler/frontend/spark/pom.xml | 2 +-
compiler/optimizer/pom.xml | 2 +-
compiler/pom.xml | 2 +-
compiler/test/pom.xml | 2 +-
conf/pom.xml | 2 +-
examples/beam/pom.xml | 2 +-
examples/pom.xml | 2 +-
examples/spark/pom.xml | 2 +-
pom.xml | 2 +-
runtime/common/pom.xml | 2 +-
runtime/driver/pom.xml | 2 +-
runtime/executor/pom.xml | 4 ++--
runtime/master/pom.xml | 2 +-
runtime/pom.xml | 2 +-
runtime/test/pom.xml | 2 +-
19 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/client/pom.xml b/client/pom.xml
index 4a31ef1..bcfe8a2 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-project</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/common/pom.xml b/common/pom.xml
index f2ce14d..e35b27f 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-project</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/compiler/backend/pom.xml b/compiler/backend/pom.xml
index 6af08f1..ea3f538 100644
--- a/compiler/backend/pom.xml
+++ b/compiler/backend/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-compiler</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/compiler/frontend/beam/pom.xml b/compiler/frontend/beam/pom.xml
index 01000f7..da50822 100644
--- a/compiler/frontend/beam/pom.xml
+++ b/compiler/frontend/beam/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-compiler</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
diff --git a/compiler/frontend/spark/pom.xml b/compiler/frontend/spark/pom.xml
index f923f77..eb2c434 100644
--- a/compiler/frontend/spark/pom.xml
+++ b/compiler/frontend/spark/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-compiler</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
diff --git a/compiler/optimizer/pom.xml b/compiler/optimizer/pom.xml
index e04056f..7a0e495 100644
--- a/compiler/optimizer/pom.xml
+++ b/compiler/optimizer/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-compiler</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/compiler/pom.xml b/compiler/pom.xml
index 060b0f7..bdd7fce 100644
--- a/compiler/pom.xml
+++ b/compiler/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<artifactId>nemo-project</artifactId>
<groupId>org.apache.nemo</groupId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/compiler/test/pom.xml b/compiler/test/pom.xml
index 7431334..c12027f 100644
--- a/compiler/test/pom.xml
+++ b/compiler/test/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-compiler</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/conf/pom.xml b/conf/pom.xml
index 3873b0b..d36f615 100644
--- a/conf/pom.xml
+++ b/conf/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-project</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/examples/beam/pom.xml b/examples/beam/pom.xml
index bf97018..1290e0b 100644
--- a/examples/beam/pom.xml
+++ b/examples/beam/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-examples</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/examples/pom.xml b/examples/pom.xml
index 66afc6e..ed66ccb 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@ under the License.
<parent>
<artifactId>nemo-project</artifactId>
<groupId>org.apache.nemo</groupId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 859cc36..11db89f 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<artifactId>nemo-examples</artifactId>
<groupId>org.apache.nemo</groupId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/pom.xml b/pom.xml
index 20cfc45..74c34b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@ under the License.
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-project</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Nemo Project</name>
diff --git a/runtime/common/pom.xml b/runtime/common/pom.xml
index f2b643d..00eed84 100644
--- a/runtime/common/pom.xml
+++ b/runtime/common/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/runtime/driver/pom.xml b/runtime/driver/pom.xml
index a9a29e4..2707c3d 100644
--- a/runtime/driver/pom.xml
+++ b/runtime/driver/pom.xml
@@ -21,7 +21,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/runtime/executor/pom.xml b/runtime/executor/pom.xml
index ae7a770..c733dd5 100644
--- a/runtime/executor/pom.xml
+++ b/runtime/executor/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
@@ -61,7 +61,7 @@ under the License.
<dependency>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime-master</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/runtime/master/pom.xml b/runtime/master/pom.xml
index 390838a..9110d0b 100644
--- a/runtime/master/pom.xml
+++ b/runtime/master/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/runtime/pom.xml b/runtime/pom.xml
index 5d2e53c..415cea6 100644
--- a/runtime/pom.xml
+++ b/runtime/pom.xml
@@ -21,7 +21,7 @@ under the License.
<parent>
<artifactId>nemo-project</artifactId>
<groupId>org.apache.nemo</groupId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/runtime/test/pom.xml b/runtime/test/pom.xml
index 4d0635e..187aa51 100644
--- a/runtime/test/pom.xml
+++ b/runtime/test/pom.xml
@@ -23,7 +23,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>