You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by ta...@apache.org on 2020/12/11 14:13:57 UTC
[incubator-nemo] branch master updated: [NEMO] Release v0.3 (#305)
This is an automated email from the ASF dual-hosted git repository.
taegeonum pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
The following commit(s) were added to refs/heads/master by this push:
new 7618046 [NEMO] Release v0.3 (#305)
7618046 is described below
commit 7618046ef702f5655cc238b52b1552fa359b350e
Author: Won Wook SONG <ws...@gmail.com>
AuthorDate: Fri Dec 11 23:13:50 2020 +0900
[NEMO] Release v0.3 (#305)
**Major changes:**
- None
**Minor changes to note:**
- Added Javadocs for the release process
- Bump up the version to 0.4-SNAPSHOT
**Tests for the changes:**
- Existing tests work.
**Other comments:**
- None
Closes #305
---
client/pom.xml | 7 +++----
common/pom.xml | 5 ++---
.../java/org/apache/nemo/common/dag/DAGBuilder.java | 6 +++---
.../executionproperty/SubPartitionSetProperty.java | 2 +-
.../apache/nemo/common/ir/vertex/LoopVertex.java | 2 ++
.../apache/nemo/common/test/ExampleTestArgs.java | 2 +-
compiler/backend/pom.xml | 5 ++---
compiler/frontend/beam/pom.xml | 5 ++---
.../beam/transform/AbstractDoFnTransform.java | 2 ++
.../frontend/beam/transform/DoFnTransform.java | 2 ++
.../frontend/beam/transform/GBKTransform.java | 12 ++++++++----
.../transform/InMemoryTimerInternalsFactory.java | 21 ++++++++++++++++++---
.../beam/transform/PushBackDoFnTransform.java | 2 ++
compiler/frontend/spark/pom.xml | 5 ++---
compiler/optimizer/pom.xml | 5 ++---
.../MapReduceDisaggregationOptimization.java | 1 -
.../reshaping/SamplingTaskSizingPass.java | 6 +++---
compiler/pom.xml | 5 ++---
compiler/test/pom.xml | 5 ++---
conf/pom.xml | 5 ++---
examples/beam/pom.xml | 7 +++----
.../nemo/examples/beam/AlternatingLeastSquare.java | 2 --
.../apache/nemo/examples/beam/BeamWordCount.java | 6 +++++-
examples/nexmark/pom.xml | 8 +++-----
examples/pom.xml | 5 ++---
examples/spark/pom.xml | 7 +++----
pom.xml | 10 +++++-----
runtime/common/pom.xml | 5 ++---
runtime/driver/pom.xml | 5 ++---
runtime/executor/pom.xml | 7 +++----
.../executor/transfer/LocalInputContext.java | 2 +-
.../executor/transfer/TransferOutputStream.java | 2 +-
runtime/master/pom.xml | 5 ++---
runtime/pom.xml | 5 ++---
runtime/test/pom.xml | 5 ++---
35 files changed, 97 insertions(+), 89 deletions(-)
diff --git a/client/pom.xml b/client/pom.xml
index e869a88..52b8f12 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-project</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
@@ -97,7 +96,7 @@ under the License.
</outputFile>
<transformers>
<!-- Required for using beam-hadoop: See https://stackoverflow.com/questions/44365545-->
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
</transformers>
</configuration>
</execution>
diff --git a/common/pom.xml b/common/pom.xml
index 87de322..3c27023 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-project</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java b/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java
index 7ec85b7..85d7ce5 100644
--- a/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java
+++ b/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java
@@ -185,7 +185,7 @@ public final class DAGBuilder<V extends Vertex, E extends Edge<V>> implements Se
* should be used when there are changes in vertices before / after SplitterVertex.
*
* CAUTION: TaskSizeSplitterVertex must only appear in IRDAG.
- * {@param originalEdge} and {@param edgeToInsert} should have same source and destination.
+ * {@code originalEdge} and {@code edgeToInsert} should have same source and destination.
*
* Relation to be Erased: originalEdge - internalEdge
* Relation to insert: edgeToInsert - newInternalEdge
@@ -250,8 +250,8 @@ public final class DAGBuilder<V extends Vertex, E extends Edge<V>> implements Se
* without replacing existing mapping relationships.
* The changes which invokes this method should not be caused by SplitterVertex itself. Therefore, this method
* should be used when there are changes in vertices before / after SplitterVertex.
- * Since {@param edgeToInsert} should also have a mapping relationship to originalVertices of SplitterVertex,
- * we give {@param edgeToReference} together to copy the mapping information. Therefore, these two parameters must
+ * Since {@code edgeToInsert} should also have a mapping relationship to originalVertices of SplitterVertex,
+ * we give {@code edgeToReference} together to copy the mapping information. Therefore, these two parameters must
* have at least one common source or destination.
*
* Relation to reference: edgeToReference - internalEdge
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/SubPartitionSetProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/SubPartitionSetProperty.java
index 4395719..655b46e 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/SubPartitionSetProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/SubPartitionSetProperty.java
@@ -25,7 +25,7 @@ import org.apache.nemo.common.ir.executionproperty.EdgeExecutionProperty;
import java.util.ArrayList;
/**
- * This property decides the hash partition set of sampled & optimized tasks in Dynamic Task Sizing Policy.
+ * This property decides the hash partition set of sampled and optimized tasks in Dynamic Task Sizing Policy.
* <p>
* Adopting this property requires changing other properties as well.
* Such execution properties include:
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java
index 011dcc1..e68fc77 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java
@@ -164,6 +164,7 @@ public class LoopVertex extends IRVertex {
/**
* Getter method for edgeWithLoopToEdgeWithInternalVertex.
+ * @return the map of edge with loop to the edge with the internal vertex.
*/
public Map<IREdge, IREdge> getEdgeWithLoopToEdgeWithInternalVertex() {
return this.edgeWithLoopToEdgeWithInternalVertex;
@@ -171,6 +172,7 @@ public class LoopVertex extends IRVertex {
/**
* Getter method for edgeWithInternalVertexToEdgeWithLoop.
+ * @return the map of edge with internal vertex to the edge with the loop.
*/
public Map<IREdge, IREdge> getEdgeWithInternalVertexToEdgeWithLoop() {
return this.edgeWithInternalVertexToEdgeWithLoop;
diff --git a/common/src/main/java/org/apache/nemo/common/test/ExampleTestArgs.java b/common/src/main/java/org/apache/nemo/common/test/ExampleTestArgs.java
index 9e0162f..6c74a5e 100644
--- a/common/src/main/java/org/apache/nemo/common/test/ExampleTestArgs.java
+++ b/common/src/main/java/org/apache/nemo/common/test/ExampleTestArgs.java
@@ -22,7 +22,7 @@ package org.apache.nemo.common.test;
* Test Arguments for Examples.
*/
public final class ExampleTestArgs {
- public static final int TIMEOUT = 240000;
+ public static final int TIMEOUT = 300000;
private static String fileBasePath;
/**
diff --git a/compiler/backend/pom.xml b/compiler/backend/pom.xml
index ef912d8..e461d4e 100644
--- a/compiler/backend/pom.xml
+++ b/compiler/backend/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-compiler</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/compiler/frontend/beam/pom.xml b/compiler/frontend/beam/pom.xml
index 1c25117..b5501d1 100644
--- a/compiler/frontend/beam/pom.xml
+++ b/compiler/frontend/beam/pom.xml
@@ -15,14 +15,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-compiler</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
index 79b31c6..59a3b1b 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
@@ -98,6 +98,8 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
* @param sideInputs side inputs
* @param options pipeline options
* @param displayData display data.
+ * @param doFnSchemaInformation doFn schema information.
+ * @param sideInputMapping side input mapping.
*/
public AbstractDoFnTransform(final DoFn<InterT, OutputT> doFn,
final Coder<InputT> inputCoder,
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
index c6b8e3b..1d29402 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
@@ -56,6 +56,8 @@ public final class DoFnTransform<InputT, OutputT> extends AbstractDoFnTransform<
* @param windowingStrategy windowing strategy
* @param options pipeline options
* @param displayData display data.
+ * @param doFnSchemaInformation doFn schema information.
+ * @param sideInputMapping side input mapping.
*/
public DoFnTransform(final DoFn<InputT, OutputT> doFn,
final Coder<InputT> inputCoder,
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GBKTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GBKTransform.java
index 1bf6cb8..4aa366f 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GBKTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GBKTransform.java
@@ -142,7 +142,7 @@ public final class GBKTransform<K, InputT, OutputT>
}
/**
- * Trigger timers that need to be fired at {@param watermark} and emit output watermark.
+ * Trigger timers that need to be fired at {@code watermark} and emit output watermark.
* @param watermark watermark
*/
@Override
@@ -263,7 +263,11 @@ public final class GBKTransform<K, InputT, OutputT>
}
}
- /** Accessor for isPartialCombining. */
+ /**
+ * Accessor for isPartialCombining.
+ *
+ * @return whether it is partial combining.
+ */
public boolean getIsPartialCombining() {
return isPartialCombining;
}
@@ -277,7 +281,7 @@ public final class GBKTransform<K, InputT, OutputT>
this.oc = oc;
}
- /** Emit output. If {@param output} is emitted on-time, save its timestamp in the output watermark map. */
+ /** Emit output. If {@code output} is emitted on-time, save its timestamp in the output watermark map. */
@Override
public final void emit(final WindowedValue<KV<K, OutputT>> output) {
// The watermark advances only in ON_TIME
@@ -301,7 +305,7 @@ public final class GBKTransform<K, InputT, OutputT>
oc.emitWatermark(watermark);
}
- /** Emit output value to {@param dstVertexId}. */
+ /** Emit output value to {@code dstVertexId}. */
@Override
public final <T> void emit(final String dstVertexId, final T output) {
oc.emit(dstVertexId, output);
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/InMemoryTimerInternalsFactory.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/InMemoryTimerInternalsFactory.java
index 0072463..ed1bb1e 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/InMemoryTimerInternalsFactory.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/InMemoryTimerInternalsFactory.java
@@ -50,7 +50,13 @@ public final class InMemoryTimerInternalsFactory<K> implements TimerInternalsFac
}
}
- /** Remove the next eligible timer in {@param timeDomain}. */
+ /**
+ * Remove the next eligible timer in {@code timeDomain}.
+ *
+ * @param timerInternal the timer internals.
+ * @param timeDomain the time domain.
+ * @return the timer data.
+ */
public TimerInternals.TimerData pollTimer(final InMemoryTimerInternals timerInternal, final TimeDomain timeDomain) {
switch (timeDomain) {
case EVENT_TIME :
@@ -64,12 +70,21 @@ public final class InMemoryTimerInternalsFactory<K> implements TimerInternalsFac
}
}
- /** Accessor for timerInternalsMap. */
+ /**
+ * Accessor for timerInternalsMap.
+ *
+ * @return the timer internals map.
+ */
public Map<K, InMemoryTimerInternals> getTimerInternalsMap() {
return timerInternalsMap;
}
- /** Helper method to check if {@param timerInternals} doesn't have any timers left. */
+ /**
+ * Helper method to check if {@code timerInternals} doesn't have any timers left.
+ *
+ * @param timerInternal the in-memory timer internals.
+ * @return whether it is empty or not.
+ */
public boolean isEmpty(final InMemoryTimerInternals timerInternal) {
for (final TimeDomain domain : TimeDomain.values()) {
if (timerInternal.getNextTimer(domain) != null) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
index 15a0873..7df9bff 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
@@ -60,6 +60,8 @@ public final class PushBackDoFnTransform<InputT, OutputT> extends AbstractDoFnTr
* @param sideInputs side inputs
* @param options pipeline options
* @param displayData display data.
+ * @param doFnSchemaInformation DoFn schema information.
+ * @param sideInputMapping side input mapping.
*/
public PushBackDoFnTransform(final DoFn<InputT, OutputT> doFn,
final Coder<InputT> inputCoder,
diff --git a/compiler/frontend/spark/pom.xml b/compiler/frontend/spark/pom.xml
index 8938b53..f135fd8 100644
--- a/compiler/frontend/spark/pom.xml
+++ b/compiler/frontend/spark/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-compiler</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
diff --git a/compiler/optimizer/pom.xml b/compiler/optimizer/pom.xml
index c6dbb79..49a546d 100644
--- a/compiler/optimizer/pom.xml
+++ b/compiler/optimizer/pom.xml
@@ -17,8 +17,7 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<build>
<plugins>
@@ -36,7 +35,7 @@ under the License.
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-compiler</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java
index 14220d7..48008d1 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java
@@ -47,7 +47,6 @@ public final class MapReduceDisaggregationOptimization {
* Main function of the example MR program.
*
* @param args arguments.
- * @throws Exception Exceptions on the way.
*/
public static void main(final String[] args) {
final IRVertex source = new EmptyComponents.EmptySourceVertex<>("Source");
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingTaskSizingPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingTaskSizingPass.java
index c35941b..dd5fe1f 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingTaskSizingPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingTaskSizingPass.java
@@ -36,17 +36,17 @@ import java.util.stream.Collectors;
/**
* Compiler pass for dynamic task size optimization. Happens only when the edge property is SHUFFLE.
- * If (size of given job) >= 1GB: enable dynamic task sizing optimization.
+ * If (size of given job) is greater than or equal to 1GB: enable dynamic task sizing optimization.
* else: break.
*
*
- * @Attributes
+ * With attributes
* PARTITIONER_PROPERTY_FOR_SMALL_JOB: PartitionerProperty for jobs in range of [1GB, 10GB) size.
* PARTITIONER_PROPERTY_FOR_MEDIUM_JOB: PartitionerProperty for jobs in range of [10GB, 100GB) size.
* PARTITIONER_PROPERTY_FOR_BIG_JOB: PartitionerProperty for jobs in range of [100GB, - ) size(No upper limit).
*
* source stage - shuffle edge - current stage - next stage
- * -> source stage - [curr stage - signal vertex] - next stage
+ * - source stage - [curr stage - signal vertex] - next stage
* where [] is a splitter vertex
*/
@Annotates({EnableDynamicTaskSizingProperty.class, PartitionerProperty.class, SubPartitionSetProperty.class,
diff --git a/compiler/pom.xml b/compiler/pom.xml
index aaf93cd..1d6b396 100644
--- a/compiler/pom.xml
+++ b/compiler/pom.xml
@@ -17,13 +17,12 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nemo-project</artifactId>
<groupId>org.apache.nemo</groupId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/compiler/test/pom.xml b/compiler/test/pom.xml
index 12f6569..54ef45a 100644
--- a/compiler/test/pom.xml
+++ b/compiler/test/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-compiler</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/conf/pom.xml b/conf/pom.xml
index e25be3c..fc7ada1 100644
--- a/conf/pom.xml
+++ b/conf/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-project</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/examples/beam/pom.xml b/examples/beam/pom.xml
index 4cca999..65651a3 100644
--- a/examples/beam/pom.xml
+++ b/examples/beam/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-examples</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
@@ -119,7 +118,7 @@ under the License.
<transformers>
<!-- Required for using beam-hadoop: See https://stackoverflow.com/questions/44365545
-->
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
</transformers>
</configuration>
</execution>
diff --git a/examples/beam/src/main/java/org/apache/nemo/examples/beam/AlternatingLeastSquare.java b/examples/beam/src/main/java/org/apache/nemo/examples/beam/AlternatingLeastSquare.java
index d87d546..3d05ce2 100644
--- a/examples/beam/src/main/java/org/apache/nemo/examples/beam/AlternatingLeastSquare.java
+++ b/examples/beam/src/main/java/org/apache/nemo/examples/beam/AlternatingLeastSquare.java
@@ -326,7 +326,6 @@ public final class AlternatingLeastSquare {
* ProcessElement method for BEAM.
*
* @param c ProcessContext.
- * @throws Exception Exception on the way.
*/
@ProcessElement
public void processElement(final ProcessContext c) {
@@ -354,7 +353,6 @@ public final class AlternatingLeastSquare {
* Main function for the ALS BEAM program.
*
* @param args arguments.
- * @throws ClassNotFoundException exception.
*/
public static void main(final String[] args) {
final Long start = System.currentTimeMillis();
diff --git a/examples/beam/src/main/java/org/apache/nemo/examples/beam/BeamWordCount.java b/examples/beam/src/main/java/org/apache/nemo/examples/beam/BeamWordCount.java
index 2d2021a..fe80521 100644
--- a/examples/beam/src/main/java/org/apache/nemo/examples/beam/BeamWordCount.java
+++ b/examples/beam/src/main/java/org/apache/nemo/examples/beam/BeamWordCount.java
@@ -153,6 +153,7 @@ public final class BeamWordCount {
/**
* By default, this example reads from a public dataset containing the text of King Lear. Set
* this option to choose a different input file or glob.
+ * @return the input file path.
*/
@Description("Path of the file to read from")
@Default.String("gs://apache-beam-samples/shakespeare/kinglear.txt")
@@ -160,7 +161,10 @@ public final class BeamWordCount {
void setInputFile(String value);
- /** Set this required option to specify where to write the output. */
+ /**
+ * Set this required option to specify where to write the output.
+ * @return the output path.
+ */
@Description("Path of the file to write to")
@Required
String getOutput();
diff --git a/examples/nexmark/pom.xml b/examples/nexmark/pom.xml
index 7ea5cce..9d6557b 100644
--- a/examples/nexmark/pom.xml
+++ b/examples/nexmark/pom.xml
@@ -17,13 +17,11 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nemo-examples</artifactId>
<groupId>org.apache.nemo</groupId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -64,7 +62,7 @@ under the License.
<transformers>
<!-- Required for using beam-hadoop: See https://stackoverflow.com/questions/44365545
-->
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
</transformers>
</configuration>
</execution>
diff --git a/examples/pom.xml b/examples/pom.xml
index 8c1bb46..5f58afc 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -17,12 +17,11 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nemo-project</artifactId>
<groupId>org.apache.nemo</groupId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 7cf23f0..7a7878c 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>nemo-examples</artifactId>
<groupId>org.apache.nemo</groupId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
@@ -93,7 +92,7 @@ under the License.
<transformers>
<!-- Required for using beam-hadoop: See https://stackoverflow.com/questions/44365545
-->
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
</transformers>
<filters>
<filter>
diff --git a/pom.xml b/pom.xml
index fc60b34..834ea5f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -17,19 +17,18 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
- <version>21</version>
+ <version>23</version>
</parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-project</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Nemo Project</name>
@@ -283,9 +282,10 @@ under the License.
<artifactId>maven-javadoc-plugin</artifactId>
<version>${maven.javadoc.version}</version>
<configuration>
- <excludePackageNames>*.org.apache.nemo.runtime.common.comm</excludePackageNames>
+ <excludePackageNames>*.org.apache.nemo.runtime.common.comm:*.compiler.frontend.spark.*</excludePackageNames>
<outputDirectory>docs/apidocs</outputDirectory>
<reportOutputDirectory>docs/apidocs</reportOutputDirectory>
+ <detectJavaApiLink>false</detectJavaApiLink>
</configuration>
<executions>
<execution>
diff --git a/runtime/common/pom.xml b/runtime/common/pom.xml
index c4ebf4d..75cfa14 100644
--- a/runtime/common/pom.xml
+++ b/runtime/common/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/runtime/driver/pom.xml b/runtime/driver/pom.xml
index db8fa1d..51c8f94 100644
--- a/runtime/driver/pom.xml
+++ b/runtime/driver/pom.xml
@@ -17,12 +17,11 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/runtime/executor/pom.xml b/runtime/executor/pom.xml
index bec9c04..2520a89 100644
--- a/runtime/executor/pom.xml
+++ b/runtime/executor/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
@@ -62,7 +61,7 @@ under the License.
<dependency>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime-master</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/LocalInputContext.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/LocalInputContext.java
index 70c55e1..570f0e6 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/LocalInputContext.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/LocalInputContext.java
@@ -32,7 +32,7 @@ import java.util.concurrent.LinkedBlockingQueue;
private boolean outputContextClosed = false;
/**
- * Creates a new local input context and connect it to {@param localOutputContext}.
+ * Creates a new local input context and connect it to {@code localOutputContext}.
* @param localOutputContext the local output context to which this local input context is connected
*/
public LocalInputContext(final LocalOutputContext localOutputContext) {
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/TransferOutputStream.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/TransferOutputStream.java
index d7dd978..dc5b9c2 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/TransferOutputStream.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/TransferOutputStream.java
@@ -28,7 +28,7 @@ public interface TransferOutputStream extends AutoCloseable {
/**
* Write an element into the output stream.
* @param element element to be sent
- * @param serializer serializer of {@param element}
+ * @param serializer serializer of {@code element}
*/
void writeElement(Object element, Serializer serializer);
diff --git a/runtime/master/pom.xml b/runtime/master/pom.xml
index 82c1ce4..5740e56 100644
--- a/runtime/master/pom.xml
+++ b/runtime/master/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
diff --git a/runtime/pom.xml b/runtime/pom.xml
index 063f1d0..71a9b45 100644
--- a/runtime/pom.xml
+++ b/runtime/pom.xml
@@ -17,12 +17,11 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nemo-project</artifactId>
<groupId>org.apache.nemo</groupId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/runtime/test/pom.xml b/runtime/test/pom.xml
index 6fd24bf..91342f3 100644
--- a/runtime/test/pom.xml
+++ b/runtime/test/pom.xml
@@ -17,14 +17,13 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nemo</groupId>
<artifactId>nemo-runtime</artifactId>
- <version>0.3-SNAPSHOT</version>
+ <version>0.4-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>