You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by wo...@apache.org on 2020/12/01 19:06:35 UTC
[incubator-nemo] 01/01: ready for the release
This is an automated email from the ASF dual-hosted git repository.
wonook pushed a commit to annotated tag nemo-project-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
commit 1dc556d5af352119ad16c3e20a5c9a80faffe532
Author: Won Wook SONG <ws...@gmail.com>
AuthorDate: Wed Dec 2 01:12:15 2020 +0900
ready for the release
---
.../java/org/apache/nemo/common/dag/DAGBuilder.java | 6 +++---
.../executionproperty/SubPartitionSetProperty.java | 2 +-
.../apache/nemo/common/ir/vertex/LoopVertex.java | 2 ++
.../beam/transform/AbstractDoFnTransform.java | 2 ++
.../frontend/beam/transform/DoFnTransform.java | 2 ++
.../frontend/beam/transform/GBKTransform.java | 12 ++++++++----
.../transform/InMemoryTimerInternalsFactory.java | 21 ++++++++++++++++++---
.../beam/transform/PushBackDoFnTransform.java | 2 ++
.../MapReduceDisaggregationOptimization.java | 1 -
.../reshaping/SamplingTaskSizingPass.java | 6 +++---
.../nemo/examples/beam/AlternatingLeastSquare.java | 2 --
.../apache/nemo/examples/beam/BeamWordCount.java | 6 +++++-
pom.xml | 5 +++--
.../executor/transfer/LocalInputContext.java | 2 +-
.../executor/transfer/TransferOutputStream.java | 2 +-
15 files changed, 51 insertions(+), 22 deletions(-)
diff --git a/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java b/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java
index 7ec85b7..85d7ce5 100644
--- a/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java
+++ b/common/src/main/java/org/apache/nemo/common/dag/DAGBuilder.java
@@ -185,7 +185,7 @@ public final class DAGBuilder<V extends Vertex, E extends Edge<V>> implements Se
* should be used when there are changes in vertices before / after SplitterVertex.
*
* CAUTION: TaskSizeSplitterVertex must only appear in IRDAG.
- * {@param originalEdge} and {@param edgeToInsert} should have same source and destination.
+ * {@code originalEdge} and {@code edgeToInsert} should have same source and destination.
*
* Relation to be Erased: originalEdge - internalEdge
* Relation to insert: edgeToInsert - newInternalEdge
@@ -250,8 +250,8 @@ public final class DAGBuilder<V extends Vertex, E extends Edge<V>> implements Se
* without replacing existing mapping relationships.
* The changes which invokes this method should not be caused by SplitterVertex itself. Therefore, this method
* should be used when there are changes in vertices before / after SplitterVertex.
- * Since {@param edgeToInsert} should also have a mapping relationship to originalVertices of SplitterVertex,
- * we give {@param edgeToReference} together to copy the mapping information. Therefore, these two parameters must
+ * Since {@code edgeToInsert} should also have a mapping relationship to originalVertices of SplitterVertex,
+ * we give {@code edgeToReference} together to copy the mapping information. Therefore, these two parameters must
* have at least one common source or destination.
*
* Relation to reference: edgeToReference - internalEdge
diff --git a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/SubPartitionSetProperty.java b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/SubPartitionSetProperty.java
index 4395719..655b46e 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/SubPartitionSetProperty.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/edge/executionproperty/SubPartitionSetProperty.java
@@ -25,7 +25,7 @@ import org.apache.nemo.common.ir.executionproperty.EdgeExecutionProperty;
import java.util.ArrayList;
/**
- * This property decides the hash partition set of sampled & optimized tasks in Dynamic Task Sizing Policy.
+ * This property decides the hash partition set of sampled and optimized tasks in Dynamic Task Sizing Policy.
* <p>
* Adopting this property requires changing other properties as well.
* Such execution properties include:
diff --git a/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java b/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java
index 011dcc1..e68fc77 100644
--- a/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java
+++ b/common/src/main/java/org/apache/nemo/common/ir/vertex/LoopVertex.java
@@ -164,6 +164,7 @@ public class LoopVertex extends IRVertex {
/**
* Getter method for edgeWithLoopToEdgeWithInternalVertex.
+ * @return the map of edge with loop to the edge with the internal vertex.
*/
public Map<IREdge, IREdge> getEdgeWithLoopToEdgeWithInternalVertex() {
return this.edgeWithLoopToEdgeWithInternalVertex;
@@ -171,6 +172,7 @@ public class LoopVertex extends IRVertex {
/**
* Getter method for edgeWithInternalVertexToEdgeWithLoop.
+ * @return the map of edge with internal vertex to the edge with the loop.
*/
public Map<IREdge, IREdge> getEdgeWithInternalVertexToEdgeWithLoop() {
return this.edgeWithInternalVertexToEdgeWithLoop;
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
index 79b31c6..59a3b1b 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
@@ -98,6 +98,8 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
* @param sideInputs side inputs
* @param options pipeline options
* @param displayData display data.
+ * @param doFnSchemaInformation doFn schema information.
+ * @param sideInputMapping side input mapping.
*/
public AbstractDoFnTransform(final DoFn<InterT, OutputT> doFn,
final Coder<InputT> inputCoder,
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
index c6b8e3b..1d29402 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
@@ -56,6 +56,8 @@ public final class DoFnTransform<InputT, OutputT> extends AbstractDoFnTransform<
* @param windowingStrategy windowing strategy
* @param options pipeline options
* @param displayData display data.
+ * @param doFnSchemaInformation doFn schema information.
+ * @param sideInputMapping side input mapping.
*/
public DoFnTransform(final DoFn<InputT, OutputT> doFn,
final Coder<InputT> inputCoder,
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GBKTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GBKTransform.java
index 1bf6cb8..4aa366f 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GBKTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GBKTransform.java
@@ -142,7 +142,7 @@ public final class GBKTransform<K, InputT, OutputT>
}
/**
- * Trigger timers that need to be fired at {@param watermark} and emit output watermark.
+ * Trigger timers that need to be fired at {@code watermark} and emit output watermark.
* @param watermark watermark
*/
@Override
@@ -263,7 +263,11 @@ public final class GBKTransform<K, InputT, OutputT>
}
}
- /** Accessor for isPartialCombining. */
+ /**
+ * Accessor for isPartialCombining.
+ *
+ * @return whether it is partial combining.
+ */
public boolean getIsPartialCombining() {
return isPartialCombining;
}
@@ -277,7 +281,7 @@ public final class GBKTransform<K, InputT, OutputT>
this.oc = oc;
}
- /** Emit output. If {@param output} is emitted on-time, save its timestamp in the output watermark map. */
+ /** Emit output. If {@code output} is emitted on-time, save its timestamp in the output watermark map. */
@Override
public final void emit(final WindowedValue<KV<K, OutputT>> output) {
// The watermark advances only in ON_TIME
@@ -301,7 +305,7 @@ public final class GBKTransform<K, InputT, OutputT>
oc.emitWatermark(watermark);
}
- /** Emit output value to {@param dstVertexId}. */
+ /** Emit output value to {@code dstVertexId}. */
@Override
public final <T> void emit(final String dstVertexId, final T output) {
oc.emit(dstVertexId, output);
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/InMemoryTimerInternalsFactory.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/InMemoryTimerInternalsFactory.java
index 0072463..ed1bb1e 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/InMemoryTimerInternalsFactory.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/InMemoryTimerInternalsFactory.java
@@ -50,7 +50,13 @@ public final class InMemoryTimerInternalsFactory<K> implements TimerInternalsFac
}
}
- /** Remove the next eligible timer in {@param timeDomain}. */
+ /**
+ * Remove the next eligible timer in {@code timeDomain}.
+ *
+ * @param timerInternal the timer internals.
+ * @param timeDomain the time domain.
+ * @return the timer data.
+ */
public TimerInternals.TimerData pollTimer(final InMemoryTimerInternals timerInternal, final TimeDomain timeDomain) {
switch (timeDomain) {
case EVENT_TIME :
@@ -64,12 +70,21 @@ public final class InMemoryTimerInternalsFactory<K> implements TimerInternalsFac
}
}
- /** Accessor for timerInternalsMap. */
+ /**
+ * Accessor for timerInternalsMap.
+ *
+ * @return the timer internals map.
+ */
public Map<K, InMemoryTimerInternals> getTimerInternalsMap() {
return timerInternalsMap;
}
- /** Helper method to check if {@param timerInternals} doesn't have any timers left. */
+ /**
+ * Helper method to check if {@code timerInternals} doesn't have any timers left.
+ *
+ * @param timerInternal the in-memory timer internals.
+ * @return whether it is empty or not.
+ */
public boolean isEmpty(final InMemoryTimerInternals timerInternal) {
for (final TimeDomain domain : TimeDomain.values()) {
if (timerInternal.getNextTimer(domain) != null) {
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
index 15a0873..7df9bff 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
@@ -60,6 +60,8 @@ public final class PushBackDoFnTransform<InputT, OutputT> extends AbstractDoFnTr
* @param sideInputs side inputs
* @param options pipeline options
* @param displayData display data.
+ * @param doFnSchemaInformation DoFn schema information.
+ * @param sideInputMapping side input mapping.
*/
public PushBackDoFnTransform(final DoFn<InputT, OutputT> doFn,
final Coder<InputT> inputCoder,
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java
index 14220d7..48008d1 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/examples/MapReduceDisaggregationOptimization.java
@@ -47,7 +47,6 @@ public final class MapReduceDisaggregationOptimization {
* Main function of the example MR program.
*
* @param args arguments.
- * @throws Exception Exceptions on the way.
*/
public static void main(final String[] args) {
final IRVertex source = new EmptyComponents.EmptySourceVertex<>("Source");
diff --git a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingTaskSizingPass.java b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingTaskSizingPass.java
index c35941b..dd5fe1f 100644
--- a/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingTaskSizingPass.java
+++ b/compiler/optimizer/src/main/java/org/apache/nemo/compiler/optimizer/pass/compiletime/reshaping/SamplingTaskSizingPass.java
@@ -36,17 +36,17 @@ import java.util.stream.Collectors;
/**
* Compiler pass for dynamic task size optimization. Happens only when the edge property is SHUFFLE.
- * If (size of given job) >= 1GB: enable dynamic task sizing optimization.
+ * If (size of given job) is greater than or equal to 1GB: enable dynamic task sizing optimization.
* else: break.
*
*
- * @Attributes
+ * With attributes
* PARTITIONER_PROPERTY_FOR_SMALL_JOB: PartitionerProperty for jobs in range of [1GB, 10GB) size.
* PARTITIONER_PROPERTY_FOR_MEDIUM_JOB: PartitionerProperty for jobs in range of [10GB, 100GB) size.
* PARTITIONER_PROPERTY_FOR_BIG_JOB: PartitionerProperty for jobs in range of [100GB, - ) size(No upper limit).
*
* source stage - shuffle edge - current stage - next stage
- * -> source stage - [curr stage - signal vertex] - next stage
+ * - source stage - [curr stage - signal vertex] - next stage
* where [] is a splitter vertex
*/
@Annotates({EnableDynamicTaskSizingProperty.class, PartitionerProperty.class, SubPartitionSetProperty.class,
diff --git a/examples/beam/src/main/java/org/apache/nemo/examples/beam/AlternatingLeastSquare.java b/examples/beam/src/main/java/org/apache/nemo/examples/beam/AlternatingLeastSquare.java
index d87d546..3d05ce2 100644
--- a/examples/beam/src/main/java/org/apache/nemo/examples/beam/AlternatingLeastSquare.java
+++ b/examples/beam/src/main/java/org/apache/nemo/examples/beam/AlternatingLeastSquare.java
@@ -326,7 +326,6 @@ public final class AlternatingLeastSquare {
* ProcessElement method for BEAM.
*
* @param c ProcessContext.
- * @throws Exception Exception on the way.
*/
@ProcessElement
public void processElement(final ProcessContext c) {
@@ -354,7 +353,6 @@ public final class AlternatingLeastSquare {
* Main function for the ALS BEAM program.
*
* @param args arguments.
- * @throws ClassNotFoundException exception.
*/
public static void main(final String[] args) {
final Long start = System.currentTimeMillis();
diff --git a/examples/beam/src/main/java/org/apache/nemo/examples/beam/BeamWordCount.java b/examples/beam/src/main/java/org/apache/nemo/examples/beam/BeamWordCount.java
index 2d2021a..fe80521 100644
--- a/examples/beam/src/main/java/org/apache/nemo/examples/beam/BeamWordCount.java
+++ b/examples/beam/src/main/java/org/apache/nemo/examples/beam/BeamWordCount.java
@@ -153,6 +153,7 @@ public final class BeamWordCount {
/**
* By default, this example reads from a public dataset containing the text of King Lear. Set
* this option to choose a different input file or glob.
+ * @return the input file path.
*/
@Description("Path of the file to read from")
@Default.String("gs://apache-beam-samples/shakespeare/kinglear.txt")
@@ -160,7 +161,10 @@ public final class BeamWordCount {
void setInputFile(String value);
- /** Set this required option to specify where to write the output. */
+ /**
+ * Set this required option to specify where to write the output.
+ * @return the output path.
+ */
@Description("Path of the file to write to")
@Required
String getOutput();
diff --git a/pom.xml b/pom.xml
index fc60b34..47d13fe 100644
--- a/pom.xml
+++ b/pom.xml
@@ -24,7 +24,7 @@ under the License.
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
- <version>21</version>
+ <version>23</version>
</parent>
<groupId>org.apache.nemo</groupId>
@@ -283,9 +283,10 @@ under the License.
<artifactId>maven-javadoc-plugin</artifactId>
<version>${maven.javadoc.version}</version>
<configuration>
- <excludePackageNames>*.org.apache.nemo.runtime.common.comm</excludePackageNames>
+ <excludePackageNames>*.org.apache.nemo.runtime.common.comm:*.compiler.frontend.spark.*</excludePackageNames>
<outputDirectory>docs/apidocs</outputDirectory>
<reportOutputDirectory>docs/apidocs</reportOutputDirectory>
+ <detectJavaApiLink>false</detectJavaApiLink>
</configuration>
<executions>
<execution>
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/LocalInputContext.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/LocalInputContext.java
index 70c55e1..570f0e6 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/LocalInputContext.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/LocalInputContext.java
@@ -32,7 +32,7 @@ import java.util.concurrent.LinkedBlockingQueue;
private boolean outputContextClosed = false;
/**
- * Creates a new local input context and connect it to {@param localOutputContext}.
+ * Creates a new local input context and connect it to {@code localOutputContext}.
* @param localOutputContext the local output context to which this local input context is connected
*/
public LocalInputContext(final LocalOutputContext localOutputContext) {
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/TransferOutputStream.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/TransferOutputStream.java
index d7dd978..dc5b9c2 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/TransferOutputStream.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/transfer/TransferOutputStream.java
@@ -28,7 +28,7 @@ public interface TransferOutputStream extends AutoCloseable {
/**
* Write an element into the output stream.
* @param element element to be sent
- * @param serializer serializer of {@param element}
+ * @param serializer serializer of {@code element}
*/
void writeElement(Object element, Serializer serializer);