You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by je...@apache.org on 2018/06/15 11:35:59 UTC
[incubator-nemo] branch master updated: [NEMO-94] Clean up legacy
TODOs (#36)
This is an automated email from the ASF dual-hosted git repository.
jeongyoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
The following commit(s) were added to refs/heads/master by this push:
new ff8a120 [NEMO-94] Clean up legacy TODOs (#36)
ff8a120 is described below
commit ff8a1202c5cb1dc8fc2538bf2195cf4c8ddf6fbf
Author: Taegeon Um <ta...@gmail.com>
AuthorDate: Fri Jun 15 20:35:49 2018 +0900
[NEMO-94] Clean up legacy TODOs (#36)
JIRA: NEMO-94: Clean up legacy TODOs
Major changes:
* Remove TODOs of resolved issues
* Re-reference the issue number
Minor changes to note:
n/a
Tests for the changes:
n/a
Other comments:
n/a
resolves NEMO-94
---
common/src/main/java/edu/snu/nemo/common/StateMachine.java | 7 +++----
.../nemo/common/ir/executionproperty/ExecutionProperty.java | 2 +-
.../nemo/compiler/frontend/beam/transform/WindowTransform.java | 2 +-
.../nemo/compiler/frontend/spark/core/SparkFrontendUtils.java | 6 +++---
.../compiler/frontend/spark/transform/CollectTransform.java | 2 +-
.../compiler/frontend/spark/transform/ReduceTransform.java | 2 +-
.../pass/compiletime/reshaping/LoopOptimizations.java | 2 +-
.../test/java/edu/snu/nemo/examples/spark/SparkJavaITCase.java | 2 +-
pom.xml | 2 +-
.../nemo/runtime/common/message/local/LocalMessageContext.java | 8 ++++----
.../nemo/runtime/common/message/ncs/NcsMessageEnvironment.java | 1 -
runtime/common/src/main/proto/ControlMessage.proto | 2 +-
.../snu/nemo/runtime/executor/datatransfer/InputReader.java | 1 -
.../main/java/edu/snu/nemo/runtime/master/RuntimeMaster.java | 10 ++++------
.../edu/snu/nemo/runtime/master/resource/ContainerManager.java | 8 +++-----
.../snu/nemo/tests/runtime/common/plan/DAGConverterTest.java | 2 --
16 files changed, 25 insertions(+), 34 deletions(-)
diff --git a/common/src/main/java/edu/snu/nemo/common/StateMachine.java b/common/src/main/java/edu/snu/nemo/common/StateMachine.java
index 5d6c529..f98c622 100644
--- a/common/src/main/java/edu/snu/nemo/common/StateMachine.java
+++ b/common/src/main/java/edu/snu/nemo/common/StateMachine.java
@@ -16,16 +16,15 @@
package edu.snu.nemo.common;
import edu.snu.nemo.common.exception.IllegalStateTransitionException;
-
-import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.*;
+
/**
* A finite state machine that can be created with user defined states and transitions.
*/
-// TODO #171: Enable StateMachine with Handlers for State Transitions
-// TODO #306: StateMachine Visualization
+// TODO #17: StateMachine Visualization
public final class StateMachine {
private static final Logger LOG = LoggerFactory.getLogger(StateMachine.class.getName());
diff --git a/common/src/main/java/edu/snu/nemo/common/ir/executionproperty/ExecutionProperty.java b/common/src/main/java/edu/snu/nemo/common/ir/executionproperty/ExecutionProperty.java
index 9c7112f..45249ae 100644
--- a/common/src/main/java/edu/snu/nemo/common/ir/executionproperty/ExecutionProperty.java
+++ b/common/src/main/java/edu/snu/nemo/common/ir/executionproperty/ExecutionProperty.java
@@ -83,7 +83,7 @@ public abstract class ExecutionProperty<T> implements Serializable {
*/
public enum Key {
// Applies to IREdge
- DataCommunicationPattern, // TODO #492: modularizing runtime components for data communication pattern.
+ DataCommunicationPattern,
DataFlowModel,
DataStore,
MetricCollection,
diff --git a/compiler/frontend/beam/src/main/java/edu/snu/nemo/compiler/frontend/beam/transform/WindowTransform.java b/compiler/frontend/beam/src/main/java/edu/snu/nemo/compiler/frontend/beam/transform/WindowTransform.java
index 3860757..f686eef 100644
--- a/compiler/frontend/beam/src/main/java/edu/snu/nemo/compiler/frontend/beam/transform/WindowTransform.java
+++ b/compiler/frontend/beam/src/main/java/edu/snu/nemo/compiler/frontend/beam/transform/WindowTransform.java
@@ -44,7 +44,7 @@ public final class WindowTransform<T> implements Transform<T, T> {
@Override
public void onData(final T element) {
- // TODO #36: Actually assign windows
+ // TODO #1: Support Beam Streaming in Compiler.
outputCollector.emit(element);
}
diff --git a/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/core/SparkFrontendUtils.java b/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/core/SparkFrontendUtils.java
index 8ad00d3..6c4c395 100644
--- a/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/core/SparkFrontendUtils.java
+++ b/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/core/SparkFrontendUtils.java
@@ -92,7 +92,7 @@ public final class SparkFrontendUtils {
final DAGBuilder<IRVertex, IREdge> builder = new DAGBuilder<>(dag);
// save result in a temporary file
- // TODO #740: remove this part, and make it properly transfer with executor.
+ // TODO #16: Implement collection of data from executor to client
final String resultFile = System.getProperty("user.dir") + "/collectresult";
final IRVertex collectVertex = new OperatorVertex(new CollectTransform<>(resultFile));
@@ -108,12 +108,12 @@ public final class SparkFrontendUtils {
JobLauncher.launchDAG(builder.build());
// Retrieve result data from file.
- // TODO #740: remove this part, and make it properly transfer with executor.
+ // TODO #16: Implement collection of data from executor to client
try {
final List<T> result = new ArrayList<>();
Integer i = 0;
- // TODO #740: remove this part, and make it properly transfer with executor.
+ // TODO #16: Implement collection of data from executor to client
File file = new File(resultFile + i);
while (file.exists()) {
try (
diff --git a/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/transform/CollectTransform.java b/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/transform/CollectTransform.java
index 6cdd7ec..c1df3ba 100644
--- a/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/transform/CollectTransform.java
+++ b/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/transform/CollectTransform.java
@@ -50,7 +50,7 @@ public final class CollectTransform<T> implements Transform<T, T> {
@Override
public void onData(final T element) {
// Write result to a temporary file.
- // TODO #740: remove this part, and make it properly transfer with executor.
+ // TODO #16: Implement collection of data from executor to client
list.add(element);
}
diff --git a/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/transform/ReduceTransform.java b/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/transform/ReduceTransform.java
index 853d8e0..c7dace1 100644
--- a/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/transform/ReduceTransform.java
+++ b/compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/transform/ReduceTransform.java
@@ -99,7 +99,7 @@ public final class ReduceTransform<T> implements Transform<T, T> {
@Override
public void close() {
// Write result to a temporary file.
- // TODO #711: remove this part, and make it properly write to sink.
+ // TODO #16: Implement collection of data from executor to client.
try {
final Kryo kryo = new Kryo();
final Output output = new Output(new FileOutputStream(filename));
diff --git a/compiler/optimizer/src/main/java/edu/snu/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopOptimizations.java b/compiler/optimizer/src/main/java/edu/snu/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopOptimizations.java
index 380a301..79f605c 100644
--- a/compiler/optimizer/src/main/java/edu/snu/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopOptimizations.java
+++ b/compiler/optimizer/src/main/java/edu/snu/nemo/compiler/optimizer/pass/compiletime/reshaping/LoopOptimizations.java
@@ -231,7 +231,7 @@ public final class LoopOptimizations {
*/
private Boolean checkEqualityOfIntPredicates(final IntPredicate predicate1, final IntPredicate predicate2,
final Integer numberToTestUntil) {
- // TODO #223: strengthen this bit of code where terminationCondition has to be checked for convergence.
+ // TODO #11: Generalize Equality of Int Predicates for Loops.
if (numberToTestUntil.equals(0)) {
return predicate1.test(numberToTestUntil) == predicate2.test(numberToTestUntil);
} else if (predicate1.test(numberToTestUntil) != predicate2.test(numberToTestUntil)) {
diff --git a/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkJavaITCase.java b/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkJavaITCase.java
index 15d2dca..8aa1d48 100644
--- a/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkJavaITCase.java
+++ b/examples/spark/src/test/java/edu/snu/nemo/examples/spark/SparkJavaITCase.java
@@ -138,7 +138,7 @@ public final class SparkJavaITCase {
final String inputFileJson = fileBasePath + peopleJson;
final String inputFileTxt = fileBasePath + peopleTxt;
- // TODO#412: Enable this after implementation of RDDs.
+ // TODO#12: Frontend support for Scala Spark.
// JobLauncher.main(builder
// .addJobId(JavaSparkSQLExample.class.getSimpleName() + "_test")
// .addUserMain(JavaSparkSQLExample.class.getCanonicalName())
diff --git a/pom.xml b/pom.xml
index 54a57f6..95939f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,7 +35,7 @@ limitations under the License.
<reef.version>0.17.0-SNAPSHOT</reef.version>
<protobuf.version>2.5.0</protobuf.version>
<hadoop.version>2.7.2</hadoop.version>
- <log4j.configuration>file://log4j.properties</log4j.configuration> <!-- TODO: issue#645 -->
+ <log4j.configuration>file://log4j.properties</log4j.configuration>
<netty.version>4.1.16.Final</netty.version>
<guava.version>19.0</guava.version>
<grpc.version>1.7.0</grpc.version>
diff --git a/runtime/common/src/main/java/edu/snu/nemo/runtime/common/message/local/LocalMessageContext.java b/runtime/common/src/main/java/edu/snu/nemo/runtime/common/message/local/LocalMessageContext.java
index 8f4f021..64ce6c9 100644
--- a/runtime/common/src/main/java/edu/snu/nemo/runtime/common/message/local/LocalMessageContext.java
+++ b/runtime/common/src/main/java/edu/snu/nemo/runtime/common/message/local/LocalMessageContext.java
@@ -28,8 +28,8 @@ final class LocalMessageContext implements MessageContext {
private Object replyMessage;
/**
- * TODO #119.
- * @param senderId TODO #119.
+ * TODO #10: Handle Method Javadocs Requirements for Checkstyle Warnings.
+ * @param senderId TODO #10: Handle Method Javadocs Requirements for Checkstyle Warnings.
*/
LocalMessageContext(final String senderId) {
this.senderId = senderId;
@@ -45,8 +45,8 @@ final class LocalMessageContext implements MessageContext {
}
/**
- * TODO #119.
- * @return TODO #119.
+ * TODO #10: Handle Method Javadocs Requirements for Checkstyle Warnings.
+ * @return TODO #10: Handle Method Javadocs Requirements for Checkstyle Warnings.
*/
public Optional<Object> getReplyMessage() {
return Optional.ofNullable(replyMessage);
diff --git a/runtime/common/src/main/java/edu/snu/nemo/runtime/common/message/ncs/NcsMessageEnvironment.java b/runtime/common/src/main/java/edu/snu/nemo/runtime/common/message/ncs/NcsMessageEnvironment.java
index 296a19e..50211b3 100644
--- a/runtime/common/src/main/java/edu/snu/nemo/runtime/common/message/ncs/NcsMessageEnvironment.java
+++ b/runtime/common/src/main/java/edu/snu/nemo/runtime/common/message/ncs/NcsMessageEnvironment.java
@@ -182,7 +182,6 @@ public final class NcsMessageEnvironment implements MessageEnvironment {
*
* Not sure these variable names are conventionally used in RPC frameworks...
* Let's revisit them when we work on
- * TODO #206: Rethink/Refactor NCS as our RPC stack
*/
enum MessageType {
Send,
diff --git a/runtime/common/src/main/proto/ControlMessage.proto b/runtime/common/src/main/proto/ControlMessage.proto
index 343b8e5..cc883dc 100644
--- a/runtime/common/src/main/proto/ControlMessage.proto
+++ b/runtime/common/src/main/proto/ControlMessage.proto
@@ -81,7 +81,7 @@ message BlockStateChangedMsg {
}
message DataSizeMetricMsg {
- // TODO #511: Refactor metric aggregation for (general) run-rime optimization.
+ // TODO #96: Modularize DataSkewPolicy to use MetricVertex and BarrierVertex.
repeated PartitionSizeEntry partitionSize = 1;
required string blockId = 2;
required string srcIRVertexId = 3;
diff --git a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/datatransfer/InputReader.java b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/datatransfer/InputReader.java
index c275e04..9129826 100644
--- a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/datatransfer/InputReader.java
+++ b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/datatransfer/InputReader.java
@@ -40,7 +40,6 @@ import java.util.stream.StreamSupport;
/**
* Represents the input data transfer to a task.
- * TODO #492: Modularize the data communication pattern.
*/
public final class InputReader extends DataTransfer {
private final int dstTaskIndex;
diff --git a/runtime/master/src/main/java/edu/snu/nemo/runtime/master/RuntimeMaster.java b/runtime/master/src/main/java/edu/snu/nemo/runtime/master/RuntimeMaster.java
index f3d75fc..a424ecc 100644
--- a/runtime/master/src/main/java/edu/snu/nemo/runtime/master/RuntimeMaster.java
+++ b/runtime/master/src/main/java/edu/snu/nemo/runtime/master/RuntimeMaster.java
@@ -291,7 +291,7 @@ public final class RuntimeMaster {
throw new RuntimeException(exception);
case DataSizeMetric:
final ControlMessage.DataSizeMetricMsg dataSizeMetricMsg = message.getDataSizeMetricMsg();
- // TODO #511: Refactor metric aggregation for (general) run-rime optimization.
+ // TODO #96: Modularize DataSkewPolicy to use MetricVertex and BarrierVertex.
accumulateBarrierMetric(dataSizeMetricMsg.getPartitionSizeList(),
dataSizeMetricMsg.getSrcIRVertexId(), dataSizeMetricMsg.getBlockId());
break;
@@ -309,8 +309,8 @@ public final class RuntimeMaster {
/**
* Accumulates the metric data for a barrier vertex.
- * TODO #511: Refactor metric aggregation for (general) run-rime optimization.
- * TODO #513: Replace MetricCollectionBarrierVertex with a Customizable IRVertex.
+ * TODO #96: Modularize DataSkewPolicy to use MetricVertex and BarrierVertex.
+ * TODO #98: Implement MetricVertex that collect metric used for dynamic optimization.
*
* @param partitionSizeInfo the size of partitions in a block to accumulate.
* @param srcVertexId the ID of the source vertex.
@@ -344,7 +344,6 @@ public final class RuntimeMaster {
}
}
- // TODO #164: Cleanup Protobuf Usage
private static TaskState.State convertTaskState(final ControlMessage.TaskStateFromExecutor state) {
switch (state) {
case READY:
@@ -379,9 +378,8 @@ public final class RuntimeMaster {
/**
* Schedules a periodic DAG logging thread.
- * TODO #58: Web UI (Real-time visualization)
* @param jobStateManager for the job the DAG should be logged.
- *
+ * TODO #20: RESTful APIs to Access Job State and Metric.
* @return the scheduled executor service.
*/
private ScheduledExecutorService scheduleDagLogging(final JobStateManager jobStateManager) {
diff --git a/runtime/master/src/main/java/edu/snu/nemo/runtime/master/resource/ContainerManager.java b/runtime/master/src/main/java/edu/snu/nemo/runtime/master/resource/ContainerManager.java
index 05a50da..32d4bd9 100644
--- a/runtime/master/src/main/java/edu/snu/nemo/runtime/master/resource/ContainerManager.java
+++ b/runtime/master/src/main/java/edu/snu/nemo/runtime/master/resource/ContainerManager.java
@@ -25,6 +25,9 @@ import org.apache.reef.driver.evaluator.AllocatedEvaluator;
import org.apache.reef.driver.evaluator.EvaluatorRequest;
import org.apache.reef.driver.evaluator.EvaluatorRequestor;
import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.annotations.Parameter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.annotation.concurrent.NotThreadSafe;
import javax.inject.Inject;
@@ -33,10 +36,6 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import org.apache.reef.tang.annotations.Parameter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* (WARNING) This class is not thread-safe.
* Only a single thread should use the methods of this class.
@@ -46,7 +45,6 @@ import org.slf4j.LoggerFactory;
* Serves as a single point of container management in Runtime.
* We define a unit of resource a container (an evaluator in REEF), and launch a single executor on each container.
*/
-// TODO #60: Specify Types in Requesting Containers
// We need an overall cleanup of this class after #60 is resolved.
@DriverSide
@NotThreadSafe
diff --git a/tests/src/test/java/edu/snu/nemo/tests/runtime/common/plan/DAGConverterTest.java b/tests/src/test/java/edu/snu/nemo/tests/runtime/common/plan/DAGConverterTest.java
index 22d875e..671fdb0 100644
--- a/tests/src/test/java/edu/snu/nemo/tests/runtime/common/plan/DAGConverterTest.java
+++ b/tests/src/test/java/edu/snu/nemo/tests/runtime/common/plan/DAGConverterTest.java
@@ -151,7 +151,6 @@ public final class DAGConverterTest {
irDAGBuilder.addVertex(v6);
irDAGBuilder.addVertex(v8);
- // TODO #13: Implement Join Node
// irDAGBuilder.addVertex(v7);
final IREdge e1 = new IREdge(DataCommunicationPatternProperty.Value.OneToOne, v1, v2);
@@ -221,7 +220,6 @@ public final class DAGConverterTest {
final Stage stage4 = sortedLogicalDAG.get(3);
final Stage stage5 = sortedLogicalDAG.get(3);
- // TODO #148: Optimize Stage Partitioner Algorithm in NemoBackend
// The following asserts depend on how stage partitioning is defined; test must be rewritten accordingly.
// assertEquals(logicalDAG.getVertices().size(), 5);
// assertEquals(logicalDAG.getIncomingEdgesOf(stage1).size(), 0);
--
To stop receiving notification emails like this one, please contact
jeongyoon@apache.org.