You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by ja...@apache.org on 2020/01/01 13:54:15 UTC
[incubator-nemo] branch master updated: [NEMO-429] SWPP TEAM 5 code
smell fix (#261)
This is an automated email from the ASF dual-hosted git repository.
jangho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
The following commit(s) were added to refs/heads/master by this push:
new fc6d222 [NEMO-429] SWPP TEAM 5 code smell fix (#261)
fc6d222 is described below
commit fc6d222d4a1f1300b60e49606ca3d6e483d2e641
Author: Haesoo Kim <ha...@gmail.com>
AuthorDate: Wed Jan 1 22:54:09 2020 +0900
[NEMO-429] SWPP TEAM 5 code smell fix (#261)
JIRA: [NEMO-429: SWPP TEAM 5 code smell fix](https://issues.apache.org/jira/projects/NEMO/issues/NEMO-429)
**Major changes:**
- Fixing code smells
Co-authored-by: Kwak taewon <ty...@gmail.com>
Co-authored-by: ttoru96 <44...@users.noreply.github.com>
Co-authored-by: Jangho Seo <ja...@jangho.io>
---
common/src/test/java/org/apache/nemo/common/DAGTest.java | 2 +-
.../org/apache/nemo/compiler/frontend/spark/sql/SparkSession.java | 4 +++-
.../nemo/compiler/frontend/spark/transform/CollectTransform.java | 3 ++-
.../java/org/apache/nemo/runtime/executor/TaskStateManager.java | 6 ++++--
4 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/common/src/test/java/org/apache/nemo/common/DAGTest.java b/common/src/test/java/org/apache/nemo/common/DAGTest.java
index fe48514..3fba1aa 100644
--- a/common/src/test/java/org/apache/nemo/common/DAGTest.java
+++ b/common/src/test/java/org/apache/nemo/common/DAGTest.java
@@ -83,7 +83,7 @@ public final class DAGTest {
assertEquals(1, dag.getChildren("2").size());
assertEquals(1, dag.getParents("5").size());
assertEquals(0, dag.getChildren("5").size());
- assertEquals(new IntegerVertex(1),dag.getVertexById("1"));
+ assertEquals(new IntegerVertex(1), dag.getVertexById("1"));
}
@Test
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/SparkSession.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/SparkSession.java
index 8aceb81..02fa3c6 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/SparkSession.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/sql/SparkSession.java
@@ -33,6 +33,7 @@ import org.apache.spark.sql.Encoder;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.sources.BaseRelation;
import org.apache.spark.sql.types.StructType;
+import scala.Serializable;
import scala.Tuple2;
import javax.naming.OperationNotSupportedException;
@@ -46,7 +47,8 @@ import java.util.stream.Stream;
/**
* A simple version of the Spark session, containing SparkContext that contains SparkConf.
*/
-public final class SparkSession extends org.apache.spark.sql.SparkSession implements NemoSparkUserFacingClass {
+public final class SparkSession extends org.apache.spark.sql.SparkSession
+ implements NemoSparkUserFacingClass, Serializable {
private final LinkedHashMap<String, Object[]> datasetCommandsList;
private final Map<String, String> initialConf;
private final AtomicBoolean isUserTriggered;
diff --git a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/CollectTransform.java b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/CollectTransform.java
index 08d8af6..f8dfec9 100644
--- a/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/CollectTransform.java
+++ b/compiler/frontend/spark/src/main/java/org/apache/nemo/compiler/frontend/spark/transform/CollectTransform.java
@@ -22,6 +22,7 @@ import org.apache.commons.lang3.SerializationUtils;
import org.apache.nemo.common.ir.OutputCollector;
import org.apache.nemo.common.ir.vertex.transform.NoWatermarkEmitTransform;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Base64;
@@ -31,7 +32,7 @@ import java.util.Base64;
*
* @param <T> type of data to collect.
*/
-public final class CollectTransform<T> extends NoWatermarkEmitTransform<T, T> {
+public final class CollectTransform<T> extends NoWatermarkEmitTransform<T, T> implements Serializable {
private final ArrayList<T> list;
private Context ctxt;
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/TaskStateManager.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/TaskStateManager.java
index 5a3eee5..9ed62a2 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/TaskStateManager.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/TaskStateManager.java
@@ -48,6 +48,8 @@ public final class TaskStateManager {
private final MetricMessageSender metricMessageSender;
private final PersistentConnectionToMasterMap persistentConnectionToMasterMap;
+ private static final String METRIC = "TaskMetric";
+
public TaskStateManager(final Task task,
final String executorId,
final PersistentConnectionToMasterMap persistentConnectionToMasterMap,
@@ -58,9 +60,9 @@ public final class TaskStateManager {
this.persistentConnectionToMasterMap = persistentConnectionToMasterMap;
this.metricMessageSender = metricMessageSender;
- metricMessageSender.send("TaskMetric", taskId,
+ metricMessageSender.send(METRIC, taskId,
"containerId", SerializationUtils.serialize(executorId));
- metricMessageSender.send("TaskMetric", taskId,
+ metricMessageSender.send(METRIC, taskId,
"scheduleAttempt", SerializationUtils.serialize(attemptIdx));
}