You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2015/05/05 01:11:30 UTC
hadoop git commit: YARN-3551. Consolidate data model change according
to the backend implementation (Zhijie Shen via sale)
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 0b1e184cc -> 557a3950b
YARN-3551. Consolidate data model change according to the backend implementation (Zhijie Shen via sale)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/557a3950
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/557a3950
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/557a3950
Branch: refs/heads/YARN-2928
Commit: 557a3950bddc837469244835f5577899080115d8
Parents: 0b1e184
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon May 4 16:10:20 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Mon May 4 16:10:20 2015 -0700
----------------------------------------------------------------------
.../mapred/TimelineServicePerformanceV2.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/timelineservice/TimelineEntity.java | 16 +--
.../records/timelineservice/TimelineMetric.java | 131 +++++++++----------
.../TestTimelineServiceRecords.java | 81 +++++++++---
.../monitor/ContainersMonitorImpl.java | 5 +-
.../TestTimelineServiceClientIntegration.java | 6 +
7 files changed, 146 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/557a3950/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
index de46617..1c2e28d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
@@ -261,7 +261,7 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
// add a metric
TimelineMetric metric = new TimelineMetric();
metric.setId("foo_metric");
- metric.setSingleData(123456789L);
+ metric.addValue(System.currentTimeMillis(), 123456789L);
entity.addMetric(metric);
// add a config
entity.addConfig("foo", "bar");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/557a3950/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3957b24..8cafca6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -58,6 +58,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3431. Sub resources of timeline entity needs to be passed to a separate
endpoint. (Zhijie Shen via junping_du)
+ YARN-3551. Consolidate data model change according to the backend
+ implementation (Zhijie Shen via sjlee)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/557a3950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 6cab753..3be7f52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -80,7 +80,7 @@ public class TimelineEntity {
private TimelineEntity real;
private Identifier identifier;
private HashMap<String, Object> info = new HashMap<>();
- private HashMap<String, Object> configs = new HashMap<>();
+ private HashMap<String, String> configs = new HashMap<>();
private Set<TimelineMetric> metrics = new HashSet<>();
private Set<TimelineEvent> events = new HashSet<>();
private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>();
@@ -213,7 +213,7 @@ public class TimelineEntity {
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "configs")
- public HashMap<String, Object> getConfigsJAXB() {
+ public HashMap<String, String> getConfigsJAXB() {
if (real == null) {
return configs;
} else {
@@ -221,7 +221,7 @@ public class TimelineEntity {
}
}
- public Map<String, Object> getConfigs() {
+ public Map<String, String> getConfigs() {
if (real == null) {
return configs;
} else {
@@ -229,19 +229,19 @@ public class TimelineEntity {
}
}
- public void setConfigs(Map<String, Object> configs) {
+ public void setConfigs(Map<String, String> configs) {
if (real == null) {
if (configs != null && !(configs instanceof HashMap)) {
- this.configs = new HashMap<String, Object>(configs);
+ this.configs = new HashMap<String, String>(configs);
} else {
- this.configs = (HashMap<String, Object>) configs;
+ this.configs = (HashMap<String, String>) configs;
}
} else {
real.setConfigs(configs);
}
}
- public void addConfigs(Map<String, Object> configs) {
+ public void addConfigs(Map<String, String> configs) {
if (real == null) {
this.configs.putAll(configs);
} else {
@@ -249,7 +249,7 @@ public class TimelineEntity {
}
}
- public void addConfig(String key, Object value) {
+ public void addConfig(String key, String value) {
if (real == null) {
configs.put(key, value);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/557a3950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index 57babbf..c897d39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -24,113 +24,104 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
+import java.util.Comparator;
import java.util.Map;
+import java.util.TreeMap;
@XmlRootElement(name = "metric")
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class TimelineMetric {
- private String id;
- private HashMap<String, Object> info = new HashMap<>();
- private Object singleData;
- private HashMap<Long, Object> timeSeries = new LinkedHashMap<>();
- private long startTime;
- private long endTime;
-
- public TimelineMetric() {
+ public static enum Type {
+ SINGLE_VALUE,
+ TIME_SERIES
}
- @XmlElement(name = "id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
+ private Type type;
+ private String id;
+ private Comparator<Long> reverseComparator = new Comparator<Long>() {
+ @Override
+ public int compare(Long l1, Long l2) {
+ return -l1.compareTo(l2);
+ }
+ };
+ private TreeMap<Long, Number> values = new TreeMap<>(reverseComparator);
- // required by JAXB
- @InterfaceAudience.Private
- @XmlElement(name = "info")
- public HashMap<String, Object> getInfoJAXB() {
- return info;
+ public TimelineMetric() {
+ this(Type.SINGLE_VALUE);
}
- public Map<String, Object> getInfo() {
- return info;
+ public TimelineMetric(Type type) {
+ this.type = type;
}
- public void setInfo(Map<String, Object> info) {
- if (info != null && !(info instanceof HashMap)) {
- this.info = new HashMap<String, Object>(info);
- } else {
- this.info = (HashMap<String, Object>) info;
- }
- }
- public void addInfo(Map<String, Object> info) {
- this.info.putAll(info);
+ @XmlElement(name = "type")
+ public Type getType() {
+ return type;
}
- public void addInfo(String key, Object value) {
- info.put(key, value);
+ public void setType(Type type) {
+ this.type = type;
}
- @XmlElement(name = "data")
- public Object getSingleData() {
- return singleData;
+ @XmlElement(name = "id")
+ public String getId() {
+ return id;
}
- public void setSingleData(Object singleData) {
- this.singleData = singleData;
+ public void setId(String id) {
+ this.id = id;
}
// required by JAXB
@InterfaceAudience.Private
- @XmlElement(name = "timeseries")
- public HashMap<Long, Object> getTimeSeriesJAXB() {
- return timeSeries;
+ @XmlElement(name = "values")
+ public TreeMap<Long, Number> getValuesJAXB() {
+ return values;
}
- public Map<Long, Object> getTimeSeries() {
- return timeSeries;
+ public Map<Long, Number> getValues() {
+ return values;
}
- public void setTimeSeries(Map<Long, Object> timeSeries) {
- if (timeSeries != null && !(timeSeries instanceof LinkedHashMap)) {
- this.timeSeries = new LinkedHashMap<Long, Object>(timeSeries);
+ public void setValues(Map<Long, Number> values) {
+ if (type == Type.SINGLE_VALUE) {
+ overwrite(values);
} else {
- this.timeSeries = (LinkedHashMap<Long, Object>) timeSeries;
+ if (values != null) {
+ this.values = new TreeMap<Long, Number>(reverseComparator);
+ this.values.putAll(values);
+ } else {
+ this.values = null;
+ }
}
}
- public void addTimeSeries(Map<Long, Object> timeSeries) {
- this.timeSeries.putAll(timeSeries);
- }
-
- public void addTimeSeriesData(long timestamp, Object value) {
- timeSeries.put(timestamp, value);
- }
-
- @XmlElement(name = "starttime")
- public long getStartTime() {
- return startTime;
- }
-
- public void setStartTime(long startTime) {
- this.startTime = startTime;
+ public void addValues(Map<Long, Number> values) {
+ if (type == Type.SINGLE_VALUE) {
+ overwrite(values);
+ } else {
+ this.values.putAll(values);
+ }
}
- @XmlElement(name = "endtime")
- public long getEndTime() {
- return endTime;
+ public void addValue(long timestamp, Number value) {
+ if (type == Type.SINGLE_VALUE) {
+ values.clear();
+ }
+ values.put(timestamp, value);
}
- public void setEndTime(long endTime) {
- this.endTime = endTime;
+ private void overwrite(Map<Long, Number> values) {
+ if (values.size() > 1) {
+ throw new IllegalArgumentException(
+ "Values cannot contain more than one point in " +
+ Type.SINGLE_VALUE + " mode");
+ }
+ this.values.clear();
+ this.values.putAll(values);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/557a3950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index caa3f3f..95b922e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -28,7 +28,11 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Test;
import org.junit.Assert;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
public class TestTimelineServiceRecords {
@@ -41,38 +45,83 @@ public class TestTimelineServiceRecords {
entity.setType("test type 1");
entity.setId("test id 1");
entity.addInfo("test info key 1", "test info value 1");
- entity.addInfo("test info key 2", "test info value 2");
+ entity.addInfo("test info key 2",
+ Arrays.asList("test info value 2", "test info value 3"));
+ entity.addInfo("test info key 3", true);
+ Assert.assertTrue(
+ entity.getInfo().get("test info key 3") instanceof Boolean);
entity.addConfig("test config key 1", "test config value 1");
entity.addConfig("test config key 2", "test config value 2");
- TimelineMetric metric1 = new TimelineMetric();
+
+ TimelineMetric metric1 =
+ new TimelineMetric(TimelineMetric.Type.TIME_SERIES);
metric1.setId("test metric id 1");
- metric1.addInfo("test info key 1", "test info value 1");
- metric1.addInfo("test info key 2", "test info value 2");
- metric1.addTimeSeriesData(1L, "test time series 1");
- metric1.addTimeSeriesData(2L, "test time series 2");
- metric1.setStartTime(0L);
- metric1.setEndTime(1L);
+ metric1.addValue(1L, 1.0F);
+ metric1.addValue(3L, 3.0D);
+ metric1.addValue(2L, 2);
+ Assert.assertEquals(TimelineMetric.Type.TIME_SERIES, metric1.getType());
+ Iterator<Map.Entry<Long, Number>> itr =
+ metric1.getValues().entrySet().iterator();
+ Map.Entry<Long, Number> entry = itr.next();
+ Assert.assertEquals(new Long(3L), entry.getKey());
+ Assert.assertEquals(new Double(3.0D), entry.getValue());
+ entry = itr.next();
+ Assert.assertEquals(new Long(2L), entry.getKey());
+ Assert.assertEquals(new Integer(2), entry.getValue());
+ entry = itr.next();
+ Assert.assertEquals(new Long(1L), entry.getKey());
+ Assert.assertEquals(new Float(1.0F), entry.getValue());
+ Assert.assertFalse(itr.hasNext());
entity.addMetric(metric1);
- TimelineMetric metric2 = new TimelineMetric();
+
+ TimelineMetric metric2 =
+ new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
metric2.setId("test metric id 1");
- metric2.addInfo("test info key 1", "test info value 1");
- metric2.addInfo("test info key 2", "test info value 2");
- metric2.setSingleData("test info value 3");
- metric1.setStartTime(0L);
- metric1.setEndTime(1L);
+ metric2.addValue(3L, (short) 3);
+ Assert.assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric2.getType());
+ Assert.assertTrue(
+ metric2.getValues().values().iterator().next() instanceof Short);
+ Map<Long, Number> points = new HashMap<>();
+ points.put(4L, 4.0D);
+ points.put(5L, 5.0D);
+ try {
+ metric2.setValues(points);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Values cannot contain more than one point in"));
+ }
+ try {
+ metric2.addValues(points);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Values cannot contain more than one point in"));
+ }
entity.addMetric(metric2);
+
TimelineEvent event1 = new TimelineEvent();
event1.setId("test event id 1");
event1.addInfo("test info key 1", "test info value 1");
- event1.addInfo("test info key 2", "test info value 2");
+ event1.addInfo("test info key 2",
+ Arrays.asList("test info value 2", "test info value 3"));
+ event1.addInfo("test info key 3", true);
+ Assert.assertTrue(
+ event1.getInfo().get("test info key 3") instanceof Boolean);
event1.setTimestamp(0L);
entity.addEvent(event1);
+
TimelineEvent event2 = new TimelineEvent();
event2.setId("test event id 2");
event2.addInfo("test info key 1", "test info value 1");
- event2.addInfo("test info key 2", "test info value 2");
+ event2.addInfo("test info key 2",
+ Arrays.asList("test info value 2", "test info value 3"));
+ event2.addInfo("test info key 3", true);
+ Assert.assertTrue(
+ event2.getInfo().get("test info key 3") instanceof Boolean);
event2.setTimestamp(1L);
entity.addEvent(event2);
+
entity.setCreatedTime(0L);
entity.setModifiedTime(1L);
entity.addRelatesToEntity("test type 2", "test id 2");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/557a3950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 3af1839..ebd43d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -544,7 +543,7 @@ public class ContainersMonitorImpl extends AbstractService implements
ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric memoryMetric = new TimelineMetric();
memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
- memoryMetric.addTimeSeriesData(currentTime, currentPmemUsage);
+ memoryMetric.addValue(currentTime, currentPmemUsage);
entity.addMetric(memoryMetric);
}
// if cpuUsageTotalCoresPercentage data is available
@@ -552,7 +551,7 @@ public class ContainersMonitorImpl extends AbstractService implements
ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric cpuMetric = new TimelineMetric();
cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
- cpuMetric.addTimeSeriesData(currentTime,
+ cpuMetric.addValue(currentTime,
cpuUsageTotalCoresPercentage);
entity.addMetric(cpuMetric);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/557a3950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 044148a..69031a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -81,6 +81,12 @@ public class TestTimelineServiceClientIntegration {
TimelineEntity entity = new TimelineEntity();
entity.setType("test entity type");
entity.setId("test entity id");
+ TimelineMetric metric =
+ new TimelineMetric(TimelineMetric.Type.TIME_SERIES);
+ metric.setId("test metric id");
+ metric.addValue(1L, 1.0D);
+ metric.addValue(2L, 2.0D);
+ entity.addMetric(metric);
client.putEntities(entity);
client.putEntitiesAsync(entity);
} finally {