You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2015/10/22 23:20:28 UTC
ambari git commit: Revert "AMBARI-13517. Ambari Server JVM crashed
after several clicks in Web UI to navigate graph timerange. Lot of chnages
for minor maint branch. (swagle)"
Repository: ambari
Updated Branches:
refs/heads/branch-2.1.2 b2bf3b79d -> 19fb525d6
Revert "AMBARI-13517. Ambari Server JVM crashed after several clicks in Web UI to navigate graph timerange. Lot of chnages for minor maint branch. (swagle)"
This reverts commit b2bf3b79df1989de526af57582981e05d8c61914.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/19fb525d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/19fb525d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/19fb525d
Branch: refs/heads/branch-2.1.2
Commit: 19fb525d692eaf14bb0f0092e10fef4f920ecce9
Parents: b2bf3b7
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Thu Oct 22 14:19:53 2015 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Thu Oct 22 14:19:53 2015 -0700
----------------------------------------------------------------------
.../metrics2/sink/timeline/TimelineMetric.java | 7 +-
.../timeline/cache/TimelineMetricsCache.java | 2 +-
.../cache/TimelineMetricsCacheTest.java | 2 +-
.../timeline/HBaseTimelineMetricStore.java | 2 +-
.../metrics/timeline/PhoenixHBaseAccessor.java | 21 ++-
.../aggregators/TimelineMetricReadHelper.java | 2 +-
.../metrics/timeline/ITClusterAggregator.java | 5 +-
.../metrics/timeline/ITMetricAggregator.java | 3 +-
.../metrics/timeline/MetricTestHelper.java | 3 +-
.../timeline/TestTimelineMetricStore.java | 5 +-
.../metrics/MetricsPaddingMethod.java | 10 +-
.../cache/TimelineMetricCacheProvider.java | 29 ++--
.../cache/TimelineMetricsCacheSizeOfEngine.java | 137 -------------------
.../timeline/MetricsPaddingMethodTest.java | 2 +-
.../cache/TimelineMetricCacheSizingTest.java | 110 ---------------
.../timeline/cache/TimelineMetricCacheTest.java | 4 +-
16 files changed, 44 insertions(+), 300 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
index e4dc423..8b8df06 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
@@ -27,7 +27,6 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.codehaus.jackson.map.annotate.JsonDeserialize;
@XmlRootElement(name = "metric")
@XmlAccessorType(XmlAccessType.NONE)
@@ -42,7 +41,7 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
private long timestamp;
private long startTime;
private String type;
- private TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+ private Map<Long, Double> metricValues = new TreeMap<Long, Double>();
// default
public TimelineMetric() {
@@ -125,11 +124,11 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
}
@XmlElement(name = "metrics")
- public TreeMap<Long, Double> getMetricValues() {
+ public Map<Long, Double> getMetricValues() {
return metricValues;
}
- public void setMetricValues(TreeMap<Long, Double> metricValues) {
+ public void setMetricValues(Map<Long, Double> metricValues) {
this.metricValues = metricValues;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java
index 9e431e2..224b490 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java
@@ -161,7 +161,7 @@ public class TimelineMetricsCache {
Double value = counterMetricLastValue.get(metricName);
double previousValue = value != null ? value : firstValue;
Map<Long, Double> metricValues = timelineMetric.getMetricValues();
- TreeMap<Long, Double> newMetricValues = new TreeMap<Long, Double>();
+ Map<Long, Double> newMetricValues = new TreeMap<Long, Double>();
for (Map.Entry<Long, Double> entry : metricValues.entrySet()) {
newMetricValues.put(entry.getKey(), entry.getValue() - previousValue);
previousValue = entry.getValue();
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java
index ad98525..4a13d63 100644
--- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java
@@ -80,7 +80,7 @@ public class TimelineMetricsCacheTest {
timelineMetric.setAppId("test serviceName");
timelineMetric.setStartTime(startTime);
timelineMetric.setType("Number");
- timelineMetric.setMetricValues(new TreeMap<Long, Double>(metricValues));
+ timelineMetric.setMetricValues(metricValues);
return timelineMetric;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index 52cef59..9c0b94d 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -275,7 +275,7 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
metric.setHostName(metricList.get(0).getHostName());
// Assumption that metrics are ordered by start time
metric.setStartTime(metricList.get(0).getStartTime());
- TreeMap<Long, Double> metricRecords = new TreeMap<Long, Double>();
+ Map<Long, Double> metricRecords = new TreeMap<Long, Double>();
for (TimelineMetric timelineMetric : metricList) {
metricRecords.putAll(timelineMetric.getMetricValues());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index 1ed2a72..9488316 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -52,7 +52,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import static java.util.concurrent.TimeUnit.SECONDS;
@@ -188,19 +187,19 @@ public class PhoenixHBaseAccessor {
return metric;
}
- private static TreeMap<Long, Double> readLastMetricValueFromJSON(String json)
- throws IOException {
- TreeMap<Long, Double> values = readMetricFromJSON(json);
- Long lastTimeStamp = values.lastKey();
+ private static Map<Long, Double> readLastMetricValueFromJSON(String json)
+ throws IOException {
+ Map<Long, Double> values = readMetricFromJSON(json);
+ Long lastTimeStamp = Collections.max(values.keySet());
- TreeMap<Long, Double> valueMap = new TreeMap<Long, Double>();
+ HashMap<Long, Double> valueMap = new HashMap<Long, Double>(1);
valueMap.put(lastTimeStamp, values.get(lastTimeStamp));
return valueMap;
}
@SuppressWarnings("unchecked")
- public static TreeMap<Long, Double> readMetricFromJSON(String json) throws IOException {
- return (TreeMap<Long, Double>) mapper.readValue(json, metricValuesTypeRef);
+ public static Map<Long, Double> readMetricFromJSON(String json) throws IOException {
+ return (Map<Long, Double>) mapper.readValue(json, metricValuesTypeRef);
}
private Connection getConnectionRetryingOnException()
@@ -468,10 +467,8 @@ public class PhoenixHBaseAccessor {
// which is thrown in hbase TimeRange.java
Throwable io = ex.getCause();
String className = null;
- if (io != null) {
- for (StackTraceElement ste : io.getStackTrace()) {
- className = ste.getClassName();
- }
+ for (StackTraceElement ste : io.getStackTrace()) {
+ className = ste.getClassName();
}
if (className != null && className.equals("TimeRange")) {
// This is "maxStamp is smaller than minStamp" exception
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
index dc27614..573e09d 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
@@ -40,7 +40,7 @@ public class TimelineMetricReadHelper {
public TimelineMetric getTimelineMetricFromResultSet(ResultSet rs)
throws SQLException, IOException {
TimelineMetric metric = getTimelineMetricCommonsFromResultSet(rs);
- TreeMap<Long, Double> sortedByTimeMetrics = new TreeMap<Long, Double>(
+ Map<Long, Double> sortedByTimeMetrics = new TreeMap<Long, Double>(
PhoenixHBaseAccessor.readMetricFromJSON(rs.getString("METRICS")));
metric.setMetricValues(sortedByTimeMetrics);
return metric;
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java
index cbf0233..b7b1737 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java
@@ -47,7 +47,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNotNull;
@@ -504,7 +503,7 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest {
metric1.setAppId("resourcemanager");
metric1.setHostName("h1");
metric1.setStartTime(1431372311811l);
- metric1.setMetricValues(new TreeMap<Long, Double>() {{
+ metric1.setMetricValues(new HashMap<Long, Double>() {{
put(1431372311811l, 1.0);
put(1431372321811l, 1.0);
put(1431372331811l, 1.0);
@@ -519,7 +518,7 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest {
metric2.setAppId("resourcemanager");
metric2.setHostName("h1");
metric2.setStartTime(1431372381810l);
- metric2.setMetricValues(new TreeMap<Long, Double>() {{
+ metric2.setMetricValues(new HashMap<Long, Double>() {{
put(1431372381810l, 1.0);
put(1431372391811l, 1.0);
put(1431372401811l, 1.0);
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java
index e9c25cf..a3640d0 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java
@@ -42,7 +42,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
-import java.util.TreeMap;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
@@ -384,7 +383,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest {
m.setHostName(host);
m.setMetricName(metricName);
m.setStartTime(startTime);
- TreeMap<Long, Double> vals = new TreeMap<Long, Double>();
+ Map<Long, Double> vals = new HashMap<Long, Double>();
vals.put(startTime + 15000l, 0.0);
vals.put(startTime + 30000l, 0.0);
vals.put(startTime + 45000l, 1.0);
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
index 37ec134..26771d7 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
-import java.util.TreeMap;
public class MetricTestHelper {
@@ -84,7 +83,7 @@ public class MetricTestHelper {
m.setInstanceId(instanceId);
m.setMetricName(metricName);
m.setStartTime(startTime);
- TreeMap<Long, Double> vals = new TreeMap<Long, Double>();
+ Map<Long, Double> vals = new HashMap<Long, Double>();
vals.put(startTime + 15000l, val);
vals.put(startTime + 30000l, val);
vals.put(startTime + 45000l, val);
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
index 4b5bfe0..b0aad57 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
@@ -25,7 +25,6 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import java.util.TreeMap;
public class TestTimelineMetricStore implements TimelineMetricStore {
@Override
@@ -45,7 +44,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore {
metric1.setInstanceId(null);
metric1.setHostName("c6401");
metric1.setStartTime(1407949812L);
- metric1.setMetricValues(new TreeMap<Long, Double>() {{
+ metric1.setMetricValues(new HashMap<Long, Double>() {{
put(1407949812L, 1.0d);
put(1407949912L, 1.8d);
put(1407950002L, 0.7d);
@@ -56,7 +55,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore {
metric2.setInstanceId("3");
metric2.setHostName("c6401");
metric2.setStartTime(1407949812L);
- metric2.setMetricValues(new TreeMap<Long, Double>() {{
+ metric2.setMetricValues(new HashMap<Long, Double>() {{
put(1407949812L, 2.5d);
put(1407949912L, 3.0d);
put(1407950002L, 0.9d);
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
index ccfb713..522c0bd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
@@ -52,7 +52,15 @@ public class MetricsPaddingMethod {
return;
}
- TreeMap<Long, Double> values = metric.getMetricValues();
+ TreeMap<Long, Double> values;
+ Map<Long, Double> metricValuesMap = metric.getMetricValues();
+ if (metricValuesMap instanceof TreeMap) {
+ values = (TreeMap<Long, Double>) metricValuesMap;
+ }
+ else {
+ // JSON dser returns LinkedHashMap that is not Navigable
+ values = new TreeMap<Long, Double>(metricValuesMap);
+ }
long dataInterval = getTimelineMetricInterval(values);
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java
index 9f268b2..6d80687 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java
@@ -23,8 +23,6 @@ import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.config.CacheConfiguration;
import net.sf.ehcache.config.PersistenceConfiguration;
-import net.sf.ehcache.config.SizeOfPolicyConfiguration;
-import net.sf.ehcache.config.SizeOfPolicyConfiguration.MaxDepthExceededBehavior;
import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.commons.lang.StringUtils;
@@ -42,7 +40,6 @@ import static net.sf.ehcache.config.PersistenceConfiguration.*;
public class TimelineMetricCacheProvider {
private TimelineMetricCache timelineMetricsCache;
private volatile boolean isCacheInitialized = false;
- public static final String TIMELINE_METRIC_CACHE_MANAGER_NAME = "timelineMetricCacheManager";
public static final String TIMELINE_METRIC_CACHE_INSTANCE_NAME = "timelineMetricCache";
Configuration configuration;
@@ -64,13 +61,8 @@ public class TimelineMetricCacheProvider {
}
System.setProperty("net.sf.ehcache.skipUpdateCheck", "true");
- // Use custom sizing engine to speed cache sizing calculations
- System.setProperty("net.sf.ehcache.sizeofengine." + TIMELINE_METRIC_CACHE_MANAGER_NAME,
- "org.apache.ambari.server.controller.metrics.timeline.cache.TimelineMetricsCacheSizeOfEngine");
-
net.sf.ehcache.config.Configuration managerConfig =
new net.sf.ehcache.config.Configuration();
- managerConfig.setName(TIMELINE_METRIC_CACHE_MANAGER_NAME);
// Set max heap available to the cache manager
managerConfig.setMaxBytesLocalHeap(configuration.getMetricsCacheManagerHeapPercent());
@@ -82,18 +74,17 @@ public class TimelineMetricCacheProvider {
configuration.getMetricCacheTTLSeconds() + ", idle = " +
configuration.getMetricCacheIdleSeconds());
- // Create a Cache specifying its configuration.
+ PersistenceConfiguration persistenceConfiguration = new PersistenceConfiguration();
+ persistenceConfiguration.setStrategy(Strategy.NONE.name());
+
+ //Create a Cache specifying its configuration.
CacheConfiguration cacheConfiguration = new CacheConfiguration()
- .name(TIMELINE_METRIC_CACHE_INSTANCE_NAME)
- .timeToLiveSeconds(configuration.getMetricCacheTTLSeconds()) // 1 hour
- .timeToIdleSeconds(configuration.getMetricCacheIdleSeconds()) // 5 minutes
- .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU)
- .sizeOfPolicy(new SizeOfPolicyConfiguration() // Set sizeOf policy to continue on max depth reached - avoid OOM
- .maxDepth(10000)
- .maxDepthExceededBehavior(MaxDepthExceededBehavior.CONTINUE))
- .eternal(false)
- .persistence(new PersistenceConfiguration()
- .strategy(Strategy.NONE.name()));
+ .name(TIMELINE_METRIC_CACHE_INSTANCE_NAME)
+ .timeToLiveSeconds(configuration.getMetricCacheTTLSeconds()) // 1 hour
+ .timeToIdleSeconds(configuration.getMetricCacheIdleSeconds()) // 5 minutes
+ .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU)
+ .eternal(false)
+ .persistence(persistenceConfiguration);
Cache cache = new Cache(cacheConfiguration);
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java
deleted file mode 100644
index d8f2e06..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.server.controller.metrics.timeline.cache;
-
-import net.sf.ehcache.pool.Size;
-import net.sf.ehcache.pool.SizeOfEngine;
-import net.sf.ehcache.pool.impl.DefaultSizeOfEngine;
-import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
-import net.sf.ehcache.pool.sizeof.SizeOf;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.util.Map;
-
-/**
- * Cache sizing engine that reduces reflective calls over the Object graph to
- * find total Heap usage.
- */
-public class TimelineMetricsCacheSizeOfEngine implements SizeOfEngine {
-
- private final static Logger LOG = LoggerFactory.getLogger(TimelineMetricsCacheSizeOfEngine.class);
- public static int DEFAULT_MAX_DEPTH = 1000;
- public static boolean DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED = false;
-
- private SizeOfEngine underlying = null;
- SizeOf reflectionSizeOf = new ReflectionSizeOf();
-
- // Optimizations
- private volatile long timelineMetricPrimitivesApproximation = 0;
-
- private TimelineMetricsCacheSizeOfEngine(SizeOfEngine underlying) {
- this.underlying = underlying;
- }
-
- public TimelineMetricsCacheSizeOfEngine() {
- this(new DefaultSizeOfEngine(DEFAULT_MAX_DEPTH, DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED));
-
- LOG.info("Creating custom sizeof engine for TimelineMetrics.");
- }
-
- @Override
- public Size sizeOf(Object key, Object value, Object container) {
- try {
- LOG.debug("BEGIN - Sizeof, key: {}, value: {}", key, value);
-
- long size = 0;
-
- if (key instanceof TimelineAppMetricCacheKey) {
- size += getTimelineMetricCacheKeySize((TimelineAppMetricCacheKey) key);
- }
-
- if (value instanceof TimelineMetricsCacheValue) {
- size += getTimelineMetricCacheValueSize((TimelineMetricsCacheValue) value);
- }
- // Mark size as not being exact
- return new Size(size, false);
- } finally {
- LOG.debug("END - Sizeof, key: {}", key);
- }
- }
-
- private long getTimelineMetricCacheKeySize(TimelineAppMetricCacheKey key) {
- long size = reflectionSizeOf.sizeOf(key.getAppId());
- size += key.getMetricNames() != null && !key.getMetricNames().isEmpty() ?
- reflectionSizeOf.deepSizeOf(1000, false, key.getMetricNames()).getCalculated() : 0;
- size += key.getSpec() != null ?
- reflectionSizeOf.deepSizeOf(1000, false, key.getSpec()).getCalculated() : 0;
- // 4 fixed longs of @TemporalInfo + reference
- size += 40;
- size += 8; // Object overhead
-
- return size;
- }
-
- private long getTimelineMetricCacheValueSize(TimelineMetricsCacheValue value) {
- long size = 16; // startTime + endTime
- Map<String, TimelineMetric> metrics = value.getTimelineMetrics();
- size += 8; // Object reference
-
- if (metrics != null) {
- for (Map.Entry<String, TimelineMetric> metricEntry : metrics.entrySet()) {
- size += reflectionSizeOf.sizeOf(metricEntry.getKey());
-
- TimelineMetric metric = metricEntry.getValue();
-
- if (timelineMetricPrimitivesApproximation == 0) {
- timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getMetricName());
- timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getAppId());
- timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getHostName());
- timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getInstanceId());
- timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getTimestamp());
- timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getStartTime());
- timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getType());
- timelineMetricPrimitivesApproximation += 8; // Object overhead
-
- LOG.debug("timelineMetricPrimitivesApproximation bytes = " + timelineMetricPrimitivesApproximation);
- }
- size += timelineMetricPrimitivesApproximation;
-
- Map<Long, Double> metricValues = metric.getMetricValues();
- if (metricValues != null && !metricValues.isEmpty()) {
- // Numeric wrapper: 12 bytes + 8 bytes Data type + 4 bytes alignment = 48 (Long, Double)
- // Tree Map: 12 bytes for header + 20 bytes for 5 object fields : pointers + 1 byte for flag = 40
- LOG.debug("Size of metric value: " + (48 + 40) * metricValues.size());
- size += (48 + 40) * metricValues.size(); // Treemap size is O(1)
- }
- }
- LOG.debug("Total Size of metric values in cache: " + size);
- }
-
- return size;
- }
-
- @Override
- public SizeOfEngine copyWith(int maxDepth, boolean abortWhenMaxDepthExceeded) {
- LOG.debug("Copying tracing sizeof engine, maxdepth: {}, abort: {}",
- maxDepth, abortWhenMaxDepthExceeded);
-
- return new TimelineMetricsCacheSizeOfEngine(
- underlying.copyWith(maxDepth, abortWhenMaxDepthExceeded));
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java
index b35295d..c30c5eb 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java
@@ -174,7 +174,7 @@ public class MetricsPaddingMethodTest {
timelineMetric.setHostName("h1");
timelineMetric.setAppId("a1");
timelineMetric.setTimestamp(now);
- TreeMap<Long, Double> inputValues = new TreeMap<Long, Double>();
+ Map<Long, Double> inputValues = new TreeMap<Long, Double>();
inputValues.put(now - 100, 1.0d);
inputValues.put(now - 200, 2.0d);
inputValues.put(now - 300, 3.0d);
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java
deleted file mode 100644
index 4dbf27a..0000000
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.server.controller.metrics.timeline.cache;
-
-import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
-import net.sf.ehcache.pool.sizeof.SizeOf;
-import org.apache.ambari.server.controller.internal.TemporalInfoImpl;
-import org.apache.ambari.server.controller.spi.TemporalInfo;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.junit.Assert;
-import org.junit.Test;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-public class TimelineMetricCacheSizingTest {
-
- SizeOf reflectionSizeOf = new ReflectionSizeOf();
-
- private TimelineMetric getSampleTimelineMetric(String metricName) {
- TimelineMetric metric = new TimelineMetric();
- metric.setMetricName(metricName);
- metric.setAppId("KAFKA_BROKER");
- metric.setInstanceId("NULL");
- metric.setHostName("my.privatehostname.of.average.length");
- metric.setTimestamp(System.currentTimeMillis());
- metric.setStartTime(System.currentTimeMillis());
- metric.setType("LONG");
-
- // JSON dser gives a LinkedHashMap
- TreeMap<Long, Double> valueMap = new TreeMap<>();
- long now = System.currentTimeMillis();
- for (int i = 0; i < 25000; i++) {
- valueMap.put(new Long(now + i), new Double(1.0 + i));
- }
-
- metric.setMetricValues(valueMap);
-
- return metric;
- }
-
- @Test
- public void testTimelineMetricCacheSizing() throws Exception {
- Set<String> metricNames = new HashSet<>();
- String metric1 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName1";
- String metric2 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName2";
- String metric3 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName3";
- String metric4 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName4";
- String metric5 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName5";
- String metric6 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName6";
-
- metricNames.add(metric1);
- metricNames.add(metric2);
- metricNames.add(metric3);
- metricNames.add(metric4);
- metricNames.add(metric5);
- metricNames.add(metric6);
-
- long now = System.currentTimeMillis();
- TemporalInfo temporalInfo = new TemporalInfoImpl(now - 1000, now, 15);
-
- TimelineAppMetricCacheKey key = new TimelineAppMetricCacheKey(
- metricNames, "KAFKA_BROKER", temporalInfo);
- // Some random spec
- key.setSpec("http://104.196.94.129:6188/ws/v1/timeline/metrics?metricNames=" +
- "jvm.JvmMetrics.MemHeapCommittedM&appId=RESOURCEMANAGER&" +
- "startTime=1439522640000&endTime=1440127440000&precision=hours");
-
- Map<String, TimelineMetric> metricMap = new HashMap<>();
- metricMap.put(metric1, getSampleTimelineMetric(metric1));
- metricMap.put(metric2, getSampleTimelineMetric(metric2));
- metricMap.put(metric3, getSampleTimelineMetric(metric3));
- metricMap.put(metric4, getSampleTimelineMetric(metric4));
- metricMap.put(metric5, getSampleTimelineMetric(metric5));
- metricMap.put(metric6, getSampleTimelineMetric(metric6));
-
- TimelineMetricsCacheValue value = new TimelineMetricsCacheValue(now - 1000, now, metricMap);
-
- TimelineMetricsCacheSizeOfEngine customSizeOfEngine = new TimelineMetricsCacheSizeOfEngine();
-
- long bytesFromReflectionEngine =
- reflectionSizeOf.deepSizeOf(1000, false, key).getCalculated() +
- reflectionSizeOf.deepSizeOf(1000, false, value).getCalculated();
-
- long bytesFromCustomSizeOfEngine = customSizeOfEngine.sizeOf(key, value, null).getCalculated();
-
- long sampleSizeInMB = bytesFromReflectionEngine / (1024 * 1024);
- long discrepancyInKB = Math.abs(bytesFromCustomSizeOfEngine - bytesFromReflectionEngine) / 1024;
-
- Assert.assertTrue("Sample size is greater that 10 MB", sampleSizeInMB > 10);
- Assert.assertTrue("Discrepancy in values is less than 10K", discrepancyInKB < 10);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/19fb525d/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
index 32ce1e4..3432e1d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
@@ -132,7 +132,7 @@ public class TimelineMetricCacheTest {
TimelineMetric timelineMetric = new TimelineMetric();
timelineMetric.setMetricName("cpu_user");
timelineMetric.setAppId("app1");
- TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+ Map<Long, Double> metricValues = new HashMap<Long, Double>();
metricValues.put(now + 100, 1.0);
metricValues.put(now + 200, 2.0);
metricValues.put(now + 300, 3.0);
@@ -297,7 +297,7 @@ public class TimelineMetricCacheTest {
final TimelineMetric timelineMetric1 = new TimelineMetric();
timelineMetric1.setMetricName("cpu_user");
timelineMetric1.setAppId("app1");
- TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
+ Map<Long, Double> metricValues = new TreeMap<Long, Double>();
metricValues.put(now - 100, 1.0);
metricValues.put(now - 200, 2.0);
metricValues.put(now - 300, 3.0);