You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2015/10/23 16:47:45 UTC
[31/50] [abbrv] ambari git commit: AMBARI-13517. Ambari Server JVM
crashed after several clicks in Web UI to navigate graph timerange. (swagle)
AMBARI-13517. Ambari Server JVM crashed after several clicks in Web UI to navigate graph timerange. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/02943430
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/02943430
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/02943430
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 0294343025e5a9bd19a69c1ca3d4c69c1ecbaa62
Parents: 3b9b7c7
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Thu Oct 22 13:38:20 2015 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Thu Oct 22 13:38:20 2015 -0700
----------------------------------------------------------------------
.../metrics2/sink/timeline/TimelineMetric.java | 7 +-
.../timeline/cache/TimelineMetricsCache.java | 2 +-
.../cache/TimelineMetricsCacheTest.java | 2 +-
.../timeline/HBaseTimelineMetricStore.java | 2 +-
.../metrics/timeline/PhoenixHBaseAccessor.java | 21 +--
.../aggregators/TimelineMetricReadHelper.java | 2 +-
.../metrics/timeline/ITClusterAggregator.java | 5 +-
.../metrics/timeline/ITMetricAggregator.java | 3 +-
.../metrics/timeline/MetricTestHelper.java | 3 +-
.../timeline/TestTimelineMetricStore.java | 5 +-
.../metrics/MetricsPaddingMethod.java | 10 +-
.../cache/TimelineMetricCacheProvider.java | 29 ++--
.../cache/TimelineMetricsCacheSizeOfEngine.java | 137 +++++++++++++++++++
.../timeline/MetricsPaddingMethodTest.java | 2 +-
.../cache/TimelineMetricCacheSizingTest.java | 110 +++++++++++++++
.../timeline/cache/TimelineMetricCacheTest.java | 4 +-
16 files changed, 300 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
index 8b8df06..e4dc423 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
@@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.codehaus.jackson.map.annotate.JsonDeserialize;
@XmlRootElement(name = "metric")
@XmlAccessorType(XmlAccessType.NONE)
@@ -41,7 +42,7 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
private long timestamp;
private long startTime;
private String type;
- private Map<Long, Double> metricValues = new TreeMap<Long, Double>();
+ private TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
// default
public TimelineMetric() {
@@ -124,11 +125,11 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
}
@XmlElement(name = "metrics")
- public Map<Long, Double> getMetricValues() {
+ public TreeMap<Long, Double> getMetricValues() {
return metricValues;
}
- public void setMetricValues(Map<Long, Double> metricValues) {
+ public void setMetricValues(TreeMap<Long, Double> metricValues) {
this.metricValues = metricValues;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java
index 77a5499..4e9e36e 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java
@@ -161,7 +161,7 @@ public class TimelineMetricsCache {
Double value = counterMetricLastValue.get(metricName);
double previousValue = value != null ? value : firstValue;
Map<Long, Double> metricValues = timelineMetric.getMetricValues();
- Map<Long, Double> newMetricValues = new TreeMap<Long, Double>();
+ TreeMap<Long, Double> newMetricValues = new TreeMap<Long, Double>();
for (Map.Entry<Long, Double> entry : metricValues.entrySet()) {
newMetricValues.put(entry.getKey(), entry.getValue() - previousValue);
previousValue = entry.getValue();
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java
index 4a13d63..ad98525 100644
--- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java
@@ -80,7 +80,7 @@ public class TimelineMetricsCacheTest {
timelineMetric.setAppId("test serviceName");
timelineMetric.setStartTime(startTime);
timelineMetric.setType("Number");
- timelineMetric.setMetricValues(metricValues);
+ timelineMetric.setMetricValues(new TreeMap<Long, Double>(metricValues));
return timelineMetric;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index 9c0b94d..52cef59 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -275,7 +275,7 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
metric.setHostName(metricList.get(0).getHostName());
// Assumption that metrics are ordered by start time
metric.setStartTime(metricList.get(0).getStartTime());
- Map<Long, Double> metricRecords = new TreeMap<Long, Double>();
+ TreeMap<Long, Double> metricRecords = new TreeMap<Long, Double>();
for (TimelineMetric timelineMetric : metricList) {
metricRecords.putAll(timelineMetric.getMetricValues());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index 9488316..1ed2a72 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -52,6 +52,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import static java.util.concurrent.TimeUnit.SECONDS;
@@ -187,19 +188,19 @@ public class PhoenixHBaseAccessor {
return metric;
}
- private static Map<Long, Double> readLastMetricValueFromJSON(String json)
- throws IOException {
- Map<Long, Double> values = readMetricFromJSON(json);
- Long lastTimeStamp = Collections.max(values.keySet());
+ private static TreeMap<Long, Double> readLastMetricValueFromJSON(String json)
+ throws IOException {
+ TreeMap<Long, Double> values = readMetricFromJSON(json);
+ Long lastTimeStamp = values.lastKey();
- HashMap<Long, Double> valueMap = new HashMap<Long, Double>(1);
+ TreeMap<Long, Double> valueMap = new TreeMap<Long, Double>();
valueMap.put(lastTimeStamp, values.get(lastTimeStamp));
return valueMap;
}
@SuppressWarnings("unchecked")
- public static Map<Long, Double> readMetricFromJSON(String json) throws IOException {
- return (Map<Long, Double>) mapper.readValue(json, metricValuesTypeRef);
+ public static TreeMap<Long, Double> readMetricFromJSON(String json) throws IOException {
+ return (TreeMap<Long, Double>) mapper.readValue(json, metricValuesTypeRef);
}
private Connection getConnectionRetryingOnException()
@@ -467,8 +468,10 @@ public class PhoenixHBaseAccessor {
// which is thrown in hbase TimeRange.java
Throwable io = ex.getCause();
String className = null;
- for (StackTraceElement ste : io.getStackTrace()) {
- className = ste.getClassName();
+ if (io != null) {
+ for (StackTraceElement ste : io.getStackTrace()) {
+ className = ste.getClassName();
+ }
}
if (className != null && className.equals("TimeRange")) {
// This is "maxStamp is smaller than minStamp" exception
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
index 573e09d..dc27614 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java
@@ -40,7 +40,7 @@ public class TimelineMetricReadHelper {
public TimelineMetric getTimelineMetricFromResultSet(ResultSet rs)
throws SQLException, IOException {
TimelineMetric metric = getTimelineMetricCommonsFromResultSet(rs);
- Map<Long, Double> sortedByTimeMetrics = new TreeMap<Long, Double>(
+ TreeMap<Long, Double> sortedByTimeMetrics = new TreeMap<Long, Double>(
PhoenixHBaseAccessor.readMetricFromJSON(rs.getString("METRICS")));
metric.setMetricValues(sortedByTimeMetrics);
return metric;
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java
index b7b1737..cbf0233 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java
@@ -47,6 +47,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNotNull;
@@ -503,7 +504,7 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest {
metric1.setAppId("resourcemanager");
metric1.setHostName("h1");
metric1.setStartTime(1431372311811l);
- metric1.setMetricValues(new HashMap<Long, Double>() {{
+ metric1.setMetricValues(new TreeMap<Long, Double>() {{
put(1431372311811l, 1.0);
put(1431372321811l, 1.0);
put(1431372331811l, 1.0);
@@ -518,7 +519,7 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest {
metric2.setAppId("resourcemanager");
metric2.setHostName("h1");
metric2.setStartTime(1431372381810l);
- metric2.setMetricValues(new HashMap<Long, Double>() {{
+ metric2.setMetricValues(new TreeMap<Long, Double>() {{
put(1431372381810l, 1.0);
put(1431372391811l, 1.0);
put(1431372401811l, 1.0);
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java
index a3640d0..e9c25cf 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java
@@ -42,6 +42,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
+import java.util.TreeMap;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
@@ -383,7 +384,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest {
m.setHostName(host);
m.setMetricName(metricName);
m.setStartTime(startTime);
- Map<Long, Double> vals = new HashMap<Long, Double>();
+ TreeMap<Long, Double> vals = new TreeMap<Long, Double>();
vals.put(startTime + 15000l, 0.0);
vals.put(startTime + 30000l, 0.0);
vals.put(startTime + 45000l, 1.0);
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
index 26771d7..37ec134 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
+import java.util.TreeMap;
public class MetricTestHelper {
@@ -83,7 +84,7 @@ public class MetricTestHelper {
m.setInstanceId(instanceId);
m.setMetricName(metricName);
m.setStartTime(startTime);
- Map<Long, Double> vals = new HashMap<Long, Double>();
+ TreeMap<Long, Double> vals = new TreeMap<Long, Double>();
vals.put(startTime + 15000l, val);
vals.put(startTime + 30000l, val);
vals.put(startTime + 45000l, val);
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
index b0aad57..4b5bfe0 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java
@@ -25,6 +25,7 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.TreeMap;
public class TestTimelineMetricStore implements TimelineMetricStore {
@Override
@@ -44,7 +45,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore {
metric1.setInstanceId(null);
metric1.setHostName("c6401");
metric1.setStartTime(1407949812L);
- metric1.setMetricValues(new HashMap<Long, Double>() {{
+ metric1.setMetricValues(new TreeMap<Long, Double>() {{
put(1407949812L, 1.0d);
put(1407949912L, 1.8d);
put(1407950002L, 0.7d);
@@ -55,7 +56,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore {
metric2.setInstanceId("3");
metric2.setHostName("c6401");
metric2.setStartTime(1407949812L);
- metric2.setMetricValues(new HashMap<Long, Double>() {{
+ metric2.setMetricValues(new TreeMap<Long, Double>() {{
put(1407949812L, 2.5d);
put(1407949912L, 3.0d);
put(1407950002L, 0.9d);
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
index 522c0bd..ccfb713 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java
@@ -52,15 +52,7 @@ public class MetricsPaddingMethod {
return;
}
- TreeMap<Long, Double> values;
- Map<Long, Double> metricValuesMap = metric.getMetricValues();
- if (metricValuesMap instanceof TreeMap) {
- values = (TreeMap<Long, Double>) metricValuesMap;
- }
- else {
- // JSON dser returns LinkedHashMap that is not Navigable
- values = new TreeMap<Long, Double>(metricValuesMap);
- }
+ TreeMap<Long, Double> values = metric.getMetricValues();
long dataInterval = getTimelineMetricInterval(values);
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java
index 6d80687..9f268b2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java
@@ -23,6 +23,8 @@ import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.config.CacheConfiguration;
import net.sf.ehcache.config.PersistenceConfiguration;
+import net.sf.ehcache.config.SizeOfPolicyConfiguration;
+import net.sf.ehcache.config.SizeOfPolicyConfiguration.MaxDepthExceededBehavior;
import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.commons.lang.StringUtils;
@@ -40,6 +42,7 @@ import static net.sf.ehcache.config.PersistenceConfiguration.*;
public class TimelineMetricCacheProvider {
private TimelineMetricCache timelineMetricsCache;
private volatile boolean isCacheInitialized = false;
+ public static final String TIMELINE_METRIC_CACHE_MANAGER_NAME = "timelineMetricCacheManager";
public static final String TIMELINE_METRIC_CACHE_INSTANCE_NAME = "timelineMetricCache";
Configuration configuration;
@@ -61,8 +64,13 @@ public class TimelineMetricCacheProvider {
}
System.setProperty("net.sf.ehcache.skipUpdateCheck", "true");
+ // Use custom sizing engine to speed cache sizing calculations
+ System.setProperty("net.sf.ehcache.sizeofengine." + TIMELINE_METRIC_CACHE_MANAGER_NAME,
+ "org.apache.ambari.server.controller.metrics.timeline.cache.TimelineMetricsCacheSizeOfEngine");
+
net.sf.ehcache.config.Configuration managerConfig =
new net.sf.ehcache.config.Configuration();
+ managerConfig.setName(TIMELINE_METRIC_CACHE_MANAGER_NAME);
// Set max heap available to the cache manager
managerConfig.setMaxBytesLocalHeap(configuration.getMetricsCacheManagerHeapPercent());
@@ -74,17 +82,18 @@ public class TimelineMetricCacheProvider {
configuration.getMetricCacheTTLSeconds() + ", idle = " +
configuration.getMetricCacheIdleSeconds());
- PersistenceConfiguration persistenceConfiguration = new PersistenceConfiguration();
- persistenceConfiguration.setStrategy(Strategy.NONE.name());
-
- //Create a Cache specifying its configuration.
+ // Create a Cache specifying its configuration.
CacheConfiguration cacheConfiguration = new CacheConfiguration()
- .name(TIMELINE_METRIC_CACHE_INSTANCE_NAME)
- .timeToLiveSeconds(configuration.getMetricCacheTTLSeconds()) // 1 hour
- .timeToIdleSeconds(configuration.getMetricCacheIdleSeconds()) // 5 minutes
- .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU)
- .eternal(false)
- .persistence(persistenceConfiguration);
+ .name(TIMELINE_METRIC_CACHE_INSTANCE_NAME)
+ .timeToLiveSeconds(configuration.getMetricCacheTTLSeconds()) // 1 hour
+ .timeToIdleSeconds(configuration.getMetricCacheIdleSeconds()) // 5 minutes
+ .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU)
+ .sizeOfPolicy(new SizeOfPolicyConfiguration() // Set sizeOf policy to continue on max depth reached - avoid OOM
+ .maxDepth(10000)
+ .maxDepthExceededBehavior(MaxDepthExceededBehavior.CONTINUE))
+ .eternal(false)
+ .persistence(new PersistenceConfiguration()
+ .strategy(Strategy.NONE.name()));
Cache cache = new Cache(cacheConfiguration);
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java
new file mode 100644
index 0000000..d8f2e06
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.metrics.timeline.cache;
+
+import net.sf.ehcache.pool.Size;
+import net.sf.ehcache.pool.SizeOfEngine;
+import net.sf.ehcache.pool.impl.DefaultSizeOfEngine;
+import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
+import net.sf.ehcache.pool.sizeof.SizeOf;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.Map;
+
+/**
+ * Cache sizing engine that reduces reflective calls over the Object graph to
+ * find total Heap usage.
+ */
+public class TimelineMetricsCacheSizeOfEngine implements SizeOfEngine {
+
+ private final static Logger LOG = LoggerFactory.getLogger(TimelineMetricsCacheSizeOfEngine.class);
+ public static int DEFAULT_MAX_DEPTH = 1000;
+ public static boolean DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED = false;
+
+ private SizeOfEngine underlying = null;
+ SizeOf reflectionSizeOf = new ReflectionSizeOf();
+
+ // Optimizations
+ private volatile long timelineMetricPrimitivesApproximation = 0;
+
+ private TimelineMetricsCacheSizeOfEngine(SizeOfEngine underlying) {
+ this.underlying = underlying;
+ }
+
+ public TimelineMetricsCacheSizeOfEngine() {
+ this(new DefaultSizeOfEngine(DEFAULT_MAX_DEPTH, DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED));
+
+ LOG.info("Creating custom sizeof engine for TimelineMetrics.");
+ }
+
+ @Override
+ public Size sizeOf(Object key, Object value, Object container) {
+ try {
+ LOG.debug("BEGIN - Sizeof, key: {}, value: {}", key, value);
+
+ long size = 0;
+
+ if (key instanceof TimelineAppMetricCacheKey) {
+ size += getTimelineMetricCacheKeySize((TimelineAppMetricCacheKey) key);
+ }
+
+ if (value instanceof TimelineMetricsCacheValue) {
+ size += getTimelineMetricCacheValueSize((TimelineMetricsCacheValue) value);
+ }
+ // Mark size as not being exact
+ return new Size(size, false);
+ } finally {
+ LOG.debug("END - Sizeof, key: {}", key);
+ }
+ }
+
+ private long getTimelineMetricCacheKeySize(TimelineAppMetricCacheKey key) {
+ long size = reflectionSizeOf.sizeOf(key.getAppId());
+ size += key.getMetricNames() != null && !key.getMetricNames().isEmpty() ?
+ reflectionSizeOf.deepSizeOf(1000, false, key.getMetricNames()).getCalculated() : 0;
+ size += key.getSpec() != null ?
+ reflectionSizeOf.deepSizeOf(1000, false, key.getSpec()).getCalculated() : 0;
+ // 4 fixed longs of @TemporalInfo + reference
+ size += 40;
+ size += 8; // Object overhead
+
+ return size;
+ }
+
+ private long getTimelineMetricCacheValueSize(TimelineMetricsCacheValue value) {
+ long size = 16; // startTime + endTime
+ Map<String, TimelineMetric> metrics = value.getTimelineMetrics();
+ size += 8; // Object reference
+
+ if (metrics != null) {
+ for (Map.Entry<String, TimelineMetric> metricEntry : metrics.entrySet()) {
+ size += reflectionSizeOf.sizeOf(metricEntry.getKey());
+
+ TimelineMetric metric = metricEntry.getValue();
+
+ if (timelineMetricPrimitivesApproximation == 0) {
+ timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getMetricName());
+ timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getAppId());
+ timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getHostName());
+ timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getInstanceId());
+ timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getTimestamp());
+ timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getStartTime());
+ timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getType());
+ timelineMetricPrimitivesApproximation += 8; // Object overhead
+
+ LOG.debug("timelineMetricPrimitivesApproximation bytes = " + timelineMetricPrimitivesApproximation);
+ }
+ size += timelineMetricPrimitivesApproximation;
+
+ Map<Long, Double> metricValues = metric.getMetricValues();
+ if (metricValues != null && !metricValues.isEmpty()) {
+ // Numeric wrapper: 12 bytes + 8 bytes Data type + 4 bytes alignment = 48 (Long, Double)
+ // Tree Map: 12 bytes for header + 20 bytes for 5 object fields : pointers + 1 byte for flag = 40
+ LOG.debug("Size of metric value: " + (48 + 40) * metricValues.size());
+ size += (48 + 40) * metricValues.size(); // Treemap size is O(1)
+ }
+ }
+ LOG.debug("Total Size of metric values in cache: " + size);
+ }
+
+ return size;
+ }
+
+ @Override
+ public SizeOfEngine copyWith(int maxDepth, boolean abortWhenMaxDepthExceeded) {
+ LOG.debug("Copying tracing sizeof engine, maxdepth: {}, abort: {}",
+ maxDepth, abortWhenMaxDepthExceeded);
+
+ return new TimelineMetricsCacheSizeOfEngine(
+ underlying.copyWith(maxDepth, abortWhenMaxDepthExceeded));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java
index c30c5eb..b35295d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java
@@ -174,7 +174,7 @@ public class MetricsPaddingMethodTest {
timelineMetric.setHostName("h1");
timelineMetric.setAppId("a1");
timelineMetric.setTimestamp(now);
- Map<Long, Double> inputValues = new TreeMap<Long, Double>();
+ TreeMap<Long, Double> inputValues = new TreeMap<Long, Double>();
inputValues.put(now - 100, 1.0d);
inputValues.put(now - 200, 2.0d);
inputValues.put(now - 300, 3.0d);
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java
new file mode 100644
index 0000000..4dbf27a
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.metrics.timeline.cache;
+
+import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
+import net.sf.ehcache.pool.sizeof.SizeOf;
+import org.apache.ambari.server.controller.internal.TemporalInfoImpl;
+import org.apache.ambari.server.controller.spi.TemporalInfo;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.junit.Assert;
+import org.junit.Test;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class TimelineMetricCacheSizingTest {
+
+ SizeOf reflectionSizeOf = new ReflectionSizeOf();
+
+ private TimelineMetric getSampleTimelineMetric(String metricName) {
+ TimelineMetric metric = new TimelineMetric();
+ metric.setMetricName(metricName);
+ metric.setAppId("KAFKA_BROKER");
+ metric.setInstanceId("NULL");
+ metric.setHostName("my.privatehostname.of.average.length");
+ metric.setTimestamp(System.currentTimeMillis());
+ metric.setStartTime(System.currentTimeMillis());
+ metric.setType("LONG");
+
+ // JSON dser gives a LinkedHashMap
+ TreeMap<Long, Double> valueMap = new TreeMap<>();
+ long now = System.currentTimeMillis();
+ for (int i = 0; i < 25000; i++) {
+ valueMap.put(new Long(now + i), new Double(1.0 + i));
+ }
+
+ metric.setMetricValues(valueMap);
+
+ return metric;
+ }
+
+ @Test
+ public void testTimelineMetricCacheSizing() throws Exception {
+ Set<String> metricNames = new HashSet<>();
+ String metric1 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName1";
+ String metric2 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName2";
+ String metric3 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName3";
+ String metric4 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName4";
+ String metric5 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName5";
+ String metric6 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName6";
+
+ metricNames.add(metric1);
+ metricNames.add(metric2);
+ metricNames.add(metric3);
+ metricNames.add(metric4);
+ metricNames.add(metric5);
+ metricNames.add(metric6);
+
+ long now = System.currentTimeMillis();
+ TemporalInfo temporalInfo = new TemporalInfoImpl(now - 1000, now, 15);
+
+ TimelineAppMetricCacheKey key = new TimelineAppMetricCacheKey(
+ metricNames, "KAFKA_BROKER", temporalInfo);
+ // Some random spec
+ key.setSpec("http://104.196.94.129:6188/ws/v1/timeline/metrics?metricNames=" +
+ "jvm.JvmMetrics.MemHeapCommittedM&appId=RESOURCEMANAGER&" +
+ "startTime=1439522640000&endTime=1440127440000&precision=hours");
+
+ Map<String, TimelineMetric> metricMap = new HashMap<>();
+ metricMap.put(metric1, getSampleTimelineMetric(metric1));
+ metricMap.put(metric2, getSampleTimelineMetric(metric2));
+ metricMap.put(metric3, getSampleTimelineMetric(metric3));
+ metricMap.put(metric4, getSampleTimelineMetric(metric4));
+ metricMap.put(metric5, getSampleTimelineMetric(metric5));
+ metricMap.put(metric6, getSampleTimelineMetric(metric6));
+
+ TimelineMetricsCacheValue value = new TimelineMetricsCacheValue(now - 1000, now, metricMap);
+
+ TimelineMetricsCacheSizeOfEngine customSizeOfEngine = new TimelineMetricsCacheSizeOfEngine();
+
+ long bytesFromReflectionEngine =
+ reflectionSizeOf.deepSizeOf(1000, false, key).getCalculated() +
+ reflectionSizeOf.deepSizeOf(1000, false, value).getCalculated();
+
+ long bytesFromCustomSizeOfEngine = customSizeOfEngine.sizeOf(key, value, null).getCalculated();
+
+ long sampleSizeInMB = bytesFromReflectionEngine / (1024 * 1024);
+ long discrepancyInKB = Math.abs(bytesFromCustomSizeOfEngine - bytesFromReflectionEngine) / 1024;
+
+ Assert.assertTrue("Sample size is greater that 10 MB", sampleSizeInMB > 10);
+ Assert.assertTrue("Discrepancy in values is less than 10K", discrepancyInKB < 10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/02943430/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
index 3432e1d..32ce1e4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java
@@ -132,7 +132,7 @@ public class TimelineMetricCacheTest {
TimelineMetric timelineMetric = new TimelineMetric();
timelineMetric.setMetricName("cpu_user");
timelineMetric.setAppId("app1");
- Map<Long, Double> metricValues = new HashMap<Long, Double>();
+ TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
metricValues.put(now + 100, 1.0);
metricValues.put(now + 200, 2.0);
metricValues.put(now + 300, 3.0);
@@ -297,7 +297,7 @@ public class TimelineMetricCacheTest {
final TimelineMetric timelineMetric1 = new TimelineMetric();
timelineMetric1.setMetricName("cpu_user");
timelineMetric1.setAppId("app1");
- Map<Long, Double> metricValues = new TreeMap<Long, Double>();
+ TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
metricValues.put(now - 100, 1.0);
metricValues.put(now - 200, 2.0);
metricValues.put(now - 300, 3.0);