[ambari] 37/39: AMBARI-23225: Ambari Server login fails due to TimelineMetricsCacheSizeOfEngine error in AMS perf branch.

[av...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git

commit 01243e2b639c08a4172b378dbcac96bb2de3fe67
Author: Aravindan Vijayan <av...@hortonworks.com>
AuthorDate: Tue Mar 13 16:23:19 2018 -0700

    AMBARI-23225: Ambari Server login fails due to TimelineMetricsCacheSizeOfEngine error in AMS perf branch.
---
 .../cache/TimelineMetricsCacheSizeOfEngine.java    | 85 ++++++++++++++++++++--
 1 file changed, 77 insertions(+), 8 deletions(-)

diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java
index 8b54017..baae751 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java
@@ -17,24 +17,61 @@
  */
 package org.apache.ambari.server.controller.metrics.timeline.cache;
 
-import org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsEhCacheSizeOfEngine;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import net.sf.ehcache.pool.Size;
+import net.sf.ehcache.pool.SizeOfEngine;
+import net.sf.ehcache.pool.impl.DefaultSizeOfEngine;
+import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
+import net.sf.ehcache.pool.sizeof.SizeOf;
+
 /**
  * Cache sizing engine that reduces reflective calls over the Object graph to
  * find total Heap usage.
  */
-public class TimelineMetricsCacheSizeOfEngine extends TimelineMetricsEhCacheSizeOfEngine {
+public class TimelineMetricsCacheSizeOfEngine implements SizeOfEngine {
 
   private final static Logger LOG = LoggerFactory.getLogger(TimelineMetricsCacheSizeOfEngine.class);
+  public static int DEFAULT_MAX_DEPTH = 1000;
+  public static boolean DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED = false;
+
+  private SizeOfEngine underlying = null;
+  SizeOf reflectionSizeOf = new ReflectionSizeOf();
+
+  // Optimizations
+  private volatile long timelineMetricPrimitivesApproximation = 0;
+
+  private long sizeOfMapEntry;
+  private long sizeOfMapEntryOverhead;
+
+  private TimelineMetricsCacheSizeOfEngine(SizeOfEngine underlying) {
+    this.underlying = underlying;
+  }
 
   public TimelineMetricsCacheSizeOfEngine() {
-    // Invoke default constructor in base class
+    this(new DefaultSizeOfEngine(DEFAULT_MAX_DEPTH, DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED));
+
+    this.sizeOfMapEntry = reflectionSizeOf.sizeOf(new Long(1)) +
+      reflectionSizeOf.sizeOf(new Double(2.0));
+
+    //SizeOfMapEntryOverhead = SizeOfMapWithOneEntry - (SizeOfEmptyMap + SizeOfOneEntry)
+    TreeMap<Long, Double> map = new TreeMap<>();
+    long emptyMapSize = reflectionSizeOf.sizeOf(map);
+    map.put(new Long(1), new Double(2.0));
+    long sizeOfMapOneEntry = reflectionSizeOf.deepSizeOf(DEFAULT_MAX_DEPTH, DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED, map).getCalculated();
+    this.sizeOfMapEntryOverhead =  sizeOfMapOneEntry - (emptyMapSize + this.sizeOfMapEntry);
+
+    LOG.info("Creating custom sizeof engine for TimelineMetrics.");
   }
 
   @Override
-  public long getSizeOfEntry(Object key, Object value) {
+  public Size sizeOf(Object key, Object value, Object container) {
     try {
       LOG.debug("BEGIN - Sizeof, key: {}, value: {}", key, value);
 
@@ -48,7 +85,7 @@ public class TimelineMetricsCacheSizeOfEngine extends TimelineMetricsEhCacheSize
         size += getTimelineMetricCacheValueSize((TimelineMetricsCacheValue) value);
       }
       // Mark size as not being exact
-      return size;
+      return new Size(size, false);
     } finally {
       LOG.debug("END - Sizeof, key: {}", key);
     }
@@ -71,13 +108,45 @@ public class TimelineMetricsCacheSizeOfEngine extends TimelineMetricsEhCacheSize
 
   private long getTimelineMetricCacheValueSize(TimelineMetricsCacheValue value) {
     long size = 16; // startTime + endTime
-
+    TimelineMetrics metrics = value.getTimelineMetrics();
     size += 8; // Object reference
 
-    size += getTimelineMetricsSize(value.getTimelineMetrics()); // TreeMap
+    if (metrics != null) {
+      for (TimelineMetric metric : metrics.getMetrics()) {
+
+        if (timelineMetricPrimitivesApproximation == 0) {
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getMetricName());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getAppId());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getHostName());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getInstanceId());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getStartTime());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getType());
+          timelineMetricPrimitivesApproximation += 8; // Object overhead
+
+          LOG.debug("timelineMetricPrimitivesApproximation bytes = {}", timelineMetricPrimitivesApproximation);
+        }
+        size += timelineMetricPrimitivesApproximation;
+
+        Map<Long, Double> metricValues = metric.getMetricValues();
+        if (metricValues != null && !metricValues.isEmpty()) {
+          // Numeric wrapper: 12 bytes + 8 bytes Data type + 4 bytes alignment = 48 (Long, Double)
+          // Tree Map: 12 bytes for header + 20 bytes for 5 object fields : pointers + 1 byte for flag = 40
+          LOG.debug("Size of metric value: {}", (sizeOfMapEntry + sizeOfMapEntryOverhead) * metricValues.size());
+          size += (sizeOfMapEntry + sizeOfMapEntryOverhead) * metricValues.size(); // Treemap size is O(1)
+        }
+      }
+      LOG.debug("Total Size of metric values in cache: {}", size);
+    }
 
     return size;
   }
 
+  @Override
+  public SizeOfEngine copyWith(int maxDepth, boolean abortWhenMaxDepthExceeded) {
+    LOG.debug("Copying tracing sizeof engine, maxdepth: {}, abort: {}",
+      maxDepth, abortWhenMaxDepthExceeded);
 
-}
+    return new TimelineMetricsCacheSizeOfEngine(
+      underlying.copyWith(maxDepth, abortWhenMaxDepthExceeded));
+  }
+}
\ No newline at end of file

-- 
To stop receiving notification emails like this one, please contact
avijayan@apache.org.