You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2018/05/11 14:06:14 UTC
[ambari] 01/03: Revert "AMBARI-23804 : Refine AMS HBase region
splitting calculation based on UUID work (Refactor Split point computation.
Allow only Murmur3Hash)."
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 7d215e0bdc519f450a127b6adbf11438c23136ad
Author: Aravindan Vijayan <av...@hortonworks.com>
AuthorDate: Fri May 11 07:03:29 2018 -0700
Revert "AMBARI-23804 : Refine AMS HBase region splitting calculation based on UUID work (Refactor Split point computation. Allow only Murmur3Hash)."
This reverts commit d4d2767ae0d9c8e3043144df52bd5d6094b61171.
---
.../metrics/core/timeline/PhoenixHBaseAccessor.java | 1 -
.../timeline/TimelineMetricSplitPointComputer.java | 19 +++++++++----------
.../discovery/TimelineMetricMetadataManager.java | 20 +++++++++++++++++++-
.../timeline/uuid/Murmur3HashUuidGenStrategy.java | 13 +------------
.../TimelineMetricSplitPointComputerTest.java | 21 +++++++++------------
5 files changed, 38 insertions(+), 36 deletions(-)
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
index c684b0a..dec7850 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
@@ -492,7 +492,6 @@ public class PhoenixHBaseAccessor {
PreparedStatement pStmt = null;
TimelineMetricSplitPointComputer splitPointComputer = new TimelineMetricSplitPointComputer(
metricsConf, hbaseConf, metadataManagerInstance);
- splitPointComputer.computeSplitPoints();
String encoding = metricsConf.get(HBASE_ENCODING_SCHEME, DEFAULT_ENCODING);
String compression = metricsConf.get(HBASE_COMPRESSION_SCHEME, DEFAULT_TABLE_COMPRESSION);
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java
index 05366cc..89bb843 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java
@@ -49,11 +49,6 @@ public class TimelineMetricSplitPointComputer {
private static final int SLAVE_EQUIDISTANT_POINTS = 50;
private static final int MASTER_EQUIDISTANT_POINTS = 5;
- private double hbaseTotalHeapsize;
- private double hbaseMemstoreUpperLimit;
- private double hbaseMemstoreFlushSize;
- private TimelineMetricMetadataManager timelineMetricMetadataManager = null;
-
private List<byte[]> precisionSplitPoints = new ArrayList<>();
private List<byte[]> aggregateSplitPoints = new ArrayList<>();
@@ -71,14 +66,18 @@ public class TimelineMetricSplitPointComputer {
slaveComponents.addAll(Arrays.asList(componentsString.split(",")));
}
- this.timelineMetricMetadataManager = timelineMetricMetadataManager;
- hbaseTotalHeapsize = metricsConf.getDouble("hbase_total_heapsize", 1024*1024*1024);
- hbaseMemstoreUpperLimit = hbaseConf.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3);
- hbaseMemstoreFlushSize = hbaseConf.getDouble("hbase.hregion.memstore.flush.size", 134217728);
+ double hbaseTotalHeapsize = metricsConf.getDouble("hbase_total_heapsize", 1024*1024*1024);
+ double hbaseMemstoreUpperLimit = hbaseConf.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5);
+ double hbaseMemstoreFlushSize = hbaseConf.getDouble("hbase.hregion.memstore.flush.size", 134217728);
+
+ computeSplitPoints(hbaseTotalHeapsize, hbaseMemstoreUpperLimit, hbaseMemstoreFlushSize, timelineMetricMetadataManager);
}
- protected void computeSplitPoints() {
+ private void computeSplitPoints(double hbaseTotalHeapsize,
+ double hbaseMemstoreUpperLimit,
+ double hbaseMemstoreFlushSize,
+ TimelineMetricMetadataManager timelineMetricMetadataManager) {
double memstoreMaxMemory = hbaseMemstoreUpperLimit * hbaseTotalHeapsize;
int maxInMemoryRegions = (int) ((memstoreMaxMemory / hbaseMemstoreFlushSize) - OTHER_TABLE_STATIC_REGIONS);
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
index 86226ec..737c2ff 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
@@ -397,6 +397,18 @@ public class TimelineMetricMetadataManager {
}
}
+ if (!HOSTED_APPS_MAP.isEmpty()) {
+ Map.Entry<String, TimelineMetricHostMetadata> entry = HOSTED_APPS_MAP.entrySet().iterator().next();
+ TimelineMetricHostMetadata timelineMetricHostMetadata = entry.getValue();
+ if (timelineMetricHostMetadata.getUuid() != null && timelineMetricHostMetadata.getUuid().length == 16) {
+ HOSTNAME_UUID_LENGTH = 16;
+ uuidGenStrategy = new MD5UuidGenStrategy();
+ } else {
+ HOSTNAME_UUID_LENGTH = 4;
+ uuidGenStrategy = new Murmur3HashUuidGenStrategy();
+ }
+ }
+
for (String host : HOSTED_APPS_MAP.keySet()) {
TimelineMetricHostMetadata timelineMetricHostMetadata = HOSTED_APPS_MAP.get(host);
if (timelineMetricHostMetadata != null && timelineMetricHostMetadata.getUuid() != null) {
@@ -411,7 +423,13 @@ public class TimelineMetricMetadataManager {
* @return the UUID generator of type org.apache.ambari.metrics.core.timeline.uuid.MetricUuidGenStrategy
*/
private MetricUuidGenStrategy getUuidStrategy(Configuration configuration) {
- return new Murmur3HashUuidGenStrategy();
+ String strategy = configuration.get(TIMELINE_METRICS_UUID_GEN_STRATEGY, "");
+ if ("md5".equalsIgnoreCase(strategy)){
+ return new MD5UuidGenStrategy();
+ } else {
+ //Default
+ return new Murmur3HashUuidGenStrategy();
+ }
}
/**
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java
index af8cee5..9418aa4 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java
@@ -24,12 +24,7 @@ import org.apache.commons.lang.StringUtils;
public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{
- /**
- * Compute Murmur3Hash 16 byte UUID for a Metric-App-Instance.
- * @param timelineClusterMetric input metric
- * @param maxLength Max length of returned UUID. (Will always be 16 for this technique)
- * @return 16 byte UUID.
- */ @Override
+ @Override
public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) {
String metricString = timelineClusterMetric.getMetricName() + timelineClusterMetric.getAppId();
@@ -40,12 +35,6 @@ public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{
return Hashing.murmur3_128().hashBytes(metricBytes).asBytes();
}
- /**
- * Compute Murmur3Hash 4 byte UUID for a String.
- * @param value String input
- * @param maxLength Max length of returned UUID. (Will always be 4 for this technique)
- * @return 4 byte UUID.
- */
@Override
public byte[] computeUuid(String value, int maxLength) {
byte[] valueBytes = value.getBytes();
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java
index 150dac2..4d663cc 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java
@@ -51,7 +51,7 @@ public class TimelineMetricSplitPointComputerTest {
expect(metricsConfMock.getDouble("hbase_total_heapsize", 1024*1024*1024)).andReturn(1024 * 1024 * 1024.0).once();
Configuration hbaseConfMock = EasyMock.createMock(Configuration.class);
- expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3)).andReturn(0.3).once();
+ expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5)).andReturn(0.5).once();
expect(hbaseConfMock.getDouble("hbase.hregion.memstore.flush.size", 134217728)).andReturn(134217728.0).once();
TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createNiceMock(TimelineMetricMetadataManager.class);
@@ -62,7 +62,6 @@ public class TimelineMetricSplitPointComputerTest {
TimelineMetricSplitPointComputer timelineMetricSplitPointComputer = new TimelineMetricSplitPointComputer(metricsConfMock,
hbaseConfMock,
metricMetadataManagerMock);
- timelineMetricSplitPointComputer.computeSplitPoints();
Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 3);
Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 1);
@@ -86,7 +85,7 @@ public class TimelineMetricSplitPointComputerTest {
expect(metricsConfMock.getDouble("hbase_total_heapsize", 1024*1024*1024)).andReturn(8589934592.0).once();
Configuration hbaseConfMock = EasyMock.createMock(Configuration.class);
- expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3)).andReturn(0.3).once();
+ expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5)).andReturn(0.5).once();
expect(hbaseConfMock.getDouble("hbase.hregion.memstore.flush.size", 134217728)).andReturn(134217728.0).once();
TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createNiceMock(TimelineMetricMetadataManager.class);
@@ -97,11 +96,10 @@ public class TimelineMetricSplitPointComputerTest {
TimelineMetricSplitPointComputer timelineMetricSplitPointComputer = new TimelineMetricSplitPointComputer(metricsConfMock,
hbaseConfMock,
metricMetadataManagerMock);
- timelineMetricSplitPointComputer.computeSplitPoints();
- Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 6);
- Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 1);
- Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 1);
+ Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 16);
+ Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 3);
+ Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 3);
}
@Test
@@ -121,7 +119,7 @@ public class TimelineMetricSplitPointComputerTest {
expect(metricsConfMock.getDouble("hbase_total_heapsize", 1024*1024*1024)).andReturn(24 * 1024 * 1024 * 1024.0).once();
Configuration hbaseConfMock = EasyMock.createMock(Configuration.class);
- expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3)).andReturn(0.3).once();
+ expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5)).andReturn(0.5).once();
expect(hbaseConfMock.getDouble("hbase.hregion.memstore.flush.size", 134217728)).andReturn(2 * 134217728.0).once();
TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createNiceMock(TimelineMetricMetadataManager.class);
@@ -132,10 +130,9 @@ public class TimelineMetricSplitPointComputerTest {
TimelineMetricSplitPointComputer timelineMetricSplitPointComputer = new TimelineMetricSplitPointComputer(metricsConfMock,
hbaseConfMock,
metricMetadataManagerMock);
- timelineMetricSplitPointComputer.computeSplitPoints();
- Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 14);
- Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 3);
- Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 3);
+ Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 28);
+ Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 6);
+ Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 6);
}
}
--
To stop receiving notification emails like this one, please contact
avijayan@apache.org.