You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2015/09/16 21:14:23 UTC
ambari git commit: AMBARI-13108. Kafka metrics result in skewed split
point distribiution. (swagle)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 e1efd557e -> 0dfe5769f
AMBARI-13108. Kafka metrics result in skewed split point distribiution. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0dfe5769
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0dfe5769
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0dfe5769
Branch: refs/heads/branch-2.1
Commit: 0dfe5769ff449c3f520a6dfb782b7256304d5cde
Parents: e1efd55
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Wed Sep 16 12:14:05 2015 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Wed Sep 16 12:14:15 2015 -0700
----------------------------------------------------------------------
.../server/configuration/Configuration.java | 2 +-
.../0.1.0/package/scripts/split_points.py | 33 ++++++++++++++++++--
2 files changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0dfe5769/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 813b3d9..e3686ac 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -422,7 +422,7 @@ public class Configuration {
private static final String TIMELINE_METRICS_REQUEST_READ_TIMEOUT = "server.timeline.metrics.cache.read.timeout.millis";
private static final String DEFAULT_TIMELINE_METRICS_REQUEST_READ_TIMEOUT = "10000";
private static final String TIMELINE_METRICS_REQUEST_INTERVAL_READ_TIMEOUT = "server.timeline.metrics.cache.interval.read.timeout.millis";
- private static final String DEFAULT_TIMELINE_METRICS_REQUEST_INTERVAL_READ_TIMEOUT = "5000";
+ private static final String DEFAULT_TIMELINE_METRICS_REQUEST_INTERVAL_READ_TIMEOUT = "10000";
private static final String TIMELINE_METRICS_REQUEST_CONNECT_TIMEOUT = "server.timeline.metrics.cache.connect.timeout.millis";
private static final String DEFAULT_TIMELINE_METRICS_REQUEST_CONNECT_TIMEOUT = "5000";
private static final String TIMELINE_METRICS_REQUEST_CATCHUP_INTERVAL = "server.timeline.metrics.cache.catchup.interval";
http://git-wip-us.apache.org/repos/asf/ambari/blob/0dfe5769/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py
index b8b38f9..a585c6f 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py
@@ -28,6 +28,8 @@ import ast
metric_filename_ext = '.txt'
# 5 regions for higher order aggregate tables
other_region_static_count = 5
+# Max equidistant points to return per service
+max_equidistant_points = 50
b_bytes = 1
k_bytes = 1 << 10 # 1024
@@ -72,6 +74,10 @@ class FindSplitPointsForAMSRegions():
self.serviceMetricsDir = serviceMetricsDir
self.services = services
self.mode = operation_mode
+ # Add host metrics if not present as input
+ if self.services and 'HOST' not in self.services:
+ self.services.append('HOST')
+
# Initialize before user
self.initialize()
@@ -90,7 +96,7 @@ class FindSplitPointsForAMSRegions():
if self.mode == 'distributed':
xmx_bytes = xmx_region_bytes
- memstore_max_mem = float(self.ams_hbase_site['hbase.regionserver.global.memstore.upperLimit']) * xmx_bytes
+ memstore_max_mem = float(self.ams_hbase_site['hbase.regionserver.global.memstore.lowerLimit']) * xmx_bytes
memstore_flush_size = format_Xmx_size_to_bytes(self.ams_hbase_site['hbase.hregion.memstore.flush.size'])
max_inmemory_regions = (memstore_max_mem / memstore_flush_size) - other_region_static_count
@@ -120,15 +126,36 @@ class FindSplitPointsForAMSRegions():
# services arg is not passed
if self.services is None or file.rstrip(metric_filename_ext) in self.services:
print 'Processing file: %s' % os.path.join(self.serviceMetricsDir, file)
+ service_metrics = set()
with open(os.path.join(self.serviceMetricsDir, file), 'r') as f:
for metric in f:
- metrics.add(metric.strip())
+ service_metrics.add(metric.strip())
+ pass
+ pass
+ metrics.update(self.find_equidistant_metrics(list(sorted(service_metrics))))
pass
pass
self.metrics = sorted(metrics)
print 'metrics length: %s' % len(self.metrics)
+ # Pick 50 metric points for each service that are equidistant from
+ # each other for a service
+ def find_equidistant_metrics(self, service_metrics):
+ equi_metrics = []
+ idx = len(service_metrics) / max_equidistant_points
+ if idx == 0:
+ return service_metrics
+ pass
+
+ index = idx
+ for i in range(0, max_equidistant_points - 1):
+ equi_metrics.append(service_metrics[index - 1])
+ index += idx
+ pass
+
+ return equi_metrics
+
def get_split_points(self):
split_points = collections.namedtuple('SplitPoints', [ 'precision', 'aggregate' ])
split_points.precision = []
@@ -159,7 +186,7 @@ class FindSplitPointsForAMSRegions():
def main(argv = None):
scriptDir = os.path.realpath(os.path.dirname(argv[0]))
serviceMetricsDir = os.path.join(scriptDir, os.pardir, 'files', 'service-metrics')
- print 'serviceMetricsDir: %s' % serviceMetricsDir
+
if os.path.exists(serviceMetricsDir):
onlyargs = argv[1:]
if len(onlyargs) < 3: