You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2016/02/10 21:37:59 UTC
[1/2] ambari git commit: AMBARI-14982 : Change AMS configs to deal
with slow writes to WASB (avijayan)
Repository: ambari
Updated Branches:
refs/heads/trunk 5e6a5556b -> 0d52ebc05
AMBARI-14982 : Change AMS configs to deal with slow writes to WASB (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/49e33511
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/49e33511
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/49e33511
Branch: refs/heads/trunk
Commit: 49e33511c4306e2c4f43916405ff223cdd901d97
Parents: 5e6a555
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Wed Feb 10 12:28:26 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Wed Feb 10 12:28:26 2016 -0800
----------------------------------------------------------------------
.../timeline/HBaseTimelineMetricStore.java | 18 +--
.../metrics/timeline/PhoenixHBaseAccessor.java | 21 ++--
.../timeline/TimelineMetricConfiguration.java | 10 ++
.../server/upgrade/UpgradeCatalog222.java | 116 ++++++++++++++++++-
.../0.1.0/configuration/ams-site.xml | 37 ++++--
.../stacks/HDP/2.0.6/services/stack_advisor.py | 5 +-
.../server/upgrade/UpgradeCatalog222Test.java | 21 +++-
.../stacks/2.2/common/test_stack_advisor.py | 5 +-
8 files changed, 203 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/49e33511/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index 5ee8b44..b5ec6e8 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -143,14 +143,16 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
aggregatorDailyThread.start();
}
- int initDelay = configuration.getTimelineMetricsServiceWatcherInitDelay();
- int delay = configuration.getTimelineMetricsServiceWatcherDelay();
- // Start the watchdog
- executorService.scheduleWithFixedDelay(
- new TimelineMetricStoreWatcher(this, configuration), initDelay, delay,
- TimeUnit.SECONDS);
- LOG.info("Started watchdog for timeline metrics store with initial " +
- "delay = " + initDelay + ", delay = " + delay);
+ if (!configuration.isTimelineMetricsServiceWatcherDisabled()) {
+ int initDelay = configuration.getTimelineMetricsServiceWatcherInitDelay();
+ int delay = configuration.getTimelineMetricsServiceWatcherDelay();
+ // Start the watchdog
+ executorService.scheduleWithFixedDelay(
+ new TimelineMetricStoreWatcher(this, configuration), initDelay, delay,
+ TimeUnit.SECONDS);
+ LOG.info("Started watchdog for timeline metrics store with initial " +
+ "delay = " + initDelay + ", delay = " + delay);
+ }
isInitialized = true;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/49e33511/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index 7fa1d2d..1c86ebb 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -221,14 +221,14 @@ public class PhoenixHBaseAccessor {
String encoding = metricsConf.get(HBASE_ENCODING_SCHEME, DEFAULT_ENCODING);
String compression = metricsConf.get(HBASE_COMPRESSION_SCHEME, DEFAULT_TABLE_COMPRESSION);
- String precisionTtl = metricsConf.get(PRECISION_TABLE_TTL, "86400"); //1 day
- String hostMinTtl = metricsConf.get(HOST_MINUTE_TABLE_TTL, "604800"); //7 days
- String hostHourTtl = metricsConf.get(HOST_HOUR_TABLE_TTL, "2592000"); //30 days
- String hostDailyTtl = metricsConf.get(HOST_DAILY_TABLE_TTL, "31536000"); //1 year
- String clusterSecTtl = metricsConf.get(CLUSTER_SECOND_TABLE_TTL, "2592000"); //7 days
- String clusterMinTtl = metricsConf.get(CLUSTER_MINUTE_TABLE_TTL, "7776000"); //30 days
- String clusterHourTtl = metricsConf.get(CLUSTER_HOUR_TABLE_TTL, "31536000"); //1 year
- String clusterDailyTtl = metricsConf.get(CLUSTER_DAILY_TABLE_TTL, "63072000"); //2 years
+ String precisionTtl = getDaysInSeconds(metricsConf.get(PRECISION_TABLE_TTL, "1")); //1 day
+ String hostMinTtl = getDaysInSeconds(metricsConf.get(HOST_MINUTE_TABLE_TTL, "7")); //7 days
+ String hostHourTtl = getDaysInSeconds(metricsConf.get(HOST_HOUR_TABLE_TTL, "30")); //30 days
+ String hostDailyTtl = getDaysInSeconds(metricsConf.get(HOST_DAILY_TABLE_TTL, "365")); //1 year
+ String clusterSecTtl = getDaysInSeconds(metricsConf.get(CLUSTER_SECOND_TABLE_TTL, "7")); //7 days
+ String clusterMinTtl = getDaysInSeconds(metricsConf.get(CLUSTER_MINUTE_TABLE_TTL, "30")); //30 days
+ String clusterHourTtl = getDaysInSeconds(metricsConf.get(CLUSTER_HOUR_TABLE_TTL, "365")); //1 year
+ String clusterDailyTtl = getDaysInSeconds(metricsConf.get(CLUSTER_DAILY_TABLE_TTL, "730")); //2 years
try {
LOG.info("Initializing metrics schema...");
@@ -1255,4 +1255,9 @@ public class PhoenixHBaseAccessor {
return metadataMap;
}
+
+ private String getDaysInSeconds(String daysString) {
+ double days = Double.valueOf(daysString.trim());
+ return String.valueOf((int)(days*86400));
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/49e33511/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index 46f61fb..cc9a2f3 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -181,6 +181,9 @@ public class TimelineMetricConfiguration {
public static final String HANDLER_THREAD_COUNT =
"timeline.metrics.service.handler.thread.count";
+ public static final String WATCHER_DISABLED =
+ "timeline.metrics.service.watcher.disabled";
+
public static final String WATCHER_INITIAL_DELAY =
"timeline.metrics.service.watcher.initial.delay";
@@ -273,6 +276,13 @@ public class TimelineMetricConfiguration {
return 20;
}
+ public boolean isTimelineMetricsServiceWatcherDisabled() {
+ if (metricsConf != null) {
+ return Boolean.parseBoolean(metricsConf.get(WATCHER_DISABLED, "false"));
+ }
+ return false;
+ }
+
public int getTimelineMetricsServiceWatcherInitDelay() {
if (metricsConf != null) {
return Integer.parseInt(metricsConf.get(WATCHER_INITIAL_DELAY, "600"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/49e33511/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
index 7e2683d..88b3151 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
/**
* Upgrade catalog for version 2.2.2.
@@ -52,8 +53,16 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
"timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier";
private static final String CLUSTER_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER =
"timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier";
-
-
+ private static final String TIMELINE_METRICS_SERVICE_WATCHER_DISBALED = "timeline.metrics.service.watcher.disabled";
+ private static final String AMS_MODE = "timeline.metrics.service.operation.mode";
+ public static final String PRECISION_TABLE_TTL = "timeline.metrics.host.aggregator.ttl";
+ public static final String HOST_MINUTE_TABLE_TTL = "timeline.metrics.host.aggregator.minute.ttl";
+ public static final String HOST_HOUR_TABLE_TTL = "timeline.metrics.host.aggregator.hourly.ttl";
+ public static final String HOST_DAILY_TABLE_TTL = "timeline.metrics.host.aggregator.daily.ttl";
+ public static final String CLUSTER_SECOND_TABLE_TTL = "timeline.metrics.cluster.aggregator.second.ttl";
+ public static final String CLUSTER_MINUTE_TABLE_TTL = "timeline.metrics.cluster.aggregator.minute.ttl";
+ public static final String CLUSTER_HOUR_TABLE_TTL = "timeline.metrics.cluster.aggregator.hourly.ttl";
+ public static final String CLUSTER_DAILY_TABLE_TTL = "timeline.metrics.cluster.aggregator.daily.ttl";
// ----- Constructors ------------------------------------------------------
@@ -175,6 +184,94 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
}
+ if (!amsSiteProperties.containsKey(TIMELINE_METRICS_SERVICE_WATCHER_DISBALED)) {
+ LOG.info("Add config " + TIMELINE_METRICS_SERVICE_WATCHER_DISBALED + " = false");
+ newProperties.put(TIMELINE_METRICS_SERVICE_WATCHER_DISBALED, String.valueOf(false));
+ }
+
+ boolean isDistributed = false;
+ if ("distributed".equals(amsSite.getProperties().get(AMS_MODE))) {
+ isDistributed = true;
+ }
+
+ if (amsSiteProperties.containsKey(PRECISION_TABLE_TTL)) {
+ String oldTtl = amsSiteProperties.get(PRECISION_TABLE_TTL);
+ String newTtl = convertToDaysIfInSeconds(oldTtl);
+ if (isDistributed) {
+ if ("86400".equals(oldTtl)) {
+ newTtl = "7.0"; // 7 days
+ }
+ }
+ newProperties.put(PRECISION_TABLE_TTL, newTtl);
+ LOG.info("Setting value of " + PRECISION_TABLE_TTL + " : " + newTtl);
+ }
+
+ if (amsSiteProperties.containsKey(HOST_MINUTE_TABLE_TTL)) {
+ String oldTtl = amsSiteProperties.get(HOST_MINUTE_TABLE_TTL);
+ String newTtl = convertToDaysIfInSeconds(oldTtl);
+ newProperties.put(HOST_MINUTE_TABLE_TTL, newTtl);
+ LOG.info("Setting value of " + HOST_MINUTE_TABLE_TTL + " : " + newTtl);
+ }
+
+ if (amsSiteProperties.containsKey(HOST_MINUTE_TABLE_TTL)) {
+ String oldTtl = amsSiteProperties.get(HOST_MINUTE_TABLE_TTL);
+ String newTtl = convertToDaysIfInSeconds(oldTtl);
+ newProperties.put(HOST_MINUTE_TABLE_TTL, newTtl);
+ LOG.info("Setting value of " + HOST_MINUTE_TABLE_TTL + " : " + newTtl);
+ }
+
+ if (amsSiteProperties.containsKey(HOST_HOUR_TABLE_TTL)) {
+ String oldTtl = amsSiteProperties.get(HOST_HOUR_TABLE_TTL);
+ String newTtl = convertToDaysIfInSeconds(oldTtl);
+ newProperties.put(HOST_HOUR_TABLE_TTL, newTtl);
+ LOG.info("Setting value of " + HOST_HOUR_TABLE_TTL + " : " + newTtl);
+ }
+
+ if (amsSiteProperties.containsKey(HOST_DAILY_TABLE_TTL)) {
+ String oldTtl = amsSiteProperties.get(HOST_DAILY_TABLE_TTL);
+ String newTtl = convertToDaysIfInSeconds(oldTtl);
+ newProperties.put(HOST_DAILY_TABLE_TTL, newTtl);
+ LOG.info("Setting value of " + HOST_DAILY_TABLE_TTL + " : " + newTtl);
+ }
+
+ if (amsSiteProperties.containsKey(CLUSTER_SECOND_TABLE_TTL)) {
+ String oldTtl = amsSiteProperties.get(CLUSTER_SECOND_TABLE_TTL);
+ String newTtl = convertToDaysIfInSeconds(oldTtl);
+
+ if ("2592000".equals(oldTtl)) {
+ newTtl = "7.0"; // 7 days
+ }
+
+ newProperties.put(CLUSTER_SECOND_TABLE_TTL, newTtl);
+ LOG.info("Setting value of " + CLUSTER_SECOND_TABLE_TTL + " : " + newTtl);
+ }
+
+ if (amsSiteProperties.containsKey(CLUSTER_MINUTE_TABLE_TTL)) {
+ String oldTtl = amsSiteProperties.get(CLUSTER_MINUTE_TABLE_TTL);
+ String newTtl = convertToDaysIfInSeconds(oldTtl);
+
+ if ("7776000".equals(oldTtl)) {
+ newTtl = "30.0"; // 30 days
+ }
+
+ newProperties.put(CLUSTER_MINUTE_TABLE_TTL, newTtl);
+ LOG.info("Setting value of " + CLUSTER_MINUTE_TABLE_TTL + " : " + newTtl);
+ }
+
+ if (amsSiteProperties.containsKey(CLUSTER_HOUR_TABLE_TTL)) {
+ String oldTtl = amsSiteProperties.get(CLUSTER_HOUR_TABLE_TTL);
+ String newTtl = convertToDaysIfInSeconds(oldTtl);
+ newProperties.put(CLUSTER_HOUR_TABLE_TTL, newTtl);
+ LOG.info("Setting value of " + CLUSTER_HOUR_TABLE_TTL + " : " + newTtl);
+ }
+
+ if (amsSiteProperties.containsKey(CLUSTER_DAILY_TABLE_TTL)) {
+ String oldTtl = amsSiteProperties.get(CLUSTER_DAILY_TABLE_TTL);
+ String newTtl = convertToDaysIfInSeconds(oldTtl);
+ newProperties.put(CLUSTER_DAILY_TABLE_TTL, newTtl);
+ LOG.info("Setting value of " + CLUSTER_DAILY_TABLE_TTL + " : " + newTtl);
+ }
+
updateConfigurationPropertiesForCluster(cluster, AMS_SITE, newProperties, true, true);
}
@@ -183,4 +280,19 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
}
}
+ private String convertToDaysIfInSeconds(String secondsString) {
+
+ int seconds = Integer.valueOf(secondsString);
+ double days = 0.0;
+
+ if (seconds >= 86400) {
+ days += TimeUnit.SECONDS.toDays(seconds);
+ }
+
+ days += ((float)seconds % 86400.0) / 86400.0;
+ days = Math.round(days * 100.0)/100.0;
+
+ return String.valueOf(days);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/49e33511/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
index a3f0028..180b43b 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
@@ -283,59 +283,65 @@
</property>
<property>
<name>timeline.metrics.host.aggregator.daily.ttl</name>
- <value>31536000</value>
+ <value>365</value>
<description>
Host based daily resolution data purge interval. Default is 1 year.
</description>
</property>
<property>
<name>timeline.metrics.host.aggregator.hourly.ttl</name>
- <value>2592000</value>
+ <value>30</value>
<description>
Host based hourly resolution data purge interval. Default is 30 days.
</description>
</property>
<property>
<name>timeline.metrics.host.aggregator.minute.ttl</name>
- <value>604800</value>
+ <value>7</value>
<description>
Host based minute resolution data purge interval. Default is 7 days.
</description>
</property>
<property>
<name>timeline.metrics.cluster.aggregator.second.ttl</name>
- <value>2592000</value>
+ <value>7</value>
<description>
Cluster wide second resolution data purge interval. Default is 7 days.
</description>
</property>
<property>
<name>timeline.metrics.cluster.aggregator.minute.ttl</name>
- <value>7776000</value>
+ <value>30</value>
<description>
Cluster wide minute resolution data purge interval. Default is 30 days.
</description>
</property>
<property>
<name>timeline.metrics.cluster.aggregator.hourly.ttl</name>
- <value>31536000</value>
+ <value>365</value>
<description>
Cluster wide hourly resolution data purge interval. Default is 1 year.
</description>
</property>
<property>
<name>timeline.metrics.cluster.aggregator.daily.ttl</name>
- <value>63072000</value>
+ <value>730</value>
<description>
Cluster wide daily resolution data purge interval. Default is 2 years.
</description>
</property>
<property>
<name>timeline.metrics.host.aggregator.ttl</name>
- <value>86400</value>
+ <value>1</value>
<description>
- 1 minute resolution data purge interval. Default is 1 day.
+ 1 minute resolution data purge interval. Default is 1 day for embedded mode and 7 days for Distributed mode.
</description>
+ <depends-on>
+ <property>
+ <type>ams-site</type>
+ <name>timeline.metrics.service.operation.mode</name>
+ </property>
+ </depends-on>
</property>
<property>
<name>timeline.metrics.hbase.data.block.encoding</name>
@@ -495,6 +501,19 @@
</description>
</property>
<property>
+ <name>timeline.metrics.service.watcher.disabled</name>
+ <value>false</value>
+ <description>
+ Disable Timeline Metric Store watcher thread. Disabled by default in AMS distributed mode.
+ </description>
+ <depends-on>
+ <property>
+ <type>ams-site</type>
+ <name>timeline.metrics.service.operation.mode</name>
+ </property>
+ </depends-on>
+ </property>
+ <property>
<name>timeline.metrics.service.watcher.initial.delay</name>
<value>600</value>
<description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/49e33511/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index a53dd8b..af21008 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -507,8 +507,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
operatingMode = services["configurations"]["ams-site"]["properties"]["timeline.metrics.service.operation.mode"]
if operatingMode == "distributed":
+ putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'true')
+ putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 7)
putAmsHbaseSiteProperty("hbase.cluster.distributed", 'true')
else:
+ putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'false')
+ putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 1)
putAmsHbaseSiteProperty("hbase.cluster.distributed", 'false')
rootDir = "file:///var/lib/ambari-metrics-collector/hbase"
@@ -564,7 +568,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 134217728)
putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", 0.35)
putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.3)
- putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 86400)
if len(amsCollectorHosts) > 1:
pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/49e33511/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
index 67b9da5..6061e06 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
@@ -133,12 +133,31 @@ public class UpgradeCatalog222Test {
{
put("timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(1));
put("timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(1));
+ put("timeline.metrics.service.operation.mode", "distributed");
+ put("timeline.metrics.host.aggregator.ttl", String.valueOf(86400));
+ put("timeline.metrics.host.aggregator.minute.ttl", String.valueOf(604800));
+ put("timeline.metrics.host.aggregator.hourly.ttl", String.valueOf(2592000));
+ put("timeline.metrics.host.aggregator.daily.ttl", String.valueOf(31536000));
+ put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(21600)); //Less than 1 day
+ put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(7776000));
+ put("timeline.metrics.cluster.aggregator.hourly.ttl", String.valueOf(31536000));
+ put("timeline.metrics.cluster.aggregator.daily.ttl", String.valueOf(63072000));
}
};
Map<String, String> newPropertiesAmsSite = new HashMap<String, String>() {
{
put("timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(2));
put("timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(2));
+ put("timeline.metrics.service.watcher.disabled", String.valueOf(false));
+ put("timeline.metrics.host.aggregator.ttl", String.valueOf(7.0));
+ put("timeline.metrics.host.aggregator.minute.ttl", String.valueOf(7.0));
+ put("timeline.metrics.host.aggregator.hourly.ttl", String.valueOf(30.0));
+ put("timeline.metrics.host.aggregator.daily.ttl", String.valueOf(365.0));
+ put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(0.25));
+ put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(30.0));
+ put("timeline.metrics.cluster.aggregator.hourly.ttl", String.valueOf(365.0));
+ put("timeline.metrics.cluster.aggregator.daily.ttl", String.valueOf(730.0));
+ put("timeline.metrics.service.operation.mode", "distributed");
}
};
EasyMockSupport easyMockSupport = new EasyMockSupport();
@@ -151,7 +170,7 @@ public class UpgradeCatalog222Test {
put("normal", cluster);
}}).once();
expect(cluster.getDesiredConfigByType("ams-site")).andReturn(mockAmsSite).atLeastOnce();
- expect(mockAmsSite.getProperties()).andReturn(oldPropertiesAmsSite).times(2);
+ expect(mockAmsSite.getProperties()).andReturn(oldPropertiesAmsSite).anyTimes();
Injector injector = easyMockSupport.createNiceMock(Injector.class);
expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes();
http://git-wip-us.apache.org/repos/asf/ambari/blob/49e33511/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 0a8f05b..7d81423 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -2052,7 +2052,8 @@ class TestHDP22StackAdvisor(TestCase):
"properties": {
"timeline.metrics.cluster.aggregate.splitpoints": " ",
"timeline.metrics.host.aggregate.splitpoints": " ",
- "timeline.metrics.host.aggregator.ttl": "86400"
+ "timeline.metrics.host.aggregator.ttl": "1",
+ 'timeline.metrics.service.watcher.disabled': 'false'
}
}
}
@@ -2230,6 +2231,8 @@ class TestHDP22StackAdvisor(TestCase):
expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512'
expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '102'
expected['ams-hbase-env']['properties']['regionserver_xmn_size'] = '384'
+ expected['ams-site']['properties']['timeline.metrics.host.aggregator.ttl'] = '7'
+ expected['ams-site']['properties']['timeline.metrics.service.watcher.disabled'] = 'true'
self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
[2/2] ambari git commit: AMBARI-14962 Auto configure AMS UI to use
AMS Datasource - Commit 3 (avijayan)
Posted by av...@apache.org.
AMBARI-14962 Auto configure AMS UI to use AMS Datasource - Commit 3 (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0d52ebc0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0d52ebc0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0d52ebc0
Branch: refs/heads/trunk
Commit: 0d52ebc05593ebce6e3ab37535bf1345df8f8492
Parents: 49e3351
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Wed Feb 10 12:29:23 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Wed Feb 10 12:29:23 2016 -0800
----------------------------------------------------------------------
.../0.1.0/package/scripts/metrics_grafana_util.py | 6 ++++--
.../0.1.0/package/templates/metrics_grafana_datasource.json.j2 | 2 +-
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0d52ebc0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
index c4a91e1..5ab40b0 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
@@ -58,8 +58,9 @@ def create_ams_datasource():
Logger.info("Ambari Metrics Grafana datasource already present. Checking Metrics Collector URL")
datasource_url = datasources_json[i]["url"]
- if datasource_url == (params.metric_collector_host + ":" + params.metric_collector_port
- + "/ws/v1/timeline/metrics") :
+ if datasource_url == (params.ams_grafana_protocol + "://"
+ + params.metric_collector_host + ":"
+ + params.metric_collector_port):
Logger.info("Metrics Collector URL validation succeeded. Skipping datasource creation")
GRAFANA_CONNECT_TRIES = 0 # No need to create datasource again
@@ -84,6 +85,7 @@ def create_ams_datasource():
try:
ams_datasource_json = Template('metrics_grafana_datasource.json.j2',
ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME,
+ ams_grafana_protocol=params.ams_grafana_protocol,
ams_collector_host=params.metric_collector_host,
ams_collector_port=params.metric_collector_port).get_content()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0d52ebc0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2
index 1b03a2d..a803da5 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2
@@ -20,7 +20,7 @@
"name": "{{ams_datasource_name}}",
"type": "ambarimetrics",
"access": "proxy",
- "url": "{{ams_collector_host}}:{{ams_collector_port}}/ws/v1/timeline/metrics",
+ "url": "{{ams_grafana_protocol}}://{{ams_collector_host}}:{{ams_collector_port}}",
"password": "",
"user": "",
"database": "",