You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2018/07/24 19:40:58 UTC
[ambari] branch trunk updated: AMBARI-24290 : Fix issues in AMS
multiple cluster support. (#1764)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new b57a763 AMBARI-24290 : Fix issues in AMS multiple cluster support. (#1764)
b57a763 is described below
commit b57a763a001efc77449dafe934412c81e4304ab6
Author: avijayanhwx <av...@hortonworks.com>
AuthorDate: Tue Jul 24 12:40:55 2018 -0700
AMBARI-24290 : Fix issues in AMS multiple cluster support. (#1764)
* AMBARI-24290 : Fix issues in AMS multiple cluster support.
* AMBARI-24290 : Fix issues in AMS multiple cluster support (2)
* AMBARI-24290 : Fix issues in AMS multiple cluster support (3)
---
.../core/timeline/HBaseTimelineMetricsService.java | 8 +++-
.../core/timeline/PhoenixHBaseAccessor.java | 8 ++++
.../core/timeline/TimelineMetricConfiguration.java | 2 +
.../TimelineMetricClusterAggregatorSecond.java | 2 +-
.../internal/AbstractProviderModule.java | 48 +++++++++++--------
.../internal/OverriddenMetricsHostProvider.java | 5 ++
.../metrics/CollectorHostDownRefreshCounter.java | 41 ++++++++++++++++
.../controller/metrics/MetricHostProvider.java | 12 +++++
.../metrics/MetricsCollectorHAClusterState.java | 15 +-----
.../metrics/MetricsCollectorHAManager.java | 56 ++++++++++++++++++++--
.../metrics/timeline/AMSPropertyProvider.java | 4 ++
.../timeline/AMSReportPropertyProvider.java | 5 ++
.../AMBARI_METRICS/0.1.0/metainfo.xml | 2 +-
.../0.1.0/package/scripts/metrics_grafana_util.py | 2 +-
.../AMBARI_METRICS/0.1.0/package/scripts/params.py | 2 -
.../DRUID/0.10.1/package/scripts/params.py | 49 +++++++++----------
.../package/alerts/alert_metrics_deviation.py | 6 +--
.../2.1/hooks/before-START/scripts/params.py | 17 ++++---
.../metrics/JMXPropertyProviderTest.java | 5 ++
.../metrics/RestMetricsPropertyProviderTest.java | 5 ++
.../ganglia/GangliaPropertyProviderTest.java | 5 ++
.../ganglia/GangliaReportPropertyProviderTest.java | 5 ++
.../metrics/timeline/AMSPropertyProviderTest.java | 5 ++
.../2.0.6/HDFS/test_alert_metrics_deviation.py | 4 +-
.../stacks/2.0.6/configs/default_ams_embedded.json | 4 +-
.../stacks/2.0.6/configs/default_with_bucket.json | 4 +-
.../stacks/2.0.6/configs/repository_file.json | 4 +-
.../test/python/stacks/2.3/configs/ats_1_5.json | 4 +-
.../python/stacks/2.5/configs/hsi_default.json | 4 +-
.../2.5/configs/hsi_default_for_restart.json | 4 +-
.../src/test/python/stacks/2.5/configs/hsi_ha.json | 4 +-
.../HDF/2.0/hooks/before-START/scripts/params.py | 21 ++++----
.../stacks/HDF/2.0/services/stack_advisor.py | 4 +-
.../ODPi/2.0/hooks/before-START/scripts/params.py | 18 +++----
.../services/HIVE/package/scripts/params_linux.py | 19 ++++----
.../stacks/ODPi/2.0/services/stack_advisor.py | 10 ++--
36 files changed, 287 insertions(+), 126 deletions(-)
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
index d768d0d..c90c93b 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
@@ -17,6 +17,7 @@
*/
package org.apache.ambari.metrics.core.timeline;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
@@ -52,6 +53,7 @@ import org.apache.ambari.metrics.core.timeline.query.Condition;
import org.apache.ambari.metrics.core.timeline.query.ConditionBuilder;
import org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL;
import org.apache.ambari.metrics.core.timeline.query.TopNCondition;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -255,6 +257,10 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
TimelineMetrics metrics = new TimelineMetrics();
List<String> transientMetricNames = new ArrayList<>();
+ if (configuration.getTimelineMetricsMultipleClusterSupport() && StringUtils.isEmpty(instanceId)) {
+ instanceId = DEFAULT_INSTANCE_ID;
+ }
+
List<byte[]> uuids = metricMetadataManager.getUuidsForGetMetricQuery(metricFunctions.keySet(),
hostnames,
applicationId,
@@ -282,7 +288,7 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
Condition condition = conditionBuilder.build();
- if (hostnames == null || hostnames.isEmpty()) {
+ if (CollectionUtils.isEmpty(hostnames)) {
metrics = hBaseAccessor.getAggregateMetricRecords(condition, metricFunctions);
} else {
metrics = hBaseAccessor.getMetricRecords(condition, metricFunctions);
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
index 0946511..c0427e7 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
@@ -29,6 +29,7 @@ import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguratio
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_SECOND_TABLE_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CONTAINER_METRICS_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DATE_TIERED_COMPACTION_POLICY;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.FIFO_COMPACTION_POLICY_CLASS;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_MAX_RETRIES;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_RESULT_LIMIT;
@@ -39,6 +40,7 @@ import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguratio
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HSTORE_COMPACTION_CLASS_KEY;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HSTORE_ENGINE_CLASS;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_EVENT_METRIC_PATTERNS;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TRANSIENT_METRIC_PATTERNS;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_DAILY_TABLE_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_HOUR_TABLE_TTL;
@@ -199,6 +201,7 @@ public class PhoenixHBaseAccessor {
private final boolean skipBlockCacheForAggregatorsEnabled;
private TimelineMetricMetadataManager metadataManagerInstance;
private Set<String> eventMetricPatterns = new HashSet<>();
+ private boolean supportMultipleClusterMetrics = false;
private Map<String, Integer> tableTTL = new HashMap<>();
@@ -258,6 +261,8 @@ public class PhoenixHBaseAccessor {
tableTTL.put(METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME, metricsConf.getInt(CLUSTER_DAILY_TABLE_TTL, 730 * 86400)); //2 years
tableTTL.put(METRIC_TRANSIENT_TABLE_NAME, metricsConf.getInt(METRICS_TRANSIENT_TABLE_TTL, 7 * 86400)); //7 days
+ this.supportMultipleClusterMetrics = Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS, "false"));
+
if (cacheEnabled) {
LOG.debug("Initialising and starting metrics cache committer thread...");
metricsCommiterThread = new MetricsCacheCommitterThread(this);
@@ -906,6 +911,9 @@ public class PhoenixHBaseAccessor {
tm.getHostName(), tm.getAppId());
if (!tm.getAppId().equals("FLUME_HANDLER")) {
+ if (supportMultipleClusterMetrics && StringUtils.isEmpty(tm.getInstanceId())) {
+ tm.setInstanceId(DEFAULT_INSTANCE_ID);
+ }
metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(), tm.getHostName());
}
}
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
index f7fd72a..9766f0c 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
@@ -300,6 +300,8 @@ public class TimelineMetricConfiguration {
public static final String HOST_APP_ID = "HOST";
+ public static final String DEFAULT_INSTANCE_ID = "DEFAULT";
+
public static final String DEFAULT_INSTANCE_PORT = "12001";
public static final String AMSHBASE_METRICS_WHITESLIST_FILE = "amshbase_metrics_whitelist";
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
index ba93792..878dcb3 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
@@ -99,7 +99,7 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
skipInterpolationMetricPatterns.addAll(getJavaMetricPatterns(skipInterpolationMetricPatternStrings));
}
- this.timelineMetricReadHelper = new TimelineMetricReadHelper(metadataManager, true);
+ this.timelineMetricReadHelper = new TimelineMetricReadHelper(metadataManager);
}
@Override
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
index 104528a..cbafea7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
@@ -99,7 +99,6 @@ public abstract class AbstractProviderModule implements ProviderModule,
private static final String PROPERTY_HDFS_HTTP_POLICY_VALUE_HTTPS_ONLY = "HTTPS_ONLY";
private static final String COLLECTOR_DEFAULT_PORT = "6188";
- private static boolean vipHostConfigPresent = false;
private static final Map<String, Map<String, String[]>> jmxDesiredProperties = new HashMap<>();
private static final Map<String, Map<String, String[]>> jmxDesiredRpcSuffixProperties = new ConcurrentHashMap<>();
@@ -374,26 +373,27 @@ public abstract class AbstractProviderModule implements ProviderModule,
return null;
}
- private String getMetricsCollectorHostName(String clusterName)
- throws SystemException {
+
+ private void checkAndAddExternalCollectorHosts(String clusterName) throws SystemException {
try {
// try to get vip properties from cluster-env
String configType = "cluster-env";
String currentConfigVersion = getDesiredConfigVersion(clusterName, configType);
String oldConfigVersion = serviceConfigVersions.get(configType);
if (!currentConfigVersion.equals(oldConfigVersion)) {
- vipHostConfigPresent = false;
serviceConfigVersions.put(configType, currentConfigVersion);
Map<String, String> configProperties = getDesiredConfigMap
(clusterName, currentConfigVersion, configType,
Collections.singletonMap("METRICS_COLLECTOR",
- new String[]{"metrics_collector_vip_host"}));
+ new String[]{"metrics_collector_external_hosts"}));
if (!configProperties.isEmpty()) {
clusterMetricserverVipHost = configProperties.get("METRICS_COLLECTOR");
- if (clusterMetricserverVipHost != null) {
- LOG.info("Setting Metrics Collector Vip Host : " + clusterMetricserverVipHost);
- vipHostConfigPresent = true;
+ if (StringUtils.isNotEmpty(clusterMetricserverVipHost)) {
+ for (String collectorHost : StringUtils.split(clusterMetricserverVipHost, ",")) {
+ metricsCollectorHAManager.addExternalMetricsCollectorHost(clusterName, collectorHost);
+ }
+ LOG.info("Setting Metrics Collector External Host : " + clusterMetricserverVipHost);
}
}
// updating the port value, because both vip properties are stored in
@@ -401,26 +401,30 @@ public abstract class AbstractProviderModule implements ProviderModule,
configProperties = getDesiredConfigMap
(clusterName, currentConfigVersion, configType,
Collections.singletonMap("METRICS_COLLECTOR",
- new String[]{"metrics_collector_vip_port"}));
+ new String[]{"metrics_collector_external_port"}));
if (!configProperties.isEmpty()) {
- clusterMetricServerVipPort = configProperties.get("METRICS_COLLECTOR");
+ clusterMetricServerVipPort = configProperties.getOrDefault("METRICS_COLLECTOR", "6188");
}
}
} catch (NoSuchParentResourceException | UnsupportedPropertyException e) {
LOG.warn("Failed to retrieve collector hostname.", e);
}
+ }
- //If vip config not present
- // If current collector host is null or if the host or the host component not live
- // Update clusterMetricCollectorMap with a live metric collector host.
- String currentCollectorHost = null;
- if (!vipHostConfigPresent) {
- currentCollectorHost = metricsCollectorHAManager.getCollectorHost(clusterName);
- }
- LOG.debug("Cluster Metrics Vip Host : {}", clusterMetricserverVipHost);
+ private String getMetricsCollectorHostName(String clusterName)
+ throws SystemException {
+
+ checkAndAddExternalCollectorHosts(clusterName);
+ String currentCollectorHost = metricsCollectorHAManager.getCollectorHost(clusterName);
+ LOG.debug("Cluster Metrics Vip Host : " + clusterMetricserverVipHost);
+
+ return currentCollectorHost;
+ }
- return (clusterMetricserverVipHost != null) ? clusterMetricserverVipHost : currentCollectorHost;
+ @Override
+ public boolean isCollectorHostExternal(String clusterName) {
+ return metricsCollectorHAManager.isExternalCollector();
}
@Override
@@ -867,6 +871,7 @@ public abstract class AbstractProviderModule implements ProviderModule,
jmxPortMap.clear();
clusterHostComponentMap = new HashMap<>();
clusterGangliaCollectorMap = new HashMap<>();
+ boolean hasMetricCollector = false;
Map<String, Cluster> clusterMap = clusters.getClusters();
if (MapUtils.isEmpty(clusterMap)) {
@@ -874,6 +879,7 @@ public abstract class AbstractProviderModule implements ProviderModule,
}
for (Cluster cluster : clusterMap.values()) {
+ hasMetricCollector = false;
String clusterName = cluster.getClusterName();
Map<String, String> hostComponentMap = clusterHostComponentMap.get(clusterName);
@@ -898,9 +904,13 @@ public abstract class AbstractProviderModule implements ProviderModule,
// If current collector host is null or if the host or the host component not live
// Update clusterMetricCollectorMap.
metricsCollectorHAManager.addCollectorHost(clusterName, hostName);
+ hasMetricCollector = true;
}
}
}
+ if (!hasMetricCollector) {
+ checkAndAddExternalCollectorHosts(clusterName);
+ }
}
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProvider.java
index cb59b2f..aabadc2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/OverriddenMetricsHostProvider.java
@@ -48,6 +48,11 @@ public class OverriddenMetricsHostProvider implements MetricHostProvider {
return getOverriddenHost(componentName).map(host -> replaceVariables(clusterName, host));
}
+ @Override
+ public boolean isCollectorHostExternal(String clusterName) {
+ return metricHostProvider.isCollectorHostExternal(clusterName);
+ }
+
private Optional<String> getOverriddenHost(String componentName) {
return Optional.ofNullable(overriddenHosts.get(componentName));
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/CollectorHostDownRefreshCounter.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/CollectorHostDownRefreshCounter.java
new file mode 100644
index 0000000..a025918
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/CollectorHostDownRefreshCounter.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.controller.metrics;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class CollectorHostDownRefreshCounter {
+
+ private int collectorDownRefreshCounterLimit = 5;
+ private AtomicInteger collectorDownRefreshCounter;
+
+ CollectorHostDownRefreshCounter(int counter) {
+ this.collectorDownRefreshCounterLimit = counter;
+ }
+
+ public boolean testRefreshCounter() {
+ collectorDownRefreshCounter.incrementAndGet();
+ if (collectorDownRefreshCounter.get() == collectorDownRefreshCounterLimit) {
+ collectorDownRefreshCounter = new AtomicInteger(0);
+ return true;
+ }
+ return false;
+ }
+
+}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricHostProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricHostProvider.java
index 86d7114..f588f68 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricHostProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricHostProvider.java
@@ -85,4 +85,16 @@ public interface MetricHostProvider {
default Optional<String> getExternalHostName(String clusterName, String componentName) {
return Optional.empty();
}
+
+
+ /**
+ * Is the collector host external to the cluster?
+ *
+ * @param clusterName the cluster name
+ *
+ * @return true if metrics server component is NOT in this cluster
+ *
+ */
+ public boolean isCollectorHostExternal(String clusterName);
+
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsCollectorHAClusterState.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsCollectorHAClusterState.java
index 231d103..bb94c9b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsCollectorHAClusterState.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsCollectorHAClusterState.java
@@ -21,7 +21,6 @@ import java.util.HashSet;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ambari.server.Role;
import org.apache.ambari.server.controller.AmbariManagementController;
@@ -40,8 +39,7 @@ public class MetricsCollectorHAClusterState {
private String clusterName;
private Set<String> liveCollectorHosts;
private Set<String> deadCollectorHosts;
- private AtomicInteger collectorDownRefreshCounter;
- private static int collectorDownRefreshCounterLimit = 5;
+ private CollectorHostDownRefreshCounter collectorDownRefreshCounter = new CollectorHostDownRefreshCounter(5);
private String currentCollectorHost = null;
@Inject
@@ -59,7 +57,6 @@ public class MetricsCollectorHAClusterState {
this.clusterName = clusterName;
this.liveCollectorHosts = new CopyOnWriteArraySet<>();
this.deadCollectorHosts = new CopyOnWriteArraySet<>();
- collectorDownRefreshCounter = new AtomicInteger(0);
}
public void addMetricsCollectorHost(String collectorHost) {
@@ -117,7 +114,7 @@ public class MetricsCollectorHAClusterState {
} else if (deadCollectorHost.equals(currentCollectorHost) && numCollectors() > 1) {
// Case 2: Event informing us that the current collector is dead. We have not refreshed it yet.
- if (testRefreshCounter()) {
+ if (collectorDownRefreshCounter.testRefreshCounter()) {
refreshCollectorHost(deadCollectorHost);
}
}
@@ -153,14 +150,6 @@ public class MetricsCollectorHAClusterState {
A refresh counter to track number of collector down events received. If it exceeds the limit,
then we go ahead and refresh the collector.
*/
- private boolean testRefreshCounter() {
- collectorDownRefreshCounter.incrementAndGet();
- if (collectorDownRefreshCounter.get() == collectorDownRefreshCounterLimit) {
- collectorDownRefreshCounter = new AtomicInteger(0);
- return true;
- }
- return false;
- }
public boolean isCollectorHostLive() {
for (String host : liveCollectorHosts) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsCollectorHAManager.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsCollectorHAManager.java
index d4b5dae..9b78199 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsCollectorHAManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsCollectorHAManager.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.controller.metrics;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import org.apache.ambari.server.controller.AmbariServer;
@@ -41,6 +42,9 @@ public class MetricsCollectorHAManager {
private static final Logger LOG =
LoggerFactory.getLogger(MetricsCollectorHAManager.class);
+ private Map<String, Map<String, Boolean>> externalMetricCollectorsState = new HashMap<>();
+ private CollectorHostDownRefreshCounter externalCollectorDownRefreshCounter = new CollectorHostDownRefreshCounter(5);
+
public MetricsCollectorHAManager() {
clusterCollectorHAState = new HashMap<>();
@@ -65,8 +69,23 @@ public class MetricsCollectorHAManager {
collectorHAClusterState.addMetricsCollectorHost(collectorHost);
}
+ public void addExternalMetricsCollectorHost(String clusterName, String collectorHost) {
+ Map<String, Boolean> hostStateMap = new HashMap<>();
+ hostStateMap.put(collectorHost, true);
+ externalMetricCollectorsState.put(clusterName, hostStateMap);
+ }
+
public String getCollectorHost(String clusterName) {
+ if (externalMetricCollectorsState.containsKey(clusterName)) {
+ for (String externalCollectorHost : externalMetricCollectorsState.get(clusterName).keySet()) {
+ if (externalMetricCollectorsState.get(clusterName).get(externalCollectorHost)) {
+ return externalCollectorHost;
+ }
+ return refreshAndReturnRandomExternalCollectorHost(clusterName);
+ }
+ }
+
if (! clusterCollectorHAState.containsKey(clusterName)) {
clusterCollectorHAState.put(clusterName, new MetricsCollectorHAClusterState(clusterName));
}
@@ -75,11 +94,22 @@ public class MetricsCollectorHAManager {
return collectorHAClusterState.getCurrentCollectorHost();
}
+ private String refreshAndReturnRandomExternalCollectorHost(String clusterName) {
+
+ Iterator<Map.Entry<String, Boolean>> itr = externalMetricCollectorsState.get(clusterName).entrySet().iterator();
+ while(itr.hasNext())
+ {
+ Map.Entry<String, Boolean> entry = itr.next();
+ entry.setValue(true);
+ }
+ itr = externalMetricCollectorsState.get(clusterName).entrySet().iterator();
+ return itr.next().getKey();
+ }
+
/**
* Handles {@link MetricsCollectorHostDownEvent}
*
- * @param event
- * the change event.
+ * @param event the change event.
*/
@Subscribe
public void onMetricsCollectorHostDownEvent(MetricsCollectorHostDownEvent event) {
@@ -87,20 +117,36 @@ public class MetricsCollectorHAManager {
LOG.debug("MetricsCollectorHostDownEvent caught, Down collector : {}", event.getCollectorHost());
String clusterName = event.getClusterName();
- MetricsCollectorHAClusterState collectorHAClusterState = clusterCollectorHAState.get(clusterName);
- collectorHAClusterState.onCollectorHostDown(event.getCollectorHost());
+ if (externalMetricCollectorsState.containsKey(clusterName)) {
+ if (externalCollectorDownRefreshCounter.testRefreshCounter()) {
+ externalMetricCollectorsState.get(clusterName).put(event.getCollectorHost(), false);
+ }
+ } else {
+ MetricsCollectorHAClusterState collectorHAClusterState = clusterCollectorHAState.get(clusterName);
+ collectorHAClusterState.onCollectorHostDown(event.getCollectorHost());
+ }
}
public boolean isEmpty() {
- return this.clusterCollectorHAState.isEmpty();
+ return this.clusterCollectorHAState.isEmpty() && externalMetricCollectorsState.isEmpty();
+ }
+
+ public boolean isExternalCollector() {
+ return !externalMetricCollectorsState.isEmpty();
}
public boolean isCollectorHostLive(String clusterName) {
+ if (!externalMetricCollectorsState.isEmpty()) {
+ return true;
+ }
MetricsCollectorHAClusterState metricsCollectorHAClusterState = this.clusterCollectorHAState.get(clusterName);
return metricsCollectorHAClusterState != null && metricsCollectorHAClusterState.isCollectorHostLive();
}
public boolean isCollectorComponentLive(String clusterName) {
+ if (!externalMetricCollectorsState.isEmpty()) {
+ return true;
+ }
MetricsCollectorHAClusterState metricsCollectorHAClusterState = this.clusterCollectorHAState.get(clusterName);
return metricsCollectorHAClusterState != null && metricsCollectorHAClusterState.isCollectorComponentAlive();
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
index 3d4ad2c..ffe8e11 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
@@ -360,6 +360,10 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider {
uriBuilder.setParameter("appId", componentName);
}
+ if (clusterName != null && hostProvider.isCollectorHostExternal(clusterName)) {
+ uriBuilder.setParameter("instanceId", clusterName);
+ }
+
if (temporalInfo != null) {
long startTime = temporalInfo.getStartTime();
if (startTime != -1) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
index d86a671..431f9ba 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
@@ -199,6 +199,11 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider {
MetricsPropertyProvider.getSetString(propertyIdMap.keySet(), -1));
uriBuilder.setParameter("appId", "HOST");
+
+ if (clusterName != null && hostProvider.isCollectorHostExternal(clusterName)) {
+ uriBuilder.setParameter("instanceId", clusterName);
+ }
+
long startTime = temporalInfo.getStartTime();
if (startTime != -1) {
uriBuilder.setParameter("startTime", String.valueOf(startTime));
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
index 5a6cd45..a12618f 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
@@ -29,7 +29,7 @@
<name>METRICS_COLLECTOR</name>
<displayName>Metrics Collector</displayName>
<category>MASTER</category>
- <cardinality>1+</cardinality>
+ <cardinality>0+</cardinality>
<versionAdvertised>false</versionAdvertised>
<reassignAllowed>true</reassignAllowed>
<timelineAppid>AMS-HBASE</timelineAppid>
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
index 3036857..d6964ad 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
@@ -307,7 +307,7 @@ def create_ams_datasource():
if new_datasource_host == "":
Logger.warning("All metric collectors are unavailable. Will use random collector as datasource host.")
- new_datasource_host = params.random_metric_collector_host
+ new_datasource_host = params.metric_collector_host
Logger.info("New datasource host will be %s" % new_datasource_host)
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 8aee8d4..81a1e2b 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -78,8 +78,6 @@ else:
metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts)
-random_metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts)
-
if 'cluster-env' in config['configurations'] and \
'metrics_collector_external_port' in config['configurations']['cluster-env']:
metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.10.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.10.1/package/scripts/params.py
index 9958123..8782a7e 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.10.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.10.1/package/scripts/params.py
@@ -85,7 +85,6 @@ druid_repo_list = config['configurations']['druid-common']['druid.extensions.rep
druid_extensions_load_list = config['configurations']['druid-common']['druid.extensions.loadList']
druid_security_extensions_load_list = config['configurations']['druid-common']['druid.security.extensions.loadList']
-
# status params
druid_pid_dir = status_params.druid_pid_dir
user_group = config['configurations']['cluster-env']['user_group']
@@ -113,7 +112,7 @@ jdk_location = config['ambariLevelParams']['jdk_location']
if 'mysql' == metadata_storage_type:
jdbc_driver_jar = default("/ambariLevelParams/custom_mysql_jdbc_name", None)
connector_curl_source = format("{jdk_location}/{jdbc_driver_jar}")
- connector_download_dir=format("{druid_extensions_dir}/mysql-metadata-storage")
+ connector_download_dir = format("{druid_extensions_dir}/mysql-metadata-storage")
downloaded_custom_connector = format("{tmp_dir}/{jdbc_driver_jar}")
check_db_connection_jar_name = "DBConnectionVerification.jar"
@@ -163,34 +162,36 @@ metric_emitter_type = "noop"
metric_collector_host = ""
metric_collector_port = ""
metric_collector_protocol = ""
-metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
-metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
-metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
+metric_truststore_path = default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
+metric_truststore_type = default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
+metric_truststore_password = default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
- metric_emitter_type = "ambari-metrics"
- if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
- metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
- else:
- metric_collector_host = ams_collector_hosts[0]
- if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
- else:
- metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "localhost:6188")
- if metric_collector_web_address.find(':') != -1:
- metric_collector_port = metric_collector_web_address.split(':')[1]
- else:
- metric_collector_port = '6188'
- if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
- metric_collector_protocol = 'https'
+ metric_emitter_type = "ambari-metrics"
+ metric_collector_host = ams_collector_hosts[0]
+ if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
+ else:
+ metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
+ if metric_collector_web_address.find(':') != -1:
+ metric_collector_port = metric_collector_web_address.split(':')[1]
else:
- metric_collector_protocol = 'http'
- pass
+ metric_collector_port = '6188'
+ if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
+ metric_collector_protocol = 'https'
+ else:
+ metric_collector_protocol = 'http'
+ pass
# Create current Hadoop Clients Libs
stack_version_unformatted = str(config['clusterLevelParams']['stack_version'])
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py
index fe2e9fc..28942b8 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py
@@ -60,8 +60,8 @@ EXECUTABLE_SEARCH_PATHS = '{{kerberos-env/executable_search_paths}}'
AMS_HTTP_POLICY = '{{ams-site/timeline.metrics.service.http.policy}}'
METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}'
-METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}'
-METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}'
+METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_external_hosts}}'
+METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_external_port}}'
CONNECTION_TIMEOUT_KEY = 'connection.timeout'
CONNECTION_TIMEOUT_DEFAULT = 5.0
@@ -173,7 +173,7 @@ def execute(configurations={}, parameters={}, host_name=None):
return (RESULT_STATE_UNKNOWN, ['{0} is a required parameter for the script'.format(HDFS_SITE_KEY)])
if METRICS_COLLECTOR_VIP_HOST_KEY in configurations and METRICS_COLLECTOR_VIP_PORT_KEY in configurations:
- collector_host = configurations[METRICS_COLLECTOR_VIP_HOST_KEY]
+ collector_host = configurations[METRICS_COLLECTOR_VIP_HOST_KEY].split(',')[0]
collector_port = int(configurations[METRICS_COLLECTOR_VIP_PORT_KEY])
else:
# ams-site/timeline.metrics.service.webapp.address is required
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/before-START/scripts/params.py
index 1058c75..7e36e04 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/before-START/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/before-START/scripts/params.py
@@ -23,16 +23,19 @@ from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_f
config = Script.get_config()
ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
+ metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts)
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
- metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
- else:
- metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts)
- if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/JMXPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/JMXPropertyProviderTest.java
index 809dc89..e375c93 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/JMXPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/JMXPropertyProviderTest.java
@@ -697,5 +697,10 @@ public class JMXPropertyProviderTest {
public boolean isCollectorComponentLive(String clusterName, MetricsService service) throws SystemException {
return false;
}
+
+ @Override
+ public boolean isCollectorHostExternal(String clusterName) {
+ return false;
+ }
}
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java
index ee18787..6bcdc08 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java
@@ -542,6 +542,11 @@ public class RestMetricsPropertyProviderTest {
public boolean isCollectorComponentLive(String clusterName, MetricsService service) throws SystemException {
return false;
}
+
+ @Override
+ public boolean isCollectorHostExternal(String clusterName) {
+ return false;
+ }
}
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaPropertyProviderTest.java
index e53cfd3..bd2ad04 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaPropertyProviderTest.java
@@ -1019,5 +1019,10 @@ public class GangliaPropertyProviderTest {
throws SystemException {
return isComponentLive;
}
+
+ @Override
+ public boolean isCollectorHostExternal(String clusterName) {
+ return false;
+ }
}
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaReportPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaReportPropertyProviderTest.java
index 6ea2c19..f46923f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaReportPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaReportPropertyProviderTest.java
@@ -132,5 +132,10 @@ public class GangliaReportPropertyProviderTest {
throws SystemException {
return true;
}
+
+ @Override
+ public boolean isCollectorHostExternal(String clusterName) {
+ return false;
+ }
}
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
index 0ac9f80..a1ff913 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProviderTest.java
@@ -1185,6 +1185,11 @@ public class AMSPropertyProviderTest {
public boolean isCollectorComponentLive(String clusterName, MetricsService service) throws SystemException {
return true;
}
+
+ @Override
+ public boolean isCollectorHostExternal(String clusterName) {
+ return false;
+ }
}
// Helper function to setup common Mocks.
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py
index 22f6d2b..21fb34a 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py
@@ -118,8 +118,8 @@ class TestAlertMetricsDeviation(RMFTestCase):
'{{hdfs-site/dfs.namenode.http-address}}': 'c6401.ambari.apache.org:50070',
'{{cluster-env/security_enabled}}': 'false',
'{{cluster-env/smokeuser}}': 'ambari-qa',
- '{{cluster-env/metrics_collector_vip_host}}': 'c6401.ambari.apache.org',
- '{{cluster-env/metrics_collector_vip_port}}': '6188',
+ '{{cluster-env/metrics_collector_external_hosts}}': 'c6401.ambari.apache.org',
+ '{{cluster-env/metrics_collector_external_port}}': '6188',
'{{hdfs-site}}': {
'dfs.datanode.address': '0.0.0.0:50010',
'dfs.namenode.rpc-address': 'c6401.ambari.apache.org:8020',
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json
index a802b74..f2ada36 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json
@@ -679,8 +679,8 @@
"user_group": "hadoop",
"repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
"repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}",
- "metrics_collector_vip_host": "c6402.ambari.apache.org",
- "metrics_collector_vip_port": "6189",
+ "metrics_collector_external_hosts": "c6402.ambari.apache.org",
+ "metrics_collector_external_port": "6189",
"override_uid" : "true"
},
"hbase-env": {
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json
index 0bd6851..712ea00 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_with_bucket.json
@@ -718,8 +718,8 @@
"user_group": "hadoop",
"repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
"repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}",
- "metrics_collector_vip_host": "c6402.ambari.apache.org",
- "metrics_collector_vip_port": "6189",
+ "metrics_collector_external_hosts": "c6402.ambari.apache.org",
+ "metrics_collector_external_port": "6189",
"override_uid" : "true",
"fetch_nonlocal_groups": "true"
},
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/repository_file.json b/ambari-server/src/test/python/stacks/2.0.6/configs/repository_file.json
index 7230971..2d514c8 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/repository_file.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/repository_file.json
@@ -770,8 +770,8 @@
"user_group": "hadoop",
"repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
"repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}",
- "metrics_collector_vip_host": "c6402.ambari.apache.org",
- "metrics_collector_vip_port": "6189",
+ "metrics_collector_external_hosts": "c6402.ambari.apache.org",
+ "metrics_collector_external_port": "6189",
"override_uid" : "true",
"fetch_nonlocal_groups": "true",
"manage_dirs_on_root": "true",
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json b/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json
index fe243e8..df99bb1 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json
@@ -676,8 +676,8 @@
"user_group": "hadoop",
"repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
"repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}",
- "metrics_collector_vip_host": "c6402.ambari.apache.org",
- "metrics_collector_vip_port": "6189",
+ "metrics_collector_external_hosts": "c6402.ambari.apache.org",
+ "metrics_collector_external_port": "6189",
"override_uid" : "true"
},
"hbase-env": {
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
index ad9b240..77db6eb 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
@@ -760,8 +760,8 @@
"user_group": "hadoop",
"repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
"repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}",
- "metrics_collector_vip_host": "c6402.ambari.apache.org",
- "metrics_collector_vip_port": "6189",
+ "metrics_collector_external_hosts": "c6402.ambari.apache.org",
+ "metrics_collector_external_port": "6189",
"override_uid" : "true",
"fetch_nonlocal_groups": "true"
},
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json
index b451949..6a24c37 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default_for_restart.json
@@ -765,8 +765,8 @@
"user_group": "hadoop",
"repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
"repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}",
- "metrics_collector_vip_host": "c6402.ambari.apache.org",
- "metrics_collector_vip_port": "6189",
+ "metrics_collector_external_hosts": "c6402.ambari.apache.org",
+ "metrics_collector_external_port": "6189",
"override_uid" : "true",
"fetch_nonlocal_groups": "true"
},
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json
index 27ff3e3..40b6b30 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_ha.json
@@ -759,8 +759,8 @@
"user_group": "hadoop",
"repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
"repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}",
- "metrics_collector_vip_host": "c6402.ambari.apache.org",
- "metrics_collector_vip_port": "6189",
+ "metrics_collector_external_hosts": "c6402.ambari.apache.org",
+ "metrics_collector_external_port": "6189",
"override_uid" : "true",
"fetch_nonlocal_groups": "true"
},
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py
index 223ae56..77df21f 100644
--- a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/hooks/before-START/scripts/params.py
@@ -98,7 +98,6 @@ has_hive_server_host = not len(hive_server_host) == 0
has_hbase_masters = not len(hbase_master_hosts) == 0
has_zk_host = not len(zk_hosts) == 0
has_ganglia_server = not len(ganglia_server_hosts) == 0
-has_metric_collector = not len(ams_collector_hosts) == 0
is_namenode_master = hostname in namenode_host
is_jtnode_master = hostname in jtnode_host
@@ -108,17 +107,21 @@ is_hbase_master = hostname in hbase_master_hosts
is_slave = hostname in slave_hosts
if has_ganglia_server:
ganglia_server_host = ganglia_server_hosts[0]
+
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
+has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
+ metric_collector_host = ams_collector_hosts[0]
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
- metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
- else:
- metric_collector_host = ams_collector_hosts[0]
- if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
- metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "localhost:6188")
+ metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
metric_collector_port = metric_collector_web_address.split(':')[1]
else:
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
index 2a6766a..e21a149 100644
--- a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/services/stack_advisor.py
@@ -590,8 +590,8 @@ class HDF20StackAdvisor(DefaultStackAdvisor):
amsCollectorHosts = self.getComponentHostNames(services, "AMBARI_METRICS", "METRICS_COLLECTOR")
if 'cluster-env' in services['configurations'] and \
- 'metrics_collector_vip_host' in services['configurations']['cluster-env']['properties']:
- metric_collector_host = services['configurations']['cluster-env']['properties']['metrics_collector_vip_host']
+ 'metrics_collector_external_hosts' in services['configurations']['cluster-env']['properties']:
+ metric_collector_host = services['configurations']['cluster-env']['properties']['metrics_collector_external_hosts']
else:
metric_collector_host = 'localhost' if len(amsCollectorHosts) == 0 else amsCollectorHosts[0]
diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/params.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/params.py
index 9425d61..d8debdf 100755
--- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/params.py
+++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/hooks/before-START/scripts/params.py
@@ -88,6 +88,12 @@ namenode_host = default("/clusterHostInfo/namenode_host", [])
zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
has_namenode = not len(namenode_host) == 0
has_resourcemanager = not len(rm_host) == 0
@@ -109,16 +115,12 @@ is_slave = hostname in slave_hosts
if has_ganglia_server:
ganglia_server_host = ganglia_server_hosts[0]
if has_metric_collector:
+ metric_collector_host = ams_collector_hosts[0]
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
- metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
- else:
- metric_collector_host = ams_collector_hosts[0]
- if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
- metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "localhost:6188")
+ metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
metric_collector_port = metric_collector_web_address.split(':')[1]
else:
diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/params_linux.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/params_linux.py
index 56d5c4d..ea386be 100755
--- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/params_linux.py
+++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/HIVE/package/scripts/params_linux.py
@@ -482,18 +482,21 @@ hive_site_config = dict(config['configurations']['hive-site'])
############# AMS related params #####################
########################################################
ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
+ metric_collector_host = ams_collector_hosts[0]
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
- metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
- else:
- metric_collector_host = ams_collector_hosts[0]
- if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
- metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "localhost:6188")
+ metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
metric_collector_port = metric_collector_web_address.split(':')[1]
else:
diff --git a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/stack_advisor.py b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/stack_advisor.py
index 418d0c0..bbf6e37 100755
--- a/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/stack_advisor.py
+++ b/contrib/management-packs/odpi-ambari-mpack/src/main/resources/stacks/ODPi/2.0/services/stack_advisor.py
@@ -581,16 +581,14 @@ class ODPi20StackAdvisor(DefaultStackAdvisor):
putAmsHbaseSiteProperty = self.putProperty(configurations, "ams-hbase-site", services)
putAmsSiteProperty = self.putProperty(configurations, "ams-site", services)
putHbaseEnvProperty = self.putProperty(configurations, "ams-hbase-env", services)
- putGrafanaProperty = self.putProperty(configurations, "ams-grafana-env", services)
putGrafanaPropertyAttribute = self.putPropertyAttribute(configurations, "ams-grafana-env")
amsCollectorHosts = self.getComponentHostNames(services, "AMBARI_METRICS", "METRICS_COLLECTOR")
-
if 'cluster-env' in services['configurations'] and \
- 'metrics_collector_vip_host' in services['configurations']['cluster-env']['properties']:
- metric_collector_host = services['configurations']['cluster-env']['properties']['metrics_collector_vip_host']
- else:
- metric_collector_host = 'localhost' if len(amsCollectorHosts) == 0 else amsCollectorHosts[0]
+ 'metrics_collector_external_hosts' in services['configurations']['cluster-env']:
+ amsCollectorHosts = services['configurations']['cluster-env']['metrics_collector_external_hosts']
+
+ metric_collector_host = 'localhost' if len(amsCollectorHosts) == 0 else amsCollectorHosts[0]
putAmsSiteProperty("timeline.metrics.service.webapp.address", str(metric_collector_host) + ":6188")