You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2018/06/26 20:17:11 UTC
[ambari] branch trunk updated: [AMBARI-24180] Ambari metrics
Service Check fails. (#1615)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6da4163 [AMBARI-24180] Ambari metrics Service Check fails. (#1615)
6da4163 is described below
commit 6da4163e1ab6e5d431a3a8393ab6f58abf166910
Author: avijayanhwx <av...@hortonworks.com>
AuthorDate: Tue Jun 26 13:17:08 2018 -0700
[AMBARI-24180] Ambari metrics Service Check fails. (#1615)
* [AMBARI-24180] Ambari metrics Service Check fails post EU . Error - 401 Authentication required in response.
* [AMBARI-24180] Ambari metrics Service Check fails.
* [AMBARI-24180] Ambari metrics Service Check fails. - 3
---
.../metrics/core/timeline/HBaseTimelineMetricsService.java | 4 +++-
.../ambari/metrics/core/timeline/PhoenixHBaseAccessor.java | 3 +++
.../ambari/metrics/core/timeline/TimelineMetricStore.java | 10 ++++++++++
.../timeline/aggregators/TimelineMetricClusterAggregator.java | 4 +++-
.../aggregators/TimelineMetricClusterAggregatorSecond.java | 4 ++++
.../timeline/aggregators/TimelineMetricHostAggregator.java | 3 +++
.../core/timeline/aggregators/TimelineMetricReadHelper.java | 10 +++++++++-
.../org/apache/ambari/metrics/webapp/TimelineWebServices.java | 8 +++++++-
.../ambari/metrics/core/timeline/TestTimelineMetricStore.java | 5 +++++
.../metrics/core/timeline/TimelineMetricStoreWatcherTest.java | 6 +++---
.../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 5 +++--
11 files changed, 53 insertions(+), 9 deletions(-)
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
index 43a468c..d768d0d 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
@@ -396,8 +396,10 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
return metricsFunctions;
}
- public void putMetricsSkipCache(TimelineMetrics metrics) throws SQLException, IOException {
+ public TimelinePutResponse putMetricsSkipCache(TimelineMetrics metrics) throws SQLException, IOException {
+ TimelinePutResponse response = new TimelinePutResponse();
hBaseAccessor.insertMetricRecordsWithMetadata(metricMetadataManager, metrics, true);
+ return response;
}
@Override
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
index 182748d..0946511 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
@@ -1105,6 +1105,9 @@ public class PhoenixHBaseAccessor {
throws SQLException, IOException {
if (condition.getPrecision().equals(Precision.SECONDS)) {
TimelineMetric metric = TIMELINE_METRIC_READ_HELPER.getTimelineMetricFromResultSet(rs);
+ if (metric == null) {
+ return;
+ }
if (f != null && f.getSuffix() != null) { //Case : Requesting "._rate" for precision data
metric.setMetricName(metric.getMetricName() + f.getSuffix());
}
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStore.java
index a723a2c..9589272 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStore.java
@@ -69,6 +69,16 @@ public interface TimelineMetricStore {
TimelinePutResponse putMetrics(TimelineMetrics metrics) throws SQLException, IOException;
/**
+ * Stores metric information to the timeline store without any buffering of data.
+ *
+ * @param metrics An {@link TimelineMetrics}.
+ * @return An {@link org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse}.
+ * @throws SQLException, IOException
+ */
+ TimelinePutResponse putMetricsSkipCache(TimelineMetrics metrics) throws SQLException, IOException;
+
+
+ /**
* Store container metric into the timeline tore
*/
TimelinePutResponse putContainerMetrics(List<ContainerMetric> metrics)
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregator.java
index 2ea5309..357e0ba 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregator.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregator.java
@@ -98,7 +98,9 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator
while (rs.next()) {
TimelineClusterMetric currentMetric = readHelper.fromResultSet(rs);
-
+ if (currentMetric == null) {
+ continue;
+ }
MetricClusterAggregate currentHostAggregate =
isClusterPrecisionInputTable ?
readHelper.getMetricClusterAggregateFromResultSet(rs) :
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
index c9998d9..ba93792 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
@@ -160,6 +160,10 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
// If rows belong to same host combine them before slicing. This
// avoids issues across rows that belong to same hosts but get
// counted as coming from different ones.
+ if (nextMetric == null) {
+ continue;
+ }
+
if (metric.equalsExceptTime(nextMetric)) {
metric.addMetricValues(nextMetric.getMetricValues());
} else {
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricHostAggregator.java
index 6f2351b..a9ee385 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricHostAggregator.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricHostAggregator.java
@@ -92,6 +92,9 @@ public class TimelineMetricHostAggregator extends AbstractTimelineAggregator {
while (rs.next()) {
TimelineMetric currentMetric =
readHelper.getTimelineMetricKeyFromResultSet(rs);
+ if (currentMetric == null) {
+ continue;
+ }
MetricHostAggregate currentHostAggregate =
readHelper.getMetricHostAggregateFromResultSet(rs);
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java
index 541cb46..9dfe4f7 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/aggregators/TimelineMetricReadHelper.java
@@ -53,6 +53,9 @@ public class TimelineMetricReadHelper {
public TimelineMetric getTimelineMetricFromResultSet(ResultSet rs)
throws SQLException, IOException {
TimelineMetric metric = getTimelineMetricCommonsFromResultSet(rs);
+ if (metric == null) {
+ return null;
+ }
TreeMap<Long, Double> sortedByTimeMetrics = PhoenixHBaseAccessor.readMetricFromJSON(rs.getString("METRICS"));
metric.setMetricValues(sortedByTimeMetrics);
return metric;
@@ -110,6 +113,9 @@ public class TimelineMetricReadHelper {
byte[] uuid = rs.getBytes("UUID");
TimelineMetric metric = metadataManagerInstance.getMetricFromUuid(uuid);
+ if (metric == null) {
+ return null;
+ }
if (ignoreInstance) {
metric.setInstanceId(null);
}
@@ -147,7 +153,9 @@ public class TimelineMetricReadHelper {
byte[] uuid = rs.getBytes("UUID");
TimelineMetric timelineMetric = metadataManagerInstance.getMetricFromUuid(uuid);
-
+ if (timelineMetric == null) {
+ return null;
+ }
return new TimelineClusterMetric(
timelineMetric.getMetricName(),
timelineMetric.getAppId(),
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
index 3bcbaf6..b3378be 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
@@ -48,6 +48,7 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.ambari.metrics.core.timeline.TimelineMetricServiceSummary;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -77,6 +78,7 @@ public class TimelineWebServices {
private static final Log LOG = LogFactory.getLog(TimelineWebServices.class);
private TimelineMetricStore timelineMetricStore;
+ private static final String SMOKETEST_METRIC_APP_ID = "amssmoketestfake";
@Inject
public TimelineWebServices(TimelineMetricStore timelineMetricStore) {
@@ -149,7 +151,11 @@ public class TimelineWebServices {
TimelineUtils.dumpTimelineRecordtoJSON(metrics, true));
}
- return timelineMetricStore.putMetrics(metrics);
+ if (CollectionUtils.isNotEmpty(metrics.getMetrics()) && metrics.getMetrics().get(0).getAppId().equals(SMOKETEST_METRIC_APP_ID)) {
+ return timelineMetricStore.putMetricsSkipCache(metrics);
+ } else {
+ return timelineMetricStore.putMetrics(metrics);
+ }
} catch (Exception e) {
LOG.error("Error saving metrics.", e);
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestTimelineMetricStore.java
index 24b6fef..cec8afa 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TestTimelineMetricStore.java
@@ -82,6 +82,11 @@ public class TestTimelineMetricStore implements TimelineMetricStore {
}
@Override
+ public TimelinePutResponse putMetricsSkipCache(TimelineMetrics metrics) throws SQLException, IOException {
+ return new TimelinePutResponse();
+ }
+
+ @Override
public TimelinePutResponse putContainerMetrics(List<ContainerMetric> metrics)
throws SQLException, IOException {
return new TimelinePutResponse();
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java
index eb64198..83e1651 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java
@@ -50,9 +50,9 @@ public class TimelineMetricStoreWatcherTest {
public void testRunPositive() throws Exception {
HBaseTimelineMetricsService metricStore = createNiceMock(HBaseTimelineMetricsService.class);
- metricStore.putMetricsSkipCache(anyObject(TimelineMetrics.class));
- expectLastCall().once();
-
+ expect(metricStore.putMetricsSkipCache(anyObject(TimelineMetrics.class)))
+ .andReturn(new TimelinePutResponse());
+
// metric found
expect(metricStore.getTimelineMetrics(EasyMock.<List<String>>anyObject(),
EasyMock.<List<String>>anyObject(), anyObject(String.class),
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
index fc57ffc..42abae7 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
@@ -378,7 +378,7 @@ def ams(name=None, action=None):
# Remove spnego configs from core-site if platform does not have python-kerberos library
truncated_core_site = {}
truncated_core_site.update(params.config['configurations']['core-site'])
- if is_spnego_enabled(params) and is_redhat_centos_6_plus() == False:
+ if is_spnego_enabled(params):
truncated_core_site.pop('hadoop.http.authentication.type')
truncated_core_site.pop('hadoop.http.filter.initializers')
@@ -407,6 +407,7 @@ def ams(name=None, action=None):
elif name == 'monitor':
+ # TODO Uncomment when SPNEGO support has been added to AMS service check and Grafana.
if is_spnego_enabled(params) and is_redhat_centos_6_plus():
try:
import kerberos
@@ -540,7 +541,7 @@ def is_spnego_enabled(params):
and 'hadoop.http.authentication.type' in params.config['configurations']['core-site'] \
and params.config['configurations']['core-site']['hadoop.http.authentication.type'] == "kerberos" \
and 'hadoop.http.filter.initializers' in params.config['configurations']['core-site'] \
- and params.config['configurations']['core-site']['hadoop.http.filter.initializers'] == "org.apache.hadoop.security.AuthenticationFilterInitializer":
+ and "org.apache.hadoop.security.AuthenticationFilterInitializer" in params.config['configurations']['core-site']['hadoop.http.filter.initializers']:
return True
return False