You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ha...@apache.org on 2020/11/09 11:13:59 UTC
[ambari] branch branch-2.7 updated: AMBARI-25563. Storm dashboards
are not showing metrics. (#3255) (dvitiuk via dgrinenko)
This is an automated email from the ASF dual-hosted git repository.
hapylestat pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new 5fd2325 AMBARI-25563. Storm dashboards are not showing metrics. (#3255) (dvitiuk via dgrinenko)
5fd2325 is described below
commit 5fd23256c7e0cbba78cee8d8883c00c34e70ff56
Author: dvitiiuk <dm...@gmail.com>
AuthorDate: Mon Nov 9 13:13:45 2020 +0200
AMBARI-25563. Storm dashboards are not showing metrics. (#3255) (dvitiuk via dgrinenko)
---
.../ambari-metrics/datasource.js | 86 +++++++++++++---------
.../core/timeline/PhoenixHBaseAccessor.java | 19 ++++-
.../timeline/query/TransientMetricCondition.java | 20 +++--
.../timeline/uuid/HashBasedUuidGenStrategy.java | 6 +-
.../core/timeline/uuid/MD5UuidGenStrategy.java | 6 +-
.../uuid/MetricUuidGenNullRestrictedStrategy.java | 71 ++++++++++++++++++
.../timeline/uuid/Murmur3HashUuidGenStrategy.java | 6 +-
.../timeline/uuid/MetricUuidGenStrategyTest.java | 14 ++++
8 files changed, 174 insertions(+), 54 deletions(-)
diff --git a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
index 9e87bb1..4d6f4c4 100644
--- a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
+++ b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
@@ -564,35 +564,47 @@ define([
//Templatized Dashboards for Storm Components
if (templateSrv.variables[0].query === "topologies" && templateSrv.variables[1] &&
templateSrv.variables[1].name === "component") {
- if (templateSrv._values) {
- var selectedTopology = templateSrv._values.topologies;
- var selectedComponent = templateSrv._values.component;
- metricsPromises.push(_.map(options.targets, function(target) {
- target.sTopology = selectedTopology;
- target.sComponent = selectedComponent;
- target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sComponent);
+ var allTopologies = templateSrv.variables.filter(function(variable) {
+ return variable.name === "topologies";
+ });
+ var allComponents = templateSrv.variables.filter(function(variable) {
+ return variable.name === "component";
+ });
+
+ var selectedTopology = (_.isEmpty(allTopologies)) ? "" : allTopologies[0].current.text;
+ var selectedComponent = (_.isEmpty(allComponents)) ? "" : allComponents[0].current.text;
+
+ metricsPromises.push(_.map(options.targets, function(target) {
+ target.sTopology = selectedTopology;
+ target.sComponent = selectedComponent;
+ target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sComponent);
return getStormData(target);
- }));
- }
+ }));
}
//Templatized Dashboard for Storm Kafka Offset
if (templateSrv.variables[0].query === "topologies" && templateSrv.variables[1] &&
templateSrv.variables[1].name === "topic") {
- if (templateSrv._values) {
- var selectedTopology = templateSrv._values.topologies;
- var selectedTopic = templateSrv._values.topic;
- metricsPromises.push(_.map(options.targets, function(target) {
- target.sTopology = selectedTopology;
- target.sTopic = selectedTopic;
- target.sPartition = options.scopedVars.partition.value;
- target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sTopic)
- .replace('*', target.sPartition);
- return getStormData(target);
- }));
- }
+ var allTopologies = templateSrv.variables.filter(function(variable) {
+ return variable.name === "topologies";
+ });
+ var allTopics = templateSrv.variables.filter(function(variable) {
+ return variable.name === "topic";
+ });
+
+ var selectedTopology = (_.isEmpty(allTopologies)) ? "" : allTopologies[0].current.text;
+ var selectedTopic = (_.isEmpty(allTopics)) ? "" : allTopics[0].current.text;
+
+ metricsPromises.push(_.map(options.targets, function(target) {
+ target.sTopology = selectedTopology;
+ target.sTopic = selectedTopic;
+ target.sPartition = options.scopedVars.partition.value;
+ target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sTopic)
+ .replace('*', target.sPartition);
+ return getStormData(target);
+ }));
}
//Templatized Dashboards for Druid
@@ -694,7 +706,7 @@ define([
this.metricFindQuery = function (query) {
var interpolated;
try {
- interpolated = query.split('.')[0];
+ interpolated = templateSrv.replace(query);
} catch (err) {
return $q.reject(err);
}
@@ -926,24 +938,28 @@ define([
if (interpolated.indexOf("stormTopic") >= 0) {
var topicName = interpolated.substring(0,interpolated.indexOf('.'));
return this.getStormEntities().then(function () {
- var topicNames = Object.keys(stormEntities[topicName]);
- return _.map(topicNames, function(names){
- return {
- text: names
- };
- });
+ if(!_.isEmpty(stormEntities) && !_.isEmpty(stormEntities[topicName])) {
+ var topicNames = Object.keys(stormEntities[topicName]);
+ return _.map(topicNames, function(names){
+ return {
+ text: names
+ };
+ });
+ } else return[];
});
}
//Templated Variables for Storm Partitions per Topic
if (interpolated.indexOf("stormPartition") >= 0) {
var topicN, topologyN;
return this.getStormEntities().then(function () {
- var partitionNames = _.uniq(stormEntities[topologyN][topicN]);
- return _.map(partitionNames, function(names){
- return {
- text: names
- };
- });
+ if(!_.isEmpty(stormEntities) && !_.isEmpty(stormEntities[topologyN]) && !_.isEmpty(stormEntities[topologyN][topicN])) {
+ var partitionNames = _.uniq(stormEntities[topologyN][topicN]);
+ return _.map(partitionNames, function(names){
+ return {
+ text: names
+ };
+ });
+ } else return [];
});
}
// Templated Variable for YARN Queues.
@@ -1028,7 +1044,7 @@ define([
});
}
- if (interpolated == 'hosts') {
+ if (interpolated.indexOf("hosts") >= 0) {
return this.suggestHosts(tComponent, templatedCluster);
} else if (interpolated == 'cluster') {
return this.suggestClusters(tComponent)
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
index b6ff202..9588a3a 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
@@ -1960,7 +1960,7 @@ public class PhoenixHBaseAccessor {
TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(metricName, appId, instanceId);
metadata.setIsPersisted(true); // Always true on retrieval
- metadata.setUuid(rs.getBytes("UUID"));
+ metadata.setUuid(checkForNull(rs.getBytes("UUID")));
metadataMap.put(key, metadata);
}
@@ -2016,7 +2016,7 @@ public class PhoenixHBaseAccessor {
true
);
- metadata.setUuid(rs.getBytes("UUID"));
+ metadata.setUuid(checkForNull(rs.getBytes("UUID")));
metadataList.add(metadata);
}
}
@@ -2152,4 +2152,19 @@ public class PhoenixHBaseAccessor {
this.metadataManagerInstance = metadataManager;
TIMELINE_METRIC_READ_HELPER = new TimelineMetricReadHelper(this.metadataManagerInstance);
}
+
+ /**
+ * Null value are being saved to DB as array of zero bytes, so we need to make back converting
+ */
+ private byte[] checkForNull(byte[] uuid) {
+ if (uuid != null) {
+ for (byte b : uuid) {
+ if (b != 0) {
+ return uuid;
+ }
+ }
+ }
+ return null;
+ }
+
}
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java
index bb43f28..71eda61 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java
@@ -61,17 +61,21 @@ public class TransientMetricCondition extends DefaultCondition {
appendConjunction = appendHostnameClause(sb, appendConjunction);
String appId = getAppId();
- if (appId.contains("%")) {
- appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID LIKE ?");
- } else {
- appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID = ?");
+ if (appId != null) {
+ if (appId.contains("%")) {
+ appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID LIKE ?");
+ } else {
+ appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID = ?");
+ }
}
String instanceId = getInstanceId();
- if (instanceId.contains("%")) {
- appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID LIKE ?");
- } else {
- appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID = ?");
+ if (instanceId != null) {
+ if (instanceId.contains("%")) {
+ appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID LIKE ?");
+ } else {
+ appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID = ?");
+ }
}
appendConjunction = append(sb, appendConjunction, getStartTime(), " SERVER_TIME >= ?");
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java
index fdfeb79..278ffd7 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java
@@ -27,7 +27,7 @@ import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
-public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy {
+public class HashBasedUuidGenStrategy extends MetricUuidGenNullRestrictedStrategy {
/**
* Computes the UUID for a timelineClusterMetric.
@@ -36,7 +36,7 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy {
* @return byte array of length 'maxlength'
*/
@Override
- public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) {
+ protected byte[] computeUuidInternal(TimelineClusterMetric timelineClusterMetric, int maxLength) {
int metricNameUuidLength = 12;
String instanceId = timelineClusterMetric.getInstanceId();
@@ -173,7 +173,7 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy {
* @return byte array of length 'maxlength'
*/
@Override
- public byte[] computeUuid(String value, int maxLength) {
+ protected byte[] computeUuidInternal(String value, int maxLength) {
if (StringUtils.isEmpty(value)) {
return null;
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MD5UuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MD5UuidGenStrategy.java
index baa2dce..0e78953 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MD5UuidGenStrategy.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MD5UuidGenStrategy.java
@@ -24,13 +24,13 @@ import java.util.UUID;
import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
import org.apache.commons.lang.StringUtils;
-public class MD5UuidGenStrategy implements MetricUuidGenStrategy {
+public class MD5UuidGenStrategy extends MetricUuidGenNullRestrictedStrategy {
public MD5UuidGenStrategy() {
}
@Override
- public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) {
+ protected byte[] computeUuidInternal(TimelineClusterMetric timelineClusterMetric, int maxLength) {
String metricString = timelineClusterMetric.getMetricName() + timelineClusterMetric.getAppId();
if (StringUtils.isNotEmpty(timelineClusterMetric.getInstanceId())) {
@@ -47,7 +47,7 @@ public class MD5UuidGenStrategy implements MetricUuidGenStrategy {
}
@Override
- public byte[] computeUuid(String value, int maxLength) {
+ protected byte[] computeUuidInternal(String value, int maxLength) {
byte[] valueBytes = value.getBytes();
UUID uuid = UUID.nameUUIDFromBytes(valueBytes);
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenNullRestrictedStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenNullRestrictedStrategy.java
new file mode 100644
index 0000000..7bc4a0b
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenNullRestrictedStrategy.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.metrics.core.timeline.uuid;
+
+import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * HBase represents null value for BINARY fields as set of zero bytes. This means that we are not able to difference
+ * byte[]{0,0,0,0} and null values in DB. So we should not generate Uuids which contains only zero bytes.
+ */
+public abstract class MetricUuidGenNullRestrictedStrategy implements MetricUuidGenStrategy {
+ private static final Log LOG = LogFactory.getLog(MetricUuidGenNullRestrictedStrategy.class);
+
+ static final int RETRY_NUMBER = 5;
+
+ @Override
+ public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) {
+ int retry_attempt = 0;
+ byte[] result = null;
+ while (retry_attempt++ < RETRY_NUMBER) {
+ result = computeUuidInternal(timelineClusterMetric, maxLength);
+ for (byte b : result) {
+ if (b != 0) {
+ return result;
+ }
+ }
+ LOG.debug("Was generated Uuid which can represent null value in DB.");
+ }
+
+ LOG.error(String.format("After %n attempts was generated Uuid which can represent null value in DB", RETRY_NUMBER));
+ return result;
+ }
+
+ @Override
+ public byte[] computeUuid(String value, int maxLength) {
+ int retry_attempt = 0;
+ byte[] result = null;
+ while (retry_attempt++ < RETRY_NUMBER) {
+ result = computeUuidInternal(value, maxLength);
+ for (byte b : result) {
+ if (b != 0) {
+ return result;
+ }
+ }
+ LOG.debug("Was generated Uuid which can represent null value in DB.");
+ }
+
+ LOG.error(String.format("After %n attempts was generated Uuid which can represent null value in DB", RETRY_NUMBER));
+ return result;
+ }
+
+ abstract byte[] computeUuidInternal(TimelineClusterMetric timelineClusterMetric, int maxLength);
+ abstract byte[] computeUuidInternal(String value, int maxLength);
+}
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java
index af8cee5..0d5e165 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java
@@ -22,7 +22,7 @@ import com.google.common.hash.Hashing;
import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
import org.apache.commons.lang.StringUtils;
-public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{
+public class Murmur3HashUuidGenStrategy extends MetricUuidGenNullRestrictedStrategy {
/**
* Compute Murmur3Hash 16 byte UUID for a Metric-App-Instance.
@@ -30,7 +30,7 @@ public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{
* @param maxLength Max length of returned UUID. (Will always be 16 for this technique)
* @return 16 byte UUID.
*/ @Override
- public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) {
+ byte[] computeUuidInternal(TimelineClusterMetric timelineClusterMetric, int maxLength) {
String metricString = timelineClusterMetric.getMetricName() + timelineClusterMetric.getAppId();
if (StringUtils.isNotEmpty(timelineClusterMetric.getInstanceId())) {
@@ -47,7 +47,7 @@ public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{
* @return 4 byte UUID.
*/
@Override
- public byte[] computeUuid(String value, int maxLength) {
+ byte[] computeUuidInternal(String value, int maxLength) {
byte[] valueBytes = value.getBytes();
return Hashing.murmur3_32().hashBytes(valueBytes).asBytes();
}
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java
index a25310b..a02e660 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java
@@ -19,6 +19,7 @@
package org.apache.ambari.metrics.core.timeline.uuid;
import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -113,6 +114,19 @@ public class MetricUuidGenStrategyTest {
testConsistencyForUuidGenStrategy(new Murmur3HashUuidGenStrategy(), 4);
}
+ @Test
+ public void testNotNullCheckForHashing() {
+ MetricUuidGenNullRestrictedStrategy strategy = EasyMock.createMockBuilder(Murmur3HashUuidGenStrategy.class)
+ .addMockedMethod("computeUuidInternal", String.class, int.class).createStrictMock();
+
+ EasyMock.expect(strategy.computeUuidInternal(EasyMock.anyString(), EasyMock.anyInt()))
+ .andReturn(new byte[]{0,0,0,0}).times(MetricUuidGenNullRestrictedStrategy.RETRY_NUMBER);
+
+ EasyMock.replay(strategy);
+ strategy.computeUuid("", 0);
+ EasyMock.verify(strategy);
+ }
+
private void testMetricCollisionsForUuidGenStrategy(MetricUuidGenStrategy strategy, int uuidLength) {
Map<TimelineMetricUuid, TimelineClusterMetric> uuids = new HashMap<>();
for (String app : metricSet.keySet()) {