You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2016/04/22 04:39:19 UTC
ambari git commit: AMBARI-15892 : Incorrect (Negative) values are
shown for memory metrics - Commit 2 (avijayan)
Repository: ambari
Updated Branches:
refs/heads/branch-2.2.2 13d69a89a -> 9d26e3774
AMBARI-15892 : Incorrect (Negative) values are shown for memory metrics - Commit 2 (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9d26e377
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9d26e377
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9d26e377
Branch: refs/heads/branch-2.2.2
Commit: 9d26e3774bdf122e5d1bd5342b687254dfbc7014
Parents: 13d69a8
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Thu Apr 21 19:37:58 2016 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Thu Apr 21 19:38:04 2016 -0700
----------------------------------------------------------------------
.../metrics/timeline/PhoenixHBaseAccessor.java | 20 ++++++++++++++++----
.../timeline/TimelineMetricConfiguration.java | 3 +++
.../server/upgrade/UpgradeCatalog222.java | 14 +++++++++++++-
.../HBASE/0.96.0.2.0/widgets.json | 2 +-
.../YARN/2.1.0.2.0/YARN_widgets.json | 8 +-------
.../stacks/HDP/2.3/services/HBASE/widgets.json | 2 +-
.../HDP/2.3/services/YARN/YARN_widgets.json | 8 +-------
.../server/upgrade/UpgradeCatalog222Test.java | 12 ++----------
8 files changed, 38 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index b4c832b..9579207 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -69,6 +69,7 @@ import java.util.concurrent.TimeUnit;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATE_TABLE_SPLIT_POINTS;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HBASE_BLOCKING_STORE_FILES;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_TABLES_DURABILITY;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_DAILY_TABLE_TTL;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_HOUR_TABLE_TTL;
@@ -390,9 +391,13 @@ public class PhoenixHBaseAccessor {
LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY +
" = " + FIFO_COMPACTION_POLICY_CLASS + " for " + tableName);
// Need to set blockingStoreFiles to 1000 for FIFO
- tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, "1000");
+ int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 1000);
+ if (blockingStoreFiles < 1000) {
+ blockingStoreFiles = 1000;
+ }
+ tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles));
LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
- " = " + 1000 + " for " + tableName);
+ " = " + blockingStoreFiles + " for " + tableName);
modifyTable = true;
}
// Set back original policy if fifo disabled
@@ -402,9 +407,16 @@ public class PhoenixHBaseAccessor {
DEFAULT_COMPACTION_POLICY_CLASS);
LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY +
" = " + DEFAULT_COMPACTION_POLICY_CLASS + " for " + tableName);
- tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, "300");
+
+ int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 300);
+ if (blockingStoreFiles > 300) {
+ LOG.warn("HBase blocking store file set too high without FIFO " +
+ "Compaction policy enabled, restoring low value = 300.");
+ blockingStoreFiles = 300;
+ }
+ tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles));
LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
- " = " + 300 + " for " + tableName);
+ " = " + blockingStoreFiles + " for " + tableName);
modifyTable = true;
}
// Change TTL setting to match user configuration
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index 1018207..a74b05e 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -219,6 +219,9 @@ public class TimelineMetricConfiguration {
public static final String TIMELINE_METRICS_TABLES_DURABILITY =
"timeline.metrics.tables.durability";
+ public static final String HBASE_BLOCKING_STORE_FILES =
+ "hbase.hstore.blockingStoreFiles";
+
public static final String HOST_APP_ID = "HOST";
private Configuration hbaseConf;
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
index 8858bd5..31a3ce5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
@@ -107,6 +107,7 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
public static final String PHOENIX_QUERY_KEEPALIVE_PROPERTY = "phoenix.query.keepAliveMs";
public static final String TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED
= "timeline.metrics.cluster.aggregator.interpolation.enabled";
+ public static final String TIMELINE_METRICS_SINK_COLLECTION_PERIOD = "timeline.metrics.sink.collection.period";
public static final String AMS_SERVICE_NAME = "AMBARI_METRICS";
public static final String AMS_COLLECTOR_COMPONENT_NAME = "METRICS_COLLECTOR";
@@ -399,6 +400,13 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
newProperties.put(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED, String.valueOf(true));
}
+ if (!amsSiteProperties.containsKey(TIMELINE_METRICS_SINK_COLLECTION_PERIOD) ||
+ "60".equals(amsSiteProperties.get(TIMELINE_METRICS_SINK_COLLECTION_PERIOD))) {
+
+ newProperties.put(TIMELINE_METRICS_SINK_COLLECTION_PERIOD, "10");
+ LOG.info("Setting value of " + TIMELINE_METRICS_SINK_COLLECTION_PERIOD + " : 10");
+ }
+
updateConfigurationPropertiesForCluster(cluster, AMS_SITE, newProperties, true, true);
}
@@ -456,7 +464,7 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
Map<String, List<String>> widgetMap = new HashMap<>();
Map<String, String> sectionLayoutMap = new HashMap<>();
- List<String> yarnSummaryWidgets = new ArrayList<>(Arrays.asList("Container Failures", "App Failures"));
+ List<String> yarnSummaryWidgets = new ArrayList<>(Arrays.asList("Container Failures", "App Failures", "Cluster Memory"));
widgetMap.put("YARN_SUMMARY", yarnSummaryWidgets);
sectionLayoutMap.put("YARN_SUMMARY", "default_yarn_dashboard");
@@ -581,6 +589,10 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
if (targetWidgetLayoutInfo != null) {
entityToUpdate.setMetrics(gson.toJson(targetWidgetLayoutInfo.getMetricsInfo()));
entityToUpdate.setWidgetValues(gson.toJson(targetWidgetLayoutInfo.getValues()));
+ if ("HBASE".equals(serviceName) && "Reads and Writes".equals(widgetName)) {
+ entityToUpdate.setDescription(targetWidgetLayoutInfo.getDescription());
+ LOG.info("Update description for HBase Reads and Writes widget");
+ }
widgetDAO.merge(entityToUpdate);
} else {
LOG.warn("Unable to find widget layout info for " + widgetName +
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json
index ede9f49..4120ff2 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json
@@ -7,7 +7,7 @@
"widgetLayoutInfo": [
{
"widget_name": "Reads and Writes",
- "description": "Count of read and write requests on all regions in the cluster.",
+ "description": "Rate (per second) of read and write requests on all regions in the cluster.",
"default_section_name": "HBASE_SUMMARY",
"widget_type": "GRAPH",
"is_visible": true,
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json
index 442d9a2..4b76a17 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json
@@ -230,18 +230,12 @@
"metric_path": "metrics/memory/mem_free._avg",
"service_name": "YARN",
"component_name": "NODEMANAGER"
- },
- {
- "name": "mem_cached._sum",
- "metric_path": "metrics/memory/mem_cached._avg",
- "service_name": "YARN",
- "component_name": "NODEMANAGER"
}
],
"values": [
{
"name": "Memory utilization",
- "value": "${((mem_total._sum - mem_free._sum - mem_cached._sum)/mem_total._sum) * 100}"
+ "value": "${((mem_total._sum - mem_free._sum)/mem_total._sum) * 100}"
}
],
"properties": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json
index d8f4e6a..ae47833 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json
@@ -7,7 +7,7 @@
"widgetLayoutInfo": [
{
"widget_name": "Reads and Writes",
- "description": "Count of read and write requests on all regions in the cluster.",
+ "description": "Rate (per second) of read and write requests on all regions in the cluster.",
"widget_type": "GRAPH",
"is_visible": true,
"metrics": [
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json
index c2acf1f..782f21d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json
@@ -259,18 +259,12 @@
"metric_path": "metrics/memory/mem_free._avg",
"service_name": "YARN",
"component_name": "NODEMANAGER"
- },
- {
- "name": "mem_cached._sum",
- "metric_path": "metrics/memory/mem_cached._avg",
- "service_name": "YARN",
- "component_name": "NODEMANAGER"
}
],
"values": [
{
"name": "Memory utilization",
- "value": "${((mem_total._sum - mem_free._sum - mem_cached._sum)/mem_total._sum) * 100}"
+ "value": "${((mem_total._sum - mem_free._sum)/mem_total._sum) * 100}"
}
],
"properties": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
index 83b995b..4302349 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
@@ -349,14 +349,10 @@ public class UpgradeCatalog222Test {
put("timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(1));
put("timeline.metrics.service.operation.mode", "distributed");
put("timeline.metrics.host.aggregator.ttl", String.valueOf(86400));
- put("timeline.metrics.host.aggregator.minute.ttl", String.valueOf(604800));
- put("timeline.metrics.host.aggregator.hourly.ttl", String.valueOf(2592000));
- put("timeline.metrics.host.aggregator.daily.ttl", String.valueOf(31536000));
put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(21600)); //Less than 1 day
put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(7776000));
- put("timeline.metrics.cluster.aggregator.hourly.ttl", String.valueOf(31536000));
- put("timeline.metrics.cluster.aggregator.daily.ttl", String.valueOf(63072000));
put("timeline.metrics.service.webapp.address", "0.0.0.0:6188");
+ put("timeline.metrics.sink.collection.period", "60");
}
};
Map<String, String> newPropertiesAmsSite = new HashMap<String, String>() {
@@ -365,16 +361,12 @@ public class UpgradeCatalog222Test {
put("timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(2));
put("timeline.metrics.service.watcher.disabled", String.valueOf(false));
put("timeline.metrics.host.aggregator.ttl", String.valueOf(3 * 86400));
- put("timeline.metrics.host.aggregator.minute.ttl", String.valueOf(7 * 86400));
- put("timeline.metrics.host.aggregator.hourly.ttl", String.valueOf(30 * 86400));
- put("timeline.metrics.host.aggregator.daily.ttl", String.valueOf(365 * 86400));
put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(21600));
put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(30 * 86400));
- put("timeline.metrics.cluster.aggregator.hourly.ttl", String.valueOf(365 * 86400));
- put("timeline.metrics.cluster.aggregator.daily.ttl", String.valueOf(730 * 86400));
put("timeline.metrics.service.operation.mode", "distributed");
put("timeline.metrics.service.webapp.address", "host1:6188");
put("timeline.metrics.cluster.aggregator.interpolation.enabled", String.valueOf(true));
+ put("timeline.metrics.sink.collection.period", "10");
}
};
EasyMockSupport easyMockSupport = new EasyMockSupport();