You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/01/02 13:26:29 UTC
ambari git commit: AMBARI-8967 Metrics: Issues when installing AMS
(dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 536901a5d -> 6dff2c162
AMBARI-8967 Metrics: Issues when installing AMS (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6dff2c16
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6dff2c16
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6dff2c16
Branch: refs/heads/trunk
Commit: 6dff2c162d435c949390689a9d6b16f1a9e7b5d6
Parents: 536901a
Author: Dmytro Sen <ds...@apache.org>
Authored: Fri Jan 2 12:57:00 2015 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Fri Jan 2 12:57:00 2015 +0200
----------------------------------------------------------------------
.../loadsimulator/net/RestMetricsSender.java | 2 +-
.../AMS/0.1.0/configuration/ams-hbase-env.xml | 136 +++++++++----------
.../AMS/0.1.0/configuration/ams-site.xml | 2 +-
.../AMS/0.1.0/package/scripts/ams.py | 6 +
.../AMS/0.1.0/package/scripts/params.py | 3 +
.../stacks/HDP/2.2/services/stack_advisor.py | 8 +-
6 files changed, 83 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
index 5130ae3..8657436 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
@@ -32,7 +32,7 @@ import java.net.ProtocolException;
public class RestMetricsSender implements MetricsSender {
private final static Logger LOG = LoggerFactory.getLogger(RestMetricsSender.class);
- private final static String COLLECTOR_URL = "http://%s:8188/ws/v1/timeline/metrics";
+ private final static String COLLECTOR_URL = "http://%s:6188/ws/v1/timeline/metrics";
private final String collectorServiceAddress;
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml
index 7a61c60..1cf80c8 100644
--- a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml
+++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml
@@ -63,74 +63,74 @@
<name>content</name>
<description>This is the jinja template for hbase-env.sh file</description>
<value>
- # Set environment variables here.
-
- # The java implementation to use. Java 1.6 required.
- export JAVA_HOME={{java64_home}}
-
- # HBase Configuration directory
- export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}
-
- # Extra Java CLASSPATH elements. Optional.
- export HBASE_CLASSPATH=${HBASE_CLASSPATH}
-
- if [ -f "/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar" ]; then
- export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar
- fi
-
- # The maximum amount of heap to use, in MB. Default is 1000.
- # export HBASE_HEAPSIZE=1000
-
- # Extra Java runtime options.
- # Below are what we set by default. May only work with SUN JVM.
- # For more on why as well as other possible settings,
- # see http://wiki.apache.org/hadoop/PerformanceTuning
- export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{hbase_log_dir}}/hs_err_pid%p.log"
- export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`"
- # Uncomment below to enable java garbage collection logging.
- # export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
-
- # Uncomment and adjust to enable JMX exporting
- # See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
- # More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
- #
- # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
- export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}"
- export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
- # export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
- # export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
-
- # File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
- export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers
-
- # Extra ssh options. Empty by default.
- # export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
-
- # Where log files are stored. $HBASE_HOME/logs by default.
- export HBASE_LOG_DIR={{hbase_log_dir}}
-
- # A string representing this instance of hbase. $USER by default.
- # export HBASE_IDENT_STRING=$USER
-
- # The scheduling priority for daemon processes. See 'man nice'.
- # export HBASE_NICENESS=10
-
- # The directory where pid files are stored. /tmp by default.
- export HBASE_PID_DIR={{hbase_pid_dir}}
-
- # Seconds to sleep between slave commands. Unset by default. This
- # can be useful in large clusters, where, e.g., slave rsyncs can
- # otherwise arrive faster than the master can service them.
- # export HBASE_SLAVE_SLEEP=0.1
-
- # Tell HBase whether it should manage it's own instance of Zookeeper or not.
- export HBASE_MANAGES_ZK=false
-
- {% if security_enabled %}
- export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}"
- export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}"
- export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}"
- {% endif %}
+# Set environment variables here.
+
+# The java implementation to use. Java 1.6 required.
+export JAVA_HOME={{java64_home}}
+
+# HBase Configuration directory
+export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}
+
+# Extra Java CLASSPATH elements. Optional.
+export HBASE_CLASSPATH=${HBASE_CLASSPATH}
+
+if [ -f "/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar" ]; then
+ export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar
+fi
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+export HBASE_HEAPSIZE={{hbase_heapsize}}
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{hbase_log_dir}}/hs_err_pid%p.log"
+export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{hbase_log_dir}}/gc.log-`date +'%Y%m%d%H%M'`"
+# Uncomment below to enable java garbage collection logging.
+# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}"
+export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
+# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored. $HBASE_HOME/logs by default.
+export HBASE_LOG_DIR={{hbase_log_dir}}
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+export HBASE_PID_DIR={{hbase_pid_dir}}
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+export HBASE_MANAGES_ZK=false
+
+{% if security_enabled %}
+export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}"
+export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}"
+{% endif %}
</value>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml
index 2f8281a..3cac412 100644
--- a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml
+++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml
@@ -47,7 +47,7 @@
</property>
<property>
<name>timeline.metrics.aggregator.checkpoint.dir</name>
- <value>/tmp</value>
+ <value>/var/lib/ambari-metrics-collector/checkpoint</value>
<description>
Directory to store aggregator checkpoints. Change to a permanent
location so that checkpoint ar not lost.
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py
index 1f09d7d..4a3a3f1 100644
--- a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py
+++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py
@@ -31,6 +31,12 @@ def ams(name=None):
recursive=True
)
+ Directory(params.ams_checkpoint_dir,
+ owner=params.ams_user,
+ group=params.user_group,
+ recursive=True
+ )
+
XmlConfig("ams-site.xml",
conf_dir=params.ams_collector_conf_dir,
configurations=params.config['configurations']['ams-site'],
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py
index afbbf4d..0932f3a 100644
--- a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py
@@ -95,7 +95,10 @@ regionserver_heapsize = config['configurations']['ams-hbase-env']['hbase_regions
regionserver_xmn_max = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_max']
regionserver_xmn_percent = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_ratio']
regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
+# For embedded mode
+hbase_heapsize = master_heapsize
+ams_checkpoint_dir = config['configurations']['ams-site']['timeline.metrics.aggregator.checkpoint.dir']
hbase_pid_dir = status_params.hbase_pid_dir
hbase_tmp_dir = config['configurations']['ams-hbase-site']['hbase.tmp.dir']
# TODO UPGRADE default, update site during upgrade
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 7f09dfa..fbfc850 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -137,13 +137,13 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
regionServerItem = self.validatorLessThenDefaultValue(properties, recommendedDefaults, "hbase_regionserver_heapsize")
masterItem = self.validatorLessThenDefaultValue(properties, recommendedDefaults, "hbase_master_heapsize")
- if regionServerItem is None and masterItem is None:
- hbase_regionserver_heapsize = formatXmxSizeToBytes(properties["hbase_regionserver_heapsize"])
+ if masterItem is None:
hbase_master_heapsize = formatXmxSizeToBytes(properties["hbase_master_heapsize"])
# TODO Add AMS Collector Xmx property to ams-env
- # Collector + HBASE Master + HBASE RegionServer HeapSize
- requiredMemory = 1073741824 + hbase_regionserver_heapsize + hbase_master_heapsize
+ # Collector 512m + HBASE Master heapsize
+ # For standalone HBase, master's heap memory is used by regionserver as well
+ requiredMemory = 536870912 + hbase_master_heapsize
amsCollectorHosts = self.getComponentHostNames(services, "AMS", "METRIC_COLLECTOR")
for collectorHostName in amsCollectorHosts: