You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by tb...@apache.org on 2013/06/20 20:09:31 UTC
svn commit: r1495115 [1/10] - in /incubator/ambari/trunk:
ambari-agent/src/main/puppet/modules/hdp-flume/
ambari-agent/src/main/puppet/modules/hdp-flume/files/
ambari-agent/src/main/puppet/modules/hdp-flume/manifests/
ambari-agent/src/main/puppet/modul...
Author: tbeerbower
Date: Thu Jun 20 18:09:29 2013
New Revision: 1495115
URL: http://svn.apache.org/r1495115
Log:
AMBARI-2437 - Merge Flume work
Added:
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/files/
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/files/flumeSmoke.sh
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/client.pp
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/init.pp
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/params.pp
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/service.pp
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/flume-env.sh.erb
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/log4j.properties.erb
incubator/ambari/trunk/ambari-server/src/main/resources/scripts/install_flume.sh
incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/
incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/configuration/
incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/configuration/global.xml
incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/metainfo.xml
incubator/ambari/trunk/ambari-server/src/test/resources/flume_ganglia_data.txt
incubator/ambari/trunk/ambari-web/app/assets/data/services/metrics/flume/
incubator/ambari/trunk/ambari-web/app/assets/data/services/metrics/flume/channelFillPct.json
incubator/ambari/trunk/ambari-web/app/assets/data/services/metrics/flume/channelSize.json
incubator/ambari/trunk/ambari-web/app/assets/data/services/metrics/flume/cpu_user.json
incubator/ambari/trunk/ambari-web/app/assets/data/services/metrics/flume/jvmGcTime.json
incubator/ambari/trunk/ambari-web/app/assets/data/services/metrics/flume/jvmMemHeapUsedM.json
incubator/ambari/trunk/ambari-web/app/assets/data/services/metrics/flume/jvmThreadsRunnable.json
incubator/ambari/trunk/ambari-web/app/assets/data/services/metrics/flume/sinkConnectionFailedCount.json
incubator/ambari/trunk/ambari-web/app/assets/data/services/metrics/flume/sinkDrainSuccessCount.json
incubator/ambari/trunk/ambari-web/app/assets/data/services/metrics/flume/sourceEventAccepted.json
incubator/ambari/trunk/ambari-web/app/models/service/flume.js
incubator/ambari/trunk/ambari-web/app/templates/main/dashboard/service/flume.hbs
incubator/ambari/trunk/ambari-web/app/views/main/dashboard/service/flume.js
incubator/ambari/trunk/ambari-web/app/views/main/service/info/metrics/flume/
incubator/ambari/trunk/ambari-web/app/views/main/service/info/metrics/flume/channel_fill_pct.js
incubator/ambari/trunk/ambari-web/app/views/main/service/info/metrics/flume/channel_size.js
incubator/ambari/trunk/ambari-web/app/views/main/service/info/metrics/flume/cpu_user.js
incubator/ambari/trunk/ambari-web/app/views/main/service/info/metrics/flume/gc.js
incubator/ambari/trunk/ambari-web/app/views/main/service/info/metrics/flume/jvm_heap.js
incubator/ambari/trunk/ambari-web/app/views/main/service/info/metrics/flume/jvm_threads_runnable.js
incubator/ambari/trunk/ambari-web/app/views/main/service/info/metrics/flume/sink_connection_failed.js
incubator/ambari/trunk/ambari-web/app/views/main/service/info/metrics/flume/sink_drain_success.js
incubator/ambari/trunk/ambari-web/app/views/main/service/info/metrics/flume/source_accepted.js
Modified:
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/manifests/params.pp
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/templates/hadoop-servicegroups.cfg.erb
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/templates/hadoop-services.cfg.erb
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
incubator/ambari/trunk/ambari-server/pom.xml
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/Role.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseProvider.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
incubator/ambari/trunk/ambari-server/src/main/resources/ganglia_properties.json
incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
incubator/ambari/trunk/ambari-web/app/assets/data/dashboard/services.json
incubator/ambari/trunk/ambari-web/app/controllers/global/update_controller.js
incubator/ambari/trunk/ambari-web/app/controllers/main/service/item.js
incubator/ambari/trunk/ambari-web/app/mappers/service_mapper.js
incubator/ambari/trunk/ambari-web/app/messages.js
incubator/ambari/trunk/ambari-web/app/models.js
incubator/ambari/trunk/ambari-web/app/models/host_component.js
incubator/ambari/trunk/ambari-web/app/models/service.js
incubator/ambari/trunk/ambari-web/app/templates/main/service/info/summary.hbs
incubator/ambari/trunk/ambari-web/app/templates/main/service/item.hbs
incubator/ambari/trunk/ambari-web/app/utils/ajax.js
incubator/ambari/trunk/ambari-web/app/utils/helper.js
incubator/ambari/trunk/ambari-web/app/views.js
incubator/ambari/trunk/ambari-web/app/views/main/service/info/summary.js
incubator/ambari/trunk/ambari-web/app/views/main/service/item.js
incubator/ambari/trunk/contrib/addons/src/addOns/nagios/scripts/nagios_alerts.php
Added: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/files/flumeSmoke.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/files/flumeSmoke.sh?rev=1495115&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/files/flumeSmoke.sh (added)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/files/flumeSmoke.sh Thu Jun 20 18:09:29 2013
@@ -0,0 +1,23 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+echo "Flume Smoke Test: Passed"
Added: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/client.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/client.pp?rev=1495115&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/client.pp (added)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/client.pp Thu Jun 20 18:09:29 2013
@@ -0,0 +1,25 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp-flume::client(
+ $service_state = $hdp::params::cluster_client_state
+) inherits hdp::params
+{
+}
Added: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/init.pp?rev=1495115&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/init.pp (added)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/init.pp Thu Jun 20 18:09:29 2013
@@ -0,0 +1,27 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp-flume(
+ $type = server,
+ $service_state = $hdp::params::cluster_service_state,
+ $opts = {}
+) inherits hdp-flume::params
+{
+}
Added: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/params.pp?rev=1495115&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/params.pp (added)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/params.pp Thu Jun 20 18:09:29 2013
@@ -0,0 +1,24 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp-flume::params() inherits hdp::params
+{
+ $flume_log_dir = hdp_default("flume_log_dir","/var/log/flume")
+}
Added: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/service.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/service.pp?rev=1495115&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/service.pp (added)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/manifests/service.pp Thu Jun 20 18:09:29 2013
@@ -0,0 +1,28 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp-flume::service(
+ $ensure = $hdp::params::cluster_service_state
+)
+{
+}
+
+
+
Added: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/flume-env.sh.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/flume-env.sh.erb?rev=1495115&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/flume-env.sh.erb (added)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/flume-env.sh.erb Thu Jun 20 18:09:29 2013
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Enviroment variables can be set here.
+#
+# #JAVA_HOME=/usr/lib/jvm/java-6-sun
+#
+# # Give Flume more memory and pre-allocate, enable remote monitoring via JMX
+JAVA_OPTS="-Xms100m -Xmx200m -Dcom.sun.management.jmxremote -Dflume.monitoring.type=ganglia -Dflume.monitoring.hosts=<%=scope.function_hdp_template_var("::hdp-flume::params::ganglia_sink")%>"
+#
+# # Note that the Flume conf directory is always included in the classpath.
+# #FLUME_CLASSPATH=""
Added: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/log4j.properties.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/log4j.properties.erb?rev=1495115&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/log4j.properties.erb (added)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-flume/templates/log4j.properties.erb Thu Jun 20 18:09:29 2013
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Define some default values that can be overridden by system properties.
+#
+# For testing, it may also be convenient to specify
+# -Dflume.root.logger=DEBUG,console when launching flume.
+
+#flume.root.logger=DEBUG,console
+flume.root.logger=INFO,LOGFILE
+flume.log.dir=<%=scope.function_hdp_template_var("::hdp-flume::params::flume_log_dir")%>
+flume.log.file=flume.log
+
+log4j.logger.org.apache.flume.lifecycle = INFO
+log4j.logger.org.jboss = WARN
+log4j.logger.org.mortbay = INFO
+log4j.logger.org.apache.avro.ipc.NettyTransceiver = WARN
+log4j.logger.org.apache.hadoop = INFO
+
+# Define the root logger to the system property "flume.root.logger".
+log4j.rootLogger=${flume.root.logger}
+
+#
+# Rolling file appender
+# Default log rotation configuration
+#
+
+log4j.appender.LOGFILE=org.apache.log4j.RollingFileAppender
+log4j.appender.LOGFILE.MaxFileSize=100MB
+log4j.appender.LOGFILE.MaxBackupIndex=10
+log4j.appender.LOGFILE.File=${flume.log.dir}/${flume.log.file}
+log4j.appender.LOGFILE.layout=org.apache.log4j.PatternLayout
+log4j.appender.LOGFILE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-5p [%t] (%C.%M:%L) %x - %m%n
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d (%t) [%p - %l] %m%n
Modified: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/manifests/params.pp?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/manifests/params.pp (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/manifests/params.pp Thu Jun 20 18:09:29 2013
@@ -75,6 +75,7 @@ class hdp-nagios::params() inherits hdp:
nagios-server => {host_member_info => 'nagios_server_host'},
jobtracker => {host_member_info => 'jtnode_host'},
ganglia-server => {host_member_info => 'ganglia_server_host'},
+ flume-servers => {host_member_info => 'flume_hosts'},
zookeeper-servers => {host_member_info => 'zookeeper_hosts'},
hbasemasters => {host_member_info => 'hbase_master_hosts'},
hiveserver => {host_member_info => 'hive_server_host'},
Modified: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/templates/hadoop-servicegroups.cfg.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/templates/hadoop-servicegroups.cfg.erb?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/templates/hadoop-servicegroups.cfg.erb (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/templates/hadoop-servicegroups.cfg.erb Thu Jun 20 18:09:29 2013
@@ -12,6 +12,11 @@ define servicegroup {
}
define servicegroup {
+ servicegroup_name FLUME
+ alias FLUME Checks
+}
+
+define servicegroup {
servicegroup_name HBASE
alias HBASE Checks
}
Modified: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/templates/hadoop-services.cfg.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/templates/hadoop-services.cfg.erb?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/templates/hadoop-services.cfg.erb (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/templates/hadoop-services.cfg.erb Thu Jun 20 18:09:29 2013
@@ -117,6 +117,8 @@ define service {
<%end-%>
<%end-%>
+
+
# GANGLIA SERVER Checks
<%if scope.function_hdp_nagios_members_exist('ganglia-server')-%>
define service {
@@ -498,6 +500,21 @@ define service {
<%end-%>
+<%if scope.function_hdp_nagios_members_exist('flume-servers')-%>
+# FLUME Checks
+define service {
+ hostgroup_name flume-servers
+ use hadoop-service
+ service_description FLUME::Flume Agent process down
+ servicegroups FLUME
+ check_command check_tcp!<%=scope.function_hdp_template_var("flume_port")%>!-w 1 -c 1
+ normal_check_interval 1
+ retry_check_interval 0.5
+ max_check_attempts 3
+}
+<%end-%>
+
+
<%if scope.function_hdp_nagios_members_exist('zookeeper-servers')-%>
# ZOOKEEPER Checks
define service {
Modified: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp Thu Jun 20 18:09:29 2013
@@ -37,6 +37,7 @@ define hdp::configfile(
$all_hosts = $hdp::params::all_hosts,
$hbase_rs_hosts = $hdp::params::hbase_rs_hosts,
$zookeeper_hosts = $hdp::params::zookeeper_hosts,
+ $flume_hosts = $hdp::params::flume_hosts,
$hbase_master_hosts = $hdp::params::hbase_master_hosts,
$hcat_server_host = $hdp::params::hcat_server_host,
$hive_server_host = $hdp::params::hive_server_host,
Modified: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp Thu Jun 20 18:09:29 2013
@@ -69,6 +69,10 @@ class hdp::params()
$zookeeper_hosts = hdp_default("zookeeper_hosts")
+ $flume_hosts = hdp_default("flume_hosts")
+
+ $flume_port = hdp_default("flume_port", "4159")
+
$hbase_master_hosts = hdp_default("hbase_master_hosts", "")
#if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py Thu Jun 20 18:09:29 2013
@@ -129,7 +129,8 @@ rolesToClass = {
'HUE_SERVICE_CHECK': 'hdp-hue::service_check',
'RESOURCEMANAGER_SERVICE_CHECK': 'hdp-yarn::resourcemanager::service_check',
'HISTORYSERVER_SERVICE_CHECK': 'hdp-yarn::historyserver::service_check',
- 'TEZ_CLIENT': 'hdp-tez::tez_client'
+ 'TEZ_CLIENT': 'hdp-tez::tez_client',
+ 'FLUME_SERVER': 'hdp-flume'
}
serviceStates = {
@@ -150,6 +151,7 @@ servicesToPidNames = {
'HISTORYSERVER': 'mapred-{USER}-historyserver.pid$',
'OOZIE_SERVER': 'oozie.pid',
'ZOOKEEPER_SERVER': 'zookeeper_server.pid',
+ 'FLUME_SERVER': 'flume-node.pid',
'TEMPLETON_SERVER': 'templeton.pid',
'NAGIOS_SERVER': 'nagios.pid',
'GANGLIA_SERVER': 'gmetad.pid',
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py Thu Jun 20 18:09:29 2013
@@ -33,7 +33,7 @@ class LiveStatus:
"HDFS", "MAPREDUCE", "GANGLIA", "HBASE",
"NAGIOS", "ZOOKEEPER", "OOZIE", "HCATALOG",
"KERBEROS", "TEMPLETON", "HIVE", "WEBHCAT",
- "YARN", "MAPREDUCE2"
+ "YARN", "MAPREDUCE2", "FLUME"
]
COMPONENTS = [
@@ -62,6 +62,9 @@ class LiveStatus:
{"serviceName" : "NAGIOS",
"componentName" : "NAGIOS_SERVER"},
+ {"serviceName" : "FLUME",
+ "componentName" : "FLUME_SERVER"},
+
{"serviceName" : "ZOOKEEPER",
"componentName" : "ZOOKEEPER_SERVER"},
Modified: incubator/ambari/trunk/ambari-server/pom.xml
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/pom.xml?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/pom.xml (original)
+++ incubator/ambari/trunk/ambari-server/pom.xml Thu Jun 20 18:09:29 2013
@@ -308,6 +308,9 @@
<source>
<location>src/main/resources/hive-schema-0.10.0.oracle.sql</location>
</source>
+ <source>
+ <location>src/main/resources/scripts/install_flume.sh</location>
+ </source>
</sources>
</mapping>
<mapping>
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/Role.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/Role.java?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/Role.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/Role.java Thu Jun 20 18:09:29 2013
@@ -76,5 +76,6 @@ public enum Role {
NODEMANAGER,
YARN_CLIENT,
HISTORYSERVER,
- TEZ_CLIENT
+ TEZ_CLIENT,
+ FLUME_SERVER
}
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java Thu Jun 20 18:09:29 2013
@@ -29,6 +29,8 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* Abstract property provider implementation for a Ganglia source.
@@ -51,16 +53,17 @@ public abstract class GangliaPropertyPro
public static final Map<String, String> GANGLIA_CLUSTER_NAME_MAP = new HashMap<String, String>();
static {
- GANGLIA_CLUSTER_NAME_MAP.put("NAMENODE", "HDPNameNode");
- GANGLIA_CLUSTER_NAME_MAP.put("DATANODE", "HDPSlaves");
- GANGLIA_CLUSTER_NAME_MAP.put("JOBTRACKER", "HDPJobTracker");
- GANGLIA_CLUSTER_NAME_MAP.put("TASKTRACKER", "HDPSlaves");
- GANGLIA_CLUSTER_NAME_MAP.put("RESOURCEMANAGER", "HDPResourceManager");
- GANGLIA_CLUSTER_NAME_MAP.put("NODEMANAGER", "HDPSlaves");
- GANGLIA_CLUSTER_NAME_MAP.put("HISTORYSERVER", "HDPHistoryServer");
- GANGLIA_CLUSTER_NAME_MAP.put("HBASE_MASTER", "HDPHBaseMaster");
- GANGLIA_CLUSTER_NAME_MAP.put("HBASE_CLIENT", "HDPSlaves");
+ GANGLIA_CLUSTER_NAME_MAP.put("NAMENODE", "HDPNameNode");
+ GANGLIA_CLUSTER_NAME_MAP.put("DATANODE", "HDPSlaves");
+ GANGLIA_CLUSTER_NAME_MAP.put("JOBTRACKER", "HDPJobTracker");
+ GANGLIA_CLUSTER_NAME_MAP.put("TASKTRACKER", "HDPSlaves");
+ GANGLIA_CLUSTER_NAME_MAP.put("RESOURCEMANAGER", "HDPResourceManager");
+ GANGLIA_CLUSTER_NAME_MAP.put("NODEMANAGER", "HDPSlaves");
+ GANGLIA_CLUSTER_NAME_MAP.put("HISTORYSERVER", "HDPHistoryServer");
+ GANGLIA_CLUSTER_NAME_MAP.put("HBASE_MASTER", "HDPHBaseMaster");
+ GANGLIA_CLUSTER_NAME_MAP.put("HBASE_CLIENT", "HDPSlaves");
GANGLIA_CLUSTER_NAME_MAP.put("HBASE_REGIONSERVER", "HDPSlaves");
+ GANGLIA_CLUSTER_NAME_MAP.put("FLUME_SERVER", "HDPSlaves");
}
protected final static Logger LOG =
@@ -208,7 +211,9 @@ public abstract class GangliaPropertyPro
new ResourceKey(getHostName(resource), gangliaClusterName);
for (String id : ids) {
- Map<String, PropertyInfo> propertyInfoMap = getPropertyInfoMap(getComponentName(resource), id);
+ Map<String, PropertyInfo> propertyInfoMap = new HashMap<String, PropertyInfo>();
+
+ boolean requestAll = getPropertyInfoMap(getComponentName(resource), id, propertyInfoMap);
for (Map.Entry<String, PropertyInfo> entry : propertyInfoMap.entrySet()) {
String propertyId = entry.getKey();
@@ -224,6 +229,7 @@ public abstract class GangliaPropertyPro
}
rrdRequest.putResource(key, resource);
rrdRequest.putPropertyId(propertyInfo.getPropertyId(), propertyId);
+ rrdRequest.setRequestAllMetrics(requestAll);
}
}
}
@@ -355,6 +361,7 @@ public abstract class GangliaPropertyPro
private final Map<String, Set<String>> metrics = new HashMap<String, Set<String>>();
private final Set<String> clusterSet = new HashSet<String>();
private final Set<String> hostSet = new HashSet<String>();
+ private boolean requestAll = false;
private RRDRequest(String clusterName, TemporalInfo temporalInfo) {
@@ -362,6 +369,10 @@ public abstract class GangliaPropertyPro
this.temporalInfo = temporalInfo;
}
+ public void setRequestAllMetrics(boolean requestAll) {
+ this.requestAll = this.requestAll | requestAll;
+ }
+
public void putResource(ResourceKey key, Resource resource) {
clusterSet.add(key.getClusterName());
hostSet.add(key.getHostName());
@@ -392,7 +403,8 @@ public abstract class GangliaPropertyPro
*/
public Collection<Resource> populateResources() throws SystemException {
- String spec = getSpec(clusterName, clusterSet, hostSet, metrics.keySet(), temporalInfo);
+ String spec = getSpec(clusterName, clusterSet, hostSet,
+ requestAll ? Collections.<String>emptySet() : metrics.keySet(), temporalInfo);
BufferedReader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(
@@ -465,13 +477,42 @@ public abstract class GangliaPropertyPro
* @param gangliaMetric the Ganglia metrics
*/
private void populateResource(Resource resource, GangliaMetric gangliaMetric) {
- Set<String> propertyIdSet = metrics.get(gangliaMetric.getMetric_name());
+ String metric_name = gangliaMetric.getMetric_name();
+
+ Set<String> propertyIdSet = metrics.get(metric_name);
+ List<String> parameterList = new LinkedList<String>();
+
+ if (propertyIdSet == null) {
+ for (Map.Entry<String, Set<String>> entry : metrics.entrySet()) {
+
+ String key = entry.getKey();
+
+ Pattern pattern = Pattern.compile(key);
+ Matcher matcher = pattern.matcher(metric_name);
+
+ if (matcher.matches()) {
+ propertyIdSet = entry.getValue();
+ // get parameters
+ for (int i = 0; i < matcher.groupCount(); ++i) {
+ parameterList.add(matcher.group(i + 1));
+ }
+ break;
+ }
+ }
+ }
if (propertyIdSet != null) {
Map<String, PropertyInfo> metricsMap = getComponentMetrics().get(getComponentName(resource));
if (metricsMap != null) {
for (String propertyId : propertyIdSet) {
if (propertyId != null) {
if (metricsMap.containsKey(propertyId)){
+ if (propertyId.matches(".*\\$\\d+.*")) {
+ int i = 1;
+ for (String param : parameterList) {
+ propertyId = propertyId.replace("$" + i, param);
+ ++i;
+ }
+ }
resource.setProperty(propertyId, getValue(gangliaMetric, temporalInfo != null));
}
}
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java Thu Jun 20 18:09:29 2013
@@ -21,7 +21,6 @@ package org.apache.ambari.server.control
import org.apache.ambari.server.controller.spi.PropertyProvider;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -73,25 +72,66 @@ public abstract class AbstractPropertyPr
* @return a map of metrics
*/
protected Map<String, PropertyInfo> getPropertyInfoMap(String componentName, String propertyId) {
+ Map<String, PropertyInfo> propertyInfoMap = new HashMap<String, PropertyInfo>();
+
+ getPropertyInfoMap(componentName, propertyId, propertyInfoMap);
+
+ return propertyInfoMap;
+ }
+
+ // TODO : added for FLUME and reg exp property ids... revisit.
+ protected boolean getPropertyInfoMap(String componentName, String propertyId, Map<String, PropertyInfo> propertyInfoMap) {
Map<String, PropertyInfo> componentMetricMap = componentMetrics.get(componentName);
+
+ propertyInfoMap.clear();
+
if (componentMetricMap == null) {
- return Collections.emptyMap();
+ return false;
}
PropertyInfo propertyInfo = componentMetricMap.get(propertyId);
if (propertyInfo != null) {
- return Collections.singletonMap(propertyId, propertyInfo);
+ propertyInfoMap.put(propertyId, propertyInfo);
+ return false;
}
+ String regExpKey = getRegExpKey(propertyId);
+
+ if (regExpKey != null) {
+ propertyInfo = componentMetricMap.get(regExpKey);
+ if (propertyInfo != null) {
+ propertyInfoMap.put(regExpKey, propertyInfo);
+ return true;
+ }
+ }
+
+ // TODO : For now, if the property info map contains any metrics with regular expressions then get back all the metrics.
+ boolean requestAllMetrics = false;
+
if (!propertyId.endsWith("/")){
propertyId += "/";
}
- Map<String, PropertyInfo> propertyInfoMap = new HashMap<String, PropertyInfo>();
+
for (Map.Entry<String, PropertyInfo> entry : componentMetricMap.entrySet()) {
if (entry.getKey().startsWith(propertyId)) {
- propertyInfoMap.put(entry.getKey(), entry.getValue());
+ String key = entry.getKey();
+ requestAllMetrics = isPatternKey(key);
+ propertyInfoMap.put(key, entry.getValue());
}
}
- return propertyInfoMap;
+
+ if (regExpKey != null) {
+ if (!regExpKey.endsWith("/")){
+ regExpKey += "/";
+ }
+
+ for (Map.Entry<String, PropertyInfo> entry : componentMetricMap.entrySet()) {
+ if (entry.getKey().startsWith(regExpKey)) {
+ requestAllMetrics = true;
+ propertyInfoMap.put(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+ return requestAllMetrics;
}
}
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseProvider.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseProvider.java?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseProvider.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseProvider.java Thu Jun 20 18:09:29 2013
@@ -27,9 +27,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* Base provider implementation for both property and resource providers.
@@ -51,6 +54,9 @@ public abstract class BaseProvider {
*/
private final Set<String> combinedIds;
+
+ private final Map<String, Pattern> patterns;
+
/**
* The logger.
*/
@@ -70,6 +76,13 @@ public abstract class BaseProvider {
this.categoryIds = PropertyHelper.getCategories(propertyIds);
this.combinedIds = new HashSet<String>(propertyIds);
this.combinedIds.addAll(this.categoryIds);
+ this.patterns = new HashMap<String, Pattern>();
+ for (String id : this.combinedIds) {
+ if (id.matches(".*\\$\\d+.*")) {
+ String pattern = id.replaceAll("\\$\\d+", "\\\\S+");
+ patterns.put(id, Pattern.compile(pattern));
+ }
+ }
}
@@ -106,12 +119,8 @@ public abstract class BaseProvider {
// we want to treat property as a category and the entries as individual properties.
Set<String> categoryProperties = new HashSet<String>();
for (String unsupportedPropertyId : unsupportedPropertyIds) {
- String category = PropertyHelper.getPropertyCategory(unsupportedPropertyId);
- while (category != null) {
- if (this.propertyIds.contains(category)) {
- categoryProperties.add(unsupportedPropertyId);
- }
- category = PropertyHelper.getPropertyCategory(category);
+ if (checkCategory(unsupportedPropertyId) || checkRegExp(unsupportedPropertyId)) {
+ categoryProperties.add(unsupportedPropertyId);
}
}
unsupportedPropertyIds.removeAll(categoryProperties);
@@ -148,16 +157,9 @@ public abstract class BaseProvider {
Set<String> unsupportedPropertyIds = new HashSet<String>(propertyIds);
unsupportedPropertyIds.removeAll(this.combinedIds);
- // Add the categories to account for map properties where the entries will not be
- // in the provider property list ids but the map (category) might be.
for (String unsupportedPropertyId : unsupportedPropertyIds) {
- String category = PropertyHelper.getPropertyCategory(unsupportedPropertyId);
- while (category != null) {
- if (this.propertyIds.contains(category)) {
- keepers.add(unsupportedPropertyId);
- break;
- }
- category = PropertyHelper.getPropertyCategory(category);
+ if (checkCategory(unsupportedPropertyId) || checkRegExp(unsupportedPropertyId)) {
+ keepers.add(unsupportedPropertyId);
}
}
propertyIds.retainAll(this.combinedIds);
@@ -167,6 +169,51 @@ public abstract class BaseProvider {
}
/**
+ * Check the categories to account for map properties where the entries will not be
+ * in the provider property list ids but the map (category) might be.
+ */
+ private boolean checkCategory(String unsupportedPropertyId) {
+ String category = PropertyHelper.getPropertyCategory(unsupportedPropertyId);
+ while (category != null) {
+ if( this.propertyIds.contains(category)) {
+ return true;
+ }
+ category = PropertyHelper.getPropertyCategory(category);
+ }
+ return false;
+ }
+
+ private boolean checkRegExp(String unsupportedPropertyId) {
+ for (Pattern pattern : patterns.values()) {
+ Matcher matcher = pattern.matcher(unsupportedPropertyId);
+ if (matcher.matches()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ protected String getRegExpKey(String id) {
+ for (Map.Entry<String, Pattern> entry : patterns.entrySet()) {
+ Pattern pattern = entry.getValue();
+
+ Matcher matcher = pattern.matcher(id);
+
+ if (matcher.matches()) {
+ return entry.getKey();
+ }
+ }
+ return null;
+ }
+
+ protected boolean isPatternKey(String id) {
+ return patterns.containsKey(id);
+ }
+
+
+ /**
+
+ /**
* Set a property value on the given resource for the given id and value.
* Make sure that the id is in the given set of requested ids.
*
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java Thu Jun 20 18:09:29 2013
@@ -148,7 +148,9 @@ public class RoleCommandOrder {
addDependency(Role.HUE_SERVER, RoleCommand.START, Role.HCAT,
RoleCommand.START);
addDependency(Role.HUE_SERVER, RoleCommand.START, Role.OOZIE_SERVER,
- RoleCommand.START);
+ RoleCommand.START);
+ addDependency(Role.FLUME_SERVER, RoleCommand.START, Role.OOZIE_SERVER,
+ RoleCommand.START);
// Service checks
addDependency(Role.HDFS_SERVICE_CHECK, RoleCommand.EXECUTE, Role.NAMENODE,
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java Thu Jun 20 18:09:29 2013
@@ -104,6 +104,7 @@ public interface Service {
GANGLIA,
ZOOKEEPER,
PIG,
- HCATALOG
+ HCATALOG,
+ FLUME
}
}
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java Thu Jun 20 18:09:29 2013
@@ -90,6 +90,7 @@ public class StageUtils {
componentToClusterInfoKeyMap.put("NODEMANAGER", "nm_hosts");
componentToClusterInfoKeyMap.put("HISTORYSERVER", "hs_host");
componentToClusterInfoKeyMap.put("ZOOKEEPER_SERVER", "zookeeper_hosts");
+ componentToClusterInfoKeyMap.put("FLUME_SERVER", "flume_hosts");
componentToClusterInfoKeyMap.put("HBASE_MASTER", "hbase_master_hosts");
componentToClusterInfoKeyMap.put("HBASE_REGIONSERVER", "hbase_rs_hosts");
componentToClusterInfoKeyMap.put("HIVE_SERVER", "hive_server_host");
Modified: incubator/ambari/trunk/ambari-server/src/main/resources/ganglia_properties.json
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/resources/ganglia_properties.json?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/resources/ganglia_properties.json (original)
+++ incubator/ambari/trunk/ambari-server/src/main/resources/ganglia_properties.json Thu Jun 20 18:09:29 2013
@@ -11811,8 +11811,294 @@
"pointInTime":true,
"temporal":true
}
+ },
+ "FLUME_SERVER":{
+ "metrics/boottime":{
+ "metric":"boottime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_aidle":{
+ "metric":"cpu_aidle",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_idle":{
+ "metric":"cpu_idle",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_nice":{
+ "metric":"cpu_nice",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_num":{
+ "metric":"cpu_num",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_speed":{
+ "metric":"cpu_speed",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_system":{
+ "metric":"cpu_system",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_user":{
+ "metric":"cpu_user",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_wio":{
+ "metric":"cpu_wio",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/disk/disk_free":{
+ "metric":"disk_free",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/disk/disk_total":{
+ "metric":"disk_total",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/disk/part_max_used":{
+ "metric":"part_max_used",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/gcCount":{
+ "metric":"jvm.metrics.gcCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/gcTimeMillis":{
+ "metric":"jvm.metrics.gcTimeMillis",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/logError":{
+ "metric":"jvm.metrics.logError",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/logFatal":{
+ "metric":"jvm.metrics.logFatal",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/logInfo":{
+ "metric":"jvm.metrics.logInfo",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/logWarn":{
+ "metric":"jvm.metrics.logWarn",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/maxMemoryM":{
+ "metric":"jvm.metrics.maxMemoryM",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/memHeapCommittedM":{
+ "metric":"jvm.metrics.memHeapCommittedM",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/memHeapUsedM":{
+ "metric":"jvm.metrics.memHeapUsedM",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/memNonHeapCommittedM":{
+ "metric":"jvm.metrics.memNonHeapCommittedM",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/memNonHeapUsedM":{
+ "metric":"jvm.metrics.memNonHeapUsedM",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsBlocked":{
+ "metric":"jvm.metrics.threadsBlocked",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsNew":{
+ "metric":"jvm.metrics.threadsNew",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsRunnable":{
+ "metric":"jvm.metrics.threadsRunnable",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsTerminated":{
+ "metric":"jvm.metrics.threadsTerminated",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsTimedWaiting":{
+ "metric":"jvm.metrics.threadsTimedWaiting",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsWaiting":{
+ "metric":"jvm.metrics.threadsWaiting",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/ChannelCapacity":{
+ "metric":"(\\w+).CHANNEL.(\\w+).ChannelCapacity",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/StartTime":{
+ "metric":"(\\w+).CHANNEL.(\\w+).StartTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/EventTakeAttemptCount":{
+ "metric":"(\\w+).CHANNEL.(\\w+).EventTakeAttemptCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/EventTakeSuccessCount":{
+ "metric":"(\\w+).CHANNEL.(\\w+).EventTakeSuccessCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/EventPutAttemptCount":{
+ "metric":"(\\w+).CHANNEL.(\\w+).EventPutAttemptCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/StopTime":{
+ "metric":"(\\w+).CHANNEL.(\\w+).StopTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/ChannelFillPercentage":{
+ "metric":"(\\w+).CHANNEL.(\\w+).ChannelFillPercentage",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/ChannelSize":{
+ "metric":"(\\w+).CHANNEL.(\\w+).ChannelSize",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/EventPutSuccessCount":{
+ "metric":"(\\w+).CHANNEL.(\\w+).EventPutSuccessCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/ConnectionCreatedCount":{
+ "metric":"(\\w+).SINK.(\\w+).ConnectionCreatedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/BatchCompleteCount":{
+ "metric":"(\\w+).SINK.(\\w+).BatchCompleteCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/EventDrainSuccessCount":{
+ "metric":"(\\w+).SINK.(\\w+).EventDrainSuccessCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/StartTime":{
+ "metric":"(\\w+).SINK.(\\w+).StartTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/EventDrainAttemptCount":{
+ "metric":"(\\w+).SINK.(\\w+).EventDrainAttemptCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/ConnectionFailedCount":{
+ "metric":"(\\w+).SINK.(\\w+).ConnectionFailedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/BatchUnderflowCount":{
+ "metric":"(\\w+).SINK.(\\w+).BatchUnderflowCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/ConnectionClosedCount":{
+ "metric":"(\\w+).SINK.(\\w+).ConnectionClosedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/StopTime":{
+ "metric":"(\\w+).SINK.(\\w+).StopTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/BatchEmptyCount":{
+ "metric":"(\\w+).SINK.(\\w+).BatchEmptyCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/AppendBatchReceivedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).AppendBatchReceivedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/AppendAcceptedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).AppendAcceptedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/StartTime":{
+ "metric":"(\\w+).SOURCE.(\\w+).StartTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/OpenConnectionCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).OpenConnectionCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/AppendBatchAcceptedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).AppendBatchAcceptedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/AppendReceivedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).AppendReceivedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/EventReceivedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).EventReceivedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/StopTime":{
+ "metric":"(\\w+).SOURCE.(\\w+).StopTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/EventAcceptedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).EventAcceptedCount",
+ "pointInTime":true,
+ "temporal":true
+ }
}
-
},
"HostComponent":{
@@ -20618,6 +20904,293 @@
"pointInTime":true,
"temporal":true
}
+ },
+ "FLUME_SERVER":{
+ "metrics/boottime":{
+ "metric":"boottime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_aidle":{
+ "metric":"cpu_aidle",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_idle":{
+ "metric":"cpu_idle",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_nice":{
+ "metric":"cpu_nice",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_num":{
+ "metric":"cpu_num",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_speed":{
+ "metric":"cpu_speed",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_system":{
+ "metric":"cpu_system",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_user":{
+ "metric":"cpu_user",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/cpu/cpu_wio":{
+ "metric":"cpu_wio",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/disk/disk_free":{
+ "metric":"disk_free",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/disk/disk_total":{
+ "metric":"disk_total",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/disk/part_max_used":{
+ "metric":"part_max_used",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/gcCount":{
+ "metric":"jvm.metrics.gcCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/gcTimeMillis":{
+ "metric":"jvm.metrics.gcTimeMillis",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/logError":{
+ "metric":"jvm.metrics.logError",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/logFatal":{
+ "metric":"jvm.metrics.logFatal",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/logInfo":{
+ "metric":"jvm.metrics.logInfo",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/logWarn":{
+ "metric":"jvm.metrics.logWarn",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/maxMemoryM":{
+ "metric":"jvm.metrics.maxMemoryM",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/memHeapCommittedM":{
+ "metric":"jvm.metrics.memHeapCommittedM",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/memHeapUsedM":{
+ "metric":"jvm.metrics.memHeapUsedM",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/memNonHeapCommittedM":{
+ "metric":"jvm.metrics.memNonHeapCommittedM",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/memNonHeapUsedM":{
+ "metric":"jvm.metrics.memNonHeapUsedM",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsBlocked":{
+ "metric":"jvm.metrics.threadsBlocked",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsNew":{
+ "metric":"jvm.metrics.threadsNew",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsRunnable":{
+ "metric":"jvm.metrics.threadsRunnable",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsTerminated":{
+ "metric":"jvm.metrics.threadsTerminated",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsTimedWaiting":{
+ "metric":"jvm.metrics.threadsTimedWaiting",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/jvm/threadsWaiting":{
+ "metric":"jvm.metrics.threadsWaiting",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/ChannelCapacity":{
+ "metric":"(\\w+).CHANNEL.(\\w+).ChannelCapacity",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/StartTime":{
+ "metric":"(\\w+).CHANNEL.(\\w+).StartTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/EventTakeAttemptCount":{
+ "metric":"(\\w+).CHANNEL.(\\w+).EventTakeAttemptCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/EventTakeSuccessCount":{
+ "metric":"(\\w+).CHANNEL.(\\w+).EventTakeSuccessCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/EventPutAttemptCount":{
+ "metric":"(\\w+).CHANNEL.(\\w+).EventPutAttemptCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/StopTime":{
+ "metric":"(\\w+).CHANNEL.(\\w+).StopTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/ChannelFillPercentage":{
+ "metric":"(\\w+).CHANNEL.(\\w+).ChannelFillPercentage",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/ChannelSize":{
+ "metric":"(\\w+).CHANNEL.(\\w+).ChannelSize",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/CHANNEL/$2/EventPutSuccessCount":{
+ "metric":"(\\w+).CHANNEL.(\\w+).EventPutSuccessCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/ConnectionCreatedCount":{
+ "metric":"(\\w+).SINK.(\\w+).ConnectionCreatedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/BatchCompleteCount":{
+ "metric":"(\\w+).SINK.(\\w+).BatchCompleteCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/EventDrainSuccessCount":{
+ "metric":"(\\w+).SINK.(\\w+).EventDrainSuccessCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/StartTime":{
+ "metric":"(\\w+).SINK.(\\w+).StartTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/EventDrainAttemptCount":{
+ "metric":"(\\w+).SINK.(\\w+).EventDrainAttemptCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/ConnectionFailedCount":{
+ "metric":"(\\w+).SINK.(\\w+).ConnectionFailedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/BatchUnderflowCount":{
+ "metric":"(\\w+).SINK.(\\w+).BatchUnderflowCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/ConnectionClosedCount":{
+ "metric":"(\\w+).SINK.(\\w+).ConnectionClosedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/StopTime":{
+ "metric":"(\\w+).SINK.(\\w+).StopTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SINK/$2/BatchEmptyCount":{
+ "metric":"(\\w+).SINK.(\\w+).BatchEmptyCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/AppendBatchReceivedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).AppendBatchReceivedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/AppendAcceptedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).AppendAcceptedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/StartTime":{
+ "metric":"(\\w+).SOURCE.(\\w+).StartTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/OpenConnectionCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).OpenConnectionCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/AppendBatchAcceptedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).AppendBatchAcceptedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/AppendReceivedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).AppendReceivedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/EventReceivedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).EventReceivedCount",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/StopTime":{
+ "metric":"(\\w+).SOURCE.(\\w+).StopTime",
+ "pointInTime":true,
+ "temporal":true
+ },
+ "metrics/flume/$1/SOURCE/$2/EventAcceptedCount":{
+ "metric":"(\\w+).SOURCE.(\\w+).EventAcceptedCount",
+ "pointInTime":true,
+ "temporal":true
+ }
}
}
}
\ No newline at end of file
Added: incubator/ambari/trunk/ambari-server/src/main/resources/scripts/install_flume.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/resources/scripts/install_flume.sh?rev=1495115&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/resources/scripts/install_flume.sh (added)
+++ incubator/ambari/trunk/ambari-server/src/main/resources/scripts/install_flume.sh Thu Jun 20 18:09:29 2013
@@ -0,0 +1,88 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+##################################################
+# Script to define, install and start FLUME service
+# after Ambari installer finishes.
+#
+# Flume service is defined, installed and started
+# via API calls.
+##################################################
+
+if (($# != 4)); then
+ echo "Usage: $0: <AMBARI_HOST> <FLUME1_INTERNAL_HOST> <FLUME2_INTERNAL_HOST> <CLUSTER_NAME>";
+ exit 1;
+fi
+
+AMBARIURL="http://$1:8080"
+USERID="admin"
+PASSWD="admin"
+
+defineService () {
+ if curl -s -u $USERID:$PASSWD "$AMBARIURL/api/v1/clusters/$1/services" | grep service_name | cut -d : -f 2 | grep -q FLUME ; then
+ echo "FLUME service already defined.";
+ else
+ echo "Defining FLUME";
+ curl -u $USERID:$PASSWD -X POST "$AMBARIURL/api/v1/clusters/$1/services" --data "[{\"ServiceInfo\":{\"service_name\":\"FLUME\"}}]";
+ fi
+}
+
+defineServiceComponent () {
+ if curl -s -u $USERID:$PASSWD "$AMBARIURL/api/v1/clusters/$1/services/FLUME" | grep components | cut -d : -f 2 | grep -q "\[ \]" ; then
+ echo "Defining FLUME_SERVER service component"
+ curl -u $USERID:$PASSWD -X POST "$AMBARIURL/api/v1/clusters/$1/services?ServiceInfo/service_name=FLUME" --data "{\"components\":[{\"ServiceComponentInfo\":{\"component_name\":\"FLUME_SERVER\"}}]}";
+ else
+ echo "FLUME_SERVER service component already defined."
+ fi
+}
+
+defineHostComponent () {
+ if ! curl -s -u $USERID:$PASSWD "$AMBARIURL/api/v1/clusters/$2/hosts/$1" | grep component_name | cut -d : -f 2 | grep -q "FLUME_SERVER" ; then
+ echo "Defining FLUME_SERVER host component on $1"
+ curl -u $USERID:$PASSWD -X POST "$AMBARIURL/api/v1/clusters/$2/hosts?Hosts/host_name=$1" --data "{\"host_components\":[{\"HostRoles\":{\"component_name\":\"FLUME_SERVER\"}}]}";
+ else
+ echo "FLUME_SERVER host component already defined on $1."
+ fi
+}
+
+installService () {
+ if curl -s -u $USERID:$PASSWD "$AMBARIURL/api/v1/clusters/$1/services/FLUME" | grep state | cut -d : -f 2 | grep -q "INIT" ; then
+ echo "Installing FLUME_SERVER service"
+ curl -u $USERID:$PASSWD -X PUT "$AMBARIURL/api/v1/clusters/$1/services?ServiceInfo/state=INIT&ServiceInfo/service_name=FLUME" --data "{\"RequestInfo\": {\"context\" :\"Install Flume Service\"}, \"Body\": {\"ServiceInfo\": {\"state\": \"INSTALLED\"}}}";
+ else
+ echo "FLUME_SERVER already installed."
+ fi
+}
+
+startService () {
+ if curl -s -u $USERID:$PASSWD "$AMBARIURL/api/v1/clusters/$1/services/FLUME" | grep state | cut -d : -f 2 | grep -q "STARTED" ; then
+ echo "FLUME_SERVER already started."
+ else
+ echo "Starting FLUME_SERVER service"
+ curl -u $USERID:$PASSWD -X PUT "$AMBARIURL/api/v1/clusters/$1/services?ServiceInfo/state=INSTALLED&ServiceInfo/service_name=FLUME" --data "{\"RequestInfo\": {\"context\" :\"Start Flume Service\"}, \"Body\": {\"ServiceInfo\": {\"state\": \"STARTED\"}}}";
+ fi
+}
+
+
+defineService $4
+defineServiceComponent $4
+defineHostComponent $2 $4
+defineHostComponent $3 $4
+installService $4
+startService $4
Added: incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/configuration/global.xml
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/configuration/global.xml?rev=1495115&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/configuration/global.xml (added)
+++ incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/configuration/global.xml Thu Jun 20 18:09:29 2013
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+</configuration>
Added: incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/metainfo.xml
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/metainfo.xml?rev=1495115&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/metainfo.xml (added)
+++ incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/FLUME/metainfo.xml Thu Jun 20 18:09:29 2013
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <user>root</user>
+ <comment>Flume is a distributed, reliable, and available system for efficiently collecting, aggregating and moving large amounts of log data from many different sources to a centralized data store.</comment>
+ <version>1.3.1.1.3.0.0</version>
+
+ <components>
+ <component>
+ <name>FLUME_SERVER</name>
+ <category>MASTER</category>
+ </component>
+ </components>
+
+</metainfo>
Modified: incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java?rev=1495115&r1=1495114&r2=1495115&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java (original)
+++ incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java Thu Jun 20 18:09:29 2013
@@ -38,6 +38,12 @@ import java.util.Set;
public class GangliaPropertyProviderTest {
private static final String PROPERTY_ID = PropertyHelper.getPropertyId("metrics/jvm", "gcCount");
+ private static final String PROPERTY_ID2 = PropertyHelper.getPropertyId("metrics/cpu", "cpu_wio");
+ private static final String FLUME_CHANNEL_CAPACITY_PROPERTY = "metrics/flume/flume/CHANNEL/c1/ChannelCapacity";
+ private static final String FLUME_CATEGORY = "metrics/flume";
+ private static final String FLUME_CATEGORY2 = "metrics/flume/flume";
+ private static final String FLUME_CATEGORY3 = "metrics/flume/flume/CHANNEL";
+ private static final String FLUME_CATEGORY4 = "metrics/flume/flume/CHANNEL/c1";
private static final String CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "cluster_name");
private static final String HOST_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "host_name");
private static final String COMPONENT_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "component_name");
@@ -191,6 +197,241 @@ public class GangliaPropertyProviderTest
}
+
+ @Test
+ public void testPopulateResources_params() throws Exception {
+ TestStreamProvider streamProvider = new TestStreamProvider("flume_ganglia_data.txt");
+ TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
+
+ GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+ PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent),
+ streamProvider,
+ hostProvider,
+ CLUSTER_NAME_PROPERTY_ID,
+ HOST_NAME_PROPERTY_ID,
+ COMPONENT_NAME_PROPERTY_ID);
+
+ // flume
+ Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+ resource.setProperty(HOST_NAME_PROPERTY_ID, "ip-10-39-113-33.ec2.internal");
+ resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "FLUME_SERVER");
+
+ // only ask for one property
+ Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+ temporalInfoMap.put(FLUME_CHANNEL_CAPACITY_PROPERTY, new TemporalInfoImpl(10L, 20L, 1L));
+ Request request = PropertyHelper.getReadRequest(Collections.singleton(FLUME_CHANNEL_CAPACITY_PROPERTY), temporalInfoMap);
+
+ Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+ Assert.assertEquals("http://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&s=10&e=20&r=1",
+ streamProvider.getLastSpec());
+
+ Assert.assertEquals(3, PropertyHelper.getProperties(resource).size());
+ Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
+ }
+
+ @Test
+ public void testPopulateResources_paramsMixed() throws Exception {
+ TestStreamProvider streamProvider = new TestStreamProvider("flume_ganglia_data.txt");
+ TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
+
+ GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+ PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent),
+ streamProvider,
+ hostProvider,
+ CLUSTER_NAME_PROPERTY_ID,
+ HOST_NAME_PROPERTY_ID,
+ COMPONENT_NAME_PROPERTY_ID);
+
+ // flume
+ Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+ resource.setProperty(HOST_NAME_PROPERTY_ID, "ip-10-39-113-33.ec2.internal");
+ resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "FLUME_SERVER");
+
+ // only ask for one property
+ Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+
+ Set<String> ids = new HashSet<String>();
+ ids.add(FLUME_CATEGORY2);
+ ids.add(PROPERTY_ID2);
+
+ Request request = PropertyHelper.getReadRequest(ids, temporalInfoMap);
+
+ Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+ Assert.assertEquals("http://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&e=now&pt=true",
+ streamProvider.getLastSpec());
+
+ Assert.assertEquals(22, PropertyHelper.getProperties(resource).size());
+ Assert.assertNotNull(resource.getPropertyValue(PROPERTY_ID2));
+ Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
+ }
+
+ @Test
+ public void testPopulateResources_paramsAll() throws Exception {
+ TestStreamProvider streamProvider = new TestStreamProvider("flume_ganglia_data.txt");
+ TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
+
+ GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+ PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent),
+ streamProvider,
+ hostProvider,
+ CLUSTER_NAME_PROPERTY_ID,
+ HOST_NAME_PROPERTY_ID,
+ COMPONENT_NAME_PROPERTY_ID);
+
+ // flume
+ Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+ resource.setProperty(HOST_NAME_PROPERTY_ID, "ip-10-39-113-33.ec2.internal");
+ resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "FLUME_SERVER");
+
+ Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+ Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet(), temporalInfoMap);
+
+ Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+ Assert.assertEquals("http://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&e=now&pt=true",
+ streamProvider.getLastSpec());
+
+ Assert.assertEquals(33, PropertyHelper.getProperties(resource).size());
+ Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
+ }
+
+ @Test
+ public void testPopulateResources_params_category1() throws Exception {
+ TestStreamProvider streamProvider = new TestStreamProvider("flume_ganglia_data.txt");
+ TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
+
+ GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+ PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent),
+ streamProvider,
+ hostProvider,
+ CLUSTER_NAME_PROPERTY_ID,
+ HOST_NAME_PROPERTY_ID,
+ COMPONENT_NAME_PROPERTY_ID);
+
+ // flume
+ Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+ resource.setProperty(HOST_NAME_PROPERTY_ID, "ip-10-39-113-33.ec2.internal");
+ resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "FLUME_SERVER");
+
+ // only ask for one property
+ Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+ temporalInfoMap.put(FLUME_CATEGORY, new TemporalInfoImpl(10L, 20L, 1L));
+ Request request = PropertyHelper.getReadRequest(Collections.singleton(FLUME_CATEGORY), temporalInfoMap);
+
+ Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+ Assert.assertEquals("http://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&s=10&e=20&r=1",
+ streamProvider.getLastSpec());
+
+ Assert.assertEquals(21, PropertyHelper.getProperties(resource).size());
+ Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
+ }
+
+ @Test
+ public void testPopulateResources_params_category2() throws Exception {
+ TestStreamProvider streamProvider = new TestStreamProvider("flume_ganglia_data.txt");
+ TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
+
+ GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+ PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent),
+ streamProvider,
+ hostProvider,
+ CLUSTER_NAME_PROPERTY_ID,
+ HOST_NAME_PROPERTY_ID,
+ COMPONENT_NAME_PROPERTY_ID);
+
+ // flume
+ Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+ resource.setProperty(HOST_NAME_PROPERTY_ID, "ip-10-39-113-33.ec2.internal");
+ resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "FLUME_SERVER");
+
+ // only ask for one property
+ Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+ temporalInfoMap.put(FLUME_CATEGORY2, new TemporalInfoImpl(10L, 20L, 1L));
+ Request request = PropertyHelper.getReadRequest(Collections.singleton(FLUME_CATEGORY2), temporalInfoMap);
+
+ Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+ Assert.assertEquals("http://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&s=10&e=20&r=1",
+ streamProvider.getLastSpec());
+
+ Assert.assertEquals(21, PropertyHelper.getProperties(resource).size());
+ Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
+ }
+
+ @Test
+ public void testPopulateResources_params_category3() throws Exception {
+ TestStreamProvider streamProvider = new TestStreamProvider("flume_ganglia_data.txt");
+ TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
+
+ GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+ PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent),
+ streamProvider,
+ hostProvider,
+ CLUSTER_NAME_PROPERTY_ID,
+ HOST_NAME_PROPERTY_ID,
+ COMPONENT_NAME_PROPERTY_ID);
+
+ // flume
+ Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+ resource.setProperty(HOST_NAME_PROPERTY_ID, "ip-10-39-113-33.ec2.internal");
+ resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "FLUME_SERVER");
+
+ // only ask for one property
+ Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+ temporalInfoMap.put(FLUME_CATEGORY3, new TemporalInfoImpl(10L, 20L, 1L));
+ Request request = PropertyHelper.getReadRequest(Collections.singleton(FLUME_CATEGORY3), temporalInfoMap);
+
+ Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+ Assert.assertEquals("http://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&s=10&e=20&r=1",
+ streamProvider.getLastSpec());
+
+ Assert.assertEquals(11, PropertyHelper.getProperties(resource).size());
+ Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
+ }
+
+ @Test
+ public void testPopulateResources_params_category4() throws Exception {
+ TestStreamProvider streamProvider = new TestStreamProvider("flume_ganglia_data.txt");
+ TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
+
+ GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+ PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent),
+ streamProvider,
+ hostProvider,
+ CLUSTER_NAME_PROPERTY_ID,
+ HOST_NAME_PROPERTY_ID,
+ COMPONENT_NAME_PROPERTY_ID);
+
+ // flume
+ Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+ resource.setProperty(HOST_NAME_PROPERTY_ID, "ip-10-39-113-33.ec2.internal");
+ resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "FLUME_SERVER");
+
+ // only ask for one property
+ Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+ temporalInfoMap.put(FLUME_CATEGORY4, new TemporalInfoImpl(10L, 20L, 1L));
+ Request request = PropertyHelper.getReadRequest(Collections.singleton(FLUME_CATEGORY4), temporalInfoMap);
+
+ Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+ Assert.assertEquals("http://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&s=10&e=20&r=1",
+ streamProvider.getLastSpec());
+
+ Assert.assertEquals(11, PropertyHelper.getProperties(resource).size());
+ Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
+ }
+
private static class TestGangliaHostProvider implements GangliaHostProvider {
@Override