You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2013/04/30 02:36:14 UTC
svn commit: r1477428 - in /incubator/ambari/trunk: ./
ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/
ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/
ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/
ambari-agent/src/main/p...
Author: swagle
Date: Tue Apr 30 00:36:13 2013
New Revision: 1477428
URL: http://svn.apache.org/r1477428
Log:
AMBARI-2046. Create ambari agent scripts for Hadoop 2.0 installation, node manager. (swagle)
Added:
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/nodemanager.pp
Modified:
incubator/ambari/trunk/CHANGES.txt
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/resourcemanager.pp
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/2.0.1/services/YARN/configuration/mapred-site.xml
Modified: incubator/ambari/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1477428&r1=1477427&r2=1477428&view=diff
==============================================================================
--- incubator/ambari/trunk/CHANGES.txt (original)
+++ incubator/ambari/trunk/CHANGES.txt Tue Apr 30 00:36:13 2013
@@ -12,6 +12,9 @@ Trunk (unreleased changes):
NEW FEATURES
+ AMBARI-2046. Create ambari agent scripts for Hadoop 2.0 installation, node
+ manager. (swagle)
+
AMBARI-2047. Create ambari agent scripts for yarn client. (swagle)
AMBARI-1679. Create ambari agent scripts for Hadoop 2.0 installation,
Modified: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp?rev=1477428&r1=1477427&r2=1477428&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp Tue Apr 30 00:36:13 2013
@@ -165,6 +165,20 @@ define hdp-hadoop::namenode::create_app_
recursive_chmod => true
}
}
+
+ if $stack_version in ("2.0.1") {
+ if ($hdp::params::yarn_log_aggregation_enabled == "true") {
+ $yarn_user = $hdp::params::yarn_user
+ $yarn_nm_app_log_dir = $hdp::params::yarn_nm_app_log_dir
+
+ hdp-hadoop::hdfs::directory{ $yarn_nm_app_log_dir:
+ service_state => $service_state,
+ owner => $yarn_user,
+ mode => '744',
+ recursive_chmod => true
+ }
+ }
+ }
}
}
Modified: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp?rev=1477428&r1=1477427&r2=1477428&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp Tue Apr 30 00:36:13 2013
@@ -77,6 +77,8 @@ class hdp-hadoop::params(
$hadoop_libexec_dir = hdp_default("hadoop_libexec_dir","/usr/lib/hadoop/libexec")
+ $mapreduce_libs_path = hdp_default("mapreduce_libs_path","/usr/lib/hadoop-mapreduce/*")
+
### compression related
if (($hdp::params::lzo_enabled == true) and ($hdp::params::snappy_enabled == true)) {
$mapred_compress_map_output = true
Modified: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb?rev=1477428&r1=1477427&r2=1477428&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb Tue Apr 30 00:36:13 2013
@@ -90,4 +90,6 @@ export HADOOP_IDENT_STRING=$USER
# Use libraries from standard classpath
JAVA_JDBC_LIBS="/usr/share/java/*"
-export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:${JAVA_JDBC_LIBS}
+#Add libraries required by nodemanager
+MAPREDUCE_LIBS=<%=scope.function_hdp_template_var("mapreduce_libs_path")%>
+export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}
Added: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/nodemanager.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/nodemanager.pp?rev=1477428&view=auto
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/nodemanager.pp (added)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/nodemanager.pp Tue Apr 30 00:36:13 2013
@@ -0,0 +1,48 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp-yarn::nodemanager(
+ $service_state = $hdp::params::cluster_service_state,
+ $opts = {}
+) inherits hdp-yarn::params
+{
+ $yarn_user = $hdp-yarn::params::yarn_user
+
+ if ($service_state == 'no_op') {
+ } elsif ($service_state in 'installed_and_configured') {
+
+ include hdp-yarn::initialize
+
+ ##Process package
+ hdp-yarn::package{'yarn-nodemanager':}
+
+ } elsif ($service_state in ['running','stopped']) {
+
+ include hdp-yarn::initialize
+
+ hdp-yarn::service{ 'nodemanager':
+ ensure => $service_state,
+ user => $yarn_user
+ }
+
+ } else {
+ hdp_fail("TODO not implemented yet: service_state = ${service_state}")
+ }
+}
\ No newline at end of file
Modified: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/resourcemanager.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/resourcemanager.pp?rev=1477428&r1=1477427&r2=1477428&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/resourcemanager.pp (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/resourcemanager.pp Tue Apr 30 00:36:13 2013
@@ -35,11 +35,6 @@ class hdp-yarn::resourcemanager(
} elsif ($service_state in ['running','stopped']) {
- if ( ($service_state == 'installed_and_configured') and
- ($security_enabled == true) and ($kerberos_install_type == "AMBARI_SET_KERBEROS") ) {
- hdp_fail("Security not yet implemented for resource manager")
- }
-
include hdp-yarn::initialize
hdp-yarn::service{ 'resourcemanager':
Modified: incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp?rev=1477428&r1=1477427&r2=1477428&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp Tue Apr 30 00:36:13 2013
@@ -36,6 +36,7 @@ class hdp::params()
$oozie-site = $configuration['oozie-site']
$sqoop-site = $configuration['sqoop-site']
$webhcat-site = $configuration['webhcat-site']
+ $yarn-site = $configuration['yarn-site']
}
##### global state defaults ####
@@ -191,6 +192,10 @@ class hdp::params()
$hive_apps_whs_dir = hdp_default("hive_apps_whs_dir", "/apps/hive/warehouse")
$webhcat_apps_dir = hdp_default("webhcat_apps_dir", "/apps/webhcat")
$hbase_hdfs_root_dir = hdp_default("hbase-site/hbase.hdfs.root.dir","/apps/hbase/data")
+
+ $yarn_nm_app_log_dir = hdp_default("yarn-site/yarn.nodemanager.remote-app-log-dir","/app-logs")
+
+ $yarn_log_aggregation_enabled = hdp_default("yarn-site/yarn.log-aggregation-enable","true")
$user_group = hdp_default("user_group","hadoop")
@@ -387,7 +392,7 @@ class hdp::params()
yarn-nodemanager => {
'ALL' => {
64 => {
- 'ALL' => ['hadoop-yarn-nodemanager', 'hadoop-yarn-proxyserver', 'hadoop-yarn-resourcemanager']
+ 'ALL' => ['hadoop-yarn-nodemanager']
}
}
},
@@ -395,7 +400,7 @@ class hdp::params()
yarn-proxyserver => {
'ALL' => {
64 => {
- 'ALL' => ['hadoop-yarn-proxyserver', 'hadoop-yarn-resourcemanager']
+ 'ALL' => ['hadoop-yarn-proxyserver']
}
}
},
@@ -403,7 +408,7 @@ class hdp::params()
yarn-resourcemanager => {
'ALL' => {
64 => {
- 'ALL' => ['hadoop-yarn-resourcemanager']
+ 'ALL' => ['hadoop-yarn-resourcemanager', 'hadoop-mapreduce']
}
}
},
Modified: incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/2.0.1/services/YARN/configuration/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/2.0.1/services/YARN/configuration/mapred-site.xml?rev=1477428&r1=1477427&r2=1477428&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/2.0.1/services/YARN/configuration/mapred-site.xml (original)
+++ incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/2.0.1/services/YARN/configuration/mapred-site.xml Tue Apr 30 00:36:13 2013
@@ -528,4 +528,10 @@ user)</description>
<description> Comma separated list of queues configured for this jobtracker.</description>
</property>
+<property>
+ <name>mapreduce.shuffle.port</name>
+ <value>8081</value>
+ <description>Default port that the ShuffleHandler will run on. ShuffleHandler is a service run at the NodeManager to facilitate transfers of intermediate Map outputs to requesting Reducers.</description>
+</property>
+
</configuration>