You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vg...@apache.org on 2011/12/28 23:46:41 UTC
svn commit: r1225365 - in /incubator/ambari/trunk: ./
agent/src/main/resources/puppet/manifests/
agent/src/main/resources/puppet/modules/hadoop/manifests/
client/src/main/java/org/apache/ambari/common/rest/entities/
controller/src/main/java/org/apache/...
Author: vgogate
Date: Wed Dec 28 22:46:40 2011
New Revision: 1225365
URL: http://svn.apache.org/viewvc?rev=1225365&view=rev
Log:
AMBARI-178. Add support for Map/Reduce component in Ambari stack
Added:
incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/UserGroup.java
incubator/ambari/trunk/controller/src/main/resources/org/apache/ambari/acd/mapreduce-0.1.0.acd
Modified:
incubator/ambari/trunk/CHANGES.txt
incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/site.pp
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/init.pp
incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/Component.java
incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/Stack.java
incubator/ambari/trunk/controller/src/main/java/org/apache/ambari/controller/Clusters.java
incubator/ambari/trunk/controller/src/main/java/org/apache/ambari/controller/StackFlattener.java
incubator/ambari/trunk/controller/src/main/resources/org/apache/ambari/stacks/puppet1-0.json
incubator/ambari/trunk/controller/src/test/java/org/apache/ambari/controller/TestStackFlattener.java
Modified: incubator/ambari/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1225365&r1=1225364&r2=1225365&view=diff
==============================================================================
--- incubator/ambari/trunk/CHANGES.txt (original)
+++ incubator/ambari/trunk/CHANGES.txt Wed Dec 28 22:46:40 2011
@@ -2,6 +2,8 @@ Ambari Change log
Release 0.1.0 - unreleased
+ AMBARI-178. Add support for Map/Reduce component in Ambari stack (vgogate)
+
AMBARI-176. Adds a first version of MapReduce ACD (ddas)
AMBARI-175. Removes the map from hostnames to heartbeat-responses. (ddas)
Modified: incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/site.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/site.pp?rev=1225365&r1=1225364&r2=1225365&view=diff
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/site.pp (original)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/site.pp Wed Dec 28 22:46:40 2011
@@ -29,28 +29,12 @@ package { "jdk":
node default {
notice($fqdn)
- /* Assign defaults */
+ /* Assign defaults. Agent is supposed to fill-in this value */
if !$ambari_stack_install_dir {
$ambari_stack_install_dir = "/var/ambari/"
}
notice ($ambari_stack_install_dir)
- /* Create group and users */
- group {$ambari_admin_group:
- ensure => present
- }
-
- /*
- * Create user for HDFS component
- * TODO: currently uid is auto selected which may cause different uids on each node..
- * No home directory specified. Need it?
- * Is user specified per role or per component?
- */
- user {$ambari_hdfs_user:
- ensure => present,
- gid => $ambari_admin_group
- }
-
/*
* Ensure cluster directory path is present
* Owned by root up to cluster directory
@@ -60,36 +44,104 @@ node default {
file {$stack_path_intermediate_dirs:
ensure => directory,
owner => root,
- group => $ambari_admin_group,
+ group => root,
mode => 755
}
+ /*
+ * Create user and groups
+ * TODO: currently uid is auto selected which may cause different uids on each node..
+ */
+ @group {$ambari_default_group:
+ ensure => present
+ }
+ @user {$ambari_default_user:
+ ensure => present,
+ gid => $ambari_default_group,
+ require => Group[$ambari_default_group]
+ }
+
+ if $ambari_hdfs_group {
+ @group {$ambari_hdfs_group:
+ ensure => present
+ }
+ } else {
+ $ambari_hdfs_group = $ambari_default_group
+ }
+
+ if $ambari_hdfs_user {
+ @user {$ambari_hdfs_user:
+ ensure => present,
+ gid => $ambari_hdfs_group,
+ require => Group[$ambari_hdfs_group]
+ }
+ } else {
+ $ambari_hdfs_user = $ambari_default_user
+ }
+
+ if $ambari_mapreduce_group {
+ @group {$ambari_mapreduce_group:
+ ensure => present
+ }
+ } else {
+ $ambari_mapreduce_group = $ambari_default_group
+ }
+
+ if $ambari_mapreduce_user {
+ @user {$ambari_mapreduce_user:
+ ensure => present,
+ gid => $ambari_mapreduce_group,
+ require => Group[$ambari_mapreduce_group]
+ }
+ } else {
+ $ambari_mapreduce_user = $ambari_default_user
+ }
+
if ($fqdn in $role_to_nodes[namenode]) {
- hadoop::namenode {"namenode":
+ hadoop::role {"namenode":
ambari_role_name => "namenode",
ambari_role_prefix => "${stack_path}/namenode",
user => $ambari_hdfs_user,
- group => $ambari_admin_group
+ group => $ambari_hdfs_group
}
}
/* hadoop.security.authentication make global variable */
if ($fqdn in $role_to_nodes[datanode]) {
- hadoop::datanode {"datanode":
+ hadoop::role {"datanode":
ambari_role_name => "datanode",
ambari_role_prefix => "${stack_path}/datanode",
user => $ambari_hdfs_user,
- group => $ambari_admin_group,
+ group => $ambari_hdfs_group,
auth_type => "simple"
}
}
- if ($fqdn in $role_to_nodes[client]) {
+ if ($fqdn in $role_to_nodes[jobtracker]) {
+ hadoop::role {"jobtracker":
+ ambari_role_name => "jobtracker",
+ ambari_role_prefix => "${stack_path}/jobtracker",
+ user => $ambari_mapreduce_user,
+ group => $ambari_mapreduce_group
+ }
+ }
+
+ /* hadoop.security.authentication make global variable */
+ if ($fqdn in $role_to_nodes[tasktracker]) {
+ hadoop::role {"tasktracker":
+ ambari_role_name => "tasktracker",
+ ambari_role_prefix => "${stack_path}/tasktracker",
+ user => $ambari_mapreduce_user,
+ group => $ambari_mapreduce_group,
+ }
+ }
+
+ if ($fqdn in $role_to_nodes['client']) {
hadoop::client {"client":
ambari_role_name => "client",
ambari_role_prefix => "${stack_path}/client",
- user => $ambari_hdfs_user,
- group => $ambari_admin_group
+ user => $ambari_default_user,
+ group => $ambari_default_group
}
}
}
Modified: incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/init.pp?rev=1225365&r1=1225364&r2=1225365&view=diff
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/init.pp (original)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/init.pp Wed Dec 28 22:46:40 2011
@@ -21,6 +21,7 @@ class hadoop {
class common {
package { "hadoop":
ensure => latest,
+ require => [Package["jdk"]],
}
package { "hadoop-native":
@@ -28,12 +29,15 @@ class hadoop {
require => [Package["hadoop"]],
}
}
-
- define datanode ($ambari_role_name = "datanode",
- $ambari_role_prefix, $user = "hdfs", $group = "hdfs", $auth_type = "simple") {
+
+ define role ($ambari_role_name = "datanode",
+ $ambari_role_prefix, $user = "hdfs", $group = "hdfs", $auth_type = "simple") {
include common
+ realize Group[$group]
+ realize User[$user]
+
/*
* Create conf directory for datanode
*/
@@ -58,56 +62,29 @@ class hadoop {
mode => 644
}
- package { "hadoop-datanode":
+ package { "hadoop-${ambari_role_name}":
ensure => latest,
- require => Package["jdk"],
+ require => [Package["hadoop"]],
}
- if ($auth_type == "kerberos") {
- package { "hadoop-sbin":
- ensure => latest,
- require => [Package["hadoop"]],
+ if ($ambari_role_name == "datanode") {
+ if ($auth_type == "kerberos") {
+ package { "hadoop-sbin":
+ ensure => latest,
+ require => [Package["hadoop"]],
+ }
}
}
}
- define namenode ($ambari_role_name = "namenode",
- $ambari_role_prefix, $user = "hdfs", $group = "hdfs") {
-
- include common
-
- $hadoop_conf_dir = "${ambari_role_prefix}/etc/hadoop"
- file {["${ambari_role_prefix}", "${ambari_role_prefix}/etc", "${ambari_role_prefix}/etc/hadoop"]:
- ensure => directory,
- owner => $user,
- group => $group,
- mode => 755
- }
- notice ($ambari_role_prefix)
- notice ($ambari_role_name)
- $files = get_files ($hadoop_conf_dir, $::hadoop_stack_conf, $ambari_role_name)
- notice($files)
-
- /* Create config files for each category */
- create_config_file {$files:
- conf_map => $::hadoop_stack_conf[$title],
- require => [Package["hadoop"]],
- owner => $user,
- group => $group,
- mode => 644
- }
-
- package { "hadoop-namenode":
- ensure => latest,
- require => Package["jdk"],
- }
- }
-
define client ($ambari_role_name = "client", $ambari_role_prefix,
$user = "hadoop", $group = "hadoop") {
include common
+ realize Group[$group]
+ realize User[$user]
+
$hadoop_conf_dir = "${ambari_role_prefix}/etc/conf"
file {["${ambari_role_prefix}", "${ambari_role_prefix}/etc", "${ambari_role_prefix}/etc/conf"]:
ensure => directory,
@@ -131,7 +108,7 @@ class hadoop {
package { ["hadoop-doc", "hadoop-source", "hadoop-debuginfo",
"hadoop-fuse", "hadoop-libhdfs", "hadoop-pipes"]:
ensure => latest,
- require => [Package["jdk"], Package["hadoop"]],
+ require => [Package["hadoop"]],
}
}
Modified: incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/Component.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/Component.java?rev=1225365&r1=1225364&r2=1225365&view=diff
==============================================================================
--- incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/Component.java (original)
+++ incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/Component.java Wed Dec 28 22:46:40 2011
@@ -30,8 +30,9 @@ import javax.xml.bind.annotation.XmlType
* Metadata information about a given component.
*/
@XmlAccessorType(XmlAccessType.FIELD)
-@XmlType(name = "CategoryType", propOrder = {
+@XmlType(name = "Component", propOrder = {
"definition",
+ "user_group",
"configuration",
"roles"
})
@@ -70,6 +71,26 @@ public class Component {
private ComponentDefinition definition;
/**
+ * Component user/group information
+ */
+ @XmlElement
+ private UserGroup user_group;
+
+ /**
+ * @return the user_group
+ */
+ public UserGroup getUser_group() {
+ return user_group;
+ }
+
+ /**
+ * @param user_group the user_group to set
+ */
+ public void setUser_group(UserGroup user_group) {
+ this.user_group = user_group;
+ }
+
+ /**
* The configuration shared between the active roles of the component.
*/
@XmlElement
@@ -87,7 +108,7 @@ public class Component {
public Component(String name, String version, String architecture,
String provider, ComponentDefinition definition,
- Configuration configuration, List<Role> roles) {
+ Configuration configuration, List<Role> roles, UserGroup user_group) {
this.name = name;
this.version = version;
this.architecture = architecture;
@@ -95,6 +116,7 @@ public class Component {
this.definition = definition;
this.configuration = configuration;
this.roles = roles;
+ this.user_group = user_group;
}
/**
@@ -123,6 +145,9 @@ public class Component {
if (other.version != null) {
this.version = other.version;
}
+ if (other.user_group != null) {
+ this.user_group = other.user_group;
+ }
}
/**
Modified: incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/Stack.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/Stack.java?rev=1225365&r1=1225364&r2=1225365&view=diff
==============================================================================
--- incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/Stack.java (original)
+++ incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/Stack.java Wed Dec 28 22:46:40 2011
@@ -97,6 +97,7 @@ import javax.xml.datatype.XMLGregorianCa
"parentRevision",
"creationTime",
"repositories",
+ "default_user_group",
"globals",
"configuration",
"components"
@@ -145,6 +146,26 @@ public class Stack {
protected List<RepositoryKind> repositories;
/**
+ * Default user group information
+ */
+ @XmlElement
+ protected UserGroup default_user_group;
+
+ /**
+ * @return the default_user_group
+ */
+ public UserGroup getDefault_user_group() {
+ return default_user_group;
+ }
+
+ /**
+ * @param default_user_group the default_user_group to set
+ */
+ public void setDefault_user_group(UserGroup default_user_group) {
+ this.default_user_group = default_user_group;
+ }
+
+ /**
* Stack Golbal variables
*/
@XmlElements({@XmlElement})
@@ -197,6 +218,7 @@ public class Stack {
this.repositories = orig.repositories;
this.globals = orig.globals;
this.revision = orig.revision;
+ this.default_user_group = orig.default_user_group;
}
/**
Added: incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/UserGroup.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/UserGroup.java?rev=1225365&view=auto
==============================================================================
--- incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/UserGroup.java (added)
+++ incubator/ambari/trunk/client/src/main/java/org/apache/ambari/common/rest/entities/UserGroup.java Wed Dec 28 22:46:40 2011
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.common.rest.entities;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlType;
+
+/**
+ * Details of the configuration for a role inside of a Stack.
+ */
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlType(name = "UserGroup", propOrder = {
+})
+@XmlRootElement
+public class UserGroup {
+
+ @XmlAttribute(required = true)
+ protected String user;
+ @XmlAttribute(required = true)
+ protected String userid;
+ @XmlAttribute(required = true)
+ protected String group;
+ @XmlAttribute(required = true)
+ protected String groupid;
+
+
+
+ /**
+ * @return the user
+ */
+ public String getUser() {
+ return user;
+ }
+ /**
+ * @param user the user to set
+ */
+ public void setUser(String user) {
+ this.user = user;
+ }
+ /**
+ * @return the userid
+ */
+ public String getUserid() {
+ return userid;
+ }
+ /**
+ * @param userid the userid to set
+ */
+ public void setUserid(String userid) {
+ this.userid = userid;
+ }
+ /**
+ * @return the group
+ */
+ public String getGroup() {
+ return group;
+ }
+ /**
+ * @param group the group to set
+ */
+ public void setGroup(String group) {
+ this.group = group;
+ }
+ /**
+ * @return the groupid
+ */
+ public String getGroupid() {
+ return groupid;
+ }
+ /**
+ * @param groupid the groupid to set
+ */
+ public void setGroupid(String groupid) {
+ this.groupid = groupid;
+ }
+}
Modified: incubator/ambari/trunk/controller/src/main/java/org/apache/ambari/controller/Clusters.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/controller/src/main/java/org/apache/ambari/controller/Clusters.java?rev=1225365&r1=1225364&r2=1225365&view=diff
==============================================================================
--- incubator/ambari/trunk/controller/src/main/java/org/apache/ambari/controller/Clusters.java (original)
+++ incubator/ambari/trunk/controller/src/main/java/org/apache/ambari/controller/Clusters.java Wed Dec 28 22:46:40 2011
@@ -17,10 +17,12 @@
*/
package org.apache.ambari.controller;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.GregorianCalendar;
+import java.util.HashMap;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
@@ -29,6 +31,8 @@ import java.util.concurrent.ConcurrentHa
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.Marshaller;
import javax.xml.datatype.DatatypeFactory;
import javax.xml.datatype.XMLGregorianCalendar;
@@ -43,6 +47,7 @@ import org.apache.ambari.common.rest.ent
import org.apache.ambari.common.rest.entities.Role;
import org.apache.ambari.common.rest.entities.RoleToNodes;
import org.apache.ambari.common.rest.entities.Stack;
+import org.apache.ambari.common.rest.entities.UserGroup;
import org.apache.ambari.datastore.DataStoreFactory;
import org.apache.ambari.datastore.DataStore;
import org.apache.ambari.resource.statemachine.ClusterFSM;
@@ -875,6 +880,17 @@ public class Clusters {
}
+ private void printStack(Stack stack, String file_path) throws Exception {
+ JAXBContext jc = JAXBContext.newInstance(org.apache.ambari.common.rest.entities.Stack.class);
+ Marshaller m = jc.createMarshaller();
+ m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
+ if (file_path == null) {
+ m.marshal(stack, System.out);
+ } else {
+ m.marshal(stack, new File(file_path));
+ }
+ }
+
/*
* Get the puppet deployment script for this cluster name/revision combo
*/
@@ -882,6 +898,7 @@ public class Clusters {
ClusterDefinition c = getClusterByName (clusterName).getClusterDefinition(revision);
Stack stack = this.flattener.flattenStack(c.getStackName(), Integer.parseInt(c.getStackRevision()));
+ printStack(stack, null);
/*
* Generate Ambari global variables
@@ -890,7 +907,7 @@ public class Clusters {
config = config + getStackConfigMapForPuppet (stack);
- config = config + getRoleToNodesMapForPuppet (c);
+ config = config + getRoleToNodesMapForPuppet (c, stack);
return config;
}
@@ -899,6 +916,7 @@ public class Clusters {
String config = "\n";
config = config + "$ambari_cluster_name" + " = " + "\"" + c.getName() + "\"\n";
+ config = config + "\n";
for (RoleToNodes rns : c.getRoleToNodesMap()) {
config = config + "$ambari_"+rns.getRoleName()+"_host" + " = " + "\"";
List<String> host_list = this.getHostnamesFromRangeExpressions(rns.getNodes());
@@ -912,6 +930,26 @@ public class Clusters {
}
config = config + "\"\n";
}
+ config = config + "\n";
+
+ /*
+ * Get the default user/group and role specific user/group information
+ */
+ config = config + "$ambari_default_user" + " = " + "\"" + stack.getDefault_user_group().getUser()+"\"\n";
+ config = config + "$ambari_default_userid" + " = " + "\"" + stack.getDefault_user_group().getUserid()+"\"\n";
+ config = config + "$ambari_default_group" + " = " + "\"" + stack.getDefault_user_group().getGroup()+"\"\n";
+ config = config + "$ambari_default_groupid" + " = " + "\"" + stack.getDefault_user_group().getGroupid()+"\"\n";
+ for (Component comp : stack.getComponents()) {
+ UserGroup ug = null;
+ if (comp.getUser_group() != null) {
+ ug = comp.getUser_group();
+ config = config + "$ambari_"+comp.getName()+"_user" + " = " + "\"" + ug.getUser()+"\"\n";
+ config = config + "$ambari_"+comp.getName()+"_userid" + " = " + "\"" + ug.getUserid()+"\"\n";
+ config = config + "$ambari_"+comp.getName()+"_group" + " = " + "\"" + ug.getGroup()+"\"\n";
+ config = config + "$ambari_"+comp.getName()+"_groupid" + " = " + "\"" + ug.getGroupid()+"\"\n";
+ }
+ }
+ config = config + "\n";
for (KeyValuePair p : stack.getGlobals()) {
config = config + "$"+p.getName() + " = " + "\"" + p.getValue() + "\"\n";
@@ -920,10 +958,12 @@ public class Clusters {
return config;
}
- private String getRoleToNodesMapForPuppet (ClusterDefinition c) throws Exception {
+ private String getRoleToNodesMapForPuppet (ClusterDefinition c, Stack stack) throws Exception {
+ HashMap<String, String> roles = new HashMap<String, String>();
String config = "\n$role_to_nodes = { ";
for (int i=0; i<c.getRoleToNodesMap().size(); i++) {
RoleToNodes roleToNodesEntry = c.getRoleToNodesMap().get(i);
+ roles.put(roleToNodesEntry.getRoleName(), null);
config = config + roleToNodesEntry.getRoleName()+ " => [";
List<String> host_list = this.getHostnamesFromRangeExpressions(roleToNodesEntry.getNodes());
for (int j=0; j<host_list.size(); j++) {
@@ -934,12 +974,22 @@ public class Clusters {
config = config + "\'"+host+"\',";
}
}
- if (i == c.getRoleToNodesMap().size()-1) {
- config = config + "] \n";
- } else {
- config = config + "], \n";
+ config = config + "], \n";
+ }
+
+ /*
+ * Add non-specified roles for puppet to work correctly
+ */
+ for (Component comp : stack.getComponents()) {
+ for (Role r : comp.getRoles()) {
+ if (!roles.containsKey(r.getName())) {
+ config = config + r.getName() + " => [ ], \n";
+ }
}
}
+ if (!roles.containsKey("client")) {
+ config = config + "client" + " => [ ], \n";
+ }
config = config + "} \n";
return config;
}
@@ -1006,11 +1056,7 @@ public class Clusters {
}
}
}
- if (k == comp.getRoles().size()-1) {
- config = config + "} \n";
- } else {
- config = config + "}, \n";
- }
+ config = config + "}, \n";
}
}
}
Modified: incubator/ambari/trunk/controller/src/main/java/org/apache/ambari/controller/StackFlattener.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/controller/src/main/java/org/apache/ambari/controller/StackFlattener.java?rev=1225365&r1=1225364&r2=1225365&view=diff
==============================================================================
--- incubator/ambari/trunk/controller/src/main/java/org/apache/ambari/controller/StackFlattener.java (original)
+++ incubator/ambari/trunk/controller/src/main/java/org/apache/ambari/controller/StackFlattener.java Wed Dec 28 22:46:40 2011
@@ -35,6 +35,7 @@ import org.apache.ambari.common.rest.ent
import org.apache.ambari.common.rest.entities.RepositoryKind;
import org.apache.ambari.common.rest.entities.Role;
import org.apache.ambari.common.rest.entities.Stack;
+import org.apache.ambari.common.rest.entities.UserGroup;
import org.apache.ambari.components.ComponentPlugin;
import org.apache.ambari.components.ComponentPluginFactory;
@@ -54,6 +55,18 @@ public class StackFlattener {
private final Stacks stacks;
private final ComponentPluginFactory plugins;
+ private UserGroup flattenUserGroup(List<Stack> stacks) {
+ UserGroup default_user_group = null;
+ for(int i=stacks.size()-1; i>=0; --i) {
+ Stack stack = stacks.get(i);
+ default_user_group = stack.getDefault_user_group();
+ if (default_user_group == null) {
+ continue;
+ }
+ }
+ return default_user_group;
+ }
+
private List<RepositoryKind> flattenRepositories(List<Stack> stacks) {
Map<String, List<String>> repositories =
new TreeMap<String, List<String>>();
@@ -235,6 +248,7 @@ public class StackFlattener {
Stack result = new Stack(stacks.get(stacks.size()-1));
result.setParentName(null);
result.setPackageRepositories(flattenRepositories(stacks));
+ result.setDefault_user_group(flattenUserGroup(stacks));
List<Component> components = new ArrayList<Component>();
result.setComponents(components);
for(String componentName: getComponents(stacks)) {
Added: incubator/ambari/trunk/controller/src/main/resources/org/apache/ambari/acd/mapreduce-0.1.0.acd
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/controller/src/main/resources/org/apache/ambari/acd/mapreduce-0.1.0.acd?rev=1225365&view=auto
==============================================================================
--- incubator/ambari/trunk/controller/src/main/resources/org/apache/ambari/acd/mapreduce-0.1.0.acd (added)
+++ incubator/ambari/trunk/controller/src/main/resources/org/apache/ambari/acd/mapreduce-0.1.0.acd Wed Dec 28 22:46:40 2011
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<component provides="mapred">
+ <requires name="hdfs"/>
+ <roles name="jobtracker"/>
+ <roles name="tasktracker"/>
+ <roles name="historyserver"/>
+ <start>
+<![CDATA[
+import os
+import sys
+
+[pgm, cluster, role] = sys.argv
+os.environ['HADOOP_CONF_DIR']=os.getcwd() + "/etc/hadoop"
+os.execlp("hadoop", "hadoop", role)
+]]>
+ </start>
+ <check runOn="jobtracker">
+<![CDATA[
+import os
+import sys
+
+os.environ['HADOOP_CONF_DIR']=os.getcwd() + "/etc/hadoop"
+sys.exit(os.system('hadoop job -list'))
+
+]]>
+ </check>
+
+ <prestart runOn="namenode">
+<![CDATA[
+import os
+import sys
+
+os.environ['HADOOP_CONF_DIR']=os.getcwd() + "/etc/hadoop"
+sys.exit(os.system('hadoop dfs -mkdir /mapred'))
+
+]]>
+ </prestart>
+</component>
Modified: incubator/ambari/trunk/controller/src/main/resources/org/apache/ambari/stacks/puppet1-0.json
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/controller/src/main/resources/org/apache/ambari/stacks/puppet1-0.json?rev=1225365&r1=1225364&r2=1225365&view=diff
==============================================================================
--- incubator/ambari/trunk/controller/src/main/resources/org/apache/ambari/stacks/puppet1-0.json (original)
+++ incubator/ambari/trunk/controller/src/main/resources/org/apache/ambari/stacks/puppet1-0.json Wed Dec 28 22:46:40 2011
@@ -4,6 +4,12 @@
"@kind":"TAR",
"urls":"http://www.apache.org/dist/hadoop/common/"
},
+ "default_user_group":{
+ "@user":"hadoop",
+ "@userid":"",
+ "@group":"hadoop",
+ "@groupid":""
+ },
"globals":[
{
"@name":"ambari_namenode_principal",
@@ -34,26 +40,6 @@
"@value":"hcat"
},
{
- "@name":"ambari_hdfs_user",
- "@value":"hdfs"
- },
- {
- "@name":"ambari_mapreduce_user",
- "@value":"mapred"
- },
- {
- "@name":"ambari_hbase_user",
- "@value":"hrt_hbase"
- },
- {
- "@name":"ambari_hcat_user",
- "@value":"hcat"
- },
- {
- "@name":"ambari_admin_group",
- "@value":"hadoop"
- },
- {
"@name":"ambari_user_realm",
"@value":"KERBEROS.EXAMPLE.COM"
},
@@ -94,6 +80,10 @@
{
"@name":"hadoop.kerberos.kinit.command",
"@value":"/usr/kerberos/bin/kinit"
+ },
+ {
+ "@name":"local.realm",
+ "@value":"<%= ambari_user_realm %>"
}
]
},
@@ -170,6 +160,171 @@
"@value":"-Xmx128m $HADOOP_CLIENT_OPTS"
}
]
+ },
+ {
+ "@name":"mapred-site.xml",
+ "property":[
+ {
+ "@name":"mapred.tasktracker.tasks.sleeptime-before-sigkill",
+ "@value":"250"
+ },
+ {
+ "@name":"mapred.system.dir",
+ "@value":"/mapred/mapredsystem"
+ },
+ {
+ "@name":"mapred.job.tracker",
+ "@value":"hrt20n24.cc1.ygridcore.net:9000"
+ },
+ {
+ "@name":"mapred.job.tracker.http.address",
+ "@value":"hrt20n24.cc1.ygridcore.net:50030"
+ },
+ {
+ "@name":"mapred.local.dir",
+ "@value":"/var/lib/hadoop/mapred1,/var/lib/hadoop/mapred2,/var/lib/hadoop/mapred3"
+ },
+ {
+ "@name":"mapreduce.cluster.administrators",
+ "@value":"hrt_mr"
+ },
+ {
+ "@name":"mapred.map.tasks.speculative.execution",
+ "@value":"false"
+ },
+ {
+ "@name":"mapred.reduce.tasks.speculative.execution",
+ "@value":"false"
+ },
+ {
+ "@name":"mapred.output.compression.type",
+ "@value":"BLOCK"
+ },
+ {
+ "@name":"jetty.connector",
+ "@value":"org.mortbay.jetty.nio.SelectChannelConnector"
+ },
+ {
+ "@name":"mapred.task.tracker.task-controller",
+ "@value":"org.apache.hadoop.mapred.DefaultTaskController"
+ },
+ {
+ "@name":"mapred.child.root.logger",
+ "@value":"INFO,TLA"
+ },
+ {
+ "@name":"mapred.child.java.opts",
+ "@value":"-server -Xmx640m -Djava.net.preferIPv4Stack=true"
+ },
+ {
+ "@name":"mapred.child.ulimit",
+ "@value":"8388608"
+ },
+ {
+ "@name":"mapred.job.tracker.persist.jobstatus.active",
+ "@value":"true"
+ },
+ {
+ "@name":"mapred.job.tracker.persist.jobstatus.dir",
+ "@value":"file:////var/log/hadoop/hrt_mr/jobstatus"
+ },
+ {
+ "@name":"mapred.job.tracker.history.completed.location",
+ "@value":"/mapred/history/done"
+ },
+ {
+ "@name":"mapred.heartbeats.in.second",
+ "@value":"200"
+ },
+ {
+ "@name":"mapreduce.tasktracker.outofband.heartbeat",
+ "@value":"true"
+ },
+ {
+ "@name":"mapred.jobtracker.maxtasks.per.job",
+ "@value":"200000"
+ },
+ {
+ "@name":"mapreduce.jobtracker.kerberos.principal",
+ "@value":"jt/_HOST@${local.realm}"
+ },
+ {
+ "@name":"mapreduce.tasktracker.kerberos.principal",
+ "@value":"tt/_HOST@${local.realm}"
+ },
+ {
+ "@name":"hadoop.job.history.user.location",
+ "@value":"none"
+ },
+ {
+ "@name":"mapreduce.jobtracker.keytab.file",
+ "@value":"/etc/security/keytabs/jt.service.keytab"
+ },
+ {
+ "@name":"mapreduce.tasktracker.keytab.file",
+ "@value":"/etc/security/keytabs/tt.service.keytab"
+ },
+ {
+ "@name":"mapreduce.jobtracker.staging.root.dir",
+ "@value":"/user"
+ },
+ {
+ "@name":"mapreduce.job.acl-modify-job",
+ "@value":""
+ },
+ {
+ "@name":"mapreduce.job.acl-view-job",
+ "@value":"Dr.Who"
+ },
+ {
+ "@name":"mapreduce.tasktracker.group",
+ "@value":"hadoop"
+ },
+ {
+ "@name":"mapred.acls.enabled",
+ "@value":"true"
+ },
+ {
+ "@name":"mapred.jobtracker.taskScheduler",
+ "@value":"org.apache.hadoop.mapred.CapacityTaskScheduler"
+ },
+ {
+ "@name":"mapred.queue.names",
+ "@value":"default"
+ },
+ {
+ "@name":"mapreduce.history.server.embedded",
+ "@value":"false"
+ },
+ {
+ "@name":"mapreduce.history.server.http.address",
+ "@value":"hrt20n24.cc1.ygridcore.net:51111"
+ },
+ {
+ "@name":"mapreduce.jobhistory.kerberos.principal",
+ "@value":"jt/_HOST@${local.realm}"
+ },
+ {
+ "@name":"mapreduce.jobhistory.keytab.file",
+ "@value":"/etc/security/keytabs/jt.service.keytab"
+ },
+ {
+ "@name":"mapred.hosts",
+ "@value":"/etc/hadoop/mapred.include"
+ },
+ {
+ "@name":"mapred.hosts.exclude",
+ "@value":"/etc/hadoop/mapred.exclude"
+ },
+ {
+ "@name":"mapred.jobtracker.retirejob.check",
+ "@value":"10000"
+ },
+ {
+ "@name":"mapred.jobtracker.retirejob.interval",
+ "@value":"0"
+ }
+ ]
}
]
},
@@ -186,6 +341,164 @@
}
},
{
+ "@name":"mapreduce",
+ "@architecture":"x86_64",
+ "@version":"0.20.205.0",
+ "@provider":"org.apache.hadoop",
+ "definition":{
+ "@provider":"org.apache.ambari",
+ "@name":"mapreduce",
+ "@version":"0.1.0"
+ },
+ "configuration":{
+ "category":[
+ {
+ "@name":"core-site.xml",
+ "property":[
+ {
+ "@name":"hadoop.security.auth_to_local",
+ "@value":"<%= ambari_auth_to_local %>"
+ },
+ {
+ "@name":"hadoop.security.groups.cache.secs",
+ "@value":"14400"
+ },
+ {
+ "@name":"hadoop.http.filter.initializers",
+ "@value":"org.apache.hadoop.http.lib.StaticUserWebFilter"
+ }
+ ]
+ },
+ {
+ "@name":"hadoop-metrics2.properties",
+ "property":{
+ "@name":"*.period",
+ "@value":"60"
+ }
+ },
+ {
+ "@name":"capacity-scheduler.xml",
+ "property":[
+ {
+ "@name":"mapred.capacity-scheduler.maximum-system-jobs",
+ "@value":"3000"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.queue.default.capacity",
+ "@value":"100"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.queue.default.maximum-capacity",
+ "@value":"-1"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.queue.default.supports-priority",
+ "@value":"false"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.queue.default.minimum-user-limit-percent",
+ "@value":"100"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.queue.default.user-limit-factor",
+ "@value":"1"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks",
+ "@value":"200000"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks-per-user",
+ "@value":"100000"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.queue.default.init-accept-jobs-factor",
+ "@value":"10"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.default-supports-priority",
+ "@value":"false"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.default-minimum-user-limit-percent",
+ "@value":"100"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.default-user-limit-factor",
+ "@value":"1"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.default-maximum-active-tasks-per-queue",
+ "@value":"200000"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.default-maximum-active-tasks-per-user",
+ "@value":"100000"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.default-init-accept-jobs-factor",
+ "@value":"10"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.init-poll-interval",
+ "@value":"5000"
+ },
+ {
+ "@name":"mapred.capacity-scheduler.init-worker-threads",
+ "@value":"5"
+ }
+ ]
+ },
+ {
+ "@name":"taskcontroller.cfg",
+ "property":[
+ {
+ "@name":"mapreduce.cluster.local.dir",
+ "@value":"/var/lib/hadoop/mapred1,/var/lib/hadoop/mapred2,/var/lib/hadoop/mapred3"
+ },
+ {
+ "@name":"mapreduce.tasktracker.group",
+ "@value":"hadoop"
+ },
+ {
+ "@name":"hadoop.log.dir",
+ "@value":"/var/log/hadoop/hrt_mr"
+ }
+ ]
+ },
+ {
+ "@name":"hadoop-env.sh",
+ "property":[
+ {
+ "@name":"HADOOP_CONF_DIR",
+ "@value":"<%= ambari_role_prefix %>/etc/hadoop"
+ },
+ {
+ "@name":"HADOOP_JOBTRACKER_OPTS",
+ "@value":"-Dsecurity.audit.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA $HADOOP_JOBTRACKER_OPTS"
+ },
+ {
+ "@name":"HADOOP_TASKTRACKER_OPTS",
+ "@value":"-Dsecurity.audit.logger=ERROR,console -Dmapred.audit.logger=ERROR,console $HADOOP_TASKTRACKER_OPTS"
+ },
+ {
+ "@name":"HADOOP_LOG_DIR",
+ "@value":"<%= ambari_role_prefix %>/logs"
+ },
+ {
+ "@name":"HADOOP_PID_DIR",
+ "@value":"<%= ambari_role_prefix %>/pids"
+ },
+ {
+ "@name":"HADOOP_IDENT_STRING",
+ "@value":"<%= ambari_cluster_name %>"
+ }
+ ]
+ }
+ ]
+ }
+ },
+ {
"@name":"hdfs",
"@provider":"org.apache.hadoop",
"definition":{
@@ -273,7 +586,7 @@
},
{
"@name":"dfs.permissions.superusergroup",
- "@value":"<%= ambari_admin_group %>"
+ "@value":"<%= ambari_hdfs_group %>"
},
{
"@name":"dfs.secondary.http.address",
@@ -337,7 +650,7 @@
"@value":"<%= ambari_cluster_name %>"
}
]
- },
+ },
{
"@name":"hadoop-metrics2.properties",
"property":{
Modified: incubator/ambari/trunk/controller/src/test/java/org/apache/ambari/controller/TestStackFlattener.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/controller/src/test/java/org/apache/ambari/controller/TestStackFlattener.java?rev=1225365&r1=1225364&r2=1225365&view=diff
==============================================================================
--- incubator/ambari/trunk/controller/src/test/java/org/apache/ambari/controller/TestStackFlattener.java (original)
+++ incubator/ambari/trunk/controller/src/test/java/org/apache/ambari/controller/TestStackFlattener.java Wed Dec 28 22:46:40 2011
@@ -32,6 +32,7 @@ import org.apache.ambari.common.rest.ent
import org.apache.ambari.common.rest.entities.RepositoryKind;
import org.apache.ambari.common.rest.entities.Role;
import org.apache.ambari.common.rest.entities.Stack;
+import org.apache.ambari.common.rest.entities.UserGroup;
import org.apache.ambari.components.ComponentPlugin;
import org.apache.ambari.components.ComponentPluginFactory;
@@ -87,12 +88,12 @@ public class TestStackFlattener {
"org.apache.ambari",
new ComponentDefinition("hdfs",
"org.apache.ambari", "0"),
- new Configuration(), new ArrayList<Role>());
+ new Configuration(), new ArrayList<Role>(), new UserGroup());
parentMapreduce = new Component("mapreduce", "0.20.205.0", "i386",
"org.apache.ambari",
new ComponentDefinition("mapreduce",
"org.apache.ambari","0"),
- new Configuration(), new ArrayList<Role>());
+ new Configuration(), new ArrayList<Role>(), new UserGroup());
List<Component> compList = new ArrayList<Component>();
parentStack.setComponents(compList);
compList.add(parentHdfs);
@@ -175,7 +176,7 @@ public class TestStackFlattener {
Configuration childHdfsConfig = new Configuration();
childStack.getComponents().add(
new Component("hdfs", null, null, null, null,
- childHdfsConfig, hdfsRoles));
+ childHdfsConfig, hdfsRoles, new UserGroup()));
Configuration nnConf = new Configuration();
hdfsRoles.add(new Role("namenode", nnConf));
setConfigParam(parentConfiguration, "ambari", "global", "global-value");