You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2012/10/24 08:46:39 UTC
svn commit: r1401568 - in /incubator/ambari/branches/AMBARI-666: ./
ambari-agent/src/main/python/ambari_agent/
ambari-server/src/main/java/org/apache/ambari/server/agent/
ambari-server/src/main/java/org/apache/ambari/server/state/
Author: mahadev
Date: Wed Oct 24 06:46:38 2012
New Revision: 1401568
URL: http://svn.apache.org/viewvc?rev=1401568&view=rev
Log:
AMBARI-905. Fix puppet site creation with flattening of execution commands send from the server. (mahadev)
Modified:
incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
Modified: incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt?rev=1401568&r1=1401567&r2=1401568&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt (original)
+++ incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt Wed Oct 24 06:46:38 2012
@@ -352,6 +352,9 @@ AMBARI-666 branch (unreleased changes)
BUG FIXES
+ AMBARI-905. Fix puppet site creation with flattening of execution commands
+ send from the server. (mahadev)
+
AMBARI-885. Fix miscellaneous issues related to Ambari Web. (yusaku)
AMBARI-879. Installer skips Confirm Hosts page of the wizard when testing
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py?rev=1401568&r1=1401567&r2=1401568&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py Wed Oct 24 06:46:38 2012
@@ -119,6 +119,10 @@ class ActionQueue(threading.Thread):
configurations = command['configurations']
result = []
commandresult = self.executor.runCommand(command)
+ status = "COMPLETED"
+ if (commandresult['exitcode'] != 0):
+ status = "FAILED"
+
# assume some puppet pluing to run these commands
roleResult = {'role' : command['role'],
'actionId' : commandId,
@@ -127,7 +131,7 @@ class ActionQueue(threading.Thread):
'stderr' : commandresult['stderr'],
'exitCode' : commandresult['exitcode'],
'serviceName' : serviceName,
- 'status' : "COMPLETED"}
+ 'status' : status}
result.append(roleResult)
pass
return result
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py?rev=1401568&r1=1401567&r2=1401568&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py Wed Oct 24 06:46:38 2012
@@ -31,7 +31,7 @@ url=https://localhost:4080
secured_url=https://localhost:8443
[agent]
-prefix=/tmp/ambari
+prefix=/tmp/ambari-agent
[stack]
installprefix=/var/ambari/
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py?rev=1401568&r1=1401567&r2=1401568&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py Wed Oct 24 06:46:38 2012
@@ -24,6 +24,13 @@ import logging
logger = logging.getLogger()
+xml_configurations_keys= ["hdfs_site", "hdfs_site", "core_site",
+ "mapred_queue_acls",
+ "hadoop_policy", "mapred_site",
+ "capacity_scheduler", "hbase_site",
+ "hbase_policy", "hive_site", "oozie_site",
+ "templeton_site"]
+
#read static imports from file and write them to manifest
def writeImports(outputFile, modulesdir, inputFileName='imports.txt'):
inputFile = open(inputFileName, 'r')
@@ -39,12 +46,13 @@ def generateManifest(parsedJson, fileNam
#reading json
hostname = parsedJson['hostname']
clusterHostInfo = parsedJson['clusterHostInfo']
- params = parsedJson['params']
+ params = parsedJson['hostLevelParams']
configurations = parsedJson['configurations']
- xmlConfigurationsKeys = parsedJson['xml_configurations_keys']
+ xmlConfigurationsKeys = xml_configurations_keys
#hostAttributes = parsedJson['hostAttributes']
- roles = parsedJson['roleCommands']
-
+ roles = [{'role' : parsedJson['role'],
+ 'cmd' : parsedJson['roleCommand'],
+ 'roleParams' : parsedJson['roleParams']}]
#writing manifest
manifest = open(fileName, 'w')
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py?rev=1401568&r1=1401567&r2=1401568&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py Wed Oct 24 06:46:38 2012
@@ -59,12 +59,12 @@ class puppetExecutor:
def runCommand(self, command):
result = {}
- taskId = "0";
+ taskId = 0;
if command.has_key("taskId"):
taskId = command['taskId']
puppetEnv = os.environ
- siteppFileName = os.path.join(self.tmpDir, "site-" + taskId + ".pp")
+ siteppFileName = os.path.join(self.tmpDir, "site-" + str(taskId) + ".pp")
generateManifest(command, siteppFileName, self.puppetModule + "/modules")
puppetcommand = self.puppetCommand(siteppFileName)
""" Run the command and make sure the output gets propagated"""
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp?rev=1401568&r1=1401567&r2=1401568&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp Wed Oct 24 06:46:38 2012
@@ -1,15 +1,15 @@
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-zookeeper/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-oozie/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-pig/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-sqoop/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-templeton/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hive/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hcat/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-mysql/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-monitor-webserver/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-zookeeper/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-oozie/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-pig/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-sqoop/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-templeton/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hive/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hcat/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-mysql/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-monitor-webserver/manifests/*.pp'
$NAMENODE= ['h2.hortonworks.com']
$DATANODE= ['h1.hortonworks.com', 'h2.hortonworks.com']
$hdfs_user="hdfs"
@@ -66,13 +66,6 @@ $task_bin_exe = "ls"
$hadoop_piddirprefix = "/tmp"
$ganglia_server_host = "localhost"
node /default/ {
- stage{1 :} -> stage{2 :} -> stage{3 :} -> stage{4 :} -> stage{5 :} -> stage{6 :} -> stage{7 :} -> stage{8 :}
+ stage{1 :}
class {'hdp-hadoop::namenode': stage => 1, service_state => installed_and_configured}
-class {'hdp-hadoop::datanode': stage => 2, service_state => installed_and_configured}
-class {'hdp-hbase::master': stage => 3, service_state => installed_and_configured}
-class {'hdp-hive::server': stage => 4, service_state => installed_and_configured}
-class {'hdp-hive::client': stage => 5, service_state => installed_and_configured}
-class {'hdp-oozie::server': stage => 6, service_state => installed_and_configured}
-class {'hdp-templeton::server': stage => 7, service_state => installed_and_configured}
-class {'hdp-templeton::client': stage => 8, service_state => installed_and_configured}
}
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json?rev=1401568&r1=1401567&r2=1401568&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json Wed Oct 24 06:46:38 2012
@@ -7,7 +7,7 @@
"NAMENODE": ["h2.hortonworks.com"],
"DATANODE": ["h1.hortonworks.com", "h2.hortonworks.com"]
},
-"params":
+"hostLevelParams":
{
"hdfs_user" : "hdfs",
"jdk_location" : "http://hdp1/downloads",
@@ -17,9 +17,6 @@
"jdk_bins" : { "32" : "jdk-6u31-linux-x64.bin", "64" : "jdk-6u31-linux-x64.bin" }
},
-"xml_configurations_keys" : ["hdfs_site", "hdfs_site", "core_site", "mapred_queue_acls",
- "hadoop_policy", "mapred_site", "capacity_scheduler", "hbase_site",
- "hbase_policy", "hive_site", "oozie_site", "templeton_site"],
"configurations" : {
"hdfs_site" : { "dfs.block.size" : "256000000", "dfs.replication" : "1" } ,
@@ -43,46 +40,8 @@
"oozie_site" : {"oozie.service.ActionService.executor.ext.classes" : "org.apache.oozie.action.hadoop.HiveActionExecutor, org.apache.oozie.action.hadoop.SqoopActionExecutor,org.apache.oozie.action.email.EmailActionExecutor,"},
"templeton_site" : {"templeton.override.enabled" : "true"}
},
-"roleCommands": [
-{
- "role": "NAMENODE",
- "cmd": "INSTALL",
- "roleParams" : {}
-},
-{
- "role": "DATANODE",
- "cmd": "INSTALL",
- "roleParams" : {}
-},
-{
- "role": "HBASE_MASTER",
- "cmd": "INSTALL",
- "roleParams" : {}
-},
-{
- "role": "HIVE_SERVER",
- "cmd": "INSTALL",
- "roleParams" : {}
-},
-{
- "role": "HIVE_CLIENT",
- "cmd": "INSTALL",
- "roleParams" : {}
-},
-{
- "role": "OOZIE_SERVER",
- "cmd": "INSTALL",
- "roleParams" : {}
-},
-{
- "role": "TEMPLETON_SERVER",
- "cmd": "INSTALL",
- "roleParams" : {}
-},
-{
- "role": "TEMPLETON_CLIENT",
- "cmd": "INSTALL",
- "roleParams" : {}
-}
-]
+
+"role": "NAMENODE",
+"roleCommand": "INSTALL",
+"roleParams" : {}
}
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java?rev=1401568&r1=1401567&r2=1401568&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java Wed Oct 24 06:46:38 2012
@@ -17,8 +17,11 @@
*/
package org.apache.ambari.server.agent;
+import org.codehaus.jackson.annotate.JsonProperty;
+
public class CommandReport {
+
String role;
String actionId;
String stdout;
@@ -27,67 +30,94 @@ public class CommandReport {
int exitCode;
private String clusterName;
private String serviceName;
+ private long taskId;
+
+ @JsonProperty("taskId")
+ public long getTaskId() {
+ return taskId;
+ }
+ @JsonProperty("taskId")
+ public void setTaskId(long taskId) {
+ this.taskId = taskId;
+ }
+
+ @JsonProperty("clusterName")
public void setClusterName(String clusterName) {
this.clusterName = clusterName;
}
+ @JsonProperty("clusterName")
public String getClusterName() {
return this.clusterName;
}
-
+
+ @JsonProperty("actionId")
public String getActionId() {
return this.actionId;
}
-
+
+ @JsonProperty("actionId")
public void setActionId(String actionId) {
this.actionId = actionId;
}
-
+
+ @JsonProperty("stderr")
public String getStdErr() {
return this.stderr;
}
-
+
+ @JsonProperty("stderr")
public void setStdErr(String stderr) {
this.stderr = stderr;
}
-
+
+ @JsonProperty("exitcode")
public int getExitCode() {
return this.exitCode;
}
-
+
+ @JsonProperty("exitcode")
public void setExitCode(int exitCode) {
this.exitCode = exitCode;
}
-
+
+ @JsonProperty("stdout")
public String getStdOut() {
return this.stdout;
}
-
+
+ @JsonProperty("stdout")
public void setStdOut(String stdout) {
this.stdout = stdout;
}
+ @JsonProperty("role")
public String getRole() {
return role;
}
-
+
+ @JsonProperty("role")
public void setRole(String role) {
this.role = role;
}
-
+
+ @JsonProperty("status")
public String getStatus() {
return status;
}
-
+
+ @JsonProperty("status")
public void setStatus(String status) {
this.status = status;
}
-
+
+ @JsonProperty("serviceName")
public String getServiceName() {
return serviceName;
}
-
+
+ @JsonProperty("serviceName")
public void setServiceName(String serviceName) {
this.serviceName = serviceName;
}
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java?rev=1401568&r1=1401567&r2=1401568&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java Wed Oct 24 06:46:38 2012
@@ -328,9 +328,9 @@ public class ServiceImpl implements Serv
+ ", clusterId=" + cluster.getClusterId()
+ ", desiredStackVersion=" + getDesiredStackVersion()
+ ", desiredState=" + getDesiredState().toString()
- + ", configs = " + getConfigVersions()
+ + ", configs = " + getConfigVersions()
+ ", components=[ ");
-
+
boolean first = true;
for(ServiceComponent sc : components.values()) {
if (!first) {