You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2012/10/22 10:09:16 UTC
svn commit: r1400792 [1/2] - in /incubator/ambari/branches/AMBARI-666: ./
ambari-agent/src/main/puppet/modules/configgenerator/manifests/
ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/
ambari-agent/src/main/puppet/modules/hdp/manifests/lib/...
Author: mahadev
Date: Mon Oct 22 08:09:14 2012
New Revision: 1400792
URL: http://svn.apache.org/viewvc?rev=1400792&view=rev
Log:
AMBARI-892. Add puppet executor at the agent to be able to run various commands from the server. (mahadev)
Added:
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/lib/
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/lib/puppet/
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/lib/puppet/parser/
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/lib/puppet/parser/functions/
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/lib/puppet/parser/functions/pkgName.rb
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py
incubator/ambari/branches/AMBARI-666/ambari-agent/src/test/python/TestPuppetExecutor.py
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactory.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/db/newcerts/.gitignore
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/stacks/
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/stacks/HDP/
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/stacks/HDP/0.1/
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/stacks/HDP/0.1/repos/
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/stacks/HDP/0.1/repos/repoinfo.xml
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/stacks/HDP/0.1/services/
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/stacks/HDP/0.1/services/HDFS/
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml
incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionDBAccessorImpl.java
Removed:
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ActionStatusDAO.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ActionStatusEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ActionStatusEntityPK.java
Modified:
incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/configgenerator/manifests/configfile.pp
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/conf/ambari.properties
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/rest/AgentResource.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/StageDAO.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ExecutionCommandEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentMappingEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RoleSuccessCriteriaEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RoleSuccessCriteriaEntityPK.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentConfigEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentHostConfigEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntity.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StageEntityPK.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/security/CertificateManager.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHostEvent.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/ambari-server-state/core-site.xml
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/Ambari-DDL.sql
incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/META-INF/persistence.xml
incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/orm/OrmTestHelper.java
incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/orm/TestOrmImpl.java
Modified: incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt (original)
+++ incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt Mon Oct 22 08:09:14 2012
@@ -12,6 +12,9 @@ AMBARI-666 branch (unreleased changes)
NEW FEATURES
+ AMBARI-892. Add puppet executor at the agent to be able to run various
+ commands from the server. (mahadev)
+
AMBARI-887. Ability to save configuration. (Nate Cole via mahadev)
AMBARI-877. Refactor resource provider implementation for changes to
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/configgenerator/manifests/configfile.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/configgenerator/manifests/configfile.pp?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/configgenerator/manifests/configfile.pp (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/configgenerator/manifests/configfile.pp Mon Oct 22 08:09:14 2012
@@ -53,9 +53,12 @@ define configgenerator::configfile ($mod
<% end -%>
</configuration>')
-file {'config':
+
+debug("Generating config: ${modulespath}/${filename}")
+
+file {"${modulespath}/${filename}":
ensure => present,
content => $configcontent,
- path => "${modulespath}/${module}/templates/${filename}",
+ path => "${modulespath}/${filename}",
}
}
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp Mon Oct 22 08:09:14 2012
@@ -40,51 +40,60 @@ class hdp-hadoop::initialize()
# Configs generation
- if has_key($configuration, 'hdp_hadoop__mapred_queue_acls') {
- configgenerator::configfile{'mapred_queue_acls_xml':
+debug('##Configs generation for hdp-hadoop')
+
+
+ if has_key($configuration, 'mapred_queue_acls') {
+ configgenerator::configfile{'mapred_queue_acls':
+ modulespath => $hdp-hadoop::params::conf_dir,
filename => 'mapred-queue-acls.xml',
module => 'hdp-hadoop',
- configuration => $configuration['hdp_hadoop__mapred_queue_acls']
+ configuration => $configuration['mapred_queue_acls']
}
}
- if has_key($configuration, 'hdp_hadoop__hadoop_policy') {
- configgenerator::configfile{'hadoop_policy_xml':
+ if has_key($configuration, 'hadoop_policy') {
+ configgenerator::configfile{'hadoop_policy':
+ modulespath => $hdp-hadoop::params::conf_dir,
filename => 'hadoop-policy.xml',
module => 'hdp-hadoop',
- configuration => $configuration['hdp_hadoop__hadoop_policy']
+ configuration => $configuration['hadoop_policy']
}
}
-
- if has_key($configuration, 'hdp_hadoop__core_site') {
- configgenerator::configfile{'core_site_xml':
+
+ if has_key($configuration, 'core_site') {
+ configgenerator::configfile{'core_site':
+ modulespath => $hdp-hadoop::params::conf_dir,
filename => 'core-site.xml',
module => 'hdp-hadoop',
- configuration => $configuration['hdp_hadoop__core_site']
+ configuration => $configuration['core_site']
}
}
- if has_key($configuration, 'hdp_hadoop__mapred_site') {
- configgenerator::configfile{'mapred_site_xml':
+ if has_key($configuration, 'mapred_site') {
+ configgenerator::configfile{'mapred_site':
+ modulespath => $hdp-hadoop::params::conf_dir,
filename => 'mapred-site.xml',
module => 'hdp-hadoop',
- configuration => $configuration['hdp_hadoop__mapred_site']
+ configuration => $configuration['mapred_site']
}
}
- if has_key($configuration, 'hdp_hadoop__capacity_scheduler') {
- configgenerator::configfile{'capacity_scheduler_xml':
+ if has_key($configuration, 'capacity_scheduler') {
+ configgenerator::configfile{'capacity_scheduler':
+ modulespath => $hdp-hadoop::params::conf_dir,
filename => 'capacity-scheduler.xml',
module => 'hdp-hadoop',
- configuration => $configuration['hdp_hadoop__capacity_scheduler']
+ configuration => $configuration['capacity_scheduler']
}
}
- if has_key($configuration, 'hdp_hadoop__hdfs_site') {
- configgenerator::configfile{'hdfs_site_xml':
+ if has_key($configuration, 'hdfs_site') {
+ configgenerator::configfile{'hdfs_site':
+ modulespath => $hdp-hadoop::params::conf_dir,
filename => 'hdfs-site.xml',
module => 'hdp-hadoop',
- configuration => $configuration['hdp_hadoop__hdfs_site']
+ configuration => $configuration['hdfs_site']
}
}
}
@@ -155,7 +164,7 @@ class hdp-hadoop(
mode => $tc_mode
}
- $template_files = ['hadoop-env.sh','core-site.xml','hadoop-policy.xml','health_check','capacity-scheduler.xml','commons-logging.properties','log4j.properties','mapred-queue-acls.xml','slaves']
+ $template_files = [ 'hadoop-env.sh', 'health_check', 'commons-logging.properties', 'log4j.properties', 'slaves']
hdp-hadoop::configfile { $template_files:
tag => 'common',
owner => $hdfs_user
@@ -166,11 +175,6 @@ class hdp-hadoop(
owner => $hdfs_user,
}
- hdp-hadoop::configfile { 'mapred-site.xml':
- tag => 'common',
- owner => $mapred_user
- }
-
Anchor['hdp-hadoop::begin'] -> Hdp-hadoop::Package<||> -> Hdp::Directory_recursive_create[$hadoop_config_dir] -> Hdp::User<|title == $hdfs_user or title == $mapred_user|>
-> Hdp-hadoop::Configfile<|tag == 'common'|> -> Anchor['hdp-hadoop::end']
Anchor['hdp-hadoop::begin'] -> Hdp::Directory_recursive_create[$logdirprefix] -> Anchor['hdp-hadoop::end']
Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/lib/puppet/parser/functions/pkgName.rb
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/lib/puppet/parser/functions/pkgName.rb?rev=1400792&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/lib/puppet/parser/functions/pkgName.rb (added)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/lib/puppet/parser/functions/pkgName.rb Mon Oct 22 08:09:14 2012
@@ -0,0 +1,191 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+# Returns package name(s) for the specific OS and architecture
+# Params:
+#
+# - name of the package
+# - OS name
+# - OS architecture
+#
+# If there are no approprite OS/architecture, it will search default entries (named as ALL)
+module Puppet::Parser::Functions
+ newfunction(:pkgName, :type => :rvalue) do |args|
+ packageName = args[0]
+ os = args[1]
+ arch = args[2]
+ ALL = 'ALL'
+
+ # First level (packages): packageName => OS hashmap
+ # Second level (OS hashmap): Architecture => real package name(s)
+ packages = {
+ 'ganglia-monitor' => {
+ 'ALL' => {
+ 64 => 'ganglia-gmond-3.2.0'
+ }
+ },
+ 'ganglia-server' => {
+ 'ALL' => {
+ 64 => 'ganglia-gmetad-3.2.0'
+ }
+ },
+ 'ganglia-gweb' => {
+ 'ALL' => {
+ 64 => 'gweb'
+ }
+ },
+ 'ganglia-hdp-gweb-addons' => {
+ ALL => {
+ 64 => 'hdp_mon_ganglia_addons'
+ }
+ },
+ 'glibc' => {
+ 'rhel6' => {
+ ALL => ['glibc','glibc.i686']
+ }
+ },
+ 'nagios-addons' => {
+ ALL => {
+ 64 => 'hdp_mon_nagios_addons'
+ }
+ },
+ 'nagios-server' => {
+ ALL => {
+ 64 => 'nagios-3.2.3'
+ }
+ },
+ 'nagios-plugins' => {
+ ALL => {
+ 64 => 'nagios-plugins-1.4.9'
+ }
+ },
+ 'nagios-fping' => {
+ ALL => {
+ 64 =>'fping'
+ }
+ },
+ 'nagios-php-pecl-json' => {
+ ALL => {
+ 64 => 'php-pecl-json.x86_64'
+ }
+ },
+ 'snmp' => {
+ ALL => {
+ 64 => ['net-snmp','net-snmp-utils'],
+ }
+ },
+ 'dashboard' => {
+ ALL => {
+ 64 => 'hdp_mon_dashboard'
+ }
+ },
+ 'templeton' => {
+ ALL => {
+ ALL => 'templeton'
+ }
+ },
+ 'oozie-client' => {
+ ALL => {
+ 64 => 'oozie-client.noarch'
+ }
+ },
+ 'oozie-server' => {
+ ALL => {
+ 64 => 'oozie.noarch'
+ }
+ },
+ 'lzo' => {
+ 'rhel5' => {
+ ALL => ['lzo','lzo.i386','lzo-devel','lzo-devel.i386']
+ },
+ 'rhel6' => {
+ ALL => ['lzo','lzo.i686','lzo-devel','lzo-devel.i686']
+ }
+ },
+ #TODO: make these two consistent on whether case of 64/32 bits
+ 'snappy' => {
+ ALL => {
+ 32 => ['snappy','snappy-devel'],
+ 64 => ['snappy','snappy-devel']
+ }
+ },
+ 'mysql' => {
+ ALL => {
+ 32 => ['mysql','mysql-server']
+ }
+ },
+ 'mysql-connector' => {
+ ALL => {
+ 64 => ['mysql-connector-java']
+ }
+ },
+ 'extjs' => {
+ ALL => {
+ 64 => ['extjs-2.2-1']
+ }
+ },
+ 'templeton-tar-hive' => {
+ ALL => {
+ 64 => ['templeton-tar-hive-0.0.1.14-1']
+ }
+ },
+ 'templeton-tar-pig' => {
+ ALL => {
+ 64 => ['templeton-tar-pig-0.0.1.14-1']
+ }
+ }
+ }
+ ########################################################
+ ########################################################
+ # seeking package hashmap
+ pkgHash = nil
+
+ if has_key(packages, packageName)
+ pkgHash = packages[packageName]
+ else
+ print "Wrong package name: " + packageName
+ return nil
+ end
+
+ # seeking os hashmap
+ osHash = nil
+
+ if has_key(pkgHash, os)
+ osHash = pkgHash[os]
+ elsif has_key(pkgHash, ALL)
+ osHash = pkgHash[ALL]
+ else
+ print "Wrong package name: " + packageName
+ return nil
+ end
+
+ #seeking arhitecture
+ result = nil
+
+ if has_key(osHash, arch)
+ result = osHash[arch]
+ elsif has_key(osHash, ALL)
+ result = osHash[ALL]
+ end
+
+ return result
+ end
+end
\ No newline at end of file
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py Mon Oct 22 08:09:14 2012
@@ -33,11 +33,14 @@ import os
import time
import subprocess
import copy
+import puppetExecutor
logger = logging.getLogger()
installScriptHash = -1
class ActionQueue(threading.Thread):
+ """ Action Queue for the agent. We pick one command at a time from the queue
+ and execute that """
global commandQueue, resultQueue
commandQueue = Queue.Queue()
resultQueue = Queue.Queue()
@@ -50,7 +53,10 @@ class ActionQueue(threading.Thread):
self._stop = threading.Event()
self.maxRetries = config.getint('command', 'maxretries')
self.sleepInterval = config.getint('command', 'sleepBetweenRetries')
-
+ self.executor = puppetExecutor.puppetExecutor(config.get('puppet', 'puppetmodules'),
+ config.get('puppet', 'puppet_home'),
+ config.get('puppet', 'facter_home'),
+ config.get('agent', 'prefix'))
def stop(self):
self._stop.set()
@@ -112,13 +118,14 @@ class ActionQueue(threading.Thread):
serviceName = command['serviceName']
configurations = command['configurations']
result = []
+ commandresult = self.executor.runCommand(command)
# assume some puppet pluing to run these commands
roleResult = {'role' : command['role'],
'actionId' : commandId,
- 'stdout' : "DONE",
+ 'stdout' : commandresult['stdout'],
'clusterName' : clusterName,
- 'stderr' : "DONE",
- 'exitCode' : 0,
+ 'stderr' : commandresult['stderr'],
+ 'exitCode' : commandresult['exitcode'],
'serviceName' : serviceName,
'status' : "COMPLETED"}
result.append(roleResult)
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py Mon Oct 22 08:09:14 2012
@@ -37,6 +37,7 @@ prefix=/tmp/ambari
installprefix=/var/ambari/
[puppet]
+puppetmodules=/var/lib/ambari-agent/puppet/
puppet_home=/root/workspace/puppet-install/puppet-2.7.9
facter_home=/root/workspace/puppet-install/facter-1.6.10
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py Mon Oct 22 08:09:14 2012
@@ -24,21 +24,19 @@ import logging
logger = logging.getLogger()
- #read static imports from file and write them to manifest
-def writeImports(outputFile, inputFileName='imports.txt'):
+#read static imports from file and write them to manifest
+def writeImports(outputFile, modulesdir, inputFileName='imports.txt'):
inputFile = open(inputFileName, 'r')
- modulesdir = os.path.abspath(os.getcwd() + "../../../puppet/modules/")
logger.info("Modules dir is " + modulesdir)
for line in inputFile:
- modulename = line.rstrip('\n')
- line = "import '" + modulesdir + "/" + modulename + "'\n"
+ modulename = line.rstrip()
+ line = "import '" + modulesdir + os.sep + modulename + "'" + os.linesep
outputFile.write(line)
inputFile.close()
-def generateManifest(inputJsonStr):
+def generateManifest(parsedJson, fileName, modulesdir):
#reading json
- parsedJson = json.loads(inputJsonStr)
hostname = parsedJson['hostname']
clusterHostInfo = parsedJson['clusterHostInfo']
params = parsedJson['params']
@@ -46,11 +44,11 @@ def generateManifest(inputJsonStr):
#hostAttributes = parsedJson['hostAttributes']
roles = parsedJson['roleCommands']
-#writing manifest
- manifest = open('site.pp', 'w')
+ #writing manifest
+ manifest = open(fileName, 'w')
#writing imports from external static file
- writeImports(manifest)
+ writeImports(outputFile=manifest, modulesdir=modulesdir)
#writing nodes
writeNodes(manifest, clusterHostInfo)
@@ -94,9 +92,25 @@ def writeNodes(outputFile, clusterHostIn
outputFile.write(']\n')
#write params
-def writeParams(outputFile, params):
- for param in params.iterkeys():
- outputFile.write('$' + param + '="' + params[param] + '"\n')
+def writeParams(outputFile, params):
+
+ for paramName in params.iterkeys():
+
+ param = params[paramName]
+ if type(param) is dict:
+
+ outputFile.write('$' + paramName + '= {\n')
+
+ coma = ''
+
+ for subParam in param.iterkeys():
+ outputFile.write(coma + '"' + subParam + '" => "' + param[subParam] + '"')
+ coma = ',\n'
+
+ outputFile.write('\n}\n')
+ else:
+ outputFile.write('$' + paramName + '="' + param + '"\n')
+
#write host attributes
def writeHostAttributes(outputFile, hostAttributes):
@@ -177,8 +191,18 @@ def writeStages(outputFile, numStages):
outputFile.write('\n')
-logging.basicConfig(level=logging.DEBUG)
-#test code
-jsonFile = open('test.json', 'r')
-jsonStr = jsonFile.read()
-generateManifest(jsonStr)
+
+def main():
+ logging.basicConfig(level=logging.DEBUG)
+ #test code
+ jsonFile = open('test.json', 'r')
+ jsonStr = jsonFile.read()
+ modulesdir = os.path.abspath(os.getcwd() + ".." + os.sep + ".." +
+ os.sep + ".." + os.sep + "puppet" +
+ os.sep + "modules" + os.sep)
+ parsedJson = json.loads(inputJsonStr)
+ generateManifest(parsedJson, 'site.pp', modulesdir)
+
+if __name__ == '__main__':
+ main()
+
Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py?rev=1400792&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py (added)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py Mon Oct 22 08:09:14 2012
@@ -0,0 +1,114 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import json
+import os.path
+import logging
+import subprocess
+from manifestGenerator import generateManifest
+import pprint
+
+logger = logging.getLogger()
+
+class puppetExecutor:
+ """ Class that executes the commands that come from the server using puppet.
+ This is the class that provides the pluggable point for executing the puppet"""
+
+ def __init__(self, puppetModule, puppetInstall, facterInstall, tmpDir):
+ self.puppetModule = puppetModule
+ self.puppetInstall = puppetInstall
+ self.facterInstall = facterInstall
+ self.tmpDir = tmpDir
+
+ def getPuppetBinary(self):
+ return os.path.join(self.puppetInstall, "bin", "puppet")
+
+ def puppetCommand(self, sitepp):
+ modules = self.puppetModule
+ puppetcommand = [];
+ puppetcommand.append(self.getPuppetBinary())
+ puppetcommand.append("apply")
+ puppetcommand.append("--confdir=" + modules)
+ puppetcommand.append("--detailed-exitcodes")
+ puppetcommand.append(sitepp)
+ return puppetcommand
+
+ def facterLib(self):
+ return self.facterInstall + "/lib/"
+ pass
+
+ def puppetLib(self):
+ return self.puppetInstall + "/lib"
+ pass
+
+ def runCommand(self, command):
+ result = {}
+ taskId = "0";
+ if command.has_key("taskId"):
+ taskId = command['taskId']
+
+ puppetEnv = os.environ
+ siteppFileName = os.path.join(self.tmpDir, "site-" + taskId + ".pp")
+ generateManifest(command, siteppFileName, self.puppetModule + "/modules")
+ puppetcommand = self.puppetCommand(siteppFileName)
+ """ Run the command and make sure the output gets propagated"""
+ rubyLib = ""
+ if os.environ.has_key("RUBYLIB"):
+ rubyLib = os.environ["RUBYLIB"]
+ logger.info("Ruby Lib env from Env " + rubyLib)
+ rubyLib = rubyLib + ":" + self.facterLib() + ":" + self.puppetLib()
+ puppetEnv["RUBYLIB"] = rubyLib
+ logger.info("Setting RUBYLIB as: " + rubyLib)
+ logger.info("Running command " + pprint.pformat(puppetcommand))
+ puppet = subprocess.Popen(puppetcommand,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=puppetEnv)
+ stderr_out = puppet.communicate()
+ error = "none"
+ if puppet.returncode != 0:
+ error = stderr_out[1]
+ result["stderr"] = error
+ logging.error("Error running puppet: " + stderr_out[1])
+ pass
+ puppetOutput = stderr_out[0]
+ result["exitcode"] = puppet.returncode
+ result["stdout"] = puppetOutput
+ logger.info("ExitCode : \n" + str(result["exitcode"]))
+ return result
+
+def main():
+ logging.basicConfig(level=logging.DEBUG)
+ #test code
+ jsonFile = open('test.json', 'r')
+ jsonStr = jsonFile.read()
+ # Below is for testing only.
+
+ puppetInstance = puppetExecutor("/root/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/",
+ "/root/workspace/puppet-install/puppet-2.7.9",
+ "/root/workspace/puppet-install/facter-1.6.10/",
+ "/tmp")
+ jsonFile = open('test.json', 'r')
+ jsonStr = jsonFile.read()
+ parsedJson = json.loads(jsonStr)
+ puppetInstance.runCommand(parsedJson)
+
+if __name__ == '__main__':
+ main()
+
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp Mon Oct 22 08:09:14 2012
@@ -1,19 +1,53 @@
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-zookeeper/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-oozie/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-pig/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-sqoop/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-templeton/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hive/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hcat/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-mysql/manifests/*.pp'
+import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-monitor-webserver/manifests/*.pp'
$NAMENODE= ['h2.hortonworks.com']
$DATANODE= ['h1.hortonworks.com', 'h2.hortonworks.com']
$hdfs_user="hdfs"
-$jdk_location="lah/blah"
+$jdk_location="http://hdp1/downloads"
+$jdk_bins= {
+"32" => "jdk-6u31-linux-x64.bin",
+"64" => "jdk-6u31-linux-x64.bin"
+}
+$java32_home="/usr/jdk64/jdk1.6.0_31"
+$java64_home="/usr/jdk64/jdk1.6.0_31"
$configuration = {
+capacity_scheduler=> {
+"mapred.capacity-scheduler.queue.default.capacity" => "100",
+"mapred.capacity-scheduler.queue.default.supports-priorit" => "false"
+},
+core_site=> {
+"fs.default.name" => "hrt8n36.cc1.ygridcore.net"
+},
+hadoop_policy=> {
+"security.client.datanode.protocol.acl" => "*",
+"security.client.protocol.acl" => "*"
+},
+mapred_queue_acls=> {
+"mapred.queue.default.acl-submit-job" => "*",
+"mapred.queue.default.acl-administer-jobs" => "*"
+},
hdfs_site=> {
"dfs.block.size" => "256000000",
"dfs.replication" => "1"
},
-core_site=> {
-"fs.default.name" => "hrt8n36.cc1.ygridcore.net"
+mapred_site=> {
+"mapred.queue.names" => "hive,pig,default",
+"mapred.jobtracker.taskScheduler" => "org.apache.hadoop.mapred.CapacityTaskScheduler"
},
}
node /default/ {
- stage{0 :} -> stage{1 :}
-class {'hdp-hadoop::datanode': stage => 1, service_state => running}
-class {'hdp-hadoop::namenode': stage => 2, service_state => installed_and_configured}
+ stage{1 :} -> stage{2 :}
+class {'hdp-hadoop::namenode': stage => 1, service_state => running}
+class {'hdp-hadoop::datanode': stage => 2, service_state => installed_and_configured}
}
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json Mon Oct 22 08:09:14 2012
@@ -10,21 +10,33 @@
"params":
{
"hdfs_user" : "hdfs",
-"jdk_location": "lah/blah"
+"jdk_location" : "http://hdp1/downloads",
+
+"java32_home" : "/usr/jdk64/jdk1.6.0_31",
+"java64_home" : "/usr/jdk64/jdk1.6.0_31",
+"jdk_bins" : { "32" : "jdk-6u31-linux-x64.bin", "64" : "jdk-6u31-linux-x64.bin" }
},
"configurations" : {
"hdfs_site" : { "dfs.block.size" : "256000000", "dfs.replication" : "1" } ,
-"core_site" : { "fs.default.name" : "hrt8n36.cc1.ygridcore.net" }
+"core_site" : { "fs.default.name" : "hrt8n36.cc1.ygridcore.net" } ,
+"mapred_queue_acls" : {"mapred.queue.default.acl-submit-job" : "*",
+ "mapred.queue.default.acl-administer-jobs" : "*"},
+"hadoop_policy" : {"security.client.protocol.acl" : "*",
+ "security.client.datanode.protocol.acl" : "*"},
+"mapred_site" : {"mapred.jobtracker.taskScheduler" : "org.apache.hadoop.mapred.CapacityTaskScheduler",
+ "mapred.queue.names" : "hive,pig,default"},
+"capacity_scheduler" : {"mapred.capacity-scheduler.queue.default.capacity" : "100",
+ "mapred.capacity-scheduler.queue.default.supports-priorit" : "false"}
},
"roleCommands": [
{
-"role" : "DATANODE",
+"role" : "NAMENODE",
"cmd": "START",
"roleParams" : {
}
},
{
-"role": "NAMENODE",
+"role": "DATANODE",
"cmd": "INSTALL",
"roleParams" : {}
}
Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/test/python/TestPuppetExecutor.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/test/python/TestPuppetExecutor.py?rev=1400792&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/test/python/TestPuppetExecutor.py (added)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/test/python/TestPuppetExecutor.py Mon Oct 22 08:09:14 2012
@@ -0,0 +1,37 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from puppetExecutor import puppetExecutor
+from pprint import pformat
+import sys
+
+class TestPuppetExecutor(TestCase):
+ def test_build(self):
+ puppetexecutor = puppetExecutor("/tmp", "/x", "/y", "/z")
+ command = puppetexecutor.puppetCommand("site.pp")
+ self.assertEquals("/x/bin/puppet", command[0], "puppet binary wrong")
+ self.assertEquals("apply", command[1], "local apply called")
+ self.assertEquals("--confdir=/tmp", command[2],"conf dir tmp")
+ self.assertEquals("--detailed-exitcodes", command[3], "make sure output \
+ correct")
+
+
+
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/conf/ambari.properties
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/conf/ambari.properties?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/conf/ambari.properties (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/conf/ambari.properties Mon Oct 22 08:09:14 2012
@@ -0,0 +1 @@
+metadata.path=src/main/resources/stacks
\ No newline at end of file
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java Mon Oct 22 08:09:14 2012
@@ -17,18 +17,55 @@
*/
package org.apache.ambari.server.actionmanager;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.persist.Transactional;
+import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.Role;
import org.apache.ambari.server.agent.CommandReport;
import com.google.inject.Singleton;
+import org.apache.ambari.server.orm.dao.*;
+import org.apache.ambari.server.orm.entities.*;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Singleton
public class ActionDBAccessorImpl implements ActionDBAccessor {
+ private static final Logger log = LoggerFactory.getLogger(ActionDBAccessorImpl.class);
+
+ @Inject
+ private ClusterDAO clusterDAO;
+ @Inject
+ private HostDAO hostDAO;
+ @Inject
+ private StageDAO stageDAO;
+ @Inject
+ private HostRoleCommandDAO hostRoleCommandDAO;
+ @Inject
+ private ExecutionCommandDAO executionCommandDAO;
+ @Inject
+ private RoleSuccessCriteriaDAO roleSuccessCriteriaDAO;
+ @Inject
+ private StageFactory stageFactory;
+ @Inject
+ private Clusters clusters;
+
+ private final long requestId;
+
+ @Inject
+ public ActionDBAccessorImpl(Injector injector) {
+ injector.injectMembers(this);
+ requestId = stageDAO.getLastRequestId();
+
- public ActionDBAccessorImpl() {
- //this.stageId = greatest stage id in the database + 1
}
/* (non-Javadoc)
@@ -36,7 +73,7 @@ public class ActionDBAccessorImpl implem
*/
@Override
public Stage getAction(String actionId) {
- return null;
+ return stageFactory.createExisting(actionId);
}
/* (non-Javadoc)
@@ -44,7 +81,11 @@ public class ActionDBAccessorImpl implem
*/
@Override
public List<Stage> getAllStages(long requestId) {
- return null;
+ List<Stage> stages = new ArrayList<Stage>();
+ for (StageEntity stageEntity : stageDAO.findByRequestId(requestId)) {
+ stages.add(stageFactory.createExisting(stageEntity));
+ }
+ return stages;
}
/* (non-Javadoc)
@@ -52,15 +93,22 @@ public class ActionDBAccessorImpl implem
*/
@Override
public void abortOperation(long requestId) {
- //Mark all pending or queued actions for this request as aborted.
+ Collection<HostRoleStatus> sourceStatuses = Arrays.asList(HostRoleStatus.QUEUED, HostRoleStatus.IN_PROGRESS, HostRoleStatus.PENDING);
+ int result = hostRoleCommandDAO.updateStatusByRequestId(requestId, HostRoleStatus.ABORTED, sourceStatuses);
+ log.info("Aborted {} commands", result);
}
/* (non-Javadoc)
* @see org.apache.ambari.server.actionmanager.ActionDBAccessor#timeoutHostRole(long, long, org.apache.ambari.server.Role)
*/
@Override
+ @Transactional
public void timeoutHostRole(String host, long requestId, long stageId, Role role) {
- // TODO Auto-generated method stub
+ List<HostRoleCommandEntity> commands = hostRoleCommandDAO.findByHostRole(host, requestId, stageId, role);
+ for (HostRoleCommandEntity command : commands) {
+ command.setStatus(HostRoleStatus.TIMEDOUT);
+ hostRoleCommandDAO.merge(command);
+ }
}
/* (non-Javadoc)
@@ -68,40 +116,110 @@ public class ActionDBAccessorImpl implem
*/
@Override
public List<Stage> getStagesInProgress() {
- return null;
+ List<Stage> stages = new ArrayList<Stage>();
+ for (StageEntity stageEntity : stageDAO.findByCommandStatuses(Arrays.asList(HostRoleStatus.IN_PROGRESS))) {
+ stages.add(stageFactory.createExisting(stageEntity));
+ }
+ return stages;
}
/* (non-Javadoc)
* @see org.apache.ambari.server.actionmanager.ActionDBAccessor#persistActions(java.util.List)
*/
@Override
+ @Transactional
public void persistActions(List<Stage> stages) {
- // TODO Auto-generated method stub
-
+ for (Stage stage : stages) {
+ StageEntity stageEntity = stage.constructNewPersistenceEntity();
+ Cluster cluster;
+ try {
+ cluster = clusters.getCluster(stage.getClusterName());
+ } catch (AmbariException e) {
+ throw new RuntimeException(e);
+ }
+ ClusterEntity clusterEntity = clusterDAO.findById(cluster.getClusterId());
+
+ stageEntity.setCluster(clusterEntity);
+ clusterEntity.getStages().add(stageEntity);
+
+ for (HostRoleCommand hostRoleCommand : stage.getOrderedHostRoleCommands()) {
+ HostRoleCommandEntity hostRoleCommandEntity = hostRoleCommand.constructNewPersistenceEntity();
+ stageEntity.getHostRoleCommands().add(hostRoleCommandEntity);
+ hostRoleCommandEntity.setStage(stageEntity);
+
+ HostEntity hostEntity = hostDAO.findByName(hostRoleCommandEntity.getHostName());
+ if (hostEntity == null) {
+ log.error("Host {} doesn't exists in database", hostRoleCommandEntity.getHostName());
+ throw new RuntimeException("Host '"+hostRoleCommandEntity.getHostName()+"' doesn't exists in database");
+ }
+ hostEntity.getHostRoleCommandEntities().add(hostRoleCommandEntity);
+ hostRoleCommandEntity.setHost(hostEntity);
+ hostRoleCommandDAO.create(hostRoleCommandEntity);
+
+ assert hostRoleCommandEntity.getTaskId() != null;
+
+ hostRoleCommand.setTaskId(hostRoleCommandEntity.getTaskId());
+ ExecutionCommandEntity executionCommandEntity = hostRoleCommand.constructExecutionCommandEntity();
+ executionCommandEntity.setHostRoleCommand(hostRoleCommandEntity);
+ hostRoleCommandEntity.setExecutionCommand(executionCommandEntity);
+
+ executionCommandDAO.create(hostRoleCommandEntity.getExecutionCommand());
+ hostRoleCommandDAO.merge(hostRoleCommandEntity);
+ hostDAO.merge(hostEntity);
+ }
+
+ for (RoleSuccessCriteriaEntity roleSuccessCriteriaEntity : stageEntity.getRoleSuccessCriterias()) {
+ roleSuccessCriteriaDAO.create(roleSuccessCriteriaEntity);
+ }
+
+ stageDAO.create(stageEntity);
+ clusterDAO.merge(clusterEntity);
+ }
}
@Override
+ @Transactional
public void updateHostRoleState(String hostname, long requestId,
long stageId, String role, CommandReport report) {
- // TODO Auto-generated method stub
-
+ List<HostRoleCommandEntity> commands = hostRoleCommandDAO.findByHostRole(hostname, requestId, stageId, Role.valueOf(role));
+ for (HostRoleCommandEntity command : commands) {
+ command.setStatus(HostRoleStatus.valueOf(report.getStatus()));
+ command.setStdOut(report.getStdOut());
+ command.setStdError(report.getStdErr());
+ command.setExitcode(report.getExitCode());
+ hostRoleCommandDAO.merge(command);
+ }
}
@Override
public void abortHostRole(String host, long requestId, long stageId, Role role) {
- // TODO Auto-generated method stub
-
+ CommandReport report = new CommandReport();
+ report.setExitCode(999);
+ report.setStdErr("Host Role in invalid state");
+ report.setStdOut("");
+ report.setStatus("ABORTED");
+ updateHostRoleState(host, requestId, stageId, role.toString(), report);
}
@Override
public long getLastPersistedRequestIdWhenInitialized() {
- // TODO Auto-generated method stub
- return 0;
+ return requestId;
}
@Override
+ @Transactional
public void hostRoleScheduled(Stage s, String hostname, String roleStr) {
- //Update start time, last update time, host role status, number of attempts
- //in the database.
+ HostRoleCommand hostRoleCommand = s.getHostRoleCommand(hostname, roleStr);
+ HostRoleCommandEntity entity = hostRoleCommandDAO.findByPK(hostRoleCommand.getTaskId());
+ if (entity != null) {
+ entity.setStartTime(hostRoleCommand.getStartTime());
+ entity.setLastAttemptTime(hostRoleCommand.getLastAttemptTime());
+ entity.setStatus(hostRoleCommand.getStatus());
+ entity.setAttemptCount(hostRoleCommand.getAttemptCount());
+ hostRoleCommandDAO.merge(entity);
+ } else {
+ throw new RuntimeException("HostRoleCommand is not persisted, cannot update:\n" + hostRoleCommand);
+ }
+
}
}
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java Mon Oct 22 08:09:14 2012
@@ -17,8 +17,20 @@
*/
package org.apache.ambari.server.actionmanager;
+import com.google.inject.Injector;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
import org.apache.ambari.server.Role;
+import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.orm.entities.ExecutionCommandEntity;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
import org.apache.ambari.server.state.ServiceComponentHostEvent;
+import org.apache.ambari.server.utils.StageUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.JAXBException;
+import java.io.IOException;
/**
* This class encapsulates the information for an task on a host for a
@@ -27,7 +39,11 @@ import org.apache.ambari.server.state.Se
* track the request.
* For the actual command refer {@link HostAction#commandToHost}
*/
-class HostRoleCommand {
+public class HostRoleCommand {
+ private static final Logger log = LoggerFactory.getLogger(HostRoleCommand.class);
+
+ private int taskId = -1;
+ private String hostName;
private final Role role;
private HostRoleStatus status = HostRoleStatus.PENDING;
private String stdout = "";
@@ -38,12 +54,91 @@ class HostRoleCommand {
private long lastAttemptTime = -1;
private short attemptCount = 0;
+ private ExecutionCommand executionCommand;
+
public HostRoleCommand(String host, Role role,
- ServiceComponentHostEvent event) {
+ ServiceComponentHostEvent event) {
+ this.hostName = host;
this.role = role;
this.event = event;
}
+ @AssistedInject
+ public HostRoleCommand(@Assisted HostRoleCommandEntity hostRoleCommandEntity, Injector injector) {
+ taskId = hostRoleCommandEntity.getTaskId();
+ this.hostName = hostRoleCommandEntity.getHostName();
+ role = hostRoleCommandEntity.getRole();
+ status = hostRoleCommandEntity.getStatus();
+ stdout = hostRoleCommandEntity.getStdOut();
+ stderr = hostRoleCommandEntity.getStdError();
+ exitCode = hostRoleCommandEntity.getExitcode();
+ startTime = hostRoleCommandEntity.getStartTime();
+ lastAttemptTime = hostRoleCommandEntity.getLastAttemptTime();
+ attemptCount = hostRoleCommandEntity.getAttemptCount();
+
+ try {
+ log.info(hostRoleCommandEntity.getEvent());
+ event = StageUtils.fromJson(hostRoleCommandEntity.getEvent(), ServiceComponentHostEvent.class);
+ executionCommand = StageUtils.stringToExecutionCommand(hostRoleCommandEntity.getExecutionCommand().getCommand());
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to parse JSON string", e);
+ }
+
+ }
+
+ HostRoleCommandEntity constructNewPersistenceEntity() {
+ HostRoleCommandEntity hostRoleCommandEntity = new HostRoleCommandEntity();
+ hostRoleCommandEntity.setHostName(hostName);
+ hostRoleCommandEntity.setRole(role);
+ hostRoleCommandEntity.setStatus(status);
+ hostRoleCommandEntity.setStdError(stderr);
+ hostRoleCommandEntity.setExitcode(exitCode);
+ hostRoleCommandEntity.setStdOut(stdout);
+ hostRoleCommandEntity.setStartTime(startTime);
+ hostRoleCommandEntity.setLastAttemptTime(lastAttemptTime);
+ hostRoleCommandEntity.setAttemptCount(attemptCount);
+
+ try {
+ hostRoleCommandEntity.setEvent(StageUtils.jaxbToString(event));
+ ExecutionCommandEntity executionCommandEntity = new ExecutionCommandEntity();
+ executionCommandEntity.setCommand(StageUtils.jaxbToString(executionCommand));
+ executionCommandEntity.setHostRoleCommand(hostRoleCommandEntity);
+ hostRoleCommandEntity.setExecutionCommand(executionCommandEntity);
+ } catch (JAXBException e) {
+ throw new RuntimeException(e);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return hostRoleCommandEntity;
+ }
+
+ ExecutionCommandEntity constructExecutionCommandEntity(){
+ try {
+ ExecutionCommandEntity executionCommandEntity = new ExecutionCommandEntity();
+ executionCommandEntity.setCommand(StageUtils.jaxbToString(executionCommand));
+ return executionCommandEntity;
+ } catch (JAXBException e) {
+ throw new RuntimeException(e);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ public int getTaskId() {
+ return taskId;
+ }
+
+ public void setTaskId(int taskId) {
+ this.taskId = taskId;
+ executionCommand.setTaskId(taskId);
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
+
public Role getRole() {
return role;
}
@@ -107,12 +202,20 @@ class HostRoleCommand {
public void incrementAttemptCount() {
this.attemptCount++;
}
-
+
+ public ExecutionCommand getExecutionCommand() {
+ return executionCommand;
+ }
+
+ public void setExecutionCommand(ExecutionCommand executionCommand) {
+ this.executionCommand = executionCommand;
+ }
+
@Override
public int hashCode() {
return role.hashCode();
}
-
+
@Override
public boolean equals(Object other) {
if (!(other instanceof HostRoleCommand)) {
@@ -121,20 +224,20 @@ class HostRoleCommand {
HostRoleCommand o = (HostRoleCommand) other;
return this.role.equals(o.role);
}
-
+
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("HostRoleCommand State:\n");
- builder.append(" Role: "+role+"\n");
- builder.append(" Status: "+status+"\n");
- builder.append(" Event: "+event+"\n");
- builder.append(" stdout: "+stdout+"\n");
- builder.append(" stderr: "+stderr+"\n");
- builder.append(" exitcode: "+exitCode+"\n");
- builder.append(" Start time: " + startTime+"\n");
- builder.append(" Last attempt time: "+lastAttemptTime+"\n");
- builder.append(" attempt count: "+ attemptCount+"\n");
+ builder.append(" Role: " + role + "\n");
+ builder.append(" Status: " + status + "\n");
+ builder.append(" Event: " + event + "\n");
+ builder.append(" stdout: " + stdout + "\n");
+ builder.append(" stderr: " + stderr + "\n");
+ builder.append(" exitcode: " + exitCode + "\n");
+ builder.append(" Start time: " + startTime + "\n");
+ builder.append(" Last attempt time: " + lastAttemptTime + "\n");
+ builder.append(" attempt count: " + attemptCount + "\n");
return builder.toString();
}
}
Added: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactory.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactory.java?rev=1400792&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactory.java (added)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommandFactory.java Mon Oct 22 08:09:14 2012
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.actionmanager;
+
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
+
+public interface HostRoleCommandFactory {
+ HostRoleCommand createExisting(HostRoleCommandEntity hostRoleCommandEntity);
+}
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java Mon Oct 22 08:09:14 2012
@@ -23,9 +23,19 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import com.google.inject.Injector;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.orm.dao.HostDAO;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.StageDAO;
+import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
+import org.apache.ambari.server.orm.entities.RoleSuccessCriteriaEntity;
+import org.apache.ambari.server.orm.entities.StageEntity;
import org.apache.ambari.server.state.ServiceComponentHostEvent;
import org.apache.ambari.server.utils.StageUtils;
import org.mortbay.log.Log;
@@ -42,7 +52,7 @@ public class Stage {
private Map<Role, Float> successFactors = new HashMap<Role, Float>();
//Map of host to host-roles
- private Map<String, Map<String, HostRoleCommand>> hostRoleCommands =
+ Map<String, Map<String, HostRoleCommand>> hostRoleCommands =
new TreeMap<String, Map<String, HostRoleCommand>>();
private Map<String, List<ExecutionCommand>> commandsToSend =
new TreeMap<String, List<ExecutionCommand>>();
@@ -53,6 +63,79 @@ public class Stage {
this.clusterName = clusterName;
}
+ /**
+ * Creates Stage existing in database
+ * @param actionId "requestId-stageId" string
+ */
+ @AssistedInject
+ public Stage(@Assisted String actionId, Injector injector) {
+ this(injector.getInstance(StageDAO.class).findByActionId(actionId), injector);
+ }
+
+ @AssistedInject
+ public Stage(@Assisted StageEntity stageEntity, Injector injector) {
+ HostRoleCommandDAO hostRoleCommandDAO = injector.getInstance(HostRoleCommandDAO.class);
+ HostDAO hostDAO = injector.getInstance(HostDAO.class);
+ HostRoleCommandFactory hostRoleCommandFactory = injector.getInstance(HostRoleCommandFactory.class);
+
+ requestId = stageEntity.getRequestId();
+ stageId = stageEntity.getStageId();
+ logDir = stageEntity.getLogInfo();
+ clusterName = stageEntity.getCluster().getClusterName();
+
+ for (HostEntity hostEntity : hostDAO.findByStage(stageEntity)) {
+ List<HostRoleCommandEntity> commands = hostRoleCommandDAO.findSortedCommandsByStageAndHost(stageEntity, hostEntity);
+ commandsToSend.put(hostEntity.getHostName(), new ArrayList<ExecutionCommand>());
+ hostRoleCommands.put(hostEntity.getHostName(), new TreeMap<String, HostRoleCommand>());
+ for (HostRoleCommandEntity command : commands) {
+ HostRoleCommand hostRoleCommand = hostRoleCommandFactory.createExisting(command);
+ hostRoleCommands.get(hostEntity.getHostName()).put(hostRoleCommand.getRole().toString(), hostRoleCommand);
+ commandsToSend.get(hostEntity.getHostName()).add(hostRoleCommand.getExecutionCommand());
+ }
+ }
+
+ for (RoleSuccessCriteriaEntity successCriteriaEntity : stageEntity.getRoleSuccessCriterias()) {
+ successFactors.put(successCriteriaEntity.getRole(), successCriteriaEntity.getSuccessFactor().floatValue());
+ }
+ }
+
+ /**
+ * Creates object to be persisted in database
+ * @return StageEntity
+ */
+ public StageEntity constructNewPersistenceEntity() {
+ StageEntity stageEntity = new StageEntity();
+ stageEntity.setRequestId(requestId);
+ stageEntity.setStageId(stageId);
+ stageEntity.setLogInfo(logDir);
+ stageEntity.setHostRoleCommands(new ArrayList<HostRoleCommandEntity>());
+ stageEntity.setRoleSuccessCriterias(new ArrayList<RoleSuccessCriteriaEntity>());
+
+ for (Role role : successFactors.keySet()) {
+ RoleSuccessCriteriaEntity roleSuccessCriteriaEntity = new RoleSuccessCriteriaEntity();
+ roleSuccessCriteriaEntity.setRole(role);
+ roleSuccessCriteriaEntity.setStage(stageEntity);
+ roleSuccessCriteriaEntity.setSuccessFactor(successFactors.get(role).doubleValue());
+ stageEntity.getRoleSuccessCriterias().add(roleSuccessCriteriaEntity);
+ }
+ return stageEntity;
+ }
+
+ public List<HostRoleCommand> getOrderedHostRoleCommands() {
+ List<HostRoleCommand> commands = new ArrayList<HostRoleCommand>();
+ //TODO trick for proper storing order, check it
+ for (String hostName : hostRoleCommands.keySet()) {
+ for (ExecutionCommand executionCommand : commandsToSend.get(hostName)) {
+ for (HostRoleCommand hostRoleCommand : hostRoleCommands.get(hostName).values()) {
+ if (hostRoleCommand.getExecutionCommand() == executionCommand) {
+ commands.add(hostRoleCommand);
+ }
+ }
+ }
+ }
+ return commands;
+ }
+
public synchronized void setStageId(long stageId) {
if (this.stageId != -1) {
throw new RuntimeException("Attempt to set stageId again! Not allowed.");
@@ -85,6 +168,7 @@ public class Stage {
+", event: "+event+", clusterName: "+clusterName+", serviceName: "+serviceName);
HostRoleCommand hrc = new HostRoleCommand(host, role, event);
ExecutionCommand cmd = new ExecutionCommand();
+ hrc.setExecutionCommand(cmd);
cmd.setHostname(host);
cmd.setClusterName(clusterName);
cmd.setServiceName(serviceName);
@@ -261,11 +345,12 @@ public class Stage {
if (hostRoleCommands.get(hostname) == null) {
hostRoleCommands.put(hostname, new TreeMap<String, HostRoleCommand>());
}
+ //TODO add reference to ExecutionCommand into HostRoleCommand
hostRoleCommands.get(hostname).put(role,
origStage.getHostRoleCommand(hostname, role));
}
- private HostRoleCommand getHostRoleCommand(String hostname, String role) {
+ HostRoleCommand getHostRoleCommand(String hostname, String role) {
return hostRoleCommands.get(hostname).get(role);
}
Added: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java?rev=1400792&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java (added)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/StageFactory.java Mon Oct 22 08:09:14 2012
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.actionmanager;
+
+import org.apache.ambari.server.orm.entities.StageEntity;
+
+public interface StageFactory {
+
+ Stage createExisting(String actionId);
+
+ Stage createExisting(StageEntity stageEntity);
+}
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java Mon Oct 22 08:09:14 2012
@@ -41,6 +41,7 @@ public class ExecutionCommand extends Ag
super(AgentCommandType.EXECUTION_COMMAND);
}
private String clusterName;
+ private int taskId;
private String commandId;
private String hostname;
private Role role;
@@ -88,7 +89,17 @@ public class ExecutionCommand extends Ag
public int hashCode() {
return (hostname + commandId + role).hashCode();
}
-
+
+ @JsonProperty("taskId")
+ public int getTaskId() {
+ return taskId;
+ }
+
+ @JsonProperty("taskId")
+ public void setTaskId(int taskId) {
+ this.taskId = taskId;
+ }
+
@JsonProperty("role")
public Role getRole() {
return role;
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/rest/AgentResource.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/rest/AgentResource.java?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/rest/AgentResource.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/agent/rest/AgentResource.java Mon Oct 22 08:09:14 2012
@@ -26,7 +26,6 @@ import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Request;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.agent.HeartBeat;
@@ -37,6 +36,7 @@ import org.apache.ambari.server.agent.Re
import org.apache.ambari.server.state.fsm.InvalidStateTransitonException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+
import com.google.inject.Inject;
/**
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java Mon Oct 22 08:09:14 2012
@@ -18,6 +18,7 @@
package org.apache.ambari.server.api.services;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.ServiceInfo;
@@ -32,11 +33,15 @@ import java.util.Map;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.ParserConfigurationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
import org.w3c.dom.Node;
import org.w3c.dom.Element;
+import org.xml.sax.SAXException;
/**
* ServiceInfo responsible getting information about cluster.
@@ -44,7 +49,36 @@ import org.w3c.dom.Element;
@Path("/metainfo/")
public class AmbariMetaInfo {
- private String CONFIG_FILE_PATH = "C:\\workspace\\stacks";//Configuration.CONFIG_FILE
+ List<StackInfo> stacksResult = new ArrayList<StackInfo>();
+
+ private final static Logger log = LoggerFactory.getLogger(AmbariMetaInfo.class);
+
+ private static final String SERVICES_FOLDER_NAME = "services";
+ private static final String SERVICE_METAINFO_FILE_NAME = "metainfo.xml";
+ private static final String SERVICE_CONFIG_FOLDER_NAME = "configs";
+ private static final String SERVICE_CONFIG_FILE_NAME_POSTFIX = "-site.xml";
+
+ private static final String REPOSITORY_FILE_NAME = "repoinfo.xml";
+ private static final String REPOSITORY_FOLDER_NAME = "repos";
+ private static final String REPOSITORY_XML_MAIN_BLOCK_NAME = "repo";
+ private static final String REPOSITORY_XML_PROPERTY_URL = "url";
+ private static final String REPOSITORY_XML_PROPERTY_OS = "os";
+ private static final String REPOSITORY_XML_PROPERTY_DESCRIPTION = "description";
+
+ private static final String METAINFO_XML_MAIN_BLOCK_NAME = "metainfo";
+ private static final String METAINFO_XML_PROPERTY_VERSION = "version";
+ private static final String METAINFO_XML_PROPERTY_USER = "user";
+ private static final String METAINFO_XML_PROPERTY_COMMENT = "comment";
+
+ private static final String PROPERTY_XML_MAIN_BLOCK_NAME = "property";
+ private static final String PROPERTY_XML_PROPERTY_NAME = "name";
+ private static final String PROPERTY_XML_PROPERTY_VALUE = "value";
+ private static final String PROPERTY_XML_PROPERTY_DESCRIPTION = "description";
+
+
+ public AmbariMetaInfo() throws Exception {
+ getConfigurationInformation();
+ }
List<ServiceInfo> getSupportedServices(String stackName, String version) {
return null;
@@ -62,12 +96,13 @@ public class AmbariMetaInfo {
return null;
}
- private List<ServiceInfo> getConfigurationInformation() throws Exception {
- List<StackInfo> stacksResult = new ArrayList<StackInfo>();
+ private void getConfigurationInformation() throws Exception {
- File stackRoot = new File(CONFIG_FILE_PATH);
- if (!stackRoot.isDirectory())
- throw new IOException("" + CONFIG_FILE_PATH + " should be a directory with stack.");
+
+ File stackRoot = new File(new Configuration().getMetadataPath());
+// File stackRoot = new File("src/main/resources/stacks");
+ if (!stackRoot.isDirectory() && !stackRoot.exists())
+ throw new IOException("" + Configuration.METADETA_DIR_PATH + " should be a directory with stack.");
File[] stacks = stackRoot.listFiles();
for (File stackFolder : stacks) {
if (stackFolder.isFile()) continue;
@@ -79,7 +114,7 @@ public class AmbariMetaInfo {
stackInfo.setVersion(stack.getName());
stacksResult.add(stackInfo);
//get repository data for current stack of techs
- File repositoryFolder = new File(stack.getAbsolutePath() + File.separator + "repos" + File.separator + "repoinfo.xml");
+ File repositoryFolder = new File(stack.getAbsolutePath() + File.separator + REPOSITORY_FOLDER_NAME + File.separator + REPOSITORY_FILE_NAME);
if (repositoryFolder.exists()) {
List<RepositoryInfo> repositoryInfoList = getRepository(repositoryFolder);
@@ -87,8 +122,8 @@ public class AmbariMetaInfo {
}
- //get services for this stack
- File servicesRootFolder = new File(stack.getAbsolutePath() + File.separator + "services");
+ //Get services for this stack
+ File servicesRootFolder = new File(stack.getAbsolutePath() + File.separator + SERVICES_FOLDER_NAME);
File[] servicesFolders = servicesRootFolder.listFiles();
if (servicesFolders != null)
@@ -98,37 +133,43 @@ public class AmbariMetaInfo {
serviceInfo.setName(serviceFolder.getName());
stackInfo.getServices().add(serviceInfo);
+ //Get metainfo data from metainfo.xml
+ File metainfoFile = new File(serviceFolder.getAbsolutePath() + File.separator + SERVICE_METAINFO_FILE_NAME);
+ if (metainfoFile.exists()) {
+ setMetaInfo(metainfoFile, serviceInfo);
+
+ }
+
//Get all properties from all "configs/*-site.xml" files
- File serviceConfigFolder = new File(serviceFolder.getAbsolutePath() + File.separator + "configs");
+ File serviceConfigFolder = new File(serviceFolder.getAbsolutePath() + File.separator + SERVICE_CONFIG_FOLDER_NAME);
File[] configFiles = serviceConfigFolder.listFiles();
for (File config : configFiles) {
- if (config.getName().endsWith("-site.xml")) {
+ if (config.getName().endsWith(SERVICE_CONFIG_FILE_NAME_POSTFIX)) {
serviceInfo.getProperties().addAll(getProperties(config));
}
-
}
}
}
}//stack root
- for (StackInfo elem : stacksResult) {
- System.out.println("###elem = \n" + elem);
- System.out.println("contain services= " + elem.getServices().size());
- }
- System.out.println(" \n\n\n ");
+//////TODO delete before final commit. Show all objects structure for debug
+// for (StackInfo elem : stacksResult) {
+// log.info("###elem = \n" + elem);
+// log.info("contain services= " + elem.getServices().size());
+// System.out.println("###elem = \n" + elem);
+// System.out.println("contain services= " + elem.getServices().size());
+// }
+// System.out.println(" \n\n\n ");
- return null;
}
public static void main(String[] args) throws Exception {
AmbariMetaInfo metadata = new AmbariMetaInfo();
- metadata.getConfigurationInformation();
-// metadata.getRepository(new File("C:\\workspace\\stacks\\HDP\\0.1\\repos\\repoinfo.xml"));
-
+// System.out.println( new Configuration().getMetadataPath() );
}
@@ -142,7 +183,7 @@ public class AmbariMetaInfo {
Document doc = dBuilder.parse(repositoryFile);
doc.getDocumentElement().normalize();
- NodeList propertyNodes = doc.getElementsByTagName("repo");
+ NodeList propertyNodes = doc.getElementsByTagName(REPOSITORY_XML_MAIN_BLOCK_NAME);
for (int index = 0; index < propertyNodes.getLength(); index++) {
@@ -151,21 +192,58 @@ public class AmbariMetaInfo {
Element property = (Element) node;
RepositoryInfo repositoryInfo = new RepositoryInfo();
- repositoryInfo.setUrl(getTagValue("url", property));
- repositoryInfo.setOs(getTagValue("os", property));
- repositoryInfo.setDescription(getTagValue("description", property));
+ repositoryInfo.setUrl(getTagValue(REPOSITORY_XML_PROPERTY_URL, property));
+ repositoryInfo.setOs(getTagValue(REPOSITORY_XML_PROPERTY_OS, property));
+ repositoryInfo.setDescription(getTagValue(REPOSITORY_XML_PROPERTY_DESCRIPTION, property));
repositorysInfo.add(repositoryInfo);
}
}
} catch (Exception e) {
e.printStackTrace();
-
}
+
return repositorysInfo;
}
- public List<PropertyInfo> getProperties(File propertyFile) {
+
+ private void setMetaInfo(File metainfoFile, ServiceInfo serviceInfo) {
+
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+
+ Document doc = null;
+ DocumentBuilder dBuilder = null;
+ try {
+ dBuilder = dbFactory.newDocumentBuilder();
+ doc = dBuilder.parse(metainfoFile);
+ } catch (SAXException e) {
+ log.error("Error while parsing metainf.xml", e);
+ } catch (IOException e) {
+ log.error("Error while open metainf.xml", e);
+ } catch (ParserConfigurationException e) {
+ log.error("Error while parsing metainf.xml", e);
+ }
+
+ doc.getDocumentElement().normalize();
+
+ NodeList metaInfoNodes = doc.getElementsByTagName(METAINFO_XML_MAIN_BLOCK_NAME);
+
+ if (metaInfoNodes.getLength() > 0) {
+ Node metaInfoNode = metaInfoNodes.item(0);
+ if (metaInfoNode.getNodeType() == Node.ELEMENT_NODE) {
+
+ Element metaInfoElem = (Element) metaInfoNode;
+
+ serviceInfo.setVersion(getTagValue(METAINFO_XML_PROPERTY_VERSION, metaInfoElem));
+ serviceInfo.setUser(getTagValue(METAINFO_XML_PROPERTY_USER, metaInfoElem));
+ serviceInfo.setComment(getTagValue(METAINFO_XML_PROPERTY_COMMENT, metaInfoElem));
+ }
+ }
+
+ }
+
+
+ private List<PropertyInfo> getProperties(File propertyFile) {
List<PropertyInfo> resultPropertyList = new ArrayList<PropertyInfo>();
try {
@@ -175,7 +253,7 @@ public class AmbariMetaInfo {
doc.getDocumentElement().normalize();
- NodeList propertyNodes = doc.getElementsByTagName("property");
+ NodeList propertyNodes = doc.getElementsByTagName(PROPERTY_XML_MAIN_BLOCK_NAME);
for (int index = 0; index < propertyNodes.getLength(); index++) {
@@ -184,9 +262,9 @@ public class AmbariMetaInfo {
Element property = (Element) node;
PropertyInfo propertyInfo = new PropertyInfo();
- propertyInfo.setName(getTagValue("name", property));
- propertyInfo.setValue(getTagValue("value", property));
- propertyInfo.setDescription(getTagValue("description", property));
+ propertyInfo.setName(getTagValue(PROPERTY_XML_PROPERTY_NAME, property));
+ propertyInfo.setValue(getTagValue(PROPERTY_XML_PROPERTY_VALUE, property));
+ propertyInfo.setDescription(getTagValue(PROPERTY_XML_PROPERTY_DESCRIPTION, property));
if (propertyInfo.getName() == null || propertyInfo.getValue() == null)
continue;
@@ -201,17 +279,24 @@ public class AmbariMetaInfo {
return resultPropertyList;
}
- private static String getTagValue(String sTag, Element rawElement) {
+
+ private String getTagValue(String sTag, Element rawElement) {
String result = null;
try {
NodeList element = rawElement.getElementsByTagName(sTag).item(0).getChildNodes();
Node value = (Node) element.item(0);
result = value.getNodeValue();
+ } catch (NullPointerException e) {
+ log.debug("There is no field like " + sTag + "in this DOM element.", e);
+ } catch (Exception e) {
+ log.error("Error while getting value from xml DOM element", e);
+ throw e;
} finally {
return result;
}
}
+
}
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java Mon Oct 22 08:09:14 2012
@@ -59,6 +59,8 @@ public class Configuration {
"security.server.passphrase_env_var";
public static final String PASSPHRASE_KEY = "security.server.passphrase";
public static final String RESOURCES_DIR_KEY = "resources.dir";
+ public static final String METADETA_DIR_PATH = "metadata.path";
+
public static final String CLIENT_SECURITY_KEY = "client.security";
public static final String LDAP_USE_SSL_KEY = "authorization.ldap.useSSL";
@@ -245,6 +247,15 @@ public class Configuration {
properties.setProperty(CLIENT_SECURITY_KEY, type.toString());
}
+ /**
+ * Gets ambari stack-path
+ * @return String
+ */
+ public String getMetadataPath() {
+ return properties.getProperty(METADETA_DIR_PATH);
+// return "src/main/resources/stacks";
+ }
+
public PersistenceType getPersistenceType() {
String value = properties.getProperty(PERSISTENCE_IN_MEMORY_KEY, PERSISTENCE_IN_MEMORY_DEFAULT);
if ("true".equalsIgnoreCase(value)) {
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java Mon Oct 22 08:09:14 2012
@@ -21,8 +21,7 @@ import com.google.gson.Gson;
import com.google.inject.Scopes;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.persist.jpa.JpaPersistModule;
-import org.apache.ambari.server.actionmanager.ActionDBAccessor;
-import org.apache.ambari.server.actionmanager.ActionDBInMemoryImpl;
+import org.apache.ambari.server.actionmanager.*;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.state.*;
import org.apache.ambari.server.state.cluster.ClusterFactory;
@@ -65,10 +64,12 @@ public class ControllerModule extends Ab
install(new FactoryModuleBuilder().implement(ServiceComponent.class, ServiceComponentImpl.class).build(ServiceComponentFactory.class));
install(new FactoryModuleBuilder().implement(ServiceComponentHost.class, ServiceComponentHostImpl.class).build(ServiceComponentHostFactory.class));
install(new FactoryModuleBuilder().implement(Config.class, ConfigImpl.class).build(ConfigFactory.class));
+ install(new FactoryModuleBuilder().build(StageFactory.class));
+ install(new FactoryModuleBuilder().build(HostRoleCommandFactory.class));
bind(Gson.class).in(Scopes.SINGLETON);
bind(Clusters.class).to(ClustersImpl.class);
- bind(ActionDBAccessor.class).to(ActionDBInMemoryImpl.class);
+ bind(ActionDBAccessor.class).to(ActionDBAccessorImpl.class);
bindConstant().annotatedWith(Names.named("schedulerSleeptime")).to(10000L);
bindConstant().annotatedWith(Names.named("actionTimeout")).to(60000L);
bind(AmbariManagementController.class)
Added: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java?rev=1400792&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java (added)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java Mon Oct 22 08:09:14 2012
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.orm.dao;
+
+import com.google.inject.Singleton;
+
+import javax.persistence.NoResultException;
+import javax.persistence.Query;
+import javax.persistence.TypedQuery;
+import java.util.Collections;
+import java.util.List;
+
+@Singleton
+public class DaoUtils {
+
+ public <T> List<T> selectList(TypedQuery<T> query, Object... parameters) {
+ setParameters(query, parameters);
+ try {
+ return query.getResultList();
+ } catch (NoResultException ignored) {
+ return Collections.emptyList();
+ }
+ }
+
+ public <T> T selectSingle(TypedQuery<T> query, Object... parameters) {
+ setParameters(query, parameters);
+ try {
+ return query.getSingleResult();
+ } catch (NoResultException ignored) {
+ return null;
+ }
+ }
+
+ public int executeUpdate(Query query, Object... parameters) {
+ setParameters(query, parameters);
+ return query.executeUpdate();
+ }
+
+ public void setParameters(Query query, Object... parameters) {
+ for (int i = 0; i < parameters.length; i++) {
+ query.setParameter(i+1, parameters[i]);
+ }
+ }
+}
Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java?rev=1400792&r1=1400791&r2=1400792&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostDAO.java Mon Oct 22 08:09:14 2012
@@ -21,12 +21,13 @@ package org.apache.ambari.server.orm.dao
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.persist.Transactional;
-import org.apache.ambari.server.orm.entities.ClusterEntity;
import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.StageEntity;
import javax.persistence.EntityManager;
import javax.persistence.NoResultException;
import javax.persistence.TypedQuery;
+import java.util.Collections;
import java.util.List;
public class HostDAO {
@@ -40,7 +41,23 @@ public class HostDAO {
public List<HostEntity> findAll() {
TypedQuery<HostEntity> query = entityManagerProvider.get().createQuery("SELECT host FROM HostEntity host", HostEntity.class);
- return query.getResultList();
+ try {
+ return query.getResultList();
+ } catch (NoResultException e) {
+ return Collections.emptyList();
+ }
+ }
+
+ public List<HostEntity> findByStage(StageEntity stageEntity) {
+ TypedQuery<HostEntity> query = entityManagerProvider.get().createQuery(
+ "SELECT DISTINCT host FROM HostEntity host JOIN host.hostRoleCommandEntities command JOIN command.stage stage " +
+ "WHERE stage=:stageEntity", HostEntity.class);
+ query.setParameter("stageEntity", stageEntity);
+ try {
+ return query.getResultList();
+ } catch (NoResultException e) {
+ return Collections.emptyList();
+ }
}
/**