You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2014/05/14 16:02:10 UTC
[02/15] AMBARI-5747 Remove facter-1.6.10, Ruby,
Puppet dependencies from pom.xml and src (dsen)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
index bc675dc..c9d3200 100644
--- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
+++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
@@ -27,7 +27,6 @@ import os
from LiveStatus import LiveStatus
from shell import shellRunner
-import PuppetExecutor
from ActualConfigHandler import ActualConfigHandler
from CommandStatusDict import CommandStatusDict
from CustomServiceOrchestrator import CustomServiceOrchestrator
@@ -62,9 +61,6 @@ class ActionQueue(threading.Thread):
COMPLETED_STATUS = 'COMPLETED'
FAILED_STATUS = 'FAILED'
- COMMAND_FORMAT_V1 = "1.0"
- COMMAND_FORMAT_V2 = "2.0"
-
def __init__(self, config, controller):
super(ActionQueue, self).__init__()
self.commandQueue = Queue.Queue()
@@ -140,33 +136,17 @@ class ActionQueue(threading.Thread):
traceback.print_exc()
logger.warn(err)
-
- def determine_command_format_version(self, command):
- """
- Returns either COMMAND_FORMAT_V1 or COMMAND_FORMAT_V2
- """
- try:
- if command['commandParams']['schema_version'] == self.COMMAND_FORMAT_V2:
- return self.COMMAND_FORMAT_V2
- else:
- return self.COMMAND_FORMAT_V1
- except KeyError:
- pass # ignore
- return self.COMMAND_FORMAT_V1 # Fallback
-
-
def execute_command(self, command):
'''
Executes commands of type EXECUTION_COMMAND
'''
clusterName = command['clusterName']
commandId = command['commandId']
- command_format = self.determine_command_format_version(command)
message = "Executing command with id = {commandId} for role = {role} of " \
- "cluster {cluster}. Command format={command_format}".format(
+ "cluster {cluster}.".format(
commandId = str(commandId), role=command['role'],
- cluster=clusterName, command_format=command_format)
+ cluster=clusterName)
logger.info(message)
logger.debug(pprint.pformat(command))
@@ -181,24 +161,13 @@ class ActionQueue(threading.Thread):
})
self.commandStatuses.put_command_status(command, in_progress_status)
# running command
- if command_format == self.COMMAND_FORMAT_V1:
- # Create a new instance of executor for the current thread
- puppetExecutor = PuppetExecutor.PuppetExecutor(
- self.config.get('puppet', 'puppetmodules'),
- self.config.get('puppet', 'puppet_home'),
- self.config.get('puppet', 'facter_home'),
- self.config.get('agent', 'prefix'), self.config)
- commandresult = puppetExecutor.runCommand(command, in_progress_status['tmpout'],
- in_progress_status['tmperr'])
- else:
- commandresult = self.customServiceOrchestrator.runCommand(command,
- in_progress_status['tmpout'], in_progress_status['tmperr'])
+ commandresult = self.customServiceOrchestrator.runCommand(command,
+ in_progress_status['tmpout'], in_progress_status['tmperr'])
# dumping results
status = self.COMPLETED_STATUS
if commandresult['exitcode'] != 0:
status = self.FAILED_STATUS
roleResult = self.commandStatuses.generate_report_template(command)
- # assume some puppet plumbing to run these commands
roleResult.update({
'stdout': commandresult['stdout'],
'stderr': commandresult['stderr'],
@@ -252,25 +221,22 @@ class ActionQueue(threading.Thread):
else:
globalConfig = {}
- command_format = self.determine_command_format_version(command)
-
livestatus = LiveStatus(cluster, service, component,
globalConfig, self.config, self.configTags)
- component_status = None
component_extra = None
- if command_format == self.COMMAND_FORMAT_V2:
- # For custom services, responsibility to determine service status is
- # delegated to python scripts
- component_status_result = self.customServiceOrchestrator.requestComponentStatus(command)
-
- if component_status_result['exitcode'] == 0:
- component_status = LiveStatus.LIVE_STATUS
- else:
- component_status = LiveStatus.DEAD_STATUS
-
- if component_status_result.has_key('structuredOut'):
- component_extra = component_status_result['structuredOut']
+
+ # For custom services, responsibility to determine service status is
+ # delegated to python scripts
+ component_status_result = self.customServiceOrchestrator.requestComponentStatus(command)
+
+ if component_status_result['exitcode'] == 0:
+ component_status = LiveStatus.LIVE_STATUS
+ else:
+ component_status = LiveStatus.DEAD_STATUS
+
+ if component_status_result.has_key('structuredOut'):
+ component_extra = component_status_result['structuredOut']
result = livestatus.build(forsed_component_status= component_status)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
index 79c7345..4f0baa9 100644
--- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
+++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
@@ -38,12 +38,6 @@ cache_dir=/var/lib/ambari-agent/cache
[services]
-[puppet]
-puppetmodules=/var/lib/ambari-agent/puppet/
-puppet_home=/root/workspace/puppet-install/puppet-2.7.9
-facter_home=/root/workspace/puppet-install/facter-1.6.10
-timeout_seconds = 600
-
[python]
custom_actions_dir = /var/lib/ambari-agent/resources/custom_actions
@@ -59,7 +53,7 @@ passphrase_env_var_name=AMBARI_PASSPHRASE
[heartbeat]
state_interval = 6
dirs=/etc/hadoop,/etc/hadoop/conf,/var/run/hadoop,/var/log/hadoop
-rpms=glusterfs,openssl,wget,net-snmp,ntpd,ruby,ganglia,nagios,glusterfs
+rpms=glusterfs,openssl,wget,net-snmp,ntpd,ganglia,nagios,glusterfs
log_lines_count=300
"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py b/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
index e491e09..f00ada2 100644
--- a/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
+++ b/ambari-agent/src/main/python/ambari_agent/CommandStatusDict.py
@@ -112,7 +112,7 @@ class CommandStatusDict():
output = grep.tail(tmpout, Grep.OUTPUT_LAST_LINES)
inprogress = self.generate_report_template(command)
inprogress.update({
- 'stdout': grep.filterMarkup(output),
+ 'stdout': output,
'stderr': tmperr,
'structuredOut': tmpstructuredout,
'exitCode': 777,
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index c68a953..e96deb5 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -26,10 +26,7 @@ import sys
from FileCache import FileCache
from AgentException import AgentException
from PythonExecutor import PythonExecutor
-from AmbariConfig import AmbariConfig
import hostname
-from LiveStatus import LiveStatus
-import manifestGenerator
logger = logging.getLogger()
@@ -48,6 +45,9 @@ class CustomServiceOrchestrator():
PRE_HOOK_PREFIX="before"
POST_HOOK_PREFIX="after"
+ HOSTS_LIST_KEY = "all_hosts"
+ PING_PORTS_KEY = "all_ping_ports"
+ AMBARI_SERVER_HOST = "ambari_server_host"
def __init__(self, config, controller):
self.config = config
@@ -197,7 +197,6 @@ class CustomServiceOrchestrator():
Converts command to json file and returns file path
"""
# Perform few modifications to stay compatible with the way in which
- # site.pp files are generated by manifestGenerator.py
public_fqdn = self.public_fqdn
command['public_hostname'] = public_fqdn
# Now, dump the json file
@@ -209,7 +208,7 @@ class CustomServiceOrchestrator():
file_path = os.path.join(self.tmp_dir, "status_command.json")
else:
task_id = command['taskId']
- command['clusterHostInfo'] = manifestGenerator.decompressClusterHostInfo(command['clusterHostInfo'])
+ command['clusterHostInfo'] = self.decompressClusterHostInfo(command['clusterHostInfo'])
file_path = os.path.join(self.tmp_dir, "command-{0}.json".format(task_id))
# Json may contain passwords, that's why we need proper permissions
if os.path.isfile(file_path):
@@ -219,3 +218,97 @@ class CustomServiceOrchestrator():
content = json.dumps(command, sort_keys = False, indent = 4)
f.write(content)
return file_path
+
+ def decompressClusterHostInfo(self, clusterHostInfo):
+ info = clusterHostInfo.copy()
+ #Pop info not related to host roles
+ hostsList = info.pop(self.HOSTS_LIST_KEY)
+ pingPorts = info.pop(self.PING_PORTS_KEY)
+ ambariServerHost = info.pop(self.AMBARI_SERVER_HOST)
+
+ decompressedMap = {}
+
+ for k,v in info.items():
+ # Convert from 1-3,5,6-8 to [1,2,3,5,6,7,8]
+ indexes = self.convertRangeToList(v)
+ # Convert from [1,2,3,5,6,7,8] to [host1,host2,host3...]
+ decompressedMap[k] = [hostsList[i] for i in indexes]
+
+ #Convert from ['1:0-2,4', '42:3,5-7'] to [1,1,1,42,1,42,42,42]
+ pingPorts = self.convertMappedRangeToList(pingPorts)
+
+ #Convert all elements to str
+ pingPorts = map(str, pingPorts)
+
+ #Add ping ports to result
+ decompressedMap[self.PING_PORTS_KEY] = pingPorts
+ #Add hosts list to result
+ decompressedMap[self.HOSTS_LIST_KEY] = hostsList
+ #Add ambari-server host to result
+ decompressedMap[self.AMBARI_SERVER_HOST] = ambariServerHost
+
+ return decompressedMap
+
+ # Converts from 1-3,5,6-8 to [1,2,3,5,6,7,8]
+ def convertRangeToList(self, list):
+
+ resultList = []
+
+ for i in list:
+
+ ranges = i.split(',')
+
+ for r in ranges:
+ rangeBounds = r.split('-')
+ if len(rangeBounds) == 2:
+
+ if not rangeBounds[0] or not rangeBounds[1]:
+ raise AgentException.AgentException("Broken data in given range, expected - ""m-n"" or ""m"", got : " + str(r))
+
+
+ resultList.extend(range(int(rangeBounds[0]), int(rangeBounds[1]) + 1))
+ elif len(rangeBounds) == 1:
+ resultList.append((int(rangeBounds[0])))
+ else:
+ raise AgentException.AgentException("Broken data in given range, expected - ""m-n"" or ""m"", got : " + str(r))
+
+ return resultList
+
+ #Converts from ['1:0-2,4', '42:3,5-7'] to [1,1,1,42,1,42,42,42]
+ def convertMappedRangeToList(self, list):
+
+ resultDict = {}
+
+ for i in list:
+ valueToRanges = i.split(":")
+ if len(valueToRanges) <> 2:
+ raise AgentException.AgentException("Broken data in given value to range, expected format - ""value:m-n"", got - " + str(i))
+ value = valueToRanges[0]
+ rangesToken = valueToRanges[1]
+
+ for r in rangesToken.split(','):
+
+ rangeIndexes = r.split('-')
+
+ if len(rangeIndexes) == 2:
+
+ if not rangeIndexes[0] or not rangeIndexes[1]:
+ raise AgentException.AgentException("Broken data in given value to range, expected format - ""value:m-n"", got - " + str(r))
+
+ start = int(rangeIndexes[0])
+ end = int(rangeIndexes[1])
+
+ for k in range(start, end + 1):
+ resultDict[k] = int(value)
+
+
+ elif len(rangeIndexes) == 1:
+ index = int(rangeIndexes[0])
+
+ resultDict[index] = int(value)
+
+
+ resultList = dict(sorted(resultDict.items())).values()
+
+ return resultList
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/main/python/ambari_agent/Grep.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Grep.py b/ambari-agent/src/main/python/ambari_agent/Grep.py
index fcd7b1f..1aaf40d 100644
--- a/ambari-agent/src/main/python/ambari_agent/Grep.py
+++ b/ambari-agent/src/main/python/ambari_agent/Grep.py
@@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import re
-
class Grep:
# How many lines from command output send to server
@@ -76,14 +74,3 @@ class Grep:
length = len(lines)
tailed = lines[length - n:]
return "".join(tailed)
-
- def filterMarkup(self, string):
- """
- Filters given string from puppet colour markup done using escape codes like [0;36m
- """
- if string is None:
- result = None
- else:
- regexp = "\x1b" + r"\[[\d;]{1,4}m"
- result = re.sub(regexp, '', string)
- return result
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py b/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py
deleted file mode 100644
index 7164a9f..0000000
--- a/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py
+++ /dev/null
@@ -1,282 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-import json
-import os.path
-import logging
-import subprocess
-import pprint
-import threading
-from threading import Thread
-
-from shell import shellRunner
-import manifestGenerator
-from RepoInstaller import RepoInstaller
-from Grep import Grep
-import shell
-
-JAVANOTVALID_MSG = "Cannot access JDK! Make sure you have permission to execute {0}/bin/java"
-
-logger = logging.getLogger()
-
-class PuppetExecutor:
-
- """ Class that executes the commands that come from the server using puppet.
- This is the class that provides the pluggable point for executing the puppet"""
-
- grep = Grep()
- NO_ERROR = "none"
-
- def __init__(self, puppetModule, puppetInstall, facterInstall, tmpDir, config):
- self.puppetModule = puppetModule
- self.puppetInstall = puppetInstall
- self.facterInstall = facterInstall
- self.tmpDir = tmpDir
- self.reposInstalled = False
- self.config = config
- self.modulesdir = self.puppetModule + "/modules"
- self.event = threading.Event()
- self.last_puppet_has_been_killed = False
- self.sh = shellRunner()
- self.puppet_timeout = config.get("puppet", "timeout_seconds")
-
- def configureEnviron(self, environ):
- if not self.config.has_option("puppet", "ruby_home"):
- return environ
- ruby_home = self.config.get("puppet", "ruby_home")
- if os.path.exists(ruby_home):
- """Only update ruby home if the config is configured"""
- path = os.environ["PATH"]
- if not ruby_home in path:
- environ["PATH"] = ruby_home + os.path.sep + "bin" + ":"+environ["PATH"]
- environ["MY_RUBY_HOME"] = ruby_home
- return environ
-
- def getPuppetBinary(self):
- puppetbin = os.path.join(self.puppetInstall, "bin", "puppet")
- if os.path.exists(puppetbin):
- return puppetbin
- else:
- logger.info("Using default puppet on the host : " + puppetbin
- + " does not exist.")
- return "puppet"
-
- def discardInstalledRepos(self):
- """
- Makes agent to forget about installed repos.
- So the next call of generate_repo_manifests() will definitely
- install repos again
- """
- self.reposInstalled = False
-
- def generate_repo_manifests(self, command, tmpDir, modulesdir, taskId):
- # Hack to only create the repo files once
- manifest_list = []
- if not self.reposInstalled:
- repoInstaller = RepoInstaller(command, tmpDir, modulesdir, taskId, self.config)
- manifest_list = repoInstaller.generate_repo_manifests()
- return manifest_list
-
- def puppetCommand(self, sitepp):
- modules = self.puppetModule
- puppetcommand = [self.getPuppetBinary(), "apply", "--confdir=" + modules, "--detailed-exitcodes", sitepp]
- return puppetcommand
-
- def facterLib(self):
- return self.facterInstall + "/lib/"
- pass
-
- def puppetLib(self):
- return self.puppetInstall + "/lib"
- pass
-
- def condenseOutput(self, stdout, stderr, retcode):
- grep = self.grep
- if stderr == self.NO_ERROR:
- result = grep.tail(stdout, grep.OUTPUT_LAST_LINES)
- else:
- result = grep.grep(stdout, "fail", grep.ERROR_LAST_LINES_BEFORE, grep.ERROR_LAST_LINES_AFTER)
- result = grep.cleanByTemplate(result, "warning")
- if result is None: # Second try
- result = grep.grep(stdout, "err", grep.ERROR_LAST_LINES_BEFORE, grep.ERROR_LAST_LINES_AFTER)
- result = grep.cleanByTemplate(result, "warning")
- filteredresult = grep.filterMarkup(result)
- return filteredresult
-
- def isSuccessfull(self, returncode):
- return not self.last_puppet_has_been_killed and (returncode == 0 or returncode == 2)
-
- def run_manifest(self, command, file, tmpoutfile, tmperrfile):
- result = {}
- taskId = 0
- timeout = command ['commandParams']['command_timeout']
- if command.has_key("taskId"):
- taskId = command['taskId']
- puppetEnv = os.environ
- #Install repos
- repo_manifest_list = self.generate_repo_manifests(command, self.tmpDir,
- self.modulesdir, taskId)
- puppetFiles = list(repo_manifest_list)
- puppetFiles.append(file)
- #Run all puppet commands, from manifest generator and for repos installation
- #Appending outputs and errors, exitcode - maximal from all
- for puppetFile in puppetFiles:
- self.runPuppetFile(puppetFile, result, puppetEnv, tmpoutfile,
- tmperrfile, timeout)
- # Check if one of the puppet command fails and error out
- if not self.isSuccessfull(result["exitcode"]):
- break
-
- if self.isSuccessfull(result["exitcode"]):
- # Check if all the repos were installed or not and reset the flag
- self.reposInstalled = True
-
- logger.info("ExitCode : " + str(result["exitcode"]))
- return result
-
- def isJavaAvailable(self, command):
- javaExecutablePath = "{0}/bin/java".format(command)
- return not self.sh.run([javaExecutablePath, '-version'])['exitCode']
-
- def runCommand(self, command, tmpoutfile, tmperrfile):
- # After installing we must have jdk available for start/stop/smoke
- if command['roleCommand'] != "INSTALL":
- java64_home = None
- if 'hostLevelParams' in command and 'java_home' in command['hostLevelParams']:
- java64_home = str(command['hostLevelParams']['java_home']).strip()
- if java64_home is None or not self.isJavaAvailable(java64_home):
- if java64_home is None:
- errMsg = "Cannot access JDK! Make sure java_home is specified in hostLevelParams"
- else:
- errMsg = JAVANOTVALID_MSG.format(java64_home)
- return {'stdout': '', 'stderr': errMsg, 'exitcode': 1}
- pass
- pass
-
- taskId = 0
- if command.has_key("taskId"):
- taskId = command['taskId']
- siteppFileName = os.path.join(self.tmpDir, "site-" + str(taskId) + ".pp")
- errMsg = manifestGenerator.generateManifest(command, siteppFileName,
- self.modulesdir, self.config)
- if not errMsg:
- result = self.run_manifest(command, siteppFileName, tmpoutfile, tmperrfile)
- else:
- result = {'stdout': '', 'stderr': errMsg, 'exitcode': 1}
- return result
-
- def runPuppetFile(self, puppetFile, result, puppetEnv, tmpoutfile,
- tmperrfile, timeout):
- """ Run the command and make sure the output gets propagated"""
- puppetcommand = self.puppetCommand(puppetFile)
- rubyLib = ""
- if os.environ.has_key("RUBYLIB"):
- rubyLib = os.environ["RUBYLIB"]
- logger.debug("RUBYLIB from Env " + rubyLib)
- if not (self.facterLib() in rubyLib):
- rubyLib = rubyLib + ":" + self.facterLib()
- if not (self.puppetLib() in rubyLib):
- rubyLib = rubyLib + ":" + self.puppetLib()
- tmpout = open(tmpoutfile, 'w')
- tmperr = open(tmperrfile, 'w')
- puppetEnv["RUBYLIB"] = rubyLib
- puppetEnv = self.configureEnviron(puppetEnv)
- logger.debug("Setting RUBYLIB as: " + rubyLib)
- logger.info("Running command " + pprint.pformat(puppetcommand))
- puppet = self.lauch_puppet_subprocess(puppetcommand, tmpout, tmperr, puppetEnv)
- logger.info("Command started with PID: " + str(puppet.pid))
- logger.debug("Launching watchdog thread")
- self.event.clear()
- self.last_puppet_has_been_killed = False
- thread = Thread(target = self.puppet_watchdog_func,
- args = (puppet, timeout))
- thread.start()
- # Waiting for process to finished or killed
- puppet.communicate()
- self.event.set()
- thread.join()
- # Building results
- error = self.NO_ERROR
- returncode = 0
- if not self.isSuccessfull(puppet.returncode):
- returncode = puppet.returncode
- error = open(tmperrfile, 'r').read()
- logging.error("Error running puppet: \n" + str(error))
- pass
- if self.last_puppet_has_been_killed:
- error = str(error) + "\n Puppet has been killed due to timeout"
- returncode = 999
- if result.has_key("stderr"):
- result["stderr"] = result["stderr"] + os.linesep + str(error)
- else:
- result["stderr"] = str(error)
- puppetOutput = open(tmpoutfile, 'r').read()
- logger.debug("Output from puppet :\n" + puppetOutput)
- logger.info("Puppet execution process with pid %s exited with code %s." %
- (str(puppet.pid), str(returncode)))
- if result.has_key("exitcode"):
- result["exitcode"] = max(returncode, result["exitcode"])
- else:
- result["exitcode"] = returncode
- condensed = self.condenseOutput(puppetOutput, error, returncode)
- if result.has_key("stdout"):
- result["stdout"] = result["stdout"] + os.linesep + str(condensed)
- else:
- result["stdout"] = str(condensed)
- return result
-
- def lauch_puppet_subprocess(self, puppetcommand, tmpout, tmperr, puppetEnv):
- """
- Creates subprocess with given parameters. This functionality was moved to separate method
- to make possible unit testing
- """
- return subprocess.Popen(puppetcommand,
- stdout=tmpout,
- stderr=tmperr,
- env=puppetEnv)
-
- def puppet_watchdog_func(self, puppet, puppet_timeout):
- self.event.wait(float(puppet_timeout))
- if puppet.returncode is None:
- logger.error("Task timed out, killing process with PID: " + str(puppet.pid))
- shell.kill_process_with_children(puppet.pid)
- self.last_puppet_has_been_killed = True
- pass
-
-
-def main():
- logging.basicConfig(level=logging.DEBUG)
- #test code
- jsonFile = open('test.json', 'r')
- jsonStr = jsonFile.read()
- # Below is for testing only.
-
- puppetInstance = PuppetExecutor("/home/centos/ambari_repo_info/ambari-agent/src/main/puppet/",
- "/usr/",
- "/root/workspace/puppet-install/facter-1.6.10/",
- "/tmp")
- jsonFile = open('test.json', 'r')
- jsonStr = jsonFile.read()
- parsedJson = json.loads(jsonStr)
- result = puppetInstance.runCommand(parsedJson, '/tmp/out.txt', '/tmp/err.txt')
- logger.debug(result)
-
-if __name__ == '__main__':
- main()
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py b/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py
deleted file mode 100644
index 78e7dc2..0000000
--- a/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import logging
-import os
-import json
-from pprint import pformat
-import ast
-from shell import shellRunner
-from manifestGenerator import writeImports
-import AmbariConfig
-
-
-PUPPET_EXT=".pp"
-
-logger = logging.getLogger()
-
-class RepoInstaller:
- def __init__(self, parsedJson, path, modulesdir, taskId, config):
- self.parsedJson = parsedJson
- self.path = path
- self.modulesdir = modulesdir
- self.taskId = taskId
- self.sh = shellRunner()
- self.config = config
-
- def prepareReposInfo(self):
- params = {}
- self.repoInfoList = []
- if self.parsedJson.has_key('hostLevelParams'):
- params = self.parsedJson['hostLevelParams']
- if params.has_key('repo_info'):
- self.repoInfoList = params['repo_info']
- logger.info("Repo List Info " + pformat(self.repoInfoList))
- if (isinstance(self.repoInfoList, basestring)):
- if (self.repoInfoList is not None and (len(self.repoInfoList) > 0)):
- self.repoInfoList = ast.literal_eval(self.repoInfoList)
- else:
- self.repoInfoList = []
-
- def generateFiles(self):
- repoPuppetFiles = []
- for repo in self.repoInfoList:
- repoFile = open(self.path + os.sep + repo['repoId'] + '-' +
- str(self.taskId) + PUPPET_EXT, 'w+')
-
- writeImports(repoFile, self.modulesdir, AmbariConfig.imports)
-
- baseUrl = ''
- mirrorList = ''
-
- if repo.has_key('baseUrl'):
- baseUrl = repo['baseUrl']
- baseUrl = baseUrl.decode('unicode-escape').encode('utf-8')
- # Hack to take care of $ signs in the repo url
- baseUrl = baseUrl.replace('$', '\$')
-
- if repo.has_key('mirrorsList'):
- mirrorList = repo['mirrorsList']
- mirrorList = mirrorList.decode('unicode-escape').encode('utf-8')
- # Hack to take care of $ signs in the repo url
- mirrorList = mirrorList.replace('$', '\$')
-
- repoFile.write('node /default/ {')
- repoFile.write('class{ "hdp-repos::process_repo" : ' + ' os_type => "' + repo['osType'] +
- '", repo_id => "' + repo['repoId'] + '", base_url => "' + baseUrl +
- '", mirror_list => "' + mirrorList +'", repo_name => "' + repo['repoName'] + '" }' )
- repoFile.write('}')
- repoFile.close()
- repoPuppetFiles.append(repoFile.name)
-
- return repoPuppetFiles
-
- def generate_repo_manifests(self):
- self.prepareReposInfo()
- repoPuppetFiles = self.generateFiles()
- return repoPuppetFiles
-
-def main():
- #Test code
- logging.basicConfig(level=logging.DEBUG)
- #test code
- jsonFile = open('test.json', 'r')
- jsonStr = jsonFile.read()
- parsedJson = json.loads(jsonStr)
- repoInstaller = RepoInstaller(parsedJson, '/tmp', '/home/centos/ambari_ws/ambari-agent/src/main/puppet/modules',0)
- repoInstaller.generate_repo_manifests()
-
-if __name__ == '__main__':
- main()
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
deleted file mode 100644
index 66ce224..0000000
--- a/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
+++ /dev/null
@@ -1,396 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import json
-import os.path
-import logging
-from datetime import datetime
-import pprint
-import AmbariConfig
-import hostname
-from ambari_agent import AgentException
-
-HOSTS_LIST_KEY = "all_hosts"
-PING_PORTS_KEY = "all_ping_ports"
-AMBARI_SERVER_HOST = "ambari_server_host"
-
-logger = logging.getLogger()
-
-non_global_configuration_types = ["hdfs-site", "core-site",
- "mapred-queue-acls",
- "hadoop-policy", "mapred-site",
- "capacity-scheduler", "hbase-site",
- "hbase-policy", "hive-site", "oozie-site",
- "webhcat-site", "hdfs-exclude-file", "hue-site",
- "yarn-site"]
-
-# Converts from 1-3,5,6-8 to [1,2,3,5,6,7,8]
-def convertRangeToList(list):
-
- resultList = []
-
- for i in list:
-
- ranges = i.split(',')
-
- for r in ranges:
- rangeBounds = r.split('-')
- if len(rangeBounds) == 2:
-
- if not rangeBounds[0] or not rangeBounds[1]:
- raise AgentException.AgentException("Broken data in given range, expected - ""m-n"" or ""m"", got : " + str(r))
-
-
- resultList.extend(range(int(rangeBounds[0]), int(rangeBounds[1]) + 1))
- elif len(rangeBounds) == 1:
- resultList.append((int(rangeBounds[0])))
- else:
- raise AgentException.AgentException("Broken data in given range, expected - ""m-n"" or ""m"", got : " + str(r))
-
- return resultList
-
-#Converts from ['1:0-2,4', '42:3,5-7'] to [1,1,1,42,1,42,42,42]
-def convertMappedRangeToList(list):
-
- resultDict = {}
-
- for i in list:
- valueToRanges = i.split(":")
- if len(valueToRanges) <> 2:
- raise AgentException.AgentException("Broken data in given value to range, expected format - ""value:m-n"", got - " + str(i))
- value = valueToRanges[0]
- rangesToken = valueToRanges[1]
-
- for r in rangesToken.split(','):
-
- rangeIndexes = r.split('-')
-
- if len(rangeIndexes) == 2:
-
- if not rangeIndexes[0] or not rangeIndexes[1]:
- raise AgentException.AgentException("Broken data in given value to range, expected format - ""value:m-n"", got - " + str(r))
-
- start = int(rangeIndexes[0])
- end = int(rangeIndexes[1])
-
- for k in range(start, end + 1):
- resultDict[k] = int(value)
-
-
- elif len(rangeIndexes) == 1:
- index = int(rangeIndexes[0])
-
- resultDict[index] = int(value)
-
-
- resultList = dict(sorted(resultDict.items())).values()
-
- return resultList
-
-def decompressClusterHostInfo(clusterHostInfo):
- info = clusterHostInfo.copy()
- #Pop info not related to host roles
- hostsList = info.pop(HOSTS_LIST_KEY)
- pingPorts = info.pop(PING_PORTS_KEY)
- ambariServerHost = info.pop(AMBARI_SERVER_HOST)
-
- decompressedMap = {}
-
- for k,v in info.items():
- # Convert from 1-3,5,6-8 to [1,2,3,5,6,7,8]
- indexes = convertRangeToList(v)
- # Convert from [1,2,3,5,6,7,8] to [host1,host2,host3...]
- decompressedMap[k] = [hostsList[i] for i in indexes]
-
- #Convert from ['1:0-2,4', '42:3,5-7'] to [1,1,1,42,1,42,42,42]
- pingPorts = convertMappedRangeToList(pingPorts)
-
- #Convert all elements to str
- pingPorts = map(str, pingPorts)
-
- #Add ping ports to result
- decompressedMap[PING_PORTS_KEY] = pingPorts
- #Add hosts list to result
- decompressedMap[HOSTS_LIST_KEY] = hostsList
- #Add ambari-server host to result
- decompressedMap[AMBARI_SERVER_HOST] = ambariServerHost
-
- return decompressedMap
-
-
-#read static imports from file and write them to manifest
-def writeImports(outputFile, modulesdir, importsList):
- logger.info("Modules dir is " + modulesdir)
- outputFile.write('#' + datetime.now().strftime('%d.%m.%Y %H:%M:%S') + os.linesep)
- for line in importsList:
- modulename = line.rstrip()
- line = "import '" + modulesdir + os.sep + modulename + "'" + os.linesep
- outputFile.write(line)
-
-
-def generateManifest(parsedJson, fileName, modulesdir, ambariconfig):
- logger.debug("JSON Received:")
- logger.debug(json.dumps(parsedJson, sort_keys=True, indent=4))
-#reading json
- hostname = parsedJson['hostname']
- clusterHostInfo = {}
- if 'clusterHostInfo' in parsedJson:
- if parsedJson['clusterHostInfo']:
- clusterHostInfo = decompressClusterHostInfo(parsedJson['clusterHostInfo'])
- params = {}
- if 'hostLevelParams' in parsedJson:
- if parsedJson['hostLevelParams']:
- params = parsedJson['hostLevelParams']
- configurations = {}
- if 'configurations' in parsedJson:
- if parsedJson['configurations']:
- configurations = parsedJson['configurations']
- nonGlobalConfigurationsKeys = non_global_configuration_types
- #hostAttributes = parsedJson['hostAttributes']
- roleParams = {}
- if 'roleParams' in parsedJson:
- if parsedJson['roleParams']:
- roleParams = parsedJson['roleParams']
- roles = [{'role' : parsedJson['role'],
- 'cmd' : parsedJson['roleCommand'],
- 'roleParams' : roleParams}]
- errMsg = ''
- try:
- #writing manifest
- manifest = open(fileName, 'w')
- #Change mode to make site.pp files readable to owner and group only
- os.chmod(fileName, 0660)
-
- #Check for Ambari Config and make sure you pick the right imports file
-
- #writing imports from external static file
- writeImports(outputFile=manifest, modulesdir=modulesdir, importsList=AmbariConfig.imports)
-
- #writing hostname
- writeHostnames(manifest)
-
- #writing nodes
- writeNodes(manifest, clusterHostInfo)
-
- #writing params from map
- writeParams(manifest, params, modulesdir)
-
- nonGlobalConfigurations = {}
- flatConfigurations = {}
-
- if configurations:
- for configKey in configurations.iterkeys():
- if configKey in nonGlobalConfigurationsKeys:
- nonGlobalConfigurations[configKey] = configurations[configKey]
- else:
- flatConfigurations[configKey] = configurations[configKey]
-
- #writing config maps
- if (nonGlobalConfigurations):
- writeNonGlobalConfigurations(manifest, nonGlobalConfigurations)
- if (flatConfigurations):
- writeFlatConfigurations(manifest, flatConfigurations)
-
- #writing host attributes
- #writeHostAttributes(manifest, hostAttributes)
-
- #writing task definitions
- writeTasks(manifest, roles, ambariconfig, clusterHostInfo, hostname)
-
-
- except TypeError:
- errMsg = 'Manifest can\'t be generated from the JSON \n' + \
- json.dumps(parsedJson, sort_keys=True, indent=4)
- logger.error(errMsg)
- finally:
- manifest.close()
-
- return errMsg
-
-def writeHostnames(outputFile):
- fqdn = hostname.hostname()
- public_fqdn = hostname.public_hostname()
- outputFile.write('$myhostname' + " = '" + fqdn + "'" + os.linesep)
- outputFile.write('$public_hostname' + " = '" + public_fqdn + "'" + os.linesep)
-
- #write nodes
-def writeNodes(outputFile, clusterHostInfo):
- if clusterHostInfo.has_key('zookeeper_hosts'):
- clusterHostInfo['zookeeper_hosts'] = sorted(clusterHostInfo['zookeeper_hosts'])
-
- for node in clusterHostInfo.iterkeys():
- outputFile.write('$' + node + '= [')
- coma = ''
-
- for value in clusterHostInfo[node]:
- outputFile.write(coma + '\'' + value + '\'')
- coma = ', '
-
- outputFile.write(']\n')
-
-#write params
-def writeParams(outputFile, params, modulesdir):
-
- for paramName in params.iterkeys():
- if paramName == 'repo_info':
- continue
-
-
- param = params[paramName]
- if type(param) is dict:
-
- outputFile.write('$' + paramName + '= {\n')
-
- coma = ''
-
- for subParam in param.iterkeys():
- outputFile.write(coma + '"' + subParam + '" => "' + param[subParam] + '"')
- coma = ',\n'
-
- outputFile.write('\n}\n')
- else:
- outputFile.write('$' + paramName + '="' + param + '"\n')
-
-
-#write host attributes
-def writeHostAttributes(outputFile, hostAttributes):
- outputFile.write('$hostAttributes={\n')
-
- coma = ''
- for attribute in hostAttributes.iterkeys():
- outputFile.write(coma + '"' + attribute + '" => "{' + hostAttributes[attribute] + '"}')
- coma = ',\n'
-
- outputFile.write('}\n')
-
-#write flat configurations
-def writeFlatConfigurations(outputFile, flatConfigs):
- flatDict = {}
- logger.debug("Generating global configurations =>\n" + pprint.pformat(flatConfigs))
- for flatConfigName in flatConfigs.iterkeys():
- for flatConfig in flatConfigs[flatConfigName].iterkeys():
- flatDict[flatConfig] = flatConfigs[flatConfigName][flatConfig]
- for gconfigKey in flatDict.iterkeys():
- outputFile.write('$' + gconfigKey + " = '" + escape(flatDict[gconfigKey]) + "'" + os.linesep)
-
-#write xml configurations
-def writeNonGlobalConfigurations(outputFile, xmlConfigs):
- outputFile.write('$configuration = {\n')
-
- for configName in xmlConfigs.iterkeys():
- config = xmlConfigs[configName]
- logger.debug("Generating " + configName + ", configurations =>\n" + pprint.pformat(config))
- outputFile.write(configName + '=> {\n')
- coma = ''
- for configParam in config.iterkeys():
- outputFile.write(coma + '"' + configParam + '" => \'' + escape(config[configParam]) + '\'')
- coma = ',\n'
-
- outputFile.write('\n},\n')
-
- outputFile.write('\n}\n')
-
-#write node tasks
-def writeTasks(outputFile, roles, ambariconfig, clusterHostInfo=None,
- hostname="localhost"):
- #reading dictionaries
- rolesToClass = AmbariConfig.rolesToClass
-
- serviceStates = AmbariConfig.serviceStates
-
- outputFile.write('node /default/ {\n ')
-
- writeStages(outputFile, len(roles) + 1)
- stageNum = 1
-
- outputFile.write('class {\'hdp\': stage => ' + str(stageNum) + '}\n')
- stageNum = stageNum + 1
- # Need to hack for zookeeper since we need
- zk_hosts = []
- for role in roles :
- rolename = role['role']
- command = role['cmd']
- taskParams = role['roleParams']
- if (rolename == 'ZOOKEEPER_SERVER'):
- zk_hosts = clusterHostInfo['zookeeper_hosts']
- # Sort the list in lexicographical order
- taskParams['myid'] = str(sorted(zk_hosts).index(hostname) + 1)
-
- taskParamsNormalized = normalizeTaskParams(taskParams)
- taskParamsPostfix = ''
-
- if len(taskParamsNormalized) > 0 :
- taskParamsPostfix = ', ' + taskParamsNormalized
-
- className = rolesToClass[rolename]
-
- if command in serviceStates:
- serviceState = serviceStates[command]
- outputFile.write('class {\'' + className + '\':' +
- ' stage => ' + str(stageNum) +
- ', service_state => ' + serviceState
- + taskParamsPostfix + '}\n')
- else:
- outputFile.write('class {\'' + className + '\':' +
- ' stage => ' + str(stageNum) +
- taskParamsPostfix + '}\n')
-
- stageNum = stageNum + 1
- outputFile.write('}\n')
-
-def normalizeTaskParams(taskParams):
- result = ''
- coma = ''
-
- for paramName in taskParams.iterkeys():
- result = coma + result + paramName + ' => ' + taskParams[paramName]
- coma = ','
-
- return result
-
-def writeStages(outputFile, numStages):
- arrow = ''
-
- for i in range(numStages):
- outputFile.write(arrow + 'stage{' + str(i + 1) + ' :}')
- arrow = ' -> '
-
- outputFile.write('\n')
-
-#Escape special characters
-def escape(param):
- return param.replace('\\', '\\\\').replace('\'', '\\\'')
-
-def main():
- logging.basicConfig(level=logging.DEBUG)
- #test code
- jsonFile = open('test.json', 'r')
- jsonStr = jsonFile.read()
- modulesdir = os.path.abspath(os.getcwd() + ".." + os.sep + ".." +
- os.sep + ".." + os.sep + "puppet" +
- os.sep + "modules" + os.sep)
- inputJsonStr = jsonStr
- parsedJson = json.loads(inputJsonStr)
- generateManifest(parsedJson, 'site.pp', modulesdir)
-
-
-if __name__ == '__main__':
- main()
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/main/python/ambari_agent/site.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/site.pp b/ambari-agent/src/main/python/ambari_agent/site.pp
deleted file mode 100644
index a5badea..0000000
--- a/ambari-agent/src/main/python/ambari_agent/site.pp
+++ /dev/null
@@ -1,92 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-zookeeper/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-oozie/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-pig/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-sqoop/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-templeton/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hive/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hcat/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-mysql/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-monitor-webserver/manifests/*.pp'
-import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-repos/manifests/*.pp'
-$NAMENODE= ['h2.hortonworks.com']
-$DATANODE= ['h1.hortonworks.com', 'h2.hortonworks.com']
-$jdk_location="http://hdp1/downloads"
-$jdk_bins= {
-"32" => "jdk-6u31-linux-x64.bin",
-"64" => "jdk-6u31-linux-x64.bin"
-}
-$hdfs_user="hdfs"
-$java32_home="/usr/jdk64/jdk1.6.0_31"
-$java64_home="/usr/jdk64/jdk1.6.0_31"
-$configuration = {
-capacity-scheduler=> {
-"mapred.capacity-scheduler.queue.default.capacity" => "100",
-"mapred.capacity-scheduler.queue.default.supports-priorit" => "false"
-},
-oozie-site=> {
-"oozie.service.ActionService.executor.ext.classes" => "org.apache.oozie.action.hadoop.HiveActionExecutor, org.apache.oozie.action.hadoop.SqoopActionExecutor,org.apache.oozie.action.email.EmailActionExecutor,"
-},
-mapred-site=> {
-"mapred.queue.names" => "hive,pig,default",
-"mapred.jobtracker.taskScheduler" => "org.apache.hadoop.mapred.CapacityTaskScheduler"
-},
-core-site=> {
-"fs.default.name" => "hrt8n36.cc1.ygridcore.net"
-},
-hbase-policy=> {
-"security.client.protocol.acl" => "*"
-},
-hbase-site=> {
-"hbase.cluster.distributed" => "true"
-},
-hdfs-site=> {
-"dfs.block.size" => "256000000",
-"dfs.replication" => "1"
-},
-hadoop-policy=> {
-"security.client.datanode.protocol.acl" => "*",
-"security.client.protocol.acl" => "*"
-},
-mapred-queue-acls=> {
-"mapred.queue.default.acl-submit-job" => "*",
-"mapred.queue.default.acl-administer-jobs" => "*"
-},
-templeton-site=> {
-"templeton.override.enabled" => "true"
-},
-hive-site=> {
-"hive.exec.scratchdir" => "/tmp"
-},
-
-}
-$security_enabled = "true"
-$task_bin_exe = "ls"
-$hadoop_piddirprefix = "/tmp"
-$ganglia_server_host = "localhost"
-node /default/ {
- stage{1 :} -> stage{2 :}
-class {'hdp': stage => 1}
-class {'hdp-hadoop::namenode': stage => 2, service_state => installed_and_configured}
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
index b6b79b1..d020ad9 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
@@ -21,7 +21,6 @@ from Queue import Queue
from unittest import TestCase
from ambari_agent.LiveStatus import LiveStatus
-from ambari_agent.PuppetExecutor import PuppetExecutor
from ambari_agent.ActionQueue import ActionQueue
from ambari_agent.AmbariConfig import AmbariConfig
import os, errno, time, pprint, tempfile, threading
@@ -243,23 +242,23 @@ class TestActionQueue(TestCase):
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
unfreeze_flag = threading.Event()
- puppet_execution_result_dict = {
+ python_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut' : ''
}
def side_effect(command, tmpoutfile, tmperrfile):
unfreeze_flag.wait()
- return puppet_execution_result_dict
+ return python_execution_result_dict
def patched_aq_execute_command(command):
# We have to perform patching for separate thread in the same thread
- with patch.object(PuppetExecutor, "runCommand") as runCommand_mock:
+ with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
### Test install/start/stop command ###
## Test successful execution with configuration tags
- puppet_execution_result_dict['status'] = 'COMPLETE'
- puppet_execution_result_dict['exitcode'] = 0
+ python_execution_result_dict['status'] = 'COMPLETE'
+ python_execution_result_dict['exitcode'] = 0
# We call method in a separate thread
execution_thread = Thread(target = patched_aq_execute_command ,
args = (self.datanode_install_command, ))
@@ -315,8 +314,8 @@ class TestActionQueue(TestCase):
self.assertEqual(len(report['reports']), 0)
## Test failed execution
- puppet_execution_result_dict['status'] = 'FAILED'
- puppet_execution_result_dict['exitcode'] = 13
+ python_execution_result_dict['status'] = 'FAILED'
+ python_execution_result_dict['exitcode'] = 13
# We call method in a separate thread
execution_thread = Thread(target = patched_aq_execute_command ,
args = (self.datanode_install_command, ))
@@ -349,8 +348,8 @@ class TestActionQueue(TestCase):
self.assertEqual(len(report['reports']), 0)
### Test upgrade command ###
- puppet_execution_result_dict['status'] = 'COMPLETE'
- puppet_execution_result_dict['exitcode'] = 0
+ python_execution_result_dict['status'] = 'COMPLETE'
+ python_execution_result_dict['exitcode'] = 0
execution_thread = Thread(target = patched_aq_execute_command ,
args = (self.datanode_upgrade_command, ))
execution_thread.start()
@@ -384,12 +383,9 @@ class TestActionQueue(TestCase):
@patch.object(CustomServiceOrchestrator, "runCommand")
@patch("CommandStatusDict.CommandStatusDict")
@patch.object(ActionQueue, "status_update_callback")
- @patch.object(ActionQueue, "determine_command_format_version")
- def test_store_configuration_tags(self, determine_command_format_version_mock,
- status_update_callback_mock,
+ def test_store_configuration_tags(self, status_update_callback_mock,
command_status_dict_mock,
cso_runCommand_mock):
- determine_command_format_version_mock.return_value = 2
custom_service_orchestrator_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
@@ -426,7 +422,6 @@ class TestActionQueue(TestCase):
self.assertEqual(expected, report['reports'][0])
@patch.object(ActionQueue, "status_update_callback")
- @patch.object(ActionQueue, "determine_command_format_version")
@patch.object(StackVersionsFileHandler, "read_stack_version")
@patch.object(CustomServiceOrchestrator, "requestComponentStatus")
@patch.object(ActionQueue, "execute_command")
@@ -435,93 +430,18 @@ class TestActionQueue(TestCase):
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
build_mock, execute_command_mock,
requestComponentStatus_mock, read_stack_version_mock,
- determine_command_format_version_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
build_mock.return_value = "dummy report"
- # Check execution ov V1 status command
- determine_command_format_version_mock.return_value = ActionQueue.COMMAND_FORMAT_V1
- actionQueue.execute_status_command(self.status_command)
- report = actionQueue.result()
- expected = 'dummy report'
- self.assertEqual(len(report['componentStatus']), 1)
- self.assertEqual(report['componentStatus'][0], expected)
- self.assertFalse(requestComponentStatus_mock.called)
- # Check execution ov V2 status command
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {'exitcode': 0}
- determine_command_format_version_mock.return_value = ActionQueue.COMMAND_FORMAT_V2
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
self.assertTrue(requestComponentStatus_mock.called)
-
-
- @patch.object(CustomServiceOrchestrator, "__init__")
- def test_determine_command_format_version(self,
- CustomServiceOrchestrator_mock):
- CustomServiceOrchestrator_mock.return_value = None
- v1_command = {
- 'commandParams': {
- 'schema_version': '1.0'
- }
- }
- v2_command = {
- 'commandParams': {
- 'schema_version': '2.0'
- }
- }
- current_command = {
- # Absent 'commandParams' section
- }
- dummy_controller = MagicMock()
- actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
- self.assertEqual(actionQueue.determine_command_format_version(v1_command),
- ActionQueue.COMMAND_FORMAT_V1)
- self.assertEqual(actionQueue.determine_command_format_version(v2_command),
- ActionQueue.COMMAND_FORMAT_V2)
- self.assertEqual(actionQueue.determine_command_format_version(current_command),
- ActionQueue.COMMAND_FORMAT_V1)
-
-
- @patch.object(ActionQueue, "determine_command_format_version")
- @patch("__builtin__.open")
- @patch.object(PuppetExecutor, "runCommand")
- @patch.object(CustomServiceOrchestrator, "runCommand")
- @patch.object(ActionQueue, "status_update_callback")
- @patch.object(CustomServiceOrchestrator, "__init__")
- def test_command_execution_depending_on_command_format(self,
- CustomServiceOrchestrator_mock,
- status_update_callback_mock,
- custom_ex_runCommand_mock,
- puppet_runCommand_mock, open_mock,
- determine_command_format_version_mock):
- CustomServiceOrchestrator_mock.return_value = None
- dummy_controller = MagicMock()
- actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
- ret = {
- 'stdout' : '',
- 'stderr' : '',
- 'exitcode': 1,
- }
- puppet_runCommand_mock.return_value = ret
- determine_command_format_version_mock.return_value = \
- ActionQueue.COMMAND_FORMAT_V1
- actionQueue.execute_command(self.datanode_install_command)
- self.assertTrue(puppet_runCommand_mock.called)
- self.assertFalse(custom_ex_runCommand_mock.called)
-
- puppet_runCommand_mock.reset_mock()
-
- custom_ex_runCommand_mock.return_value = ret
- determine_command_format_version_mock.return_value = \
- ActionQueue.COMMAND_FORMAT_V2
- actionQueue.execute_command(self.datanode_install_command)
- self.assertFalse(puppet_runCommand_mock.called)
- self.assertTrue(custom_ex_runCommand_mock.called)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
index d972928..ed74232 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
@@ -37,7 +37,6 @@ import sys
from AgentException import AgentException
from FileCache import FileCache
from LiveStatus import LiveStatus
-import manifestGenerator
class TestCustomServiceOrchestrator(TestCase):
@@ -67,7 +66,7 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertTrue(dummy_controller.registration_listeners.append.called)
- @patch.object(manifestGenerator, 'decompressClusterHostInfo')
+ @patch.object(CustomServiceOrchestrator, 'decompressClusterHostInfo')
@patch("hostname.public_hostname")
@patch("os.path.isfile")
@patch("os.unlink")
@@ -217,7 +216,7 @@ class TestCustomServiceOrchestrator(TestCase):
run_file_mock.reset_mock()
# unknown script type case
- command['commandParams']['script_type'] = "PUPPET"
+ command['commandParams']['script_type'] = "SOME_TYPE"
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
self.assertEqual(ret['exitcode'], 1)
self.assertFalse(run_file_mock.called)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/test/python/ambari_agent/TestGrep.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestGrep.py b/ambari-agent/src/test/python/ambari_agent/TestGrep.py
index f379960..206e6e8 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestGrep.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestGrep.py
@@ -95,13 +95,6 @@ debug: Processing report from ambari-dmi with processor Puppet::Reports::Store
desired = ''
self.assertEquals(fragment, desired, 'Grep tail function contains bug in index arithmetics')
- def test_filterMarkup(self):
- string = """[0;36mnotice: /Stage[main]/Hdp-hadoop/Hdp-hadoop::Package[hadoop]/Hdp::Package[hadoop 64]/Hdp::Package::Process_pkg[hadoop 64]/Package[hadoop-libhdfs]/ensure: created[0m"""
- desired="""notice: /Stage[main]/Hdp-hadoop/Hdp-hadoop::Package[hadoop]/Hdp::Package[hadoop 64]/Hdp::Package::Process_pkg[hadoop 64]/Package[hadoop-libhdfs]/ensure: created"""
- filtered = self.grep.filterMarkup(string)
- #sys.stderr.write(filtered)
- self.assertEquals(filtered, desired)
-
def tearDown(self):
pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/test/python/ambari_agent/TestManifestGenerator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestManifestGenerator.py b/ambari-agent/src/test/python/ambari_agent/TestManifestGenerator.py
deleted file mode 100644
index b3163e2..0000000
--- a/ambari-agent/src/test/python/ambari_agent/TestManifestGenerator.py
+++ /dev/null
@@ -1,230 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-import os, sys, StringIO
-from ambari_agent.AgentException import AgentException
-
-from unittest import TestCase
-from ambari_agent import manifestGenerator
-import ambari_agent.AmbariConfig
-import tempfile
-import json
-import shutil
-from ambari_agent.AmbariConfig import AmbariConfig
-from mock.mock import patch, MagicMock, call
-
-
-class TestManifestGenerator(TestCase):
-
- def setUp(self):
- # disable stdout
- out = StringIO.StringIO()
- sys.stdout = out
-
- self.dir = tempfile.mkdtemp()
- self.config = AmbariConfig()
- jsonCommand = file('../../main/python/ambari_agent/test.json').read()
- self.parsedJson = json.loads(jsonCommand)
-
-
- def tearDown(self):
- shutil.rmtree(self.dir)
-
- # enable stdout
- sys.stdout = sys.__stdout__
-
-
- def testWriteImports(self):
- tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
- print tmpFileName
- tmpFile = file(tmpFileName, 'r+')
-
- manifestGenerator.writeImports(tmpFile, '../../main/puppet/modules', self.config.getImports())
- tmpFile.seek(0)
- print tmpFile.read()
- tmpFile.close()
-
- def testEscape(self):
- shouldBe = '\\\'\\\\'
- result = manifestGenerator.escape('\'\\')
- self.assertEqual(result, shouldBe)
-
-
- def test_writeNodes(self):
- tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
- tmpFile = file(tmpFileName, 'r+')
-
- clusterHostInfo = self.parsedJson['clusterHostInfo']
- clusterHostInfo['zookeeper_hosts'] = ["h1.hortonworks.com", "h2.hortonworks.com"]
- manifestGenerator.writeNodes(tmpFile, clusterHostInfo)
- tmpFile.seek(0)
- print tmpFile.read()
- tmpFile.close()
- os.remove(tmpFileName)
-
- def test_writeNodes_failed(self):
- tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
- tmpFile = file(tmpFileName, 'r+')
-
- clusterHostInfo = self.parsedJson['clusterHostInfo']
- clusterHostInfo.update({u'ZOOKEEPER':[None]})
- clusterHostInfo['zookeeper_hosts'] = ["h1.hortonworks.com", "h2.hortonworks.com"]
- self.assertRaises(TypeError, manifestGenerator.writeNodes, tmpFile, clusterHostInfo)
- tmpFile.seek(0)
- print tmpFile.read()
- tmpFile.close()
- os.remove(tmpFileName)
-
- def test_writeHostAttributes(self):
- tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
- tmpFile = file(tmpFileName, 'r+')
-
- hostAttributes = {'HostAttr1' : '1', 'HostAttr2' : '2'}
- manifestGenerator.writeHostAttributes(tmpFile, hostAttributes)
- tmpFile.seek(0)
- print tmpFile.read()
- tmpFile.close()
- os.remove(tmpFileName)
-
-
- def test_writeTasks(self):
- tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
- tmpFile = file(tmpFileName, 'r+')
- roles = [{'role' : 'ZOOKEEPER_SERVER',
- 'cmd' : 'NONE',
- 'roleParams' : {'someRoleParams': '-x'}}]
- clusterHostInfo = self.parsedJson['clusterHostInfo']
- clusterHostInfo['zookeeper_hosts'] = ["h1.hortonworks.com", "h2.hortonworks.com"]
- manifestGenerator.writeTasks(tmpFile, roles, self.config, clusterHostInfo, "h1.hortonworks.com")
- tmpFile.seek(0)
- print tmpFile.read()
- tmpFile.close()
- os.remove(tmpFileName)
-
- def testConvertRangeToList(self):
-
- rangesList = ["1-3", "4", "6", "7-9"]
- list = manifestGenerator.convertRangeToList(rangesList)
- self.assertEqual(sorted(list), sorted([1,2,3,4,6,7,8,9]))
-
- rangesList = ["5", "4"]
- list = manifestGenerator.convertRangeToList(rangesList)
- self.assertEqual(list, [5,4])
-
- exceptionWasTrown = False
- try:
- rangesList = ["0", "-2"]
- list = manifestGenerator.convertRangeToList(rangesList)
- except AgentException, err:
- #Expected
- exceptionWasTrown = True
-
- self.assertTrue(exceptionWasTrown)
-
- exceptionWasTrown = False
- try:
- rangesList = ["0", "-"]
- list = manifestGenerator.convertRangeToList(rangesList)
- except AgentException, err:
- #Expected
- exceptionWasTrown = True
- self.assertTrue(exceptionWasTrown)
-
- exceptionWasTrown = False
- try:
- rangesList = ["0", "2-"]
- list = manifestGenerator.convertRangeToList(rangesList)
- except AgentException, err:
- #Expected
- exceptionWasTrown = True
- self.assertTrue(exceptionWasTrown)
-
- def testConvertMappedRangeToList(self):
- mappedRangedList = ["1:0-2,5", "2:3,4"]
- list = manifestGenerator.convertMappedRangeToList(mappedRangedList)
- self.assertEqual(list, [1,1,1,2,2,1])
-
- mappedRangedList = ["7:0"]
- list = manifestGenerator.convertMappedRangeToList(mappedRangedList)
- self.assertEqual(list, [7])
-
- exceptionWasTrown = False
- mappedRangedList = ["7:0-"]
- try:
- list = manifestGenerator.convertMappedRangeToList(mappedRangedList)
- except AgentException, err:
- #Expected
- exceptionWasTrown = True
- self.assertTrue(exceptionWasTrown)
-
-
- exceptionWasTrown = False
- mappedRangedList = ["7:-"]
- try:
- list = manifestGenerator.convertMappedRangeToList(mappedRangedList)
- except AgentException, err:
- #Expected
- exceptionWasTrown = True
- self.assertTrue(exceptionWasTrown)
-
- exceptionWasTrown = False
- mappedRangedList = ["7:-1"]
- try:
- list = manifestGenerator.convertMappedRangeToList(mappedRangedList)
- except AgentException, err:
- #Expected
- exceptionWasTrown = True
- self.assertTrue(exceptionWasTrown)
-
- def testDecompressClusterHostInfo(self):
-
- all_hosts_key = "all_hosts"
- all_ping_ports_key = "all_ping_ports"
- ambari_server_host_key = "ambari_server_host"
- info = { "jtnode_host" : ["5"],
- "hbase_master_hosts" : ["5"],
- all_hosts_key: ["h8", "h9", "h5", "h4", "h7", "h6", "h1", "h3", "h2", "h10"],
- "namenode_host" : ["6"],
- "mapred_tt_hosts" : ["0", "7-9", "2","3", "5"],
- "slave_hosts" : ["3", "0", "1", "5-9"],
- "snamenode_host" : ["8"],
- all_ping_ports_key: ["8670:1,5-8", "8673:9", "8672:0,4", "8671:2,3"],
- "hbase_rs_hosts" : ["3", "1", "5", "8", "9"],
- ambari_server_host_key: ["h0"]
- }
-
- decompressedInfo = manifestGenerator.decompressClusterHostInfo(info)
-
- self.assertTrue(decompressedInfo.has_key(all_hosts_key))
-
- self.assertEquals(info.pop(all_hosts_key), decompressedInfo.get(all_hosts_key))
-
- self.assertEquals(['8672', '8670', '8671', '8671', '8672',
- '8670', '8670', '8670', '8670', '8673'],
- decompressedInfo.get(all_ping_ports_key))
-
- self.assertEquals(["h0"], decompressedInfo.get(ambari_server_host_key))
-
- for k,v in decompressedInfo.items():
- if k is all_ping_ports_key:
- continue # Skip checking this list
- # Check that list contains only host names
- non_converted = [x for x in v if not x.startswith("h")]
- self.assertEquals(0, len(non_converted))
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/test/python/ambari_agent/TestPuppetExecutor.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestPuppetExecutor.py b/ambari-agent/src/test/python/ambari_agent/TestPuppetExecutor.py
deleted file mode 100644
index ee7f9d8..0000000
--- a/ambari-agent/src/test/python/ambari_agent/TestPuppetExecutor.py
+++ /dev/null
@@ -1,275 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from unittest import TestCase
-from PuppetExecutor import PuppetExecutor
-from RepoInstaller import RepoInstaller
-from Grep import Grep
-from pprint import pformat
-import socket, threading, tempfile
-import os, time
-import sys
-import json
-from AmbariConfig import AmbariConfig
-from mock.mock import patch, MagicMock, call
-from threading import Thread
-from shell import shellRunner
-import manifestGenerator
-
-class TestPuppetExecutor(TestCase):
-
-
- def test_build(self):
- puppetexecutor = PuppetExecutor("/tmp", "/x", "/y", "/z", AmbariConfig().getConfig())
- command = puppetexecutor.puppetCommand("site.pp")
- self.assertEquals("puppet", command[0], "puppet binary wrong")
- self.assertEquals("apply", command[1], "local apply called")
- self.assertEquals("--confdir=/tmp", command[2],"conf dir tmp")
- self.assertEquals("--detailed-exitcodes", command[3], "make sure output \
- correct")
-
- @patch.object(shellRunner,'run')
- def test_isJavaAvailable(self, cmdrun_mock):
- puppetInstance = PuppetExecutor("/tmp", "/x", "/y", '/tmpdir',
- AmbariConfig().getConfig())
- command = {'configurations':{'global':{'java64_home':'/usr/jdk/jdk123'}}}
-
- cmdrun_mock.return_value = {'exitCode': 1, 'output': 'Command not found', 'error': ''}
- self.assertEquals(puppetInstance.isJavaAvailable(command), False)
-
- cmdrun_mock.return_value = {'exitCode': 0, 'output': 'OK', 'error': ''}
- self.assertEquals(puppetInstance.isJavaAvailable(command), True)
-
- @patch.object(manifestGenerator, 'generateManifest')
- @patch.object(PuppetExecutor, 'isJavaAvailable')
- @patch.object(PuppetExecutor, 'runPuppetFile')
- def test_run_command(self, runPuppetFileMock, isJavaAvailableMock, generateManifestMock):
- tmpdir = tempfile.gettempdir()
- puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir, AmbariConfig().getConfig())
- jsonFile = open('../../main/python/ambari_agent/test.json', 'r')
- jsonStr = jsonFile.read()
- parsedJson = json.loads(jsonStr)
- parsedJson["taskId"] = 1
- def side_effect1(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile, timeout):
- result["exitcode"] = 0
- runPuppetFileMock.side_effect = side_effect1
- generateManifestMock.return_value = ''
- puppetInstance.reposInstalled = False
- isJavaAvailableMock.return_value = True
- res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
- self.assertEquals(res["exitcode"], 0)
- self.assertTrue(puppetInstance.reposInstalled)
-
- def side_effect2(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile, timeout):
- result["exitcode"] = 999
- runPuppetFileMock.side_effect = side_effect2
- puppetInstance.reposInstalled = False
- isJavaAvailableMock.return_value = True
- res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
- self.assertEquals(res["exitcode"], 999)
- self.assertFalse(puppetInstance.reposInstalled)
-
- generateManifestMock.return_value = 'error during manifest generation'
- res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
- self.assertTrue(generateManifestMock.called)
- self.assertEquals(res["exitcode"], 1)
- generateManifestMock.return_value = ''
-
- def side_effect2(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile):
- result["exitcode"] = 0
- runPuppetFileMock.side_effect = side_effect2
- puppetInstance.reposInstalled = False
- isJavaAvailableMock.return_value = False
- parsedJson['roleCommand'] = "START"
- parsedJson['hostLevelParams'] = {'java_home':'/usr/jdk/jdk123'}
- res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
-
- JAVANOTVALID_MSG = "Cannot access JDK! Make sure you have permission to execute {0}/bin/java"
- errMsg = JAVANOTVALID_MSG.format('/usr/jdk/jdk123')
- self.assertEquals(res["exitcode"], 1)
- self.assertEquals(res["stderr"], errMsg)
- self.assertFalse(puppetInstance.reposInstalled)
-
- parsedJson['hostLevelParams'] = {'random':{'name1':'value2'}}
- res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
- self.assertEquals(res["exitcode"], 1)
- self.assertEquals(res["stderr"], "Cannot access JDK! Make sure java_home is specified in hostLevelParams")
-
- @patch("os.path.exists")
- def test_configure_environ(self, osPathExistsMock):
- config = AmbariConfig().getConfig()
- tmpdir = tempfile.gettempdir()
- puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir, config)
- environ = puppetInstance.configureEnviron({})
- self.assertEquals(environ, {})
-
- config.set('puppet','ruby_home',"test/ruby_home")
- puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir, config)
- osPathExistsMock.return_value = True
- environ = puppetInstance.configureEnviron({"PATH" : "test_path"})
- self.assertEquals(environ["PATH"], "test/ruby_home/bin:test_path")
- self.assertEquals(environ["MY_RUBY_HOME"], "test/ruby_home")
-
- def test_condense_bad2(self):
- puppetexecutor = PuppetExecutor("/tmp", "/x", "/y", "/z", AmbariConfig().getConfig())
- grep = Grep()
- puppetexecutor.grep = grep
- grep.ERROR_LAST_LINES_BEFORE = 2
- grep.ERROR_LAST_LINES_AFTER = 3
- string_err = open('ambari_agent' + os.sep + 'dummy_puppet_output_error2.txt', 'r').read().replace("\n", os.linesep)
- result = puppetexecutor.condenseOutput(string_err, '', 1)
- stripped_string = string_err.strip()
- lines = stripped_string.splitlines(True)
- d = lines[1:6]
- d = grep.cleanByTemplate("".join(d).strip(), "warning").splitlines(True)
- result_check = True
- for l in d:
- result_check &= grep.filterMarkup(l) in result
- self.assertEquals(result_check, True, "Failed to condence fail log")
- self.assertEquals(('warning' in result.lower()), False, "Failed to condence fail log")
- self.assertEquals(len(result.splitlines(True)), 5, "Failed to condence fail log")
-
- def test_condense_bad3(self):
- puppetexecutor = PuppetExecutor("/tmp", "/x", "/y", "/z", AmbariConfig().getConfig())
- grep = Grep()
- puppetexecutor.grep = grep
- string_err = open('ambari_agent' + os.sep + 'dummy_puppet_output_error3.txt', 'r').read().replace("\n", os.linesep)
- result = puppetexecutor.condenseOutput(string_err, '', 1)
- stripped_string = string_err.strip()
- lines = stripped_string.splitlines(True)
- #sys.stderr.write(result)
- d = lines[0:31]
- d = grep.cleanByTemplate("".join(d).strip(), "warning").splitlines(True)
- result_check = True
- for l in d:
- result_check &= grep.filterMarkup(l) in result
- self.assertEquals(result_check, True, "Failed to condence fail log")
- self.assertEquals(('warning' in result.lower()), False, "Failed to condence fail log")
- self.assertEquals(len(result.splitlines(True)), 19, "Failed to condence fail log")
-
- def test_condense_good(self):
- puppetexecutor = PuppetExecutor("/tmp", "/x", "/y", "/z", AmbariConfig().getConfig())
- grep = Grep()
- puppetexecutor.grep = grep
- grep.OUTPUT_LAST_LINES = 2
- string_good = open('ambari_agent' + os.sep + 'dummy_puppet_output_good.txt', 'r').read().replace("\n", os.linesep)
- result = puppetexecutor.condenseOutput(string_good, PuppetExecutor.NO_ERROR, 0)
- stripped_string = string_good.strip()
- lines = stripped_string.splitlines(True)
- result_check = lines[45].strip() in result and lines[46].strip() in result
- self.assertEquals(result_check, True, "Failed to condence output log")
- self.assertEquals(len(result.splitlines(True)), 2, "Failed to condence output log")
-
- @patch("shell.kill_process_with_children")
- def test_watchdog_1(self, kill_process_with_children_mock):
- """
- Tests whether watchdog works
- """
- subproc_mock = self.Subprocess_mockup()
- config = AmbariConfig().getConfig()
- executor_mock = self.PuppetExecutor_mock("/home/centos/ambari_repo_info/ambari-agent/src/main/puppet/",
- "/usr/",
- "/root/workspace/puppet-install/facter-1.6.10/",
- "/tmp", config, subproc_mock)
- _, tmpoutfile = tempfile.mkstemp()
- _, tmperrfile = tempfile.mkstemp()
- result = { }
- puppetEnv = { "RUBYLIB" : ""}
- kill_process_with_children_mock.side_effect = lambda pid : subproc_mock.terminate()
- subproc_mock.returncode = None
- timeout = "0.1"
- thread = Thread(target = executor_mock.runPuppetFile, args = ("fake_puppetFile",
- result, puppetEnv, tmpoutfile, tmperrfile, timeout))
- thread.start()
- time.sleep(0.1)
- subproc_mock.finished_event.wait()
- self.assertEquals(subproc_mock.was_terminated, True, "Subprocess should be terminated due to timeout")
-
-
- def test_watchdog_2(self):
- """
- Tries to catch false positive watchdog invocations
- """
- subproc_mock = self.Subprocess_mockup()
- config = AmbariConfig().getConfig()
- executor_mock = self.PuppetExecutor_mock("/home/centos/ambari_repo_info/ambari-agent/src/main/puppet/",
- "/usr/",
- "/root/workspace/puppet-install/facter-1.6.10/",
- "/tmp", config, subproc_mock)
- _, tmpoutfile = tempfile.mkstemp()
- _, tmperrfile = tempfile.mkstemp()
- result = { }
- puppetEnv = { "RUBYLIB" : ""}
- subproc_mock.returncode = 0
- timeout = "5"
- thread = Thread(target = executor_mock.runPuppetFile, args = ("fake_puppetFile",
- result, puppetEnv, tmpoutfile, tmperrfile, timeout))
- thread.start()
- time.sleep(0.1)
- subproc_mock.should_finish_event.set()
- subproc_mock.finished_event.wait()
- self.assertEquals(subproc_mock.was_terminated, False, "Subprocess should not be terminated before timeout")
- self.assertEquals(subproc_mock.returncode, 0, "Subprocess should not be terminated before timeout")
-
-
- class PuppetExecutor_mock(PuppetExecutor):
-
- def __init__(self, puppetModule, puppetInstall, facterInstall, tmpDir, config, subprocess_mockup):
- self.subprocess_mockup = subprocess_mockup
- PuppetExecutor.__init__(self, puppetModule, puppetInstall, facterInstall, tmpDir, config)
- pass
-
- def lauch_puppet_subprocess(self, puppetcommand, tmpout, tmperr, puppetEnv):
- self.subprocess_mockup.tmpout = tmpout
- self.subprocess_mockup.tmperr = tmperr
- return self.subprocess_mockup
-
- def runShellKillPgrp(self, puppet):
- puppet.terminate() # note: In real code, subprocess.terminate() is not called
- pass
-
- class Subprocess_mockup():
-
- returncode = 0
-
- started_event = threading.Event()
- should_finish_event = threading.Event()
- finished_event = threading.Event()
- was_terminated = False
- tmpout = None
- tmperr = None
- pid=-1
-
- def communicate(self):
- self.started_event.set()
- self.tmpout.write("Dummy output")
- self.tmpout.flush()
-
- self.tmperr.write("Dummy err")
- self.tmperr.flush()
- self.should_finish_event.wait()
- self.finished_event.set()
- pass
-
- def terminate(self):
- self.was_terminated = True
- self.returncode = 17
- self.should_finish_event.set()
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/test/python/ambari_agent/TestPuppetExecutorManually.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestPuppetExecutorManually.py b/ambari-agent/src/test/python/ambari_agent/TestPuppetExecutorManually.py
deleted file mode 100644
index 5299edb..0000000
--- a/ambari-agent/src/test/python/ambari_agent/TestPuppetExecutorManually.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from unittest import TestCase
-from ambari_agent.PuppetExecutor import PuppetExecutor
-from pprint import pformat
-import socket
-import os
-import sys
-import logging
-from AmbariConfig import AmbariConfig
-import tempfile
-
-FILEPATH="runme.pp"
-logger = logging.getLogger()
-
-class TestPuppetExecutor(TestCase):
-
- def test_run(self):
- """
- Used to run arbitrary puppet manifest. Test tries to find puppet manifest 'runme.pp' and runs it.
- Test does not make any assertions
- """
- if not os.path.isfile(FILEPATH):
- return
-
- logger.info("***** RUNNING " + FILEPATH + " *****")
- cwd = os.getcwd()
- puppetexecutor = PuppetExecutor(cwd, "/x", "/y", "/tmp", AmbariConfig().getConfig())
- result = {}
- puppetEnv = os.environ
- _, tmpoutfile = tempfile.mkstemp()
- _, tmperrfile = tempfile.mkstemp()
- result = puppetexecutor.runPuppetFile(FILEPATH, result, puppetEnv, tmpoutfile, tmperrfile)
- logger.info("*** Puppet output: " + str(result['stdout']))
- logger.info("*** Puppet errors: " + str(result['stderr']))
- logger.info("*** Puppet retcode: " + str(result['exitcode']))
- logger.info("****** DONE *****")
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/test/python/ambari_agent/TestRepoInstaller.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestRepoInstaller.py b/ambari-agent/src/test/python/ambari_agent/TestRepoInstaller.py
deleted file mode 100644
index 87683f9..0000000
--- a/ambari-agent/src/test/python/ambari_agent/TestRepoInstaller.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from unittest import TestCase
-from ambari_agent.RepoInstaller import RepoInstaller
-import tempfile
-import json, os
-import shutil
-from ambari_agent.AmbariConfig import AmbariConfig
-from mock.mock import patch, MagicMock, call
-
-class TestRepoInstaller(TestCase):
-
- def setUp(self):
- self.dir = tempfile.mkdtemp()
- jsonCommand = file('../../main/python/ambari_agent/test.json').read()
- self.parsedJson= json.loads(jsonCommand)
- self.config = AmbariConfig().getConfig()
- self.repoInstaller = RepoInstaller(self.parsedJson, self.dir, '../../main/puppet/modules', 1, self.config)
-
- pass
-
- def tearDown(self):
- shutil.rmtree(self.dir)
- pass
-
-
- def test_prepare_repos_info(self):
- localParsedJson = json.loads('{"hostLevelParams" : {"repo_info" : {"test" : "test"}}}')
- localRepoInstaller = RepoInstaller(localParsedJson, self.dir, '../../main/puppet/modules', 1, self.config)
- localRepoInstaller.prepareReposInfo()
- self.assertEquals(localRepoInstaller.repoInfoList['test'], "test")
-
- localParsedJson = json.loads('{"hostLevelParams" : {"repo_info" : "1"}}')
- localRepoInstaller = RepoInstaller(localParsedJson, self.dir, '../../main/puppet/modules', 1, self.config)
- localRepoInstaller.prepareReposInfo()
- self.assertEquals(localRepoInstaller.repoInfoList, 1)
-
- localParsedJson = json.loads('{"hostLevelParams" : {"repo_info" : ""}}')
- localRepoInstaller = RepoInstaller(localParsedJson, self.dir, '../../main/puppet/modules', 1, self.config)
- localRepoInstaller.prepareReposInfo()
- self.assertEquals(localRepoInstaller.repoInfoList, [])
-
-
- def test_generate_files(self):
- localParsedJson = json.loads('{"hostLevelParams": { "repo_info" : [{"baseUrl":"http://public-repo-1.hortonworks.com/HDP-1.1.1.16/repos/centos5"\
- ,"osType":"centos5","repoId":"HDP-1.1.1.16_TEST","repoName":"HDP_TEST", "mirrorsList":"http://mirrors.fedoraproject.org/mirrorlist"}]}}')
- localRepoInstaller = RepoInstaller(localParsedJson, self.dir, '../../main/puppet/modules', 1, self.config)
- localRepoInstaller.prepareReposInfo()
- result = localRepoInstaller.generateFiles()
- self.assertTrue(result[0].endswith("HDP-1.1.1.16_TEST-1.pp"))
-
- @patch.object(RepoInstaller, 'prepareReposInfo')
- @patch.object(RepoInstaller, 'generateFiles')
- def testInstallRepos(self, generateFilesMock, prepareReposInfoMock):
- result = self.repoInstaller.generate_repo_manifests()
- self.assertTrue(prepareReposInfoMock.called)
- self.assertTrue(generateFilesMock.called)
- print('generate_repo_manifests result: ' + result.__str__())
- pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-agent/src/test/python/ambari_agent/examples/ControllerTester.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/examples/ControllerTester.py b/ambari-agent/src/test/python/ambari_agent/examples/ControllerTester.py
index b5aa404..a3163f4 100644
--- a/ambari-agent/src/test/python/ambari_agent/examples/ControllerTester.py
+++ b/ambari-agent/src/test/python/ambari_agent/examples/ControllerTester.py
@@ -26,14 +26,14 @@ from mock.mock import patch, MagicMock, call
from ambari_agent.AmbariConfig import AmbariConfig
import Queue
import logging
-from ambari_agent import PuppetExecutor, PythonExecutor
+from ambari_agent import PythonExecutor
logger=logging.getLogger()
queue = Queue.Queue()
-# Set to True to replace python and puppet calls with mockups
-disable_python_and_puppet = True
+# Set to True to replace python calls with mockups
+disable_python = True
agent_version = "1.3.0"
@@ -113,24 +113,16 @@ responseId = Int(0)
def main():
- if disable_python_and_puppet:
- with patch.object(PuppetExecutor.PuppetExecutor, 'run_manifest') \
- as run_manifest_method:
- run_manifest_method.side_effect = \
- lambda command, file, tmpout, tmperr: {
- 'exitcode' : 0,
- 'stdout' : "Simulated run of pp %s" % file,
- 'stderr' : 'None'
- }
- with patch.object(PythonExecutor.PythonExecutor, 'run_file') \
+ if disable_python:
+ with patch.object(PythonExecutor.PythonExecutor, 'run_file') \
as run_file_py_method:
- run_file_py_method.side_effect = \
- lambda command, file, tmpoutfile, tmperrfile: {
- 'exitcode' : 0,
- 'stdout' : "Simulated run of py %s" % file,
- 'stderr' : 'None'
- }
- run_simulation()
+ run_file_py_method.side_effect = \
+ lambda command, file, tmpoutfile, tmperrfile: {
+ 'exitcode' : 0,
+ 'stdout' : "Simulated run of py %s" % file,
+ 'stderr' : 'None'
+ }
+ run_simulation()
else:
run_simulation()
http://git-wip-us.apache.org/repos/asf/ambari/blob/3d1171b0/ambari-client/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-client/pom.xml b/ambari-client/pom.xml
index 86a0837..038422f 100755
--- a/ambari-client/pom.xml
+++ b/ambari-client/pom.xml
@@ -153,7 +153,6 @@
<exclude>src/examples/*</exclude>
<exclude>src/test/python/dummy*.txt</exclude>
<exclude>src/main/python/ambari_client/imports.txt</exclude>
- <exclude>src/main/puppet/modules/stdlib/**</exclude>
<exclude>**/*.erb</exclude>
<exclude>**/*.json</exclude>
</excludes>