You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2012/10/16 09:01:35 UTC
svn commit: r1398674 - in /incubator/ambari/branches/AMBARI-666:
AMBARI-666-CHANGES.txt
ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
ambari-agent/src/main/python/ambari_agent/site.pp
Author: mahadev
Date: Tue Oct 16 07:01:35 2012
New Revision: 1398674
URL: http://svn.apache.org/viewvc?rev=1398674&view=rev
Log:
AMBARI-868. Clean up site.pp generation on the agent and remove the imports in the sample site.pp. (mahadev)
Modified:
incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp
Modified: incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt?rev=1398674&r1=1398673&r2=1398674&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt (original)
+++ incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt Tue Oct 16 07:01:35 2012
@@ -12,6 +12,9 @@ AMBARI-666 branch (unreleased changes)
NEW FEATURES
+ AMBARI-868. Clean up site.pp generation on the agent and remove the imports
+ in the sample site.pp. (mahadev)
+
AMBARI-862. API query against /clusters doesn't return any data.
(John Speidel via mahadev)
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py?rev=1398674&r1=1398673&r2=1398674&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py Tue Oct 16 07:01:35 2012
@@ -1,184 +1,184 @@
-#!/usr/bin/env python2.6
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import json
-import os.path
-import logging
-
-logger = logging.getLogger()
-
- #read static imports from file and write them to manifest
-def writeImports(outputFile, inputFileName='imports.txt'):
- inputFile = open(inputFileName, 'r')
- modulesdir = os.path.abspath(os.getcwd() + "../../../puppet/modules/")
- logger.info("Modules dir is " + modulesdir)
- for line in inputFile:
- modulename = line.rstrip('\n')
- line = "import " + "\"" + modulesdir + "/" + modulename + "\"\n"
- outputFile.write(line)
-
- inputFile.close()
-
-def generateManifest(inputJsonStr):
-#reading json
- parsedJson = json.loads(inputJsonStr)
- hostname = parsedJson['hostname']
- clusterHostInfo = parsedJson['clusterHostInfo']
- params = parsedJson['params']
- configurations = parsedJson['configurations']
- #hostAttributes = parsedJson['hostAttributes']
- roles = parsedJson['roleCommands']
-
-#writing manifest
- manifest = open('site.pp', 'w')
-
- #writing imports from external static file
- writeImports(manifest)
-
- #writing nodes
- writeNodes(manifest, clusterHostInfo)
-
- #writing params from map
- writeParams(manifest, params)
-
- #writing config maps
- writeConfigurations(manifest, configurations)
-
- #writing host attributes
- #writeHostAttributes(manifest, hostAttributes)
-
- #writing task definitions
- writeTasks(manifest, roles)
-
- manifest.close()
-
-
- #read dictionary
-def readDict(file, separator='='):
- result = dict()
-
- for line in file :
- dictTuple = line.partition(separator)
- result[dictTuple[0].strip()] = dictTuple[2].strip()
-
- return result
-
-
- #write nodes
-def writeNodes(outputFile, clusterHostInfo):
- for node in clusterHostInfo.iterkeys():
- outputFile.write('$' + node + '= [')
- coma = ''
-
- for value in clusterHostInfo[node]:
- outputFile.write(coma + '\'' + value + '\'')
- coma = ', '
-
- outputFile.write(']\n')
-
-#write params
-def writeParams(outputFile, params):
- for param in params.iterkeys():
- outputFile.write('$' + param + '="' + params[param] + '"\n')
-
-#write host attributes
-def writeHostAttributes(outputFile, hostAttributes):
- outputFile.write('$hostAttributes={\n')
-
- coma = ''
- for attribute in hostAttributes.iterkeys():
- outputFile.write(coma + '"' + attribute + '" => "{' + hostAttributes[attribute] + '"}')
- coma = ',\n'
-
- outputFile.write('}\n')
-
-#write configurations
-def writeConfigurations(outputFile, configs):
- outputFile.write('$configuration = {\n')
-
- for configName in configs.iterkeys():
- outputFile.write(configName + '=> {\n')
- config = configs[configName]
-
- coma = ''
- for configParam in config.iterkeys():
- outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')
- coma = ',\n'
-
- outputFile.write('\n},\n')
-
- outputFile.write('\n}\n')
-
-#write node tasks
-def writeTasks(outputFile, roles):
- #reading dictionaries
- rolesToClassFile = open('rolesToClass.dict', 'r')
- rolesToClass = readDict(rolesToClassFile)
- rolesToClassFile.close()
-
- serviceStatesFile = open('serviceStates.dict', 'r')
- serviceStates = readDict(serviceStatesFile)
- serviceStatesFile.close()
-
- outputFile.write('node /default/ {\n ')
- writeStages(outputFile, len(roles))
- stageNum = 1
-
- for role in roles :
- rolename = role['role']
- command = role['cmd']
- taskParams = role['roleParams']
- taskParamsNormalized = normalizeTaskParams(taskParams)
- taskParamsPostfix = ''
-
- if len(taskParamsNormalized) > 0 :
- taskParamsPostfix = ', ' + taskParamsNormalized
-
- className = rolesToClass[rolename]
- serviceState = serviceStates[command]
-
- outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) +
- ', service_state => ' + serviceState + taskParamsPostfix + '}\n')
- stageNum = stageNum + 1
- outputFile.write('}\n')
-def normalizeTaskParams(taskParams):
- result = ''
- coma = ''
-
- for paramName in taskParams.iterkeys():
- result = coma + result + paramName + ' => ' + taskParams[paramName]
- coma = ','
-
- return result
-
-def writeStages(outputFile, numStages):
- arrow = ''
-
- for i in range(numStages):
- outputFile.write(arrow + 'stage{' + str(i) + ' :}')
- arrow = ' -> '
-
- outputFile.write('\n')
-
-logging.basicConfig(level=logging.DEBUG)
-#test code
-jsonFile = open('test.json', 'r')
-jsonStr = jsonFile.read()
-generateManifest(jsonStr)
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import json
+import os.path
+import logging
+
+logger = logging.getLogger()
+
+ #read static imports from file and write them to manifest
+def writeImports(outputFile, inputFileName='imports.txt'):
+ inputFile = open(inputFileName, 'r')
+ modulesdir = os.path.abspath(os.getcwd() + "../../../puppet/modules/")
+ logger.info("Modules dir is " + modulesdir)
+ for line in inputFile:
+ modulename = line.rstrip('\n')
+ line = "import '" + modulesdir + "/" + modulename + "'\n"
+ outputFile.write(line)
+
+ inputFile.close()
+
+def generateManifest(inputJsonStr):
+#reading json
+ parsedJson = json.loads(inputJsonStr)
+ hostname = parsedJson['hostname']
+ clusterHostInfo = parsedJson['clusterHostInfo']
+ params = parsedJson['params']
+ configurations = parsedJson['configurations']
+ #hostAttributes = parsedJson['hostAttributes']
+ roles = parsedJson['roleCommands']
+
+#writing manifest
+ manifest = open('site.pp', 'w')
+
+ #writing imports from external static file
+ writeImports(manifest)
+
+ #writing nodes
+ writeNodes(manifest, clusterHostInfo)
+
+ #writing params from map
+ writeParams(manifest, params)
+
+ #writing config maps
+ writeConfigurations(manifest, configurations)
+
+ #writing host attributes
+ #writeHostAttributes(manifest, hostAttributes)
+
+ #writing task definitions
+ writeTasks(manifest, roles)
+
+ manifest.close()
+
+
+ #read dictionary
+def readDict(file, separator='='):
+ result = dict()
+
+ for line in file :
+ dictTuple = line.partition(separator)
+ result[dictTuple[0].strip()] = dictTuple[2].strip()
+
+ return result
+
+
+ #write nodes
+def writeNodes(outputFile, clusterHostInfo):
+ for node in clusterHostInfo.iterkeys():
+ outputFile.write('$' + node + '= [')
+ coma = ''
+
+ for value in clusterHostInfo[node]:
+ outputFile.write(coma + '\'' + value + '\'')
+ coma = ', '
+
+ outputFile.write(']\n')
+
+#write params
+def writeParams(outputFile, params):
+ for param in params.iterkeys():
+ outputFile.write('$' + param + '="' + params[param] + '"\n')
+
+#write host attributes
+def writeHostAttributes(outputFile, hostAttributes):
+ outputFile.write('$hostAttributes={\n')
+
+ coma = ''
+ for attribute in hostAttributes.iterkeys():
+ outputFile.write(coma + '"' + attribute + '" => "{' + hostAttributes[attribute] + '"}')
+ coma = ',\n'
+
+ outputFile.write('}\n')
+
+#write configurations
+def writeConfigurations(outputFile, configs):
+ outputFile.write('$configuration = {\n')
+
+ for configName in configs.iterkeys():
+ outputFile.write(configName + '=> {\n')
+ config = configs[configName]
+
+ coma = ''
+ for configParam in config.iterkeys():
+ outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')
+ coma = ',\n'
+
+ outputFile.write('\n},\n')
+
+ outputFile.write('\n}\n')
+
+#write node tasks
+def writeTasks(outputFile, roles):
+ #reading dictionaries
+ rolesToClassFile = open('rolesToClass.dict', 'r')
+ rolesToClass = readDict(rolesToClassFile)
+ rolesToClassFile.close()
+
+ serviceStatesFile = open('serviceStates.dict', 'r')
+ serviceStates = readDict(serviceStatesFile)
+ serviceStatesFile.close()
+
+ outputFile.write('node /default/ {\n ')
+ writeStages(outputFile, len(roles))
+ stageNum = 1
+
+ for role in roles :
+ rolename = role['role']
+ command = role['cmd']
+ taskParams = role['roleParams']
+ taskParamsNormalized = normalizeTaskParams(taskParams)
+ taskParamsPostfix = ''
+
+ if len(taskParamsNormalized) > 0 :
+ taskParamsPostfix = ', ' + taskParamsNormalized
+
+ className = rolesToClass[rolename]
+ serviceState = serviceStates[command]
+
+ outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) +
+ ', service_state => ' + serviceState + taskParamsPostfix + '}\n')
+ stageNum = stageNum + 1
+ outputFile.write('}\n')
+def normalizeTaskParams(taskParams):
+ result = ''
+ coma = ''
+
+ for paramName in taskParams.iterkeys():
+ result = coma + result + paramName + ' => ' + taskParams[paramName]
+ coma = ','
+
+ return result
+
+def writeStages(outputFile, numStages):
+ arrow = ''
+
+ for i in range(numStages):
+ outputFile.write(arrow + 'stage{' + str(i) + ' :}')
+ arrow = ' -> '
+
+ outputFile.write('\n')
+
+logging.basicConfig(level=logging.DEBUG)
+#test code
+jsonFile = open('test.json', 'r')
+jsonStr = jsonFile.read()
+generateManifest(jsonStr)
Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp?rev=1398674&r1=1398673&r2=1398674&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp (original)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp Tue Oct 16 07:01:35 2012
@@ -1,27 +1,15 @@
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hadoop/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hbase/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-zookeeper/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-oozie/manifests/*.pp""
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-pig/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-sqoop/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-templeton/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hive/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hcat/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-mysql/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-monitor-webserver/manifests/*.pp"
$NAMENODE= ['h2.hortonworks.com']
$DATANODE= ['h1.hortonworks.com', 'h2.hortonworks.com']
$hdfs_user="hdfs"
$jdk_location="lah/blah"
$configuration = {
-$hdfs_site=> {
+hdfs_site=> {
"dfs.block.size" => "256000000",
"dfs.replication" => "1"
-}
-$core_site=> {
+},
+core_site=> {
"fs.default.name" => "hrt8n36.cc1.ygridcore.net"
-}
+},
}
node /default/ {