You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2008/03/21 09:48:57 UTC
svn commit: r639571 - in /hadoop/core/trunk/src/contrib/hod/testing:
testHadoop.py testHodRing.py
Author: ddas
Date: Fri Mar 21 01:48:56 2008
New Revision: 639571
URL: http://svn.apache.org/viewvc?rev=639571&view=rev
Log:
HADOOP-2936. Added the missed testcases in the original commit.
Added:
hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py
hadoop/core/trunk/src/contrib/hod/testing/testHodRing.py
Added: hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py?rev=639571&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py (added)
+++ hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py Fri Mar 21 01:48:56 2008
@@ -0,0 +1,122 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re, threading, time
+
+myDirectory = os.path.realpath(sys.argv[0])
+rootDirectory = re.sub("/testing/.*", "", myDirectory)
+
+sys.path.append(rootDirectory)
+
+from testing.lib import BaseTestSuite
+
+excludes = []
+
+import tempfile, getpass
+from xml.dom import minidom
+
+from hodlib.Hod.hadoop import hadoopConfig
+
+# All test-case classes should have the naming convention test_.*
+class test_hadoopConfig(unittest.TestCase):
+ def setUp(self):
+ self.__hadoopConfig = hadoopConfig()
+ self.rootDir = '/tmp/hod-%s' % getpass.getuser()
+ if not os.path.exists(self.rootDir):
+ os.mkdir(self.rootDir)
+ self.testingDir = tempfile.mkdtemp( dir=self.rootDir,
+ prefix='HadoopTestSuite.test_hadoopConfig')
+ self.confDir = tempfile.mkdtemp(dir=self.rootDir,
+ prefix='HadoopTestSuite.test_hadoopConfig')
+ self.tempDir = '/tmp/hod-%s/something' % getpass.getuser()
+ self.hadoopSite = os.path.join(self.confDir,'hadoop-site.xml')
+ self.numNodes = 4
+ self.hdfsAddr = 'nosuchhost1.apache.org:50505'
+ self.mapredAddr = 'nosuchhost2.apache.org:50506'
+ self.finalServerParams = {
+ 'mapred.child.java.opts' : '-Xmx1024m',
+ 'mapred.compress.map.output' : 'false',
+ }
+ self.serverParams = {
+ 'mapred.userlog.limit' : '200',
+ 'mapred.userlog.retain.hours' : '10',
+ 'mapred.reduce.parallel.copies' : '20',
+ }
+ self.clientParams = {
+ 'mapred.tasktracker.tasks.maximum' : '2',
+ 'io.sort.factor' : '100',
+ 'io.sort.mb' : '200',
+ 'mapred.userlog.limit.kb' : '1024',
+ 'io.file.buffer.size' : '262144',
+ }
+ self.clusterFactor = 1.9
+ self.mySysDir = '/user/' + getpass.getuser() + '/mapredsystem'
+ pass
+
+ def testSuccess(self):
+ self.__hadoopConfig.gen_site_conf(
+ confDir = self.confDir,\
+ tempDir = self.tempDir,\
+ numNodes = self.numNodes,\
+ hdfsAddr = self.hdfsAddr,\
+ mrSysDir = self.mySysDir,\
+ mapredAddr = self.mapredAddr,\
+ clientParams = self.clientParams,\
+ serverParams = self.serverParams,\
+ finalServerParams = self.finalServerParams,\
+ clusterFactor = self.clusterFactor
+
+ )
+ xmldoc = minidom.parse(self.hadoopSite)
+ xmldoc = xmldoc.childNodes[0] # leave out xml spec
+ properties = xmldoc.childNodes # children of tag configuration
+ keyvals = {}
+ for prop in properties:
+ if not isinstance(prop,minidom.Comment):
+ # ---------- tag -------------------- -value elem-- data --
+ name = prop.getElementsByTagName('name')[0].childNodes[0].data
+ value = prop.getElementsByTagName('value')[0].childNodes[0].data
+ keyvals[name] = value
+
+ # fs.default.name should start with hdfs://
+ assert(keyvals['fs.default.name'].startswith('hdfs://'))
+
+ # TODO other tests
+ pass
+
+ def tearDown(self):
+ if os.path.exists(self.hadoopSite): os.unlink(self.hadoopSite)
+ if os.path.exists(self.confDir) : os.rmdir(self.confDir)
+ if os.path.exists(self.testingDir) : os.rmdir(self.testingDir)
+ pass
+
+class HadoopTestSuite(BaseTestSuite):
+ def __init__(self):
+ # suite setup
+ BaseTestSuite.__init__(self, __name__, excludes)
+ pass
+
+ def cleanUp(self):
+ # suite tearDown
+ pass
+
+def RunHadoopTests():
+ suite = HadoopTestSuite()
+ testResult = suite.runTests()
+ suite.cleanUp()
+ return testResult
+
+if __name__ == "__main__":
+ RunHadoopTests()
Added: hadoop/core/trunk/src/contrib/hod/testing/testHodRing.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/testing/testHodRing.py?rev=639571&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/testing/testHodRing.py (added)
+++ hadoop/core/trunk/src/contrib/hod/testing/testHodRing.py Fri Mar 21 01:48:56 2008
@@ -0,0 +1,117 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re, threading, time
+
+myDirectory = os.path.realpath(sys.argv[0])
+rootDirectory = re.sub("/testing/.*", "", myDirectory)
+
+sys.path.append(rootDirectory)
+
+from testing.lib import BaseTestSuite
+
+excludes = []
+
+import tempfile, getpass, logging
+from xml.dom import minidom
+
+from hodlib.Hod.hadoop import hadoopConfig
+from hodlib.HodRing.hodRing import CommandDesc, HadoopCommand
+
+# All test-case classes should have the naming convention test_.*
+class test_HadoopCommand(unittest.TestCase):
+ def setUp(self):
+ self.rootDir = '/tmp/hod-%s' % getpass.getuser()
+ self.id = 0
+ self.desc = None
+ self.tempDir = os.path.join(self.rootDir,'test_HadoopCommand_tempDir')
+ self.pkgDir = os.path.join(self.rootDir,'test_HadoopCommand_pkgDir')
+ self.log = logging.getLogger() # TODO Use MockLogger
+ self.javaHome = '/usr/java/bin/'
+ self.mrSysDir = '/user/' + getpass.getuser() + '/mapredsystem'
+
+ self.attrs = {}
+ self.finalAttrs = {
+ 'fs.default.name': 'nohost.apache.com:56366',
+ 'mapred.child.java.opts' : '-Xmx1024m',
+ 'mapred.compress.map.output' : 'false',
+ }
+ self.attrs = {
+ 'mapred.userlog.limit' : '200',
+ 'mapred.userlog.retain.hours' : '10',
+ 'mapred.reduce.parallel.copies' : '20',
+ }
+ self.desc = CommandDesc(
+ {
+ 'name' : 'dummyHadoop',
+ 'program' : 'bin/hadoop',
+ 'pkgdirs' : self.pkgDir,
+ 'final-attrs' : self.finalAttrs,
+ 'attrs' : self.attrs,
+ }, self.log
+ )
+ # TODO
+ # 4th arg to HadoopCommand 'tardir' is not used at all. Instead pkgdir is
+ # specified through HadoopCommand.run(pkgdir). This could be changed so
+ # that pkgdir is specified at the time of object creation.
+ # END OF TODO
+ self.hadoopCommand = HadoopCommand(self.id, self.desc, self.tempDir,
+ self.pkgDir, self.log, self.javaHome,
+ self.mrSysDir, restart=True)
+ self.hadoopSite = os.path.join( self.hadoopCommand.confdir,
+ 'hadoop-site.xml')
+ pass
+
+ def test_createHadoopSiteXml(self):
+ self.hadoopCommand._createHadoopSiteXml()
+ xmldoc = minidom.parse(self.hadoopSite)
+ xmldoc = xmldoc.childNodes[0] # leave out xml spec
+ properties = xmldoc.childNodes # children of tag configuration
+ keyvals = {}
+ for prop in properties:
+ if not isinstance(prop,minidom.Comment):
+ # ---------- tag -------------------- -value elem-- data --
+ name = prop.getElementsByTagName('name')[0].childNodes[0].data
+ value = prop.getElementsByTagName('value')[0].childNodes[0].data
+ keyvals[name] = value
+
+ # fs.default.name should start with hdfs://
+ assert(keyvals['fs.default.name'].startswith('hdfs://'))
+
+ # TODO other tests
+ pass
+
+ def tearDown(self):
+ pass
+
+class HodRingTestSuite(BaseTestSuite):
+ def __init__(self):
+ # suite setup
+ BaseTestSuite.__init__(self, __name__, excludes)
+ pass
+
+ def cleanUp(self):
+ # suite tearDown
+ pass
+
+def RunHodRingTests():
+ # modulename_suite
+ suite = HodRingTestSuite()
+ testResult = suite.runTests()
+ suite.cleanUp()
+ return testResult
+
+if __name__ == "__main__":
+ RunHodRingTests()