You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2008/03/14 06:14:56 UTC
svn commit: r636995 - in /hadoop/core/trunk: ./ src/contrib/hod/
src/contrib/hod/testing/
Author: ddas
Date: Thu Mar 13 22:14:54 2008
New Revision: 636995
URL: http://svn.apache.org/viewvc?rev=636995&view=rev
Log:
HADOOP-2775. Adds unit test framework for HOD. Contributed by Vinod Kumar Vavilapalli.
Added:
hadoop/core/trunk/src/contrib/hod/testing/
hadoop/core/trunk/src/contrib/hod/testing/__init__.py
hadoop/core/trunk/src/contrib/hod/testing/lib.py
hadoop/core/trunk/src/contrib/hod/testing/main.py
hadoop/core/trunk/src/contrib/hod/testing/testModule.py
hadoop/core/trunk/src/contrib/hod/testing/testRingmasterRPCs.py
hadoop/core/trunk/src/contrib/hod/testing/testXmlrpc.py
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/hod/build.xml
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=636995&r1=636994&r2=636995&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Mar 13 22:14:54 2008
@@ -92,6 +92,9 @@
HADOOP-2981. Update README.txt to reflect the upcoming use of
cryptography. (omalley)
+ HADOOP-2775. Adds unit test framework for HOD.
+ (Vinod Kumar Vavilapalli via ddas).
+
OPTIMIZATIONS
HADOOP-2790. Fixed inefficient method hasSpeculativeTask by removing
Modified: hadoop/core/trunk/src/contrib/hod/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/build.xml?rev=636995&r1=636994&r2=636995&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/build.xml (original)
+++ hadoop/core/trunk/src/contrib/hod/build.xml Thu Mar 13 22:14:54 2008
@@ -32,4 +32,45 @@
</copy>
<chmod dir="${dist.dir}/contrib/${name}/bin" perm="a+x" includes="*"/>
</target>
+
+ <target name="test" depends="compile" description="Run HOD unit tests">
+ <antcall target="python.pathcheck"/>
+ <antcall target="checkAndRunTests"/>
+ </target>
+
+ <target name="checkAndRunTests" if="python.home">
+ <!-- Check python version now -->
+ <exec executable="/bin/sh" outputproperty="hodtest.pythonVersion">
+ <arg value="-c" />
+ <arg value="${python.home}/python -V" />
+ </exec>
+ <condition property="python.versionmatched">
+ <!--- Currently check for only 2.5.1 -->
+ <equals arg1="${hodtest.pythonVersion}" arg2="Python 2.5.1" />
+ </condition>
+ <antcall target="python.versioncheck"/>
+ <antcall target="runtests"/>
+ </target>
+
+ <target name="python.pathcheck" unless="python.home">
+ <echo message="'python.home' is not defined. Please pass -Dpython.home=<Path to Python> to Ant on the command-line."/>
+ </target>
+
+ <target name="runtests" if="python.versionmatched">
+ <echo message="Using Python at : ${python.home}" />
+ <echo message="Version : ${hodtest.pythonVersion}"/>
+ <exec executable="/bin/sh" resultproperty="hodtest.failedTests">
+ <arg value="-c" />
+ <arg value="${python.home}/python ${build.dir}/testing/main.py" />
+ </exec>
+ <condition property="hodtest.success">
+ <equals arg1="${hodtest.failedTests}" arg2="0"/>
+ </condition>
+ <fail message="TestCases failed. ${hodtest.failedTests} failed to run successfully." unless="hodtest.success"/>
+ </target>
+
+ <target name="python.versioncheck" unless="python.versionmatched">
+ <echo message="Need Python version 2.5.1. You specified ${hodtest.pythonVersion}"/>
+ </target>
+
</project>
Added: hadoop/core/trunk/src/contrib/hod/testing/__init__.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/testing/__init__.py?rev=636995&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/testing/__init__.py (added)
+++ hadoop/core/trunk/src/contrib/hod/testing/__init__.py Thu Mar 13 22:14:54 2008
@@ -0,0 +1,15 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
Added: hadoop/core/trunk/src/contrib/hod/testing/lib.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/testing/lib.py?rev=636995&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/testing/lib.py (added)
+++ hadoop/core/trunk/src/contrib/hod/testing/lib.py Thu Mar 13 22:14:54 2008
@@ -0,0 +1,64 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, re, sys
+
+class BaseTestSuite():
+ def __init__(self, name, excludes):
+ self.name = name
+ self.excludes = excludes
+ pass
+
+ def runTests(self):
+ # Create a runner
+ self.runner = unittest.TextTestRunner()
+
+ # Get all the test-case classes
+ # From module import *
+ mod = __import__(self.name, fromlist=['*'])
+ modItemsList = dir(mod)
+
+ allsuites = []
+
+ # Create all the test suites
+ for modItem in modItemsList:
+ if re.search(r"^test_", modItem):
+ # Yes this is a test class
+ if modItem not in self.excludes:
+ test_class = getattr(mod, modItem)
+ allsuites.append(unittest.makeSuite(test_class))
+
+ # Create a master suite to be run.
+ alltests = unittest.TestSuite(tuple(allsuites))
+
+ # Run the master test suite.
+ runner = self.runner.run(alltests)
+ if(runner.wasSuccessful()): return 0
+ printLine( "%s test(s) failed." % runner.failures.__len__())
+ printLine( "%s test(s) threw errors." % runner.errors.__len__())
+ return runner.failures.__len__() + runner.errors.__len__()
+
+ def cleanUp(self):
+ # suite tearDown
+ pass
+
+def printLine(str):
+ print >>sys.stderr, str
+
+def printSeparator():
+ str = ""
+ for i in range(0,79):
+ str = str + "*"
+ print >>sys.stderr, "\n", str, "\n"
Added: hadoop/core/trunk/src/contrib/hod/testing/main.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/testing/main.py?rev=636995&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/testing/main.py (added)
+++ hadoop/core/trunk/src/contrib/hod/testing/main.py Thu Mar 13 22:14:54 2008
@@ -0,0 +1,84 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re
+
+myPath = os.path.realpath(sys.argv[0])
+rootDirectory = re.sub("/testing/.*", "", myPath)
+testingDir = os.path.join(rootDirectory, "testing")
+
+sys.path.append(rootDirectory)
+
+from testing.lib import printSeparator, printLine
+
+moduleList = []
+allList = []
+excludes = [
+ 'Xmlrpc'
+ ]
+
+# Build a module list by scanning through all files in testingDir
+for file in os.listdir(testingDir):
+ if(re.search(r".py$", file) and re.search(r"^test", file)):
+ # All .py files with names starting in 'test'
+ module = re.sub(r"^test","",file)
+ module = re.sub(r".py$","",module)
+ allList.append(module)
+ if module not in excludes:
+ moduleList.append(module)
+
+printLine("All testcases - %s" % allList)
+printLine("Excluding the testcases - %s" % excludes)
+printLine("Executing the testcases - %s" % moduleList)
+
+testsResult = 0
+# Now import each of these modules and start calling the corresponding
+#testSuite methods
+for moduleBaseName in moduleList:
+ try:
+ module = "testing.test" + moduleBaseName
+ suiteCaller = "Run" + moduleBaseName + "Tests"
+ printSeparator()
+ printLine("Running %s" % suiteCaller)
+
+ # Import the corresponding test cases module
+ imported_module = __import__(module , fromlist=[suiteCaller] )
+
+ # Call the corresponding suite method now
+ testRes = getattr(imported_module, suiteCaller)()
+ testsResult = testsResult + testRes
+ printLine("Finished %s. TestSuite Result : %s\n" % \
+ (suiteCaller, testRes))
+ except ImportError, i:
+ # Failed to import a test module
+ printLine(i)
+ testsResult = testsResult + 1
+ pass
+ except AttributeError, n:
+ # Failed to get suiteCaller from a test module
+ printLine(n)
+ testsResult = testsResult + 1
+ pass
+ except Exception, e:
+ # Test module suiteCaller threw some exception
+ printLine("%s failed. \nReason : %s" % (suiteCaller, e))
+ printLine("Skipping %s" % suiteCaller)
+ testsResult = testsResult + 1
+ pass
+
+if testsResult != 0:
+ printSeparator()
+ printLine("Total testcases with failure or error : %s" % testsResult)
+sys.exit(testsResult)
Added: hadoop/core/trunk/src/contrib/hod/testing/testModule.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/testing/testModule.py?rev=636995&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/testing/testModule.py (added)
+++ hadoop/core/trunk/src/contrib/hod/testing/testModule.py Thu Mar 13 22:14:54 2008
@@ -0,0 +1,88 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re, threading, time
+
+myDirectory = os.path.realpath(sys.argv[0])
+rootDirectory = re.sub("/testing/.*", "", myDirectory)
+
+sys.path.append(rootDirectory)
+
+from testing.lib import BaseTestSuite
+
+excludes = ['test_MINITEST3']
+
+# All test-case classes should have the naming convention test_.*
+class test_MINITEST1(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ # All testMethods have to have their names start with 'test'
+ def testSuccess(self):
+ pass
+
+ def testFailure(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+class test_MINITEST2(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ # All testMethods have to have their names start with 'test'
+ def testSuccess(self):
+ pass
+
+ def testFailure(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+class test_MINITEST3(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ # All testMethods have to have their names start with 'test'
+ def testSuccess(self):
+ pass
+
+ def testFailure(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+class ModuleTestSuite(BaseTestSuite):
+ def __init__(self):
+ # suite setup
+ BaseTestSuite.__init__(self, __name__, excludes)
+ pass
+
+ def cleanUp(self):
+ # suite tearDown
+ pass
+
+def RunModuleTests():
+ # modulename_suite
+ suite = ModuleTestSuite()
+ testResult = suite.runTests()
+ suite.cleanUp()
+ return testResult
+
+if __name__ == "__main__":
+ RunModuleTests()
Added: hadoop/core/trunk/src/contrib/hod/testing/testRingmasterRPCs.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/testing/testRingmasterRPCs.py?rev=636995&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/testing/testRingmasterRPCs.py (added)
+++ hadoop/core/trunk/src/contrib/hod/testing/testRingmasterRPCs.py Thu Mar 13 22:14:54 2008
@@ -0,0 +1,131 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re, threading, time
+
+import logging
+
+myDirectory = os.path.realpath(sys.argv[0])
+rootDirectory = re.sub("/testing/.*", "", myDirectory)
+
+sys.path.append(rootDirectory)
+
+from testing.lib import BaseTestSuite
+
+excludes = ['test_MINITEST1', 'test_MINITEST2']
+
+from hodlib.GridServices import *
+from hodlib.Common.desc import ServiceDesc
+from hodlib.RingMaster.ringMaster import _LogMasterSources
+
+# All test-case classes should have the naming convention test_.*
+class test_MINITEST1(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ # All testMethods have to have their names start with 'test'
+ def testSuccess(self):
+ pass
+
+ def testFailure(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+class test_MINITEST2(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ # All testMethods have to have their names start with 'test'
+ def testSuccess(self):
+ pass
+
+ def testFailure(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+class test_GetCommand(unittest.TestCase):
+ def setUp(self):
+ self.config = {
+ 'hod': {},
+ 'resource_manager': {
+ 'id': 'torque',
+ 'batch-home': '/home/y/'
+ },
+ 'ringmaster': {
+ 'max-connect' : 2
+ },
+ 'hodring': {
+ },
+ 'gridservice-mapred': {
+ 'id': 'mapred'
+ } ,
+ 'gridservice-hdfs': {
+ 'id': 'hdfs'
+ },
+ 'servicedesc' : {} ,
+ 'nodepooldesc': {} ,
+ }
+
+ hdfsDesc = self.config['servicedesc']['hdfs'] = ServiceDesc(self.config['gridservice-hdfs'])
+ mrDesc = self.config['servicedesc']['mapred'] = ServiceDesc(self.config['gridservice-mapred'])
+
+ # API : serviceObj = service(desc, workDirs, reqNodes, version)
+ self.hdfs = Hdfs(hdfsDesc, [], 0, 17)
+ self.hdfsExternal = HdfsExternal(hdfsDesc, [], 17)
+ self.mr = MapReduce(mrDesc, [],1, 17)
+ self.mrExternal = MapReduceExternal(mrDesc, [], 17)
+
+ self.log = logging.getLogger()
+ pass
+
+ # All testMethods have to have their names start with 'test'
+ def testBothInternal(self):
+ self.serviceDict = {}
+ self.serviceDict[self.hdfs.getName()] = self.hdfs
+ self.serviceDict[self.mr.getName()] = self.mr
+ self.rpcSet = _LogMasterSources(self.serviceDict, self.config, None, self.log, None)
+
+ cmdList = self.rpcSet.getCommand('localhost')
+ self.assertEquals(cmdList.__len__(), 2)
+ self.assertEquals(cmdList[0].dict['argv'][0], 'namenode')
+ self.assertEquals(cmdList[1].dict['argv'][0], 'namenode')
+ pass
+
+ def tearDown(self):
+ pass
+
+class RingmasterRPCsTestSuite(BaseTestSuite):
+ def __init__(self):
+ # suite setup
+ BaseTestSuite.__init__(self, __name__, excludes)
+ pass
+
+ def cleanUp(self):
+ # suite tearDown
+ pass
+
+def RunRingmasterRPCsTests():
+ # modulename_suite
+ suite = RingmasterRPCsTestSuite()
+ testResult = suite.runTests()
+ suite.cleanUp()
+ return testResult
+
+if __name__ == "__main__":
+ RunRingmasterRPCsTests()
Added: hadoop/core/trunk/src/contrib/hod/testing/testXmlrpc.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/testing/testXmlrpc.py?rev=636995&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/testing/testXmlrpc.py (added)
+++ hadoop/core/trunk/src/contrib/hod/testing/testXmlrpc.py Thu Mar 13 22:14:54 2008
@@ -0,0 +1,99 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re, threading, time
+
+myDirectory = os.path.realpath(sys.argv[0])
+rootDirectory = re.sub("/testing/.*", "", myDirectory)
+
+sys.path.append(rootDirectory)
+
+from hodlib.Common.xmlrpc import hodXRClient
+from hodlib.Common.socketServers import hodXMLRPCServer
+
+from testing.lib import BaseTestSuite
+
+excludes = []
+
+class test_HodXRClient(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ # All testMethods have to have their names start with 'test'
+ def testSuccess(self):
+ client = hodXRClient('http://localhost:5555', retryRequests=False)
+ self.assertEqual(client.testing(), True)
+ pass
+
+ def testFailure(self):
+ client = hodXRClient('http://localhost:5555', retryRequests=False)
+ # client.noMethod()
+ self.assertRaises(Exception, client.noMethod)
+ pass
+
+ def testTimeout(self):
+ """HOD should raise Exception when rpc call times out"""
+ client = hodXRClient('http://localhost:567823', retryRequests=False)
+ # client.noMethod()
+ self.assertRaises(Exception, client.testing)
+ pass
+
+ def testInterrupt(self):
+ """ HOD should raise HodInterruptException when interrupted"""
+ from hodlib.Common.util import hodInterrupt, HodInterruptException
+
+ def interrupt():
+ client = hodXRClient('http://localhost:59087')
+ thread = threading.Thread(name='testinterrupt', target=client.testing)
+ thread.start()
+ time.sleep(1)
+ hodInterrupt.setFlag()
+ thread.join()
+
+ self.assertRaises(HodInterruptException, interrupt)
+ pass
+
+ def tearDown(self):
+ pass
+
+class XmlrpcTestSuite(BaseTestSuite):
+ def __init__(self):
+ # suite setup
+ BaseTestSuite.__init__(self, __name__, excludes)
+
+ def rpcCall():
+ return True
+
+ self.server = hodXMLRPCServer('localhost', ['5555'])
+ self.server.register_function(rpcCall, 'testing')
+ self.thread = threading.Thread(name="server",
+ target=self.server._serve_forever)
+ self.thread.start()
+ time.sleep(1) # give some time to start server
+
+ def cleanUp(self):
+ # suite tearDown
+ self.server.stop()
+ self.thread.join()
+
+def RunXmlrpcTests():
+ # modulename_suite
+ suite = XmlrpcTestSuite()
+ testResult = suite.runTests()
+ suite.cleanUp()
+ return testResult
+
+if __name__ == "__main__":
+ RunXmlrpcTests()