You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ni...@apache.org on 2008/03/05 17:36:43 UTC
svn commit: r633918 - in /hadoop/core/branches/branch-0.16: ./
src/contrib/hod/bin/ src/contrib/hod/hodlib/Common/
src/contrib/hod/hodlib/Hod/ src/contrib/hod/hodlib/HodRing/
src/contrib/hod/support/ src/docs/src/documentation/content/xdocs/
Author: nigel
Date: Wed Mar 5 08:36:36 2008
New Revision: 633918
URL: http://svn.apache.org/viewvc?rev=633918&view=rev
Log:
Merge of -r 633914:633915 from trunk to branch 0.16 to fix HADOOP-2925.
Modified:
hadoop/core/branches/branch-0.16/CHANGES.txt
hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hod
hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodcleanup
hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodring
hadoop/core/branches/branch-0.16/src/contrib/hod/bin/ringmaster
hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/util.py
hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Hod/hadoop.py
hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/HodRing/hodRing.py
hadoop/core/branches/branch-0.16/src/contrib/hod/support/logcondense.py
hadoop/core/branches/branch-0.16/src/docs/src/documentation/content/xdocs/hod_config_guide.xml
Modified: hadoop/core/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/CHANGES.txt?rev=633918&r1=633917&r2=633918&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.16/CHANGES.txt Wed Mar 5 08:36:36 2008
@@ -113,6 +113,10 @@
datnodes flushes the block file buffered output stream before
sending a positive ack for the packet back to the client. (dhruba)
+ HADOOP-2925. Fix HOD to create the mapred system directory using a
+ naming convention that will avoid clashes in multi-user shared
+ cluster scenario. (Hemanth Yamijala via nigel)
+
Release 0.16.0 - 2008-02-07
INCOMPATIBLE CHANGES
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hod
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hod?rev=633918&r1=633917&r2=633918&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hod (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hod Wed Mar 5 08:36:36 2008
@@ -346,7 +346,10 @@
False, 2, False, True),
('cmd-retry-interval', 'pos_float','interval to spread retries for getting commands',
- False, 2, False, True))
+ False, 2, False, True),
+
+ ('mapred-system-dir-root', 'string', 'Root under which mapreduce system directory names are generated by HOD.',
+ False, '/mapredsystem', False, False))
}
defOrder = [ 'hod', 'ringmaster', 'hodring', 'resource_manager',
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodcleanup
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodcleanup?rev=633918&r1=633917&r2=633918&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodcleanup (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodcleanup Wed Mar 5 08:36:36 2008
@@ -123,7 +123,7 @@
# END LAME HACK
(head, tail) = os.path.split(archiveFile)
- destFile = os.path.join(hdfsURIMatch.group(2), conf['user-id'], conf['service-id'], tail)
+ destFile = os.path.join(hdfsURIMatch.group(2), conf['user-id'], 'hod-logs', conf['service-id'], tail)
log.info("copying archive %s to DFS %s ..." % (archiveFile, destFile))
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodring
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodring?rev=633918&r1=633917&r2=633918&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodring (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodring Wed Mar 5 08:36:36 2008
@@ -130,7 +130,10 @@
False, 2, False, True),
('cmd-retry-interval', 'pos_float','interval to spread retries for getting commands',
- False, 2, False, True))
+ False, 2, False, True),
+
+ ('mapred-system-dir-root', 'string', 'Root under which mapreduce system directory names are generated by HOD.',
+ False, '/mapredsystem', False, False))
}
if __name__ == '__main__':
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/bin/ringmaster
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/bin/ringmaster?rev=633918&r1=633917&r2=633918&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/bin/ringmaster (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/bin/ringmaster Wed Mar 5 08:36:36 2008
@@ -261,8 +261,10 @@
False, 2, False, True),
('cmd-retry-interval', 'pos_float','interval to spread retries for getting commands',
- False, 2, False, True))
-
+ False, 2, False, True),
+
+ ('mapred-system-dir-root', 'string', 'Root under which mapreduce system directory names are generated by HOD.',
+ False, '/mapredsystem', False, False))
}
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/util.py
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/util.py?rev=633918&r1=633917&r2=633918&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/util.py (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/util.py Wed Mar 5 08:36:36 2008
@@ -250,6 +250,9 @@
dict[splits[0]] = splits[1]
return dict
+def getMapredSystemDirectory(mrSysDirRoot, userid, jobid):
+ return os.path.join(mrSysDirRoot, userid, 'mapredsystem', jobid)
+
class HodInterrupt:
def __init__(self):
self.HodInterruptFlag = False
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Hod/hadoop.py
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Hod/hadoop.py?rev=633918&r1=633917&r2=633918&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Hod/hadoop.py (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Hod/hadoop.py Wed Mar 5 08:36:36 2008
@@ -57,7 +57,7 @@
return prop
- def gen_site_conf(self, confDir, tempDir, numNodes, hdfsAddr,\
+ def gen_site_conf(self, confDir, tempDir, numNodes, hdfsAddr, mrSysDir,\
mapredAddr=None, clientParams=None, serverParams=None,\
finalServerParams=None, clusterFactor=None):
if not mapredAddr:
@@ -76,9 +76,7 @@
'hadoop.tmp.dir' : confDir, \
'dfs.client.buffer.dir' : tempDir, }
- mapredAddrSplit = mapredAddr.split(":")
- mapredsystem = os.path.join('/mapredsystem', mapredAddrSplit[0])
- paramsDict['mapred.system.dir'] = mapredsystem
+ paramsDict['mapred.system.dir'] = mrSysDir
# mapred-default.xml is no longer used now.
numred = int(math.floor(clusterFactor * (int(numNodes) - 1)))
@@ -515,8 +513,10 @@
os.makedirs(tempDir)
tempDir = os.path.join( tempDir, self.__cfg['hod']['userid']\
+ "." + self.jobId )
+ mrSysDir = getMapredSystemDirectory(self.__cfg['hodring']['mapred-system-dir-root'],\
+ self.__cfg['hod']['userid'], self.jobId)
self.__hadoopCfg.gen_site_conf(clusterDir, tempDir, min,\
- hdfsAddr, mapredAddr, clientParams,\
+ hdfsAddr, mrSysDir, mapredAddr, clientParams,\
serverParams, finalServerParams,\
clusterFactor)
self.__log.info("hadoop-site.xml at %s" % clusterDir)
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/HodRing/hodRing.py
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/HodRing/hodRing.py?rev=633918&r1=633917&r2=633918&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/HodRing/hodRing.py (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/HodRing/hodRing.py Wed Mar 5 08:36:36 2008
@@ -25,7 +25,7 @@
from pprint import pformat
from optparse import OptionParser
from urlparse import urlparse
-from hodlib.Common.util import local_fqdn, parseEquals
+from hodlib.Common.util import local_fqdn, parseEquals, getMapredSystemDirectory
from hodlib.Common.tcp import tcpSocket, tcpError
binfile = sys.path[0]
@@ -150,10 +150,12 @@
class HadoopCommand:
"""Runs a single hadoop command"""
- def __init__(self, id, desc, tempdir, tardir, log, javahome, restart=False):
+ def __init__(self, id, desc, tempdir, tardir, log, javahome,
+ mrSysDir, restart=False):
self.desc = desc
self.log = log
self.javahome = javahome
+ self.__mrSysDir = mrSysDir
self.program = desc.getProgram()
self.name = desc.getName()
self.workdirs = desc.getWorkDirs()
@@ -218,7 +220,7 @@
self.filledInKeyVals.append(keyvalpair)
if ( v == "fillindir"):
- v = os.path.join('/mapredsystem', local_fqdn())
+ v = self.__mrSysDir
pass
prop = None
@@ -509,8 +511,11 @@
id = 0
for desc in self._cfg['commanddesc']:
self.log.debug(pprint.pformat(desc.dict))
+ mrSysDir = getMapredSystemDirectory(self._cfg['mapred-system-dir-root'],
+ self._cfg['userid'], self._cfg['service-id'])
+ self.log.debug('mrsysdir is %s' % mrSysDir)
cmd = HadoopCommand(id, desc, self.__tempDir, self.__pkgDir, self.log,
- self._cfg['java-home'], restart)
+ self._cfg['java-home'], mrSysDir, restart)
self.__hadoopLogDirs.append(cmd.logdir)
self.log.debug("hadoop log directory: %s" % self.__hadoopLogDirs)
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/support/logcondense.py
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/support/logcondense.py?rev=633918&r1=633917&r2=633918&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/support/logcondense.py (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/support/logcondense.py Wed Mar 5 08:36:36 2008
@@ -125,7 +125,7 @@
for line in stdout:
m = re.match("^(.*?)\s.*$", line)
filename = m.group(1)
- # file name format: <prefix>/<user>/<jobid>/[0-1]-[jobtracker|tasktracker|datanode|namenode|]-hostname-YYYYMMDDtime-random.tar.gz
+ # file name format: <prefix>/<user>/hod-logs/<jobid>/[0-1]-[jobtracker|tasktracker|datanode|namenode|]-hostname-YYYYMMDDtime-random.tar.gz
# first strip prefix:
if filename.startswith(options.log):
filename = filename.lstrip(options.log)
@@ -135,7 +135,7 @@
continue
# Now get other details from filename.
- k = re.match("/(.*)/(.*)/.*-.*-([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9]).*$", filename)
+ k = re.match("/(.*)/.*/(.*)/.*-.*-([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9]).*$", filename)
if k:
username = k.group(1)
jobid = k.group(2)
@@ -159,7 +159,7 @@
if (diff.days > options.days):
desttodel = filename
if not toPurge.has_key(jobid):
- toPurge[jobid] = options.log.rstrip("/") + "/" + username + "/" + jobid
+ toPurge[jobid] = options.log.rstrip("/") + "/" + username + "/hod-logs/" + jobid
for job in toPurge.keys():
for prefix in deletedNamePrefixes:
Modified: hadoop/core/branches/branch-0.16/src/docs/src/documentation/content/xdocs/hod_config_guide.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/docs/src/documentation/content/xdocs/hod_config_guide.xml?rev=633918&r1=633917&r2=633918&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/docs/src/documentation/content/xdocs/hod_config_guide.xml (original)
+++ hadoop/core/branches/branch-0.16/src/docs/src/documentation/content/xdocs/hod_config_guide.xml Wed Mar 5 08:36:36 2008
@@ -85,6 +85,10 @@
be picked for use to run an HTTP server.</li>
<li>java-home: Location of Java to be used by Hadoop.</li>
+ <li>syslog-address: Address to which a syslog daemon is bound to. The format
+ of the value is host:port. If configured, HOD log messages
+ will be logged to syslog using this value.</li>
+
</ul>
</section>
@@ -200,6 +204,47 @@
JobTracker and TaskTrackers</li>
<li>final-server-params: Same as above, except they will be marked final.</li>
+ </ul>
+ </section>
+
+ <section>
+ <title>3.7 hodring options</title>
+
+ <ul>
+ <li>mapred-system-dir-root: Directory in the DFS under which HOD will
+ generate sub-directory names and pass the full path
+ as the value of the 'mapred.system.dir' configuration
+ parameter to Hadoop daemons. The format of the full
+ path will be value-of-this-option/userid/mapredsystem/cluster-id.
+ Note that the directory specified here should be such
+ that all users can create directories under this, if
+ permissions are enabled in HDFS. Setting the value of
+ this option to /user will make HOD use the user's
+ home directory to generate the mapred.system.dir value.</li>
+
+ <li>log-destination-uri: URL describing a path in an external, static DFS or the
+ cluster node's local file system where HOD will upload
+ Hadoop logs when a cluster is deallocated. To specify a
+ DFS path, use the format 'hdfs://path'. To specify a
+ cluster node's local file path, use the format 'file://path'.
+
+ When clusters are deallocated by HOD, the hadoop logs will
+ be deleted as part of HOD's cleanup process. In order to
+ persist these logs, you can use this configuration option.
+
+ The format of the path is
+ value-of-this-option/userid/hod-logs/cluster-id
+
+ Note that the directory you specify here must be such that all
+ users can create sub-directories under this. Setting this value
+ to hdfs://user will make the logs come in the user's home directory
+ in DFS.</li>
+
+ <li>pkgs: Installation directory, under which bin/hadoop executable is located. This will
+ be used by HOD to upload logs if a HDFS URL is specified in log-destination-uri
+ option. Note that this is useful if the users are using a tarball whose version
+ may differ from the external, static HDFS version.</li>
+
</ul>
</section>
</section>