You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yh...@apache.org on 2008/03/28 09:40:23 UTC
svn commit: r642139 - in /hadoop/core/trunk/src/contrib/hod:
hodlib/GridServices/hdfs.py hodlib/GridServices/mapred.py
hodlib/Hod/hadoop.py testing/testHadoop.py
Author: yhemanth
Date: Fri Mar 28 01:40:08 2008
New Revision: 642139
URL: http://svn.apache.org/viewvc?rev=642139&view=rev
Log:
HADOOP-3103. Sets hadoop.tmp.dir to path under hod.temp-dir instead of cluster directory. Contributed by Vinod Kumar Vavilapalli.
Modified:
hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py
hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/mapred.py
hadoop/core/trunk/src/contrib/hod/hodlib/Hod/hadoop.py
hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py
Modified: hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py?rev=642139&r1=642138&r2=642139&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py (original)
+++ hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py Fri Mar 28 01:40:08 2008
@@ -149,6 +149,7 @@
def _setWorkDirs(self, workDirs, envs, attrs, parentDirs, subDir):
namedir = None
+ hadooptmpdir = None
datadir = []
for p in parentDirs:
@@ -156,6 +157,9 @@
workDirs.append(os.path.join(p, subDir))
dir = os.path.join(p, subDir, 'dfs-data')
datadir.append(dir)
+ if not hadooptmpdir:
+ # Not used currently, generating hadooptmpdir just in case
+ hadooptmpdir = os.path.join(p, subDir, 'hadoop-tmp')
if not namedir:
namedir = os.path.join(p, subDir, 'dfs-name')
@@ -165,6 +169,7 @@
# FIXME!! use csv
attrs['dfs.name.dir'] = namedir
+ attrs['hadoop.tmp.dir'] = hadooptmpdir
attrs['dfs.data.dir'] = ','.join(datadir)
# FIXME -- change dfs.client.buffer.dir
envs['HADOOP_ROOT_LOGGER'] = "INFO,DRFA"
Modified: hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/mapred.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/mapred.py?rev=642139&r1=642138&r2=642139&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/mapred.py (original)
+++ hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/mapred.py Fri Mar 28 01:40:08 2008
@@ -153,6 +153,7 @@
local = []
system = None
temp = None
+ hadooptmpdir = None
dfsclient = []
for p in parentDirs:
@@ -164,6 +165,9 @@
system = os.path.join(p, subDir, 'mapred-system')
if not temp:
temp = os.path.join(p, subDir, 'mapred-temp')
+ if not hadooptmpdir:
+ # Not used currently, generating hadooptmpdir just in case
+ hadooptmpdir = os.path.join(p, subDir, 'hadoop-tmp')
dfsclientdir = os.path.join(p, subDir, 'dfs-client')
dfsclient.append(dfsclientdir)
workDirs.append(dfsclientdir)
@@ -172,6 +176,7 @@
attrs['mapred.system.dir'] = 'fillindir'
attrs['mapred.temp.dir'] = temp
attrs['dfs.client.buffer.dir'] = ','.join(dfsclient)
+ attrs['hadoop.tmp.dir'] = hadooptmpdir
envs['HADOOP_ROOT_LOGGER'] = "INFO,DRFA"
Modified: hadoop/core/trunk/src/contrib/hod/hodlib/Hod/hadoop.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/hodlib/Hod/hadoop.py?rev=642139&r1=642138&r2=642139&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/hodlib/Hod/hadoop.py (original)
+++ hadoop/core/trunk/src/contrib/hod/hodlib/Hod/hadoop.py Fri Mar 28 01:40:08 2008
@@ -73,8 +73,10 @@
description = {}
paramsDict = { 'mapred.job.tracker' : mapredAddr , \
'fs.default.name' : "hdfs://" + hdfsAddr, \
- 'hadoop.tmp.dir' : confDir, \
- 'dfs.client.buffer.dir' : tempDir, }
+ 'hadoop.tmp.dir' : tempDir, \
+ 'dfs.client.buffer.dir' : os.path.join(tempDir, 'dfs',
+ 'tmp'),
+ }
paramsDict['mapred.system.dir'] = mrSysDir
Modified: hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py?rev=642139&r1=642138&r2=642139&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py (original)
+++ hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py Fri Mar 28 01:40:08 2008
@@ -92,6 +92,9 @@
# fs.default.name should start with hdfs://
assert(keyvals['fs.default.name'].startswith('hdfs://'))
+ assert(keyvals['hadoop.tmp.dir'] == self.tempDir)
+ assert(keyvals['dfs.client.buffer.dir'] == os.path.join(self.tempDir,
+ 'dfs', 'tmp'))
# TODO other tests
pass