You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/03/17 21:21:54 UTC
svn commit: r1082677 [17/38] - in /hadoop/mapreduce/branches/MR-279: ./
assembly/ ivy/ mr-client/ mr-client/hadoop-mapreduce-client-app/
mr-client/hadoop-mapreduce-client-app/src/
mr-client/hadoop-mapreduce-client-app/src/main/ mr-client/hadoop-mapredu...
Modified: hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/TaskTracker.java Thu Mar 17 20:21:13 2011
@@ -99,6 +99,7 @@ import org.apache.hadoop.mapreduce.util.
import org.apache.hadoop.mapreduce.util.MemoryCalculatorPlugin;
import org.apache.hadoop.mapreduce.util.ProcfsBasedProcessTree;
import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.mapreduce.util.ServerConfigUtil;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
@@ -145,7 +146,7 @@ public class TaskTracker
static enum State {NORMAL, STALE, INTERRUPTED, DENIED}
static{
- ConfigUtil.loadResources();
+ ServerConfigUtil.loadResources();
}
public static final Log LOG =
@@ -243,7 +244,6 @@ public class TaskTracker
static final String JARSDIR = "jars";
static final String LOCAL_SPLIT_FILE = "split.dta";
static final String LOCAL_SPLIT_META_FILE = "split.info";
- static final String JOBFILE = "job.xml";
static final String JOB_TOKEN_FILE="jobToken"; //localized file
static final String TT_PRIVATE_DIR = "ttprivate";
@@ -485,7 +485,7 @@ public class TaskTracker
}
static String getLocalJobConfFile(String user, String jobid) {
- return getLocalJobDir(user, jobid) + Path.SEPARATOR + TaskTracker.JOBFILE;
+ return getLocalJobDir(user, jobid) + Path.SEPARATOR + Constants.JOBFILE;
}
static String getPrivateDirJobConfFile(String user, String jobid) {
@@ -500,7 +500,7 @@ public class TaskTracker
static String getTaskConfFile(String user, String jobid, String taskid,
boolean isCleanupAttempt) {
return getLocalTaskDir(user, jobid, taskid, isCleanupAttempt)
- + Path.SEPARATOR + TaskTracker.JOBFILE;
+ + Path.SEPARATOR + Constants.JOBFILE;
}
static String getPrivateDirTaskScriptLocation(String user, String jobid,
@@ -750,7 +750,7 @@ public class TaskTracker
mapEventsFetcher.start();
Class<? extends ResourceCalculatorPlugin> clazz =
- fConf.getClass(TT_RESOURCE_CALCULATOR_PLUGIN,
+ fConf.getClass(RESOURCE_CALCULATOR_PLUGIN,
null, ResourceCalculatorPlugin.class);
resourceCalculatorPlugin = ResourceCalculatorPlugin
.getResourceCalculatorPlugin(clazz, fConf);
@@ -1398,7 +1398,7 @@ public class TaskTracker
fConf = conf;
maxMapSlots = conf.getInt(TT_MAP_SLOTS, 2);
maxReduceSlots = conf.getInt(TT_REDUCE_SLOTS, 2);
- this.jobTrackAddr = JobTracker.getAddress(conf);
+ this.jobTrackAddr = Master.getMasterAddress(conf);
InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(
conf.get(TT_HTTP_ADDRESS, "0.0.0.0:50060"));
String httpBindAddress = infoSocAddr.getHostName();
@@ -2165,7 +2165,7 @@ public class TaskTracker
}
private void addToTaskQueue(LaunchTaskAction action) {
- if (action.getTask().isMapTask()) {
+ if (action.getTask().getTask().isMapTask()) {
mapLauncher.addToTaskQueue(action);
} else {
reduceLauncher.addToTaskQueue(action);
@@ -2301,10 +2301,11 @@ public class TaskTracker
}
private TaskInProgress registerTask(LaunchTaskAction action,
TaskLauncher launcher) {
- Task t = action.getTask();
+ TTTask ttTask = action.getTask();
+ Task t = ttTask.getTask();
LOG.info("LaunchTaskAction (registerTask): " + t.getTaskID() +
" task's state:" + t.getState());
- TaskInProgress tip = new TaskInProgress(t, this.fConf, launcher);
+ TaskInProgress tip = new TaskInProgress(ttTask, this.fConf, launcher);
synchronized (this) {
tasks.put(t.getTaskID(), tip);
runningTasks.put(t.getTaskID(), tip);
@@ -2455,6 +2456,7 @@ public class TaskTracker
// its TaskStatus, and the TaskRunner.
///////////////////////////////////////////////////////
class TaskInProgress {
+ final TTTask ttTask;
Task task;
long lastProgressReport;
StringBuffer diagnosticInfo = new StringBuffer();
@@ -2485,12 +2487,13 @@ public class TaskTracker
/**
*/
- public TaskInProgress(Task task, JobConf conf) {
- this(task, conf, null);
+ public TaskInProgress(TTTask ttTask, JobConf conf) {
+ this(ttTask, conf, null);
}
- public TaskInProgress(Task task, JobConf conf, TaskLauncher launcher) {
- this.task = task;
+ public TaskInProgress(TTTask ttTask, JobConf conf, TaskLauncher launcher) {
+ this.ttTask = ttTask;
+ this.task = ttTask.getTask();
this.launcher = launcher;
this.lastProgressReport = System.currentTimeMillis();
this.ttConf = conf;
@@ -2585,7 +2588,7 @@ public class TaskTracker
if (this.taskStatus.getRunState() == TaskStatus.State.UNASSIGNED) {
this.taskStatus.setRunState(TaskStatus.State.RUNNING);
}
- setTaskRunner(task.createRunner(TaskTracker.this, this, rjob));
+ setTaskRunner(ttTask.createRunner(TaskTracker.this, this, rjob));
this.runner.start();
this.taskStatus.setStartTime(System.currentTimeMillis());
} else {
Modified: hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java Thu Mar 17 20:21:13 2011
@@ -29,6 +29,8 @@ import org.apache.hadoop.ipc.RemoteExcep
import org.apache.hadoop.mapred.AdminOperationsProtocol;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapred.Master;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.UserGroupInformation;
@@ -151,14 +153,14 @@ public class MRAdmin extends Configured
// should be JT's one.
JobConf jConf = new JobConf(conf);
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY,
- jConf.get(JobTracker.JT_USER_NAME, ""));
+ jConf.get(MRConfig.MASTER_USER_NAME, ""));
// Create the client
RefreshAuthorizationPolicyProtocol refreshProtocol =
(RefreshAuthorizationPolicyProtocol)
RPC.getProxy(RefreshAuthorizationPolicyProtocol.class,
RefreshAuthorizationPolicyProtocol.versionID,
- JobTracker.getAddress(conf), getUGI(conf), conf,
+ Master.getMasterAddress(conf), getUGI(conf), conf,
NetUtils.getSocketFactory(conf,
RefreshAuthorizationPolicyProtocol.class));
@@ -183,14 +185,14 @@ public class MRAdmin extends Configured
// should be JT's one.
JobConf jConf = new JobConf(conf);
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY,
- jConf.get(JobTracker.JT_USER_NAME, ""));
+ jConf.get(MRConfig.MASTER_USER_NAME, ""));
// Create the client
RefreshUserMappingsProtocol refreshProtocol =
(RefreshUserMappingsProtocol)
RPC.getProxy(RefreshUserMappingsProtocol.class,
RefreshUserMappingsProtocol.versionID,
- JobTracker.getAddress(conf), getUGI(conf), conf,
+ Master.getMasterAddress(conf), getUGI(conf), conf,
NetUtils.getSocketFactory(conf,
RefreshUserMappingsProtocol.class));
@@ -215,14 +217,14 @@ public class MRAdmin extends Configured
// should be JT's one.
JobConf jConf = new JobConf(conf);
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY,
- jConf.get(JobTracker.JT_USER_NAME, ""));
+ jConf.get(MRConfig.MASTER_USER_NAME, ""));
// Create the client
RefreshUserMappingsProtocol refreshProtocol =
(RefreshUserMappingsProtocol)
RPC.getProxy(RefreshUserMappingsProtocol.class,
RefreshUserMappingsProtocol.versionID,
- JobTracker.getAddress(conf), getUGI(conf), conf,
+ Master.getMasterAddress(conf), getUGI(conf), conf,
NetUtils.getSocketFactory(conf,
RefreshUserMappingsProtocol.class));
@@ -241,7 +243,7 @@ public class MRAdmin extends Configured
(AdminOperationsProtocol)
RPC.getProxy(AdminOperationsProtocol.class,
AdminOperationsProtocol.versionID,
- JobTracker.getAddress(conf), getUGI(conf), conf,
+ Master.getMasterAddress(conf), getUGI(conf), conf,
NetUtils.getSocketFactory(conf,
AdminOperationsProtocol.class));
@@ -266,7 +268,7 @@ public class MRAdmin extends Configured
(AdminOperationsProtocol)
RPC.getProxy(AdminOperationsProtocol.class,
AdminOperationsProtocol.versionID,
- JobTracker.getAddress(conf), getUGI(conf), conf,
+ Master.getMasterAddress(conf), getUGI(conf), conf,
NetUtils.getSocketFactory(conf,
AdminOperationsProtocol.class));
Modified: hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/filecache/TaskDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/filecache/TaskDistributedCacheManager.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/filecache/TaskDistributedCacheManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/filecache/TaskDistributedCacheManager.java Thu Mar 17 20:21:13 2011
@@ -151,13 +151,13 @@ public class TaskDistributedCacheManager
this.cacheFiles.addAll(
CacheFile.makeCacheFiles(DistributedCache.getCacheFiles(taskConf),
DistributedCache.getFileTimestamps(taskConf),
- TrackerDistributedCacheManager.getFileVisibilities(taskConf),
+ DistributedCache.getFileVisibilities(taskConf),
DistributedCache.getFileClassPaths(taskConf),
CacheFile.FileType.REGULAR));
this.cacheFiles.addAll(
CacheFile.makeCacheFiles(DistributedCache.getCacheArchives(taskConf),
DistributedCache.getArchiveTimestamps(taskConf),
- TrackerDistributedCacheManager.getArchiveVisibilities(taskConf),
+ DistributedCache.getArchiveVisibilities(taskConf),
DistributedCache.getArchiveClassPaths(taskConf),
CacheFile.FileType.ARCHIVE));
}
@@ -215,13 +215,13 @@ public class TaskDistributedCacheManager
if (!localArchives.isEmpty()) {
// TODO verify
// DistributedCache.addLocalArchives(taskConf,
- TrackerDistributedCacheManager.setLocalArchives(taskConf,
+ DistributedCache.setLocalArchives(taskConf,
stringifyPathList(localArchives));
}
if (!localFiles.isEmpty()) {
// TODO verify
// DistributedCache.addLocalFiles(taskConf, stringifyPathList(localFiles));
- TrackerDistributedCacheManager.setLocalFiles(taskConf,
+ DistributedCache.setLocalFiles(taskConf,
stringifyPathList(localFiles));
}
Modified: hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/filecache/TrackerDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/filecache/TrackerDistributedCacheManager.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/filecache/TrackerDistributedCacheManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/filecache/TrackerDistributedCacheManager.java Thu Mar 17 20:21:13 2011
@@ -332,22 +332,6 @@ public class TrackerDistributedCacheMana
}
/**
- * Returns {@link FileStatus} of a given cache file on hdfs.
- *
- * @param conf configuration
- * @param cache cache file
- * @return {@link FileStatus} of a given cache file on hdfs
- * @throws IOException
- */
- static FileStatus getFileStatus(Configuration conf, URI cache)
- throws IOException {
- FileSystem fileSystem = FileSystem.get(cache, conf);
- Path filePath = new Path(cache.getPath());
-
- return fileSystem.getFileStatus(filePath);
- }
-
- /**
* Returns mtime of a given cache file on hdfs.
*
* @param conf configuration
@@ -357,52 +341,7 @@ public class TrackerDistributedCacheMana
*/
static long getTimestamp(Configuration conf, URI cache)
throws IOException {
- return getFileStatus(conf, cache).getModificationTime();
- }
-
- /**
- * Returns a boolean to denote whether a cache file is visible to all(public)
- * or not
- * @param conf
- * @param uri
- * @return true if the path in the uri is visible to all, false otherwise
- * @throws IOException
- */
- static boolean isPublic(Configuration conf, URI uri) throws IOException {
- FileSystem fs = FileSystem.get(uri, conf);
- Path current = new Path(uri.getPath());
- //the leaf level file should be readable by others
- if (!checkPermissionOfOther(fs, current, FsAction.READ)) {
- return false;
- }
- current = current.getParent();
- while (current != null) {
- //the subdirs in the path should have execute permissions for others
- if (!checkPermissionOfOther(fs, current, FsAction.EXECUTE)) {
- return false;
- }
- current = current.getParent();
- }
- return true;
- }
- /**
- * Checks for a given path whether the Other permissions on it
- * imply the permission in the passed FsAction
- * @param fs
- * @param path
- * @param action
- * @return true if the path in the uri is visible to all, false otherwise
- * @throws IOException
- */
- private static boolean checkPermissionOfOther(FileSystem fs, Path path,
- FsAction action) throws IOException {
- FileStatus status = fs.getFileStatus(path);
- FsPermission perms = status.getPermission();
- FsAction otherAction = perms.getOtherAction();
- if (otherAction.implies(action)) {
- return true;
- }
- return false;
+ return DistributedCache.getFileStatus(conf, cache).getModificationTime();
}
private Path checkCacheStatusValidity(Configuration conf,
@@ -724,261 +663,6 @@ public class TrackerDistributedCacheMana
}
/**
- * Determines timestamps of files to be cached, and stores those
- * in the configuration. This is intended to be used internally by JobClient
- * after all cache files have been added.
- *
- * This is an internal method!
- *
- * @param job Configuration of a job.
- * @throws IOException
- */
- public static void determineTimestamps(Configuration job) throws IOException {
- URI[] tarchives = DistributedCache.getCacheArchives(job);
- if (tarchives != null) {
- FileStatus status = getFileStatus(job, tarchives[0]);
- StringBuilder archiveFileSizes =
- new StringBuilder(String.valueOf(status.getLen()));
- StringBuilder archiveTimestamps =
- new StringBuilder(String.valueOf(status.getModificationTime()));
- for (int i = 1; i < tarchives.length; i++) {
- status = getFileStatus(job, tarchives[i]);
- archiveFileSizes.append(",");
- archiveFileSizes.append(String.valueOf(status.getLen()));
- archiveTimestamps.append(",");
- archiveTimestamps.append(String.valueOf(status.getModificationTime()));
- }
- job.set(MRJobConfig.CACHE_ARCHIVES_SIZES, archiveFileSizes.toString());
- setArchiveTimestamps(job, archiveTimestamps.toString());
- }
-
- URI[] tfiles = DistributedCache.getCacheFiles(job);
- if (tfiles != null) {
- FileStatus status = getFileStatus(job, tfiles[0]);
- StringBuilder fileSizes =
- new StringBuilder(String.valueOf(status.getLen()));
- StringBuilder fileTimestamps = new StringBuilder(String.valueOf(
- status.getModificationTime()));
- for (int i = 1; i < tfiles.length; i++) {
- status = getFileStatus(job, tfiles[i]);
- fileSizes.append(",");
- fileSizes.append(String.valueOf(status.getLen()));
- fileTimestamps.append(",");
- fileTimestamps.append(String.valueOf(status.getModificationTime()));
- }
- job.set(MRJobConfig.CACHE_FILES_SIZES, fileSizes.toString());
- setFileTimestamps(job, fileTimestamps.toString());
- }
- }
-
- /**
- * For each archive or cache file - get the corresponding delegation token
- * @param job
- * @param credentials
- * @throws IOException
- */
- public static void getDelegationTokens(Configuration job,
- Credentials credentials) throws IOException {
- URI[] tarchives = DistributedCache.getCacheArchives(job);
- URI[] tfiles = DistributedCache.getCacheFiles(job);
-
- int size = (tarchives!=null? tarchives.length : 0) + (tfiles!=null ? tfiles.length :0);
- Path[] ps = new Path[size];
-
- int i = 0;
- if (tarchives != null) {
- for (i=0; i < tarchives.length; i++) {
- ps[i] = new Path(tarchives[i].toString());
- }
- }
-
- if (tfiles != null) {
- for(int j=0; j< tfiles.length; j++) {
- ps[i+j] = new Path(tfiles[j].toString());
- }
- }
-
- TokenCache.obtainTokensForNamenodes(credentials, ps, job);
- }
-
- /**
- * Determines the visibilities of the distributed cache files and
- * archives. The visibility of a cache path is "public" if the leaf component
- * has READ permissions for others, and the parent subdirs have
- * EXECUTE permissions for others
- * @param job
- * @throws IOException
- */
- public static void determineCacheVisibilities(Configuration job)
- throws IOException {
- URI[] tarchives = DistributedCache.getCacheArchives(job);
- if (tarchives != null) {
- StringBuilder archiveVisibilities =
- new StringBuilder(String.valueOf(isPublic(job, tarchives[0])));
- for (int i = 1; i < tarchives.length; i++) {
- archiveVisibilities.append(",");
- archiveVisibilities.append(String.valueOf(isPublic(job, tarchives[i])));
- }
- setArchiveVisibilities(job, archiveVisibilities.toString());
- }
- URI[] tfiles = DistributedCache.getCacheFiles(job);
- if (tfiles != null) {
- StringBuilder fileVisibilities =
- new StringBuilder(String.valueOf(isPublic(job, tfiles[0])));
- for (int i = 1; i < tfiles.length; i++) {
- fileVisibilities.append(",");
- fileVisibilities.append(String.valueOf(isPublic(job, tfiles[i])));
- }
- setFileVisibilities(job, fileVisibilities.toString());
- }
- }
-
- private static boolean[] parseBooleans(String[] strs) {
- if (null == strs) {
- return null;
- }
- boolean[] result = new boolean[strs.length];
- for(int i=0; i < strs.length; ++i) {
- result[i] = Boolean.parseBoolean(strs[i]);
- }
- return result;
- }
-
- /**
- * Get the booleans on whether the files are public or not. Used by
- * internal DistributedCache and MapReduce code.
- * @param conf The configuration which stored the timestamps
- * @return a string array of booleans
- * @throws IOException
- */
- public static boolean[] getFileVisibilities(Configuration conf) {
- return parseBooleans(conf.getStrings(MRJobConfig.CACHE_FILE_VISIBILITIES));
- }
-
- /**
- * Get the booleans on whether the archives are public or not. Used by
- * internal DistributedCache and MapReduce code.
- * @param conf The configuration which stored the timestamps
- * @return a string array of booleans
- */
- public static boolean[] getArchiveVisibilities(Configuration conf) {
- return parseBooleans(conf.getStrings(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES));
- }
-
- /**
- * This method checks if there is a conflict in the fragment names
- * of the uris. Also makes sure that each uri has a fragment. It
- * is only to be called if you want to create symlinks for
- * the various archives and files. May be used by user code.
- * @param uriFiles The uri array of urifiles
- * @param uriArchives the uri array of uri archives
- */
- public static boolean checkURIs(URI[] uriFiles, URI[] uriArchives) {
- if ((uriFiles == null) && (uriArchives == null)) {
- return true;
- }
- // check if fragment is null for any uri
- // also check if there are any conflicts in fragment names
- Set<String> fragments = new HashSet<String>();
-
- // iterate over file uris
- if (uriFiles != null) {
- for (int i = 0; i < uriFiles.length; i++) {
- String fragment = uriFiles[i].getFragment();
- if (fragment == null) {
- return false;
- }
- String lowerCaseFragment = fragment.toLowerCase();
- if (fragments.contains(lowerCaseFragment)) {
- return false;
- }
- fragments.add(lowerCaseFragment);
- }
- }
-
- // iterate over archive uris
- if (uriArchives != null) {
- for (int i = 0; i < uriArchives.length; i++) {
- String fragment = uriArchives[i].getFragment();
- if (fragment == null) {
- return false;
- }
- String lowerCaseFragment = fragment.toLowerCase();
- if (fragments.contains(lowerCaseFragment)) {
- return false;
- }
- fragments.add(lowerCaseFragment);
- }
- }
- return true;
- }
-
- /**
- * This is to check the public/private visibility of the archives to be
- * localized.
- *
- * @param conf Configuration which stores the timestamp's
- * @param booleans comma separated list of booleans (true - public)
- * The order should be the same as the order in which the archives are added.
- */
- static void setArchiveVisibilities(Configuration conf, String booleans) {
- conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, booleans);
- }
-
- /**
- * This is to check the public/private visibility of the files to be localized
- *
- * @param conf Configuration which stores the timestamp's
- * @param booleans comma separated list of booleans (true - public)
- * The order should be the same as the order in which the files are added.
- */
- static void setFileVisibilities(Configuration conf, String booleans) {
- conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, booleans);
- }
-
- /**
- * This is to check the timestamp of the archives to be localized.
- *
- * @param conf Configuration which stores the timestamp's
- * @param timestamps comma separated list of timestamps of archives.
- * The order should be the same as the order in which the archives are added.
- */
- static void setArchiveTimestamps(Configuration conf, String timestamps) {
- conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, timestamps);
- }
-
- /**
- * This is to check the timestamp of the files to be localized.
- *
- * @param conf Configuration which stores the timestamp's
- * @param timestamps comma separated list of timestamps of files.
- * The order should be the same as the order in which the files are added.
- */
- static void setFileTimestamps(Configuration conf, String timestamps) {
- conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, timestamps);
- }
-
- /**
- * Set the conf to contain the location for localized archives.
- *
- * @param conf The conf to modify to contain the localized caches
- * @param str a comma separated list of local archives
- */
- static void setLocalArchives(Configuration conf, String str) {
- conf.set(MRJobConfig.CACHE_LOCALARCHIVES, str);
- }
-
- /**
- * Set the conf to contain the location for localized files.
- *
- * @param conf The conf to modify to contain the localized caches
- * @param str a comma separated list of local files
- */
- static void setLocalFiles(Configuration conf, String str) {
- conf.set(MRJobConfig.CACHE_LOCALFILES, str);
- }
-
- /**
* A thread to check and cleanup the unused files periodically
*/
private class CleanupThread extends Thread {
Modified: hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/server/jobtracker/JTConfig.java Thu Mar 17 20:21:13 2011
@@ -30,7 +30,11 @@ import org.apache.hadoop.mapreduce.MRCon
@InterfaceStability.Evolving
public interface JTConfig extends MRConfig {
// JobTracker configuration parameters
- public static final String JT_IPC_ADDRESS = "mapreduce.jobtracker.address";
+ /**
+ * deprecated Use {@link MRConfig#MASTER_ADDRESS} instead
+ */
+ @Deprecated
+ public static final String JT_IPC_ADDRESS = MASTER_ADDRESS;
public static final String JT_HTTP_ADDRESS =
"mapreduce.jobtracker.http.address";
public static final String JT_IPC_HANDLER_COUNT =
@@ -102,9 +106,6 @@ public interface JTConfig extends MRConf
"mapreduce.jobtracker.maxmapmemory.mb";
public static final String JT_MAX_REDUCEMEMORY_MB =
"mapreduce.jobtracker.maxreducememory.mb";
- public static final String JT_MAX_JOB_SPLIT_METAINFO_SIZE =
- "mapreduce.jobtracker.split.metainfo.maxsize";
- public static final String JT_USER_NAME = "mapreduce.jobtracker.kerberos.principal";
public static final String JT_KEYTAB_FILE =
"mapreduce.jobtracker.keytab.file";
}
Modified: hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java Thu Mar 17 20:21:13 2011
@@ -51,8 +51,6 @@ public interface TTConfig extends MRConf
"mapreduce.tasktracker.taskcontroller";
public static final String TT_CONTENTION_TRACKING =
"mapreduce.tasktracker.contention.tracking";
- public static final String TT_STATIC_RESOLUTIONS =
- "mapreduce.tasktracker.net.static.resolutions";
public static final String TT_HTTP_THREADS =
"mapreduce.tasktracker.http.threads";
public static final String TT_HOST_NAME = "mapreduce.tasktracker.host.name";
@@ -64,22 +62,22 @@ public interface TTConfig extends MRConf
"mapreduce.tasktracker.dns.nameserver";
public static final String TT_MAX_TASK_COMPLETION_EVENTS_TO_POLL =
"mapreduce.tasktracker.events.batchsize";
- public static final String TT_INDEX_CACHE =
- "mapreduce.tasktracker.indexcache.mb";
public static final String TT_INSTRUMENTATION =
"mapreduce.tasktracker.instrumentation";
public static final String TT_MAP_SLOTS =
"mapreduce.tasktracker.map.tasks.maximum";
- public static final String TT_MAP_INPUT_SPLITINFO =
- "mapreduce.tasktracker.map.input.splitinfo";
/**
* @deprecated Use {@link #TT_RESOURCE_CALCULATOR_PLUGIN} instead
*/
@Deprecated
public static final String TT_MEMORY_CALCULATOR_PLUGIN =
"mapreduce.tasktracker.memorycalculatorplugin";
+ /**
+ * @deprecated Use {@link MRConfig#RESOURCE_CALCULATOR_PLUGIN} instead
+ */
+ @Deprecated
public static final String TT_RESOURCE_CALCULATOR_PLUGIN =
- "mapreduce.tasktracker.resourcecalculatorplugin";
+ RESOURCE_CALCULATOR_PLUGIN;
public static final String TT_REDUCE_SLOTS =
"mapreduce.tasktracker.reduce.tasks.maximum";
public static final String TT_MEMORY_MANAGER_MONITORING_INTERVAL =
@@ -102,4 +100,11 @@ public interface TTConfig extends MRConf
"mapreduce.tasktracker.userlogcleanup.sleeptime";
public static final String TT_DISTRIBUTED_CACHE_CHECK_PERIOD =
"mapreduce.tasktracker.distributedcache.checkperiod";
+
+ /**
+ * @deprecated Use {@link MRConfig#JOB_INDEX_CACHE} instead
+ */
+ @Deprecated
+ public static final String TT_INDEX_CACHE = JOB_INDEX_CACHE;
+
}
Added: hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/util/ServerConfigUtil.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/util/ServerConfigUtil.java?rev=1082677&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/util/ServerConfigUtil.java (added)
+++ hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapreduce/util/ServerConfigUtil.java Thu Mar 17 20:21:13 2011
@@ -0,0 +1,152 @@
+package org.apache.hadoop.mapreduce.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+
+/**
+ * Place holder for deprecated keys in the framework
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class ServerConfigUtil {
+
+
+ /**
+ * Adds all the deprecated keys. Loads mapred-default.xml and mapred-site.xml
+ */
+ public static void loadResources() {
+ addDeprecatedKeys();
+ Configuration.addDefaultResource("mapred-default.xml");
+ Configuration.addDefaultResource("mapred-site.xml");
+ }
+
+ /**
+ * Adds deprecated keys and the corresponding new keys to the Configuration
+ */
+ private static void addDeprecatedKeys() {
+ Configuration.addDeprecation("mapred.temp.dir",
+ new String[] {MRConfig.TEMP_DIR});
+ Configuration.addDeprecation("mapred.local.dir",
+ new String[] {MRConfig.LOCAL_DIR});
+ Configuration.addDeprecation("mapred.cluster.map.memory.mb",
+ new String[] {MRConfig.MAPMEMORY_MB});
+ Configuration.addDeprecation("mapred.cluster.reduce.memory.mb",
+ new String[] {MRConfig.REDUCEMEMORY_MB});
+ Configuration.addDeprecation("mapred.acls.enabled",
+ new String[] {MRConfig.MR_ACLS_ENABLED});
+
+ Configuration.addDeprecation("mapred.cluster.max.map.memory.mb",
+ new String[] {JTConfig.JT_MAX_MAPMEMORY_MB});
+ Configuration.addDeprecation("mapred.cluster.max.reduce.memory.mb",
+ new String[] {JTConfig.JT_MAX_REDUCEMEMORY_MB});
+
+ Configuration.addDeprecation("mapred.cluster.average.blacklist.threshold",
+ new String[] {JTConfig.JT_AVG_BLACKLIST_THRESHOLD});
+ Configuration.addDeprecation("hadoop.job.history.location",
+ new String[] {JTConfig.JT_JOBHISTORY_LOCATION});
+ Configuration.addDeprecation(
+ "mapred.job.tracker.history.completed.location",
+ new String[] {JTConfig.JT_JOBHISTORY_COMPLETED_LOCATION});
+ Configuration.addDeprecation("mapred.jobtracker.job.history.block.size",
+ new String[] {JTConfig.JT_JOBHISTORY_BLOCK_SIZE});
+ Configuration.addDeprecation("mapred.job.tracker.jobhistory.lru.cache.size",
+ new String[] {JTConfig.JT_JOBHISTORY_CACHE_SIZE});
+ Configuration.addDeprecation("mapred.hosts",
+ new String[] {JTConfig.JT_HOSTS_FILENAME});
+ Configuration.addDeprecation("mapred.hosts.exclude",
+ new String[] {JTConfig.JT_HOSTS_EXCLUDE_FILENAME});
+ Configuration.addDeprecation("mapred.system.dir",
+ new String[] {JTConfig.JT_SYSTEM_DIR});
+ Configuration.addDeprecation("mapred.max.tracker.blacklists",
+ new String[] {JTConfig.JT_MAX_TRACKER_BLACKLISTS});
+ Configuration.addDeprecation("mapred.job.tracker.http.address",
+ new String[] {JTConfig.JT_HTTP_ADDRESS});
+ Configuration.addDeprecation("mapred.job.tracker.handler.count",
+ new String[] {JTConfig.JT_IPC_HANDLER_COUNT});
+ Configuration.addDeprecation("mapred.jobtracker.restart.recover",
+ new String[] {JTConfig.JT_RESTART_ENABLED});
+ Configuration.addDeprecation("mapred.jobtracker.taskScheduler",
+ new String[] {JTConfig.JT_TASK_SCHEDULER});
+ Configuration.addDeprecation(
+ "mapred.jobtracker.taskScheduler.maxRunningTasksPerJob",
+ new String[] {JTConfig.JT_RUNNINGTASKS_PER_JOB});
+ Configuration.addDeprecation("mapred.jobtracker.instrumentation",
+ new String[] {JTConfig.JT_INSTRUMENTATION});
+ Configuration.addDeprecation("mapred.jobtracker.maxtasks.per.job",
+ new String[] {JTConfig.JT_TASKS_PER_JOB});
+ Configuration.addDeprecation("mapred.heartbeats.in.second",
+ new String[] {JTConfig.JT_HEARTBEATS_IN_SECOND});
+ Configuration.addDeprecation("mapred.job.tracker.persist.jobstatus.active",
+ new String[] {JTConfig.JT_PERSIST_JOBSTATUS});
+ Configuration.addDeprecation("mapred.job.tracker.persist.jobstatus.hours",
+ new String[] {JTConfig.JT_PERSIST_JOBSTATUS_HOURS});
+ Configuration.addDeprecation("mapred.job.tracker.persist.jobstatus.dir",
+ new String[] {JTConfig.JT_PERSIST_JOBSTATUS_DIR});
+ Configuration.addDeprecation("mapred.permissions.supergroup",
+ new String[] {MRConfig.MR_SUPERGROUP});
+ Configuration.addDeprecation("mapreduce.jobtracker.permissions.supergroup",
+ new String[] {MRConfig.MR_SUPERGROUP});
+ Configuration.addDeprecation("mapred.task.cache.levels",
+ new String[] {JTConfig.JT_TASKCACHE_LEVELS});
+ Configuration.addDeprecation("mapred.jobtracker.taskalloc.capacitypad",
+ new String[] {JTConfig.JT_TASK_ALLOC_PAD_FRACTION});
+ Configuration.addDeprecation("mapred.jobinit.threads",
+ new String[] {JTConfig.JT_JOBINIT_THREADS});
+ Configuration.addDeprecation("mapred.tasktracker.expiry.interval",
+ new String[] {JTConfig.JT_TRACKER_EXPIRY_INTERVAL});
+ Configuration.addDeprecation("mapred.job.tracker.retiredjobs.cache.size",
+ new String[] {JTConfig.JT_RETIREJOB_CACHE_SIZE});
+ Configuration.addDeprecation("mapred.job.tracker.retire.jobs",
+ new String[] {JTConfig.JT_RETIREJOBS});
+ Configuration.addDeprecation("mapred.healthChecker.interval",
+ new String[] {TTConfig.TT_HEALTH_CHECKER_INTERVAL});
+ Configuration.addDeprecation("mapred.healthChecker.script.args",
+ new String[] {TTConfig.TT_HEALTH_CHECKER_SCRIPT_ARGS});
+ Configuration.addDeprecation("mapred.healthChecker.script.path",
+ new String[] {TTConfig.TT_HEALTH_CHECKER_SCRIPT_PATH});
+ Configuration.addDeprecation("mapred.healthChecker.script.timeout",
+ new String[] {TTConfig.TT_HEALTH_CHECKER_SCRIPT_TIMEOUT});
+ Configuration.addDeprecation("mapred.local.dir.minspacekill",
+ new String[] {TTConfig.TT_LOCAL_DIR_MINSPACE_KILL});
+ Configuration.addDeprecation("mapred.local.dir.minspacestart",
+ new String[] {TTConfig.TT_LOCAL_DIR_MINSPACE_START});
+ Configuration.addDeprecation("mapred.task.tracker.http.address",
+ new String[] {TTConfig.TT_HTTP_ADDRESS});
+ Configuration.addDeprecation("mapred.task.tracker.report.address",
+ new String[] {TTConfig.TT_REPORT_ADDRESS});
+ Configuration.addDeprecation("mapred.task.tracker.task-controller",
+ new String[] {TTConfig.TT_TASK_CONTROLLER});
+ Configuration.addDeprecation("mapred.tasktracker.dns.interface",
+ new String[] {TTConfig.TT_DNS_INTERFACE});
+ Configuration.addDeprecation("mapred.tasktracker.dns.nameserver",
+ new String[] {TTConfig.TT_DNS_NAMESERVER});
+ Configuration.addDeprecation("mapred.tasktracker.events.batchsize",
+ new String[] {TTConfig.TT_MAX_TASK_COMPLETION_EVENTS_TO_POLL});
+ Configuration.addDeprecation("mapred.tasktracker.instrumentation",
+ new String[] {TTConfig.TT_INSTRUMENTATION});
+ Configuration.addDeprecation("mapred.tasktracker.map.tasks.maximum",
+ new String[] {TTConfig.TT_MAP_SLOTS});
+ Configuration.addDeprecation("mapred.tasktracker.reduce.tasks.maximum",
+ new String[] {TTConfig.TT_REDUCE_SLOTS});
+ Configuration.addDeprecation(
+ "mapred.tasktracker.taskmemorymanager.monitoring-interval",
+ new String[] {TTConfig.TT_MEMORY_MANAGER_MONITORING_INTERVAL});
+ Configuration.addDeprecation(
+ "mapred.tasktracker.tasks.sleeptime-before-sigkill",
+ new String[] {TTConfig.TT_SLEEP_TIME_BEFORE_SIG_KILL});
+ Configuration.addDeprecation("slave.host.name",
+ new String[] {TTConfig.TT_HOST_NAME});
+ Configuration.addDeprecation("tasktracker.http.threads",
+ new String[] {TTConfig.TT_HTTP_THREADS});
+ Configuration.addDeprecation("local.cache.size",
+ new String[] {TTConfig.TT_LOCAL_CACHE_SIZE});
+ Configuration.addDeprecation("tasktracker.contention.tracking",
+ new String[] {TTConfig.TT_CONTENTION_TRACKING});
+ }
+}
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java Thu Mar 17 20:21:13 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
import org.apache.hadoop.mapreduce.split.JobSplit;
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
+import org.apache.hadoop.mapreduce.util.HostUtil;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
/**
@@ -68,8 +69,7 @@ public class FakeObjectUtilities {
public ClusterStatus getClusterStatus(boolean detailed) {
return new ClusterStatus(
taskTrackers().size() - getBlacklistedTrackerCount(),
- getBlacklistedTrackerCount(), 0, 0, 0, totalSlots/2, totalSlots/2,
- JobTracker.State.RUNNING, 0);
+ getBlacklistedTrackerCount(), 0, 0, 0, totalSlots/2, totalSlots/2, 0);
}
public void setNumSlots(int totalSlots) {
@@ -169,14 +169,14 @@ public class FakeObjectUtilities {
public TaskAttemptID findMapTask(String trackerName)
throws IOException {
return findTask(trackerName,
- JobInProgress.convertTrackerNameToHostName(trackerName),
+ HostUtil.convertTrackerNameToHostName(trackerName),
nonLocalMaps, nonLocalRunningMaps, TaskType.MAP);
}
public TaskAttemptID findReduceTask(String trackerName)
throws IOException {
return findTask(trackerName,
- JobInProgress.convertTrackerNameToHostName(trackerName),
+ HostUtil.convertTrackerNameToHostName(trackerName),
nonRunningReduces, runningReduces, TaskType.REDUCE);
}
@@ -192,7 +192,7 @@ public class FakeObjectUtilities {
private void makeRunning(TaskAttemptID taskId, TaskInProgress tip,
String taskTracker) {
addRunningTaskToTIP(tip, taskId, new TaskTrackerStatus(taskTracker,
- JobInProgress.convertTrackerNameToHostName(taskTracker)), true);
+ HostUtil.convertTrackerNameToHostName(taskTracker)), true);
TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId,
0.0f, 1, TaskStatus.State.RUNNING, "", "", taskTracker,
@@ -241,7 +241,7 @@ public class FakeObjectUtilities {
throws IOException {
if (status == null) {
status = new TaskTrackerStatus(tracker,
- JobInProgress.convertTrackerNameToHostName(tracker));
+ HostUtil.convertTrackerNameToHostName(tracker));
}
jt.heartbeat(status, false, initialContact, acceptNewTasks, responseId);
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java Thu Mar 17 20:21:13 2011
@@ -29,6 +29,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.JobTracker.State;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
@@ -40,6 +41,7 @@ import org.apache.hadoop.net.NetworkTopo
import org.apache.hadoop.net.StaticMapping;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.tools.rumen.LoggedJob.JobType;
/**
* This class creates a single-process Map-Reduce cluster for junit testing.
@@ -108,6 +110,10 @@ public class MiniMRCluster {
return tracker;
}
+ public JobTracker.State getState() {
+ return tracker.getState();
+ }
+
/**
* Create the job tracker and run it.
*/
@@ -313,6 +319,18 @@ public class MiniMRCluster {
}
}
+ public void waitForJT() {
+ while (jobTracker.getState() != State.RUNNING) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ie) {}
+ }
+ }
+
+ public JobTracker.State getJobTrackerState() {
+ return jobTracker.getState();
+ }
+
/**
* Wait until the system is idle.
*/
@@ -649,7 +667,7 @@ public class MiniMRCluster {
ClusterStatus status = null;
if (jobTracker.isUp()) {
status = jobTracker.getJobTracker().getClusterStatus(false);
- while (jobTracker.isActive() && status.getJobTrackerState()
+ while (jobTracker.isActive() && jobTracker.getState()
== JobTracker.State.INITIALIZING) {
try {
LOG.info("JobTracker still initializing. Waiting.");
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestClusterStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestClusterStatus.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestClusterStatus.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestClusterStatus.java Thu Mar 17 20:21:13 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
+import org.apache.hadoop.mapreduce.util.HostUtil;
import junit.extensions.TestSetup;
import junit.framework.Test;
@@ -102,7 +103,7 @@ public class TestClusterStatus extends T
private TaskTrackerStatus getTTStatus(String trackerName,
List<TaskStatus> taskStatuses) {
return new TaskTrackerStatus(trackerName,
- JobInProgress.convertTrackerNameToHostName(trackerName), 0,
+ HostUtil.convertTrackerNameToHostName(trackerName), 0,
taskStatuses, 0, mapSlotsPerTracker, reduceSlotsPerTracker);
}
@@ -217,10 +218,10 @@ public class TestClusterStatus extends T
TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]);
TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]);
TaskTrackerStatus status1 = new TaskTrackerStatus(
- trackers[0],JobInProgress.convertTrackerNameToHostName(
+ trackers[0],HostUtil.convertTrackerNameToHostName(
trackers[0]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
TaskTrackerStatus status2 = new TaskTrackerStatus(
- trackers[1],JobInProgress.convertTrackerNameToHostName(
+ trackers[1],HostUtil.convertTrackerNameToHostName(
trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
tt1.setStatus(status1);
tt2.setStatus(status2);
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobInProgress.java Thu Mar 17 20:21:13 2011
@@ -316,8 +316,7 @@ public class TestJobInProgress extends T
// Should be invoked numMaps + numReds times by different TIP objects
verify(jspy, times(4)).setFirstTaskLaunchTime(any(TaskInProgress.class));
- ClusterStatus cspy = spy(new ClusterStatus(4, 0, 0, 0, 0, 4, 4,
- JobTracker.State.RUNNING, 0));
+ ClusterStatus cspy = spy(new ClusterStatus(4, 0, 0, 0, 0, 4, 4, 0));
JobInProgress.JobSummary.logJobSummary(jspy, cspy);
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java Thu Mar 17 20:21:13 2011
@@ -155,8 +155,7 @@ public class TestJobQueueTaskScheduler e
10 * 60 * 1000,
maps, reduces,
numTrackers * maxMapTasksPerTracker,
- numTrackers * maxReduceTasksPerTracker,
- JobTracker.State.RUNNING);
+ numTrackers * maxReduceTasksPerTracker);
}
@Override
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java Thu Mar 17 20:21:13 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapred.UtilsFor
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
+import org.apache.hadoop.mapreduce.util.HostUtil;
@SuppressWarnings("deprecation")
public class TestJobTrackerInstrumentation extends TestCase {
@@ -89,7 +90,7 @@ public class TestJobTrackerInstrumentati
private TaskTrackerStatus getTTStatus(String trackerName,
List<TaskStatus> taskStatuses) {
return new TaskTrackerStatus(trackerName,
- JobInProgress.convertTrackerNameToHostName(trackerName), 0,
+ HostUtil.convertTrackerNameToHostName(trackerName), 0,
taskStatuses, 0, mapSlotsPerTracker, reduceSlotsPerTracker);
}
@@ -287,7 +288,7 @@ public class TestJobTrackerInstrumentati
//Set task tracker objects for reservation.
TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]);
TaskTrackerStatus status2 = new TaskTrackerStatus(
- trackers[1],JobInProgress.convertTrackerNameToHostName(
+ trackers[1],HostUtil.convertTrackerNameToHostName(
trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
tt2.setStatus(status2);
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJvmManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJvmManager.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJvmManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestJvmManager.java Thu Mar 17 20:21:13 2011
@@ -116,9 +116,11 @@ public class TestJvmManager {
JobConf taskConf = new JobConf(ttConf);
TaskAttemptID attemptID = new TaskAttemptID("test", 0, TaskType.MAP, 0, 0);
Task task = new MapTask(null, attemptID, 0, null, MAP_SLOTS);
+ TTTask ttTask = new TTMapTask((MapTask)task);
task.setUser(user);
task.setConf(taskConf);
- TaskInProgress tip = tt.new TaskInProgress(task, taskConf);
+ TaskInProgress tip =
+ tt.new TaskInProgress(ttTask, taskConf);
File pidFile = new File(TEST_DIR, "pid");
RunningJob rjob = new RunningJob(attemptID.getJobID());
TaskController taskController = new DefaultTaskController();
@@ -126,7 +128,7 @@ public class TestJvmManager {
rjob.distCacheMgr =
new TrackerDistributedCacheManager(ttConf).
newTaskDistributedCacheManager(attemptID.getJobID(), taskConf);
- final TaskRunner taskRunner = task.createRunner(tt, tip, rjob);
+ final TaskRunner taskRunner = ttTask.createRunner(tt, tip, rjob);
// launch a jvm which sleeps for 60 seconds
final Vector<String> vargs = new Vector<String>(2);
vargs.add(writeScript("SLEEP", "sleep 60\n", pidFile).getAbsolutePath());
@@ -196,10 +198,11 @@ public class TestJvmManager {
// launch another jvm and see it finishes properly
attemptID = new TaskAttemptID("test", 0, TaskType.MAP, 0, 1);
task = new MapTask(null, attemptID, 0, null, MAP_SLOTS);
+ ttTask = new TTMapTask((MapTask)task);
task.setUser(user);
task.setConf(taskConf);
- tip = tt.new TaskInProgress(task, taskConf);
- TaskRunner taskRunner2 = task.createRunner(tt, tip, rjob);
+ tip = tt.new TaskInProgress(ttTask, taskConf);
+ TaskRunner taskRunner2 = ttTask.createRunner(tt, tip, rjob);
// build dummy vargs to call ls
Vector<String> vargs2 = new Vector<String>(1);
vargs2.add(writeScript("LS", "ls", pidFile).getAbsolutePath());
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java Thu Mar 17 20:21:13 2011
@@ -44,6 +44,7 @@ import org.apache.hadoop.mapred.lib.Iden
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.mapreduce.util.HostUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
@@ -446,7 +447,7 @@ public class TestNodeRefresh extends Tes
// find the tracker to decommission
String hostToDecommission =
- JobInProgress.convertTrackerNameToHostName(
+ HostUtil.convertTrackerNameToHostName(
jt.getBlacklistedTrackers()[0].getTaskTrackerName());
LOG.info("Decommissioning host " + hostToDecommission);
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestParallelInitialization.java Thu Mar 17 20:21:13 2011
@@ -105,8 +105,7 @@ public class TestParallelInitialization
10 * 60 * 1000,
maps, reduces,
numTrackers * maxMapTasksPerTracker,
- numTrackers * maxReduceTasksPerTracker,
- JobTracker.State.RUNNING);
+ numTrackers * maxReduceTasksPerTracker);
}
public int getNumberOfUniqueHosts() {
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java Thu Mar 17 20:21:13 2011
@@ -131,7 +131,7 @@ public class TestRecoveryManager extends
// check if the jobtracker came up or not
assertEquals("JobTracker crashed!",
- JobTracker.State.RUNNING, status.getJobTrackerState());
+ JobTracker.State.RUNNING, mr.getJobTrackerState());
// assert the no of recovered jobs
assertEquals("No of recovered jobs not correct",
@@ -250,7 +250,7 @@ public class TestRecoveryManager extends
// start the jobtracker
LOG.info("Starting jobtracker");
mr.startJobTracker();
- UtilsForTests.waitForJobTracker(jc);
+ mr.waitForJT();
jobtracker = mr.getJobTrackerRunner().getJobTracker();
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestSetupTaskScheduling.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestSetupTaskScheduling.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestSetupTaskScheduling.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestSetupTaskScheduling.java Thu Mar 17 20:21:13 2011
@@ -26,6 +26,7 @@ import org.apache.hadoop.mapred.FakeObje
import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.split.JobSplit;
+import org.apache.hadoop.mapreduce.util.HostUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskType;
@@ -207,7 +208,7 @@ public class TestSetupTaskScheduling ext
List<TaskStatus> reports) {
TaskTrackerStatus ttStatus =
new TaskTrackerStatus(tracker,
- JobInProgress.convertTrackerNameToHostName(tracker),
+ HostUtil.convertTrackerNameToHostName(tracker),
0, reports, 0, 2, 2);
return ttStatus;
}
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java Thu Mar 17 20:21:13 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapred.lib.Iden
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MapReduceTestUtil;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.HostUtil;
public class TestTaskFail extends TestCase {
private static String taskLog = "Task attempt log";
@@ -141,7 +142,7 @@ public class TestTaskFail extends TestCa
// access the logs from web url
TaskTrackerStatus ttStatus = jt.getTaskTracker(
tip.machineWhereTaskRan(attemptId)).getStatus();
- String tasklogUrl = TaskLogServlet.getTaskLogUrl("localhost",
+ String tasklogUrl = HostUtil.getTaskLogUrl("localhost",
String.valueOf(ttStatus.getHttpPort()), attemptId.toString()) +
"&filter=STDERR";
assertEquals(HttpURLConnection.HTTP_OK, TestWebUIAuthorization
@@ -158,7 +159,7 @@ public class TestTaskFail extends TestCa
// access the cleanup attempt's logs from web url
ttStatus = jt.getTaskTracker(tip.machineWhereCleanupRan(attemptId))
.getStatus();
- String cleanupTasklogUrl = TaskLogServlet.getTaskLogUrl("localhost",
+ String cleanupTasklogUrl = HostUtil.getTaskLogUrl("localhost",
String.valueOf(ttStatus.getHttpPort()), attemptId.toString())
+ "&filter=STDERR&cleanup=true";
assertEquals(HttpURLConnection.HTTP_OK, TestWebUIAuthorization
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java Thu Mar 17 20:21:13 2011
@@ -39,6 +39,7 @@ import org.apache.hadoop.mapred.TaskTrac
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
+import org.apache.hadoop.mapreduce.util.HostUtil;
public class TestTaskTrackerBlacklisting extends TestCase {
@@ -129,7 +130,7 @@ public class TestTaskTrackerBlacklisting
String tracker = entry.getKey();
if (failures.intValue() >= this.getJobConf()
.getMaxTaskFailuresPerTracker()) {
- blackListedTrackers.add(JobInProgress
+ blackListedTrackers.add(HostUtil
.convertTrackerNameToHostName(tracker));
}
}
@@ -162,7 +163,7 @@ public class TestTaskTrackerBlacklisting
boolean initialContact)
throws IOException {
for (String tracker : trackers) {
- TaskTrackerStatus tts = new TaskTrackerStatus(tracker, JobInProgress
+ TaskTrackerStatus tts = new TaskTrackerStatus(tracker, HostUtil
.convertTrackerNameToHostName(tracker));
if (status != null) {
TaskTrackerHealthStatus healthStatus = tts.getHealthStatus();
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java Thu Mar 17 20:21:13 2011
@@ -87,6 +87,7 @@ public class TestTaskTrackerLocalization
protected JobID jobId;
protected TaskAttemptID taskId;
protected Task task;
+ protected TTTask ttTask;
protected String[] localDirs;
protected static LocalDirAllocator lDirAlloc =
new LocalDirAllocator(MRConfig.LOCAL_DIR);
@@ -178,7 +179,7 @@ public class TestTaskTrackerLocalization
// mimic register task
// create the tip
- tip = tracker.new TaskInProgress(task, trackerFConf);
+ tip = tracker.new TaskInProgress(ttTask, trackerFConf);
}
private void startTracker() throws IOException {
@@ -225,6 +226,7 @@ public class TestTaskTrackerLocalization
task = new MapTask(jobConfFile.toURI().toString(), taskId, 1, null, 1);
task.setConf(jobConf); // Set conf. Set user name in particular.
task.setUser(jobConf.getUser());
+ ttTask = new TTMapTask((MapTask)task);
}
protected UserGroupInformation getJobOwner() throws IOException {
@@ -623,7 +625,7 @@ public class TestTaskTrackerLocalization
new TrackerDistributedCacheManager(trackerFConf).
newTaskDistributedCacheManager(jobId, trackerFConf);
- TaskRunner runner = task.createRunner(tracker, tip, rjob);
+ TaskRunner runner = ttTask.createRunner(tracker, tip, rjob);
tip.setTaskRunner(runner);
// /////// Few more methods being tested
@@ -964,7 +966,7 @@ public class TestTaskTrackerLocalization
createTask();
task.setTaskCleanupTask();
// register task
- tip = tracker.new TaskInProgress(task, trackerFConf);
+ tip = tracker.new TaskInProgress(ttTask, trackerFConf);
// localize the job again.
rjob = tracker.localizeJob(tip);
@@ -1005,7 +1007,7 @@ public class TestTaskTrackerLocalization
createTask();
task.setTaskCleanupTask();
// register task
- tip = tracker.new TaskInProgress(task, trackerFConf);
+ tip = tracker.new TaskInProgress(ttTask, trackerFConf);
// localize the job again.
rjob = tracker.localizeJob(tip);
@@ -1031,7 +1033,7 @@ public class TestTaskTrackerLocalization
task.setTaskCleanupTask();
// register task
- tip = tracker.new TaskInProgress(task, trackerFConf);
+ tip = tracker.new TaskInProgress(ttTask, trackerFConf);
// localize the job.
RunningJob rjob = tracker.localizeJob(tip);
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java Thu Mar 17 20:21:13 2011
@@ -27,6 +27,7 @@ import org.apache.hadoop.mapreduce.Clust
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
+import org.apache.hadoop.mapreduce.util.HostUtil;
import junit.extensions.TestSetup;
import junit.framework.Test;
@@ -100,13 +101,13 @@ public class TestTrackerReservation exte
TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]);
TaskTracker tt3 = jobTracker.getTaskTracker(trackers[2]);
TaskTrackerStatus status1 = new TaskTrackerStatus(
- trackers[0],JobInProgress.convertTrackerNameToHostName(
+ trackers[0],HostUtil.convertTrackerNameToHostName(
trackers[0]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
TaskTrackerStatus status2 = new TaskTrackerStatus(
- trackers[1],JobInProgress.convertTrackerNameToHostName(
+ trackers[1],HostUtil.convertTrackerNameToHostName(
trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
TaskTrackerStatus status3 = new TaskTrackerStatus(
- trackers[1],JobInProgress.convertTrackerNameToHostName(
+ trackers[1],HostUtil.convertTrackerNameToHostName(
trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
tt1.setStatus(status1);
tt2.setStatus(status2);
@@ -169,7 +170,7 @@ public class TestTrackerReservation exte
.getBlackListedTrackers().size());
assertTrue("Tracker 1 not blacklisted for the job", job
.getBlackListedTrackers().contains(
- JobInProgress.convertTrackerNameToHostName(trackers[0])));
+ HostUtil.convertTrackerNameToHostName(trackers[0])));
assertEquals("Job didnt complete successfully complete", job.getStatus()
.getRunState(), JobStatus.SUCCEEDED);
assertEquals("Reservation for the job not released: Maps",
@@ -207,13 +208,13 @@ public class TestTrackerReservation exte
TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]);
TaskTracker tt3 = jobTracker.getTaskTracker(trackers[2]);
TaskTrackerStatus status1 = new TaskTrackerStatus(
- trackers[0],JobInProgress.convertTrackerNameToHostName(
+ trackers[0],HostUtil.convertTrackerNameToHostName(
trackers[0]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
TaskTrackerStatus status2 = new TaskTrackerStatus(
- trackers[1],JobInProgress.convertTrackerNameToHostName(
+ trackers[1],HostUtil.convertTrackerNameToHostName(
trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
TaskTrackerStatus status3 = new TaskTrackerStatus(
- trackers[1],JobInProgress.convertTrackerNameToHostName(
+ trackers[1],HostUtil.convertTrackerNameToHostName(
trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
tt1.setStatus(status1);
tt2.setStatus(status2);
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java Thu Mar 17 20:21:13 2011
@@ -40,6 +40,7 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.mapreduce.util.HostUtil;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
import org.apache.hadoop.security.Groups;
@@ -357,12 +358,12 @@ public class TestWebUIAuthorization exte
// validate access to tasklogs - STDOUT and STDERR. SYSLOGs are not
// generated for the 1 map sleep job in the test case.
- String stdoutURL = TaskLogServlet.getTaskLogUrl("localhost",
+ String stdoutURL = HostUtil.getTaskLogUrl("localhost",
Integer.toString(attemptsMap.get(attempt).getHttpPort()),
attempt.toString()) + "&filter=" + TaskLog.LogName.STDOUT;
validateViewJob(stdoutURL, "GET");
- String stderrURL = TaskLogServlet.getTaskLogUrl("localhost",
+ String stderrURL = HostUtil.getTaskLogUrl("localhost",
Integer.toString(attemptsMap.get(attempt).getHttpPort()),
attempt.toString()) + "&filter=" + TaskLog.LogName.STDERR;
validateViewJob(stderrURL, "GET");
@@ -385,11 +386,11 @@ public class TestWebUIAuthorization exte
tipsMap.get(tip).getAllTaskAttempts();
for (TaskAttemptID attempt : attemptsMap.keySet()) {
- String stdoutURL = TaskLogServlet.getTaskLogUrl("localhost",
+ String stdoutURL = HostUtil.getTaskLogUrl("localhost",
Integer.toString(attemptsMap.get(attempt).getHttpPort()),
attempt.toString()) + "&filter=" + TaskLog.LogName.STDOUT;;
- String stderrURL = TaskLogServlet.getTaskLogUrl("localhost",
+ String stderrURL = HostUtil.getTaskLogUrl("localhost",
Integer.toString(attemptsMap.get(attempt).getHttpPort()),
attempt.toString()) + "&filter=" + TaskLog.LogName.STDERR;
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java Thu Mar 17 20:21:13 2011
@@ -243,22 +243,6 @@ public class UtilsForTests {
}
/**
- * Wait for the jobtracker to be RUNNING.
- */
- static void waitForJobTracker(JobClient jobClient) {
- while (true) {
- try {
- ClusterStatus status = jobClient.getClusterStatus();
- while (status.getJobTrackerState() != JobTracker.State.RUNNING) {
- waitFor(100);
- status = jobClient.getClusterStatus();
- }
- break; // means that the jt is ready
- } catch (IOException ioe) {}
- }
- }
-
- /**
* Waits until all the jobs at the jobtracker complete.
*/
static void waitTillDone(JobClient jobClient) throws IOException {
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java Thu Mar 17 20:21:13 2011
@@ -165,8 +165,8 @@ public class TestTrackerDistributedCache
JobID jobid = new JobID("jt",1);
DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf);
DistributedCache.addFileToClassPath(secondCacheFile, subConf);
- TrackerDistributedCacheManager.determineTimestamps(subConf);
- TrackerDistributedCacheManager.determineCacheVisibilities(subConf);
+ DistributedCache.determineTimestamps(subConf);
+ DistributedCache.determineCacheVisibilities(subConf);
// ****** End of imitating JobClient code
Path jobFile = new Path(TEST_ROOT_DIR, "job.xml");
@@ -251,8 +251,8 @@ public class TestTrackerDistributedCache
conf1.set("user.name", userName);
DistributedCache.addCacheFile(secondCacheFile.toUri(), conf1);
- TrackerDistributedCacheManager.determineTimestamps(conf1);
- TrackerDistributedCacheManager.determineCacheVisibilities(conf1);
+ DistributedCache.determineTimestamps(conf1);
+ DistributedCache.determineCacheVisibilities(conf1);
// Task localizing for first job
TaskDistributedCacheManager handle = manager
@@ -279,8 +279,8 @@ public class TestTrackerDistributedCache
// add a file that is never localized
DistributedCache.addCacheFile(thirdCacheFile.toUri(), conf2);
- TrackerDistributedCacheManager.determineTimestamps(conf2);
- TrackerDistributedCacheManager.determineCacheVisibilities(conf2);
+ DistributedCache.determineTimestamps(conf2);
+ DistributedCache.determineCacheVisibilities(conf2);
// Task localizing for second job
// localization for the "firstCacheFile" will fail.
@@ -388,8 +388,8 @@ public class TestTrackerDistributedCache
job1.setUser(userName);
job1.addCacheFile(cacheFile.toUri());
Configuration conf1 = job1.getConfiguration();
- TrackerDistributedCacheManager.determineTimestamps(conf1);
- TrackerDistributedCacheManager.determineCacheVisibilities(conf1);
+ DistributedCache.determineTimestamps(conf1);
+ DistributedCache.determineCacheVisibilities(conf1);
// Task localizing for job
TaskDistributedCacheManager handle = manager
@@ -570,8 +570,8 @@ public class TestTrackerDistributedCache
createPrivateTempFile(thirdCacheFile);
createPrivateTempFile(fourthCacheFile);
DistributedCache.setCacheFiles(new URI[]{thirdCacheFile.toUri()}, conf2);
- TrackerDistributedCacheManager.determineCacheVisibilities(conf2);
- TrackerDistributedCacheManager.determineTimestamps(conf2);
+ DistributedCache.determineCacheVisibilities(conf2);
+ DistributedCache.determineTimestamps(conf2);
stat = fs.getFileStatus(thirdCacheFile);
CacheFile cfile3 = new CacheFile(thirdCacheFile.toUri(),
CacheFile.FileType.REGULAR, false,
@@ -597,8 +597,8 @@ public class TestTrackerDistributedCache
localfs.exists(thirdLocalCache));
DistributedCache.setCacheFiles(new URI[]{fourthCacheFile.toUri()}, conf2);
DistributedCache.setLocalFiles(conf2, thirdCacheFile.toUri().toString());
- TrackerDistributedCacheManager.determineCacheVisibilities(conf2);
- TrackerDistributedCacheManager.determineTimestamps(conf2);
+ DistributedCache.determineCacheVisibilities(conf2);
+ DistributedCache.determineTimestamps(conf2);
Path fourthLocalCache = manager.getLocalCache(fourthCacheFile.toUri(), conf2,
TaskTracker.getPrivateDistributedCacheDir(userName),
fs.getFileStatus(fourthCacheFile), false,
@@ -736,8 +736,8 @@ public class TestTrackerDistributedCache
Configuration subConf = new Configuration(myConf);
subConf.set(MRJobConfig.USER_NAME, userName);
DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf);
- TrackerDistributedCacheManager.determineTimestamps(subConf);
- TrackerDistributedCacheManager.determineCacheVisibilities(subConf);
+ DistributedCache.determineTimestamps(subConf);
+ DistributedCache.determineCacheVisibilities(subConf);
// ****** End of imitating JobClient code
// ****** Imitate TaskRunner code.
@@ -808,8 +808,8 @@ public class TestTrackerDistributedCache
Configuration subConf2 = new Configuration(myConf);
subConf2.set(MRJobConfig.USER_NAME, userName);
DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf2);
- TrackerDistributedCacheManager.determineTimestamps(subConf2);
- TrackerDistributedCacheManager.determineCacheVisibilities(subConf2);
+ DistributedCache.determineTimestamps(subConf2);
+ DistributedCache.determineCacheVisibilities(subConf2);
handle =
manager.newTaskDistributedCacheManager(new JobID("jt", 2), subConf2);
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestURIFragments.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestURIFragments.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestURIFragments.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestURIFragments.java Thu Mar 17 20:21:13 2011
@@ -30,83 +30,83 @@ public class TestURIFragments {
*/
@Test
public void testURIs() throws URISyntaxException {
- assertTrue(TrackerDistributedCacheManager.checkURIs(null, null));
+ assertTrue(DistributedCache.checkURIs(null, null));
// uris with no fragments
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] { new URI(
+ assertFalse(DistributedCache.checkURIs(new URI[] { new URI(
"file://foo/bar/myCacheFile.txt") }, null));
- assertFalse(TrackerDistributedCacheManager.checkURIs(null,
+ assertFalse(DistributedCache.checkURIs(null,
new URI[] { new URI("file://foo/bar/myCacheArchive.txt") }));
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertFalse(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#file"),
new URI("file://foo/bar/myCacheFile2.txt") }, null));
- assertFalse(TrackerDistributedCacheManager.checkURIs(null, new URI[] {
+ assertFalse(DistributedCache.checkURIs(null, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt"),
new URI("file://foo/bar/myCacheArchive2.txt#archive") }));
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] { new URI(
+ assertFalse(DistributedCache.checkURIs(new URI[] { new URI(
"file://foo/bar/myCacheFile.txt") }, new URI[] { new URI(
"file://foo/bar/myCacheArchive.txt") }));
// conflicts in fragment names
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertFalse(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#file"),
new URI("file://foo/bar/myCacheFile2.txt#file") }, null));
- assertFalse(TrackerDistributedCacheManager.checkURIs(null, new URI[] {
+ assertFalse(DistributedCache.checkURIs(null, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt#archive"),
new URI("file://foo/bar/myCacheArchive2.txt#archive") }));
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] { new URI(
+ assertFalse(DistributedCache.checkURIs(new URI[] { new URI(
"file://foo/bar/myCacheFile.txt#cache") }, new URI[] { new URI(
"file://foo/bar/myCacheArchive.txt#cache") }));
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertFalse(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#file1"),
new URI("file://foo/bar/myCacheFile2.txt#file2") }, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt#archive"),
new URI("file://foo/bar/myCacheArchive2.txt#archive") }));
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertFalse(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#file"),
new URI("file://foo/bar/myCacheFile2.txt#file") }, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt#archive1"),
new URI("file://foo/bar/myCacheArchive2.txt#archive2") }));
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertFalse(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#file1"),
new URI("file://foo/bar/myCacheFile2.txt#cache") }, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt#cache"),
new URI("file://foo/bar/myCacheArchive2.txt#archive2") }));
// test ignore case
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertFalse(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#file"),
new URI("file://foo/bar/myCacheFile2.txt#FILE") }, null));
- assertFalse(TrackerDistributedCacheManager.checkURIs(null, new URI[] {
+ assertFalse(DistributedCache.checkURIs(null, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt#archive"),
new URI("file://foo/bar/myCacheArchive2.txt#ARCHIVE") }));
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] { new URI(
+ assertFalse(DistributedCache.checkURIs(new URI[] { new URI(
"file://foo/bar/myCacheFile.txt#cache") }, new URI[] { new URI(
"file://foo/bar/myCacheArchive.txt#CACHE") }));
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertFalse(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#file1"),
new URI("file://foo/bar/myCacheFile2.txt#file2") }, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt#ARCHIVE"),
new URI("file://foo/bar/myCacheArchive2.txt#archive") }));
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertFalse(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#FILE"),
new URI("file://foo/bar/myCacheFile2.txt#file") }, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt#archive1"),
new URI("file://foo/bar/myCacheArchive2.txt#archive2") }));
- assertFalse(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertFalse(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#file1"),
new URI("file://foo/bar/myCacheFile2.txt#CACHE") }, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt#cache"),
new URI("file://foo/bar/myCacheArchive2.txt#archive2") }));
// allowed uri combinations
- assertTrue(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertTrue(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#file1"),
new URI("file://foo/bar/myCacheFile2.txt#file2") }, null));
- assertTrue(TrackerDistributedCacheManager.checkURIs(null, new URI[] {
+ assertTrue(DistributedCache.checkURIs(null, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt#archive1"),
new URI("file://foo/bar/myCacheArchive2.txt#archive2") }));
- assertTrue(TrackerDistributedCacheManager.checkURIs(new URI[] {
+ assertTrue(DistributedCache.checkURIs(new URI[] {
new URI("file://foo/bar/myCacheFile1.txt#file1"),
new URI("file://foo/bar/myCacheFile2.txt#file2") }, new URI[] {
new URI("file://foo/bar/myCacheArchive1.txt#archive1"),
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java Thu Mar 17 20:21:13 2011
@@ -37,9 +37,9 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.SleepJob;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -180,7 +180,7 @@ public class TestBinaryTokenFile {
String nnUri = dfsCluster.getURI(0).toString();
jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri);
// job tracker principla id..
- jConf.set(JTConfig.JT_USER_NAME, "jt_id");
+ jConf.set(MRConfig.MASTER_USER_NAME, "jt_id");
// using argument to pass the file name
String[] args = {
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java?rev=1082677&r1=1082676&r2=1082677&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java Thu Mar 17 20:21:13 2011
@@ -48,8 +48,10 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Master;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.SleepJob;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
@@ -227,7 +229,7 @@ public class TestTokenCache {
String nnUri = dfsCluster.getURI(0).toString();
jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri);
// job tracker principla id..
- jConf.set(JTConfig.JT_USER_NAME, "jt_id/foo@BAR");
+ jConf.set(MRConfig.MASTER_USER_NAME, "jt_id/foo@BAR");
// using argument to pass the file name
String[] args = {
@@ -306,7 +308,7 @@ public class TestTokenCache {
DelegationTokenSecretManager dtSecretManager =
dfsCluster.getNamesystem().getDelegationTokenSecretManager();
String renewer = "renewer/foo@BAR";
- jConf.set(JTConfig.JT_USER_NAME,renewer);
+ jConf.set(MRConfig.MASTER_USER_NAME,renewer);
DelegationTokenIdentifier dtId =
new DelegationTokenIdentifier(new Text("user"), new Text(renewer), null);
final Token<DelegationTokenIdentifier> t =
@@ -372,9 +374,9 @@ public class TestTokenCache {
String domainName = "@BAR";
Configuration conf = new Configuration();
conf.set(JTConfig.JT_IPC_ADDRESS, hostName + ":8888");
- conf.set(JTConfig.JT_USER_NAME, serviceName + SecurityUtil.HOSTNAME_PATTERN
+ conf.set(MRConfig.MASTER_USER_NAME, serviceName + SecurityUtil.HOSTNAME_PATTERN
+ domainName);
assertEquals("Failed to substitute HOSTNAME_PATTERN with hostName",
- serviceName + hostName + domainName, TokenCache.getJTPrincipal(conf));
+ serviceName + hostName + domainName, Master.getMasterPrincipal(conf));
}
}