You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/05/11 07:54:20 UTC
svn commit: r1101755 - in /hadoop/mapreduce/branches/MR-279: ./
yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/
yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/
yarn/yarn-ser...
Author: vinodkv
Date: Wed May 11 05:54:20 2011
New Revision: 1101755
URL: http://svn.apache.org/viewvc?rev=1101755&view=rev
Log:
Removal of stale application-log dirs from NM local disks. Contributed by Vinod Kumar Vavilapalli.
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.h
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/main.c
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1101755&r1=1101754&r2=1101755&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Wed May 11 05:54:20 2011
@@ -3,6 +3,8 @@ Hadoop MapReduce Change Log
Trunk (unreleased changes)
MAPREDUCE-279
+ Removal of stale application-log dirs from NM local disks. (vinodkv)
+
Fix TestQueueMetrics. (Luke Lu via sharad)
Fix assembly to add mapreduce shell scripts to the assembly package. (mahadev)
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c?rev=1101755&r1=1101754&r2=1101755&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c Wed May 11 05:54:20 2011
@@ -1060,17 +1060,3 @@ int delete_as_user(const char *user,
}
return ret;
}
-
-/**
- * delete a given log directory
- */
-int delete_log_directory(const char *subdir) {
- // TODO: FIXME. This isn't used at all!!!!!
- char* log_subdir = get_job_log_directory(subdir, subdir); // TODO: FIX
- int ret = -1;
- if (log_subdir != NULL) {
- ret = delete_path(log_subdir, strchr(subdir, '/') == NULL);
- }
- free(log_subdir);
- return ret;
-}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.h
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.h?rev=1101755&r1=1101754&r2=1101755&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.h (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.h Wed May 11 05:54:20 2011
@@ -25,7 +25,6 @@ enum command {
LAUNCH_TASK_JVM = 1,
SIGNAL_TASK = 2,
DELETE_AS_USER = 3,
- DELETE_LOG_AS_USER = 4
};
enum errorcodes {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/main.c
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/main.c?rev=1101755&r1=1101754&r2=1101755&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/main.c (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/main.c Wed May 11 05:54:20 2011
@@ -45,8 +45,6 @@ void display_usage(FILE *stream) {
SIGNAL_TASK);
fprintf(stream, " delete as user: %2d relative-path\n",
DELETE_AS_USER);
- fprintf(stream, " delete log: %2d relative-path\n",
- DELETE_LOG_AS_USER);
}
int main(int argc, char **argv) {
@@ -179,10 +177,6 @@ int main(int argc, char **argv) {
exit_code= delete_as_user(user_detail->pw_name, dir_to_be_deleted,
argv + optind);
break;
- case DELETE_LOG_AS_USER:
- dir_to_be_deleted = argv[optind++];
- exit_code= delete_log_directory(dir_to_be_deleted);
- break;
default:
exit_code = INVALID_COMMAND_PROVIDED;
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1101755&r1=1101754&r2=1101755&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Wed May 11 05:54:20 2011
@@ -189,7 +189,7 @@ public class DefaultContainerExecutor ex
return;
}
for (Path baseDir : baseDirs) {
- Path del = new Path(baseDir, subDir);
+ Path del = subDir == null ? baseDir : new Path(baseDir, subDir);
LOG.info("Deleting path : " + del);
lfs.delete(del, true);
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java?rev=1101755&r1=1101754&r2=1101755&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java Wed May 11 05:54:20 2011
@@ -109,7 +109,7 @@ public class DeletionService extends Abs
return;
}
for (Path baseDir : baseDirs) {
- Path del = new Path(baseDir, subDir);
+ Path del = subDir == null? baseDir : new Path(baseDir, subDir);
LOG.debug("NM deleting path : " + del);
try {
lfs.delete(del, true);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1101755&r1=1101754&r2=1101755&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java Wed May 11 05:54:20 2011
@@ -220,27 +220,21 @@ public class LinuxContainerExecutor exte
}
@Override
- public void deleteAsUser(String user, Path subDir, Path... baseDirs)
- throws IOException, InterruptedException {
-
- if (baseDirs == null || baseDirs.length == 0) {
- LOG.info("Deleting absolute path : " + subDir);
- deleteAsUser(user, subDir);
- return;
- }
- for (Path baseDir : baseDirs) {
- Path del = new Path(baseDir, subDir);
- LOG.info("Deleting path : " + del);
- deleteAsUser(user, del);
- }
- }
-
- private void deleteAsUser(String user, Path dir) {
+ public void deleteAsUser(String user, Path dir, Path... baseDirs) {
List<String> command = new ArrayList<String>(
Arrays.asList(containerExecutorExe,
user,
Integer.toString(Commands.DELETE_AS_USER.getValue()),
- dir.toUri().getPath()));
+ dir == null ? "" : dir.toUri().getPath()));
+ if (baseDirs == null || baseDirs.length == 0) {
+ LOG.info("Deleting absolute path : " + dir);
+ } else {
+ for (Path baseDir : baseDirs) {
+ Path del = dir == null ? baseDir : new Path(baseDir, dir);
+ LOG.info("Deleting path : " + del);
+ command.add(baseDir.toUri().getPath());
+ }
+ }
String[] commandArray = command.toArray(new String[command.size()]);
ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
LOG.info(" -- DEBUG -- deleteAsUser: " + Arrays.toString(commandArray));
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1101755&r1=1101754&r2=1101755&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java Wed May 11 05:54:20 2011
@@ -184,7 +184,6 @@ public class ContainerLocalizer {
}
ExecutorService createDownloadThreadPool() {
- // TODO: Only Single thread?
return Executors.newSingleThreadExecutor();
}
@@ -333,7 +332,6 @@ public class ContainerLocalizer {
}
}
- // TODO: Won't there be a race for creating these dirs.
private static void initDirs(Configuration conf, String user, String appId,
FileContext lfs, List<Path> localDirs) throws IOException {
if (null == localDirs || 0 == localDirs.size()) {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java?rev=1101755&r1=1101754&r2=1101755&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java Wed May 11 05:54:20 2011
@@ -128,11 +128,14 @@ public class AppLogAggregatorImpl implem
}
// Remove the local app-log-dirs
+ Path[] localAppLogDirs = new Path[this.rootLogDirs.length];
+ int index = 0;
for (String rootLogDir : this.rootLogDirs) {
- File localAppLogDir = new File(rootLogDir, this.applicationId);
- this.delService.delete(this.userUgi.getShortUserName(), new Path(
- localAppLogDir.getAbsolutePath()), new Path[] {});
+ localAppLogDirs[index] = new Path(rootLogDir, this.applicationId);
+ index++;
}
+ this.delService.delete(this.userUgi.getShortUserName(), null,
+ localAppLogDirs);
if (this.writer != null) {
this.writer.closeWriter();
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1101755&r1=1101754&r2=1101755&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Wed May 11 05:54:20 2011
@@ -125,6 +125,8 @@ public class TestLogAggregationService e
Assert.assertFalse(new File(containerLogDir, fileType).exists());
}
+ Assert.assertFalse(app1LogDir.exists());
+
Assert.assertTrue(new File(logAggregationService
.getRemoteNodeLogFileForApp(application1).toUri().getPath()).exists());
}