You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2017/06/05 20:31:12 UTC
[23/50] [abbrv] hadoop git commit: YARN-6366. Refactor the
NodeManager DeletionService to support additional DeletionTask types.
Contributed by Shane Kumpf.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14c8b4ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
index 5f9b883..c1df562 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.nodemanager;
import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.isNull;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -28,6 +27,7 @@ import java.io.File;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -58,19 +58,17 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
-import org.apache.hadoop.yarn.server.nodemanager.DeletionService.FileDeletionTask;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionMatcher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.mockito.ArgumentMatcher;
public class TestNodeManagerReboot {
@@ -195,19 +193,18 @@ public class TestNodeManagerReboot {
// restart the NodeManager
restartNM(MAX_TRIES);
checkNumOfLocalDirs();
-
- verify(delService, times(1)).delete(
- (String) isNull(),
- argThat(new PathInclude(ResourceLocalizationService.NM_PRIVATE_DIR
- + "_DEL_")));
- verify(delService, times(1)).delete((String) isNull(),
- argThat(new PathInclude(ContainerLocalizer.FILECACHE + "_DEL_")));
- verify(delService, times(1)).scheduleFileDeletionTask(
- argThat(new FileDeletionInclude(user, null,
- new String[] { destinationFile })));
- verify(delService, times(1)).scheduleFileDeletionTask(
- argThat(new FileDeletionInclude(null, ContainerLocalizer.USERCACHE
- + "_DEL_", new String[] {})));
+
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, null,
+ new Path(ResourceLocalizationService.NM_PRIVATE_DIR + "_DEL_"), null)));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, null, new Path(ContainerLocalizer.FILECACHE + "_DEL_"),
+ null)));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, user, null, Arrays.asList(new Path(destinationFile)))));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, null, new Path(ContainerLocalizer.USERCACHE + "_DEL_"),
+ new ArrayList<Path>())));
// restart the NodeManager again
// this time usercache directory should be empty
@@ -329,72 +326,4 @@ public class TestNodeManagerReboot {
return conf;
}
}
-
- class PathInclude extends ArgumentMatcher<Path> {
-
- final String part;
-
- PathInclude(String part) {
- this.part = part;
- }
-
- @Override
- public boolean matches(Object o) {
- return ((Path) o).getName().indexOf(part) != -1;
- }
- }
-
- class FileDeletionInclude extends ArgumentMatcher<FileDeletionTask> {
- final String user;
- final String subDirIncludes;
- final String[] baseDirIncludes;
-
- public FileDeletionInclude(String user, String subDirIncludes,
- String [] baseDirIncludes) {
- this.user = user;
- this.subDirIncludes = subDirIncludes;
- this.baseDirIncludes = baseDirIncludes;
- }
-
- @Override
- public boolean matches(Object o) {
- FileDeletionTask fd = (FileDeletionTask)o;
- if (fd.getUser() == null && user != null) {
- return false;
- } else if (fd.getUser() != null && user == null) {
- return false;
- } else if (fd.getUser() != null && user != null) {
- return fd.getUser().equals(user);
- }
- if (!comparePaths(fd.getSubDir(), subDirIncludes)) {
- return false;
- }
- if (baseDirIncludes == null && fd.getBaseDirs() != null) {
- return false;
- } else if (baseDirIncludes != null && fd.getBaseDirs() == null ) {
- return false;
- } else if (baseDirIncludes != null && fd.getBaseDirs() != null) {
- if (baseDirIncludes.length != fd.getBaseDirs().size()) {
- return false;
- }
- for (int i =0 ; i < baseDirIncludes.length; i++) {
- if (!comparePaths(fd.getBaseDirs().get(i), baseDirIncludes[i])) {
- return false;
- }
- }
- }
- return true;
- }
-
- public boolean comparePaths(Path p1, String p2) {
- if (p1 == null && p2 != null){
- return false;
- } else if (p1 != null && p2 == null) {
- return false;
- } else if (p1 != null && p2 != null ){
- return p1.toUri().getPath().contains(p2.toString());
- }
- return true;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14c8b4ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java
new file mode 100644
index 0000000..69e01bc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.impl.pb;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.recovery.DeletionTaskRecoveryInfo;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DeletionTask;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DeletionTaskType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionTask;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Test conversion to {@link DeletionTask}.
+ */
+public class TestNMProtoUtils {
+
+ @Test
+ public void testConvertProtoToDeletionTask() throws Exception {
+ DeletionService deletionService = mock(DeletionService.class);
+ DeletionServiceDeleteTaskProto.Builder protoBuilder =
+ DeletionServiceDeleteTaskProto.newBuilder();
+ int id = 0;
+ protoBuilder.setId(id);
+ DeletionServiceDeleteTaskProto proto = protoBuilder.build();
+ DeletionTask deletionTask =
+ NMProtoUtils.convertProtoToDeletionTask(proto, deletionService);
+ assertEquals(DeletionTaskType.FILE, deletionTask.getDeletionTaskType());
+ assertEquals(id, deletionTask.getTaskId());
+ }
+
+ @Test
+ public void testConvertProtoToFileDeletionTask() throws Exception {
+ DeletionService deletionService = mock(DeletionService.class);
+ int id = 0;
+ String user = "user";
+ Path subdir = new Path("subdir");
+ Path basedir = new Path("basedir");
+ DeletionServiceDeleteTaskProto.Builder protoBuilder =
+ DeletionServiceDeleteTaskProto.newBuilder();
+ protoBuilder
+ .setId(id)
+ .setUser("user")
+ .setSubdir(subdir.getName())
+ .addBasedirs(basedir.getName());
+ DeletionServiceDeleteTaskProto proto = protoBuilder.build();
+ DeletionTask deletionTask =
+ NMProtoUtils.convertProtoToFileDeletionTask(proto, deletionService, id);
+ assertEquals(DeletionTaskType.FILE.name(),
+ deletionTask.getDeletionTaskType().name());
+ assertEquals(id, deletionTask.getTaskId());
+ assertEquals(subdir, ((FileDeletionTask) deletionTask).getSubDir());
+ assertEquals(basedir,
+ ((FileDeletionTask) deletionTask).getBaseDirs().get(0));
+ }
+
+ @Test
+ public void testConvertProtoToDeletionTaskRecoveryInfo() throws Exception {
+ long delTime = System.currentTimeMillis();
+ List<Integer> successorTaskIds = Arrays.asList(1);
+ DeletionTask deletionTask = mock(DeletionTask.class);
+ DeletionTaskRecoveryInfo info =
+ new DeletionTaskRecoveryInfo(deletionTask, successorTaskIds, delTime);
+ assertEquals(deletionTask, info.getTask());
+ assertEquals(successorTaskIds, info.getSuccessorTaskIds());
+ assertEquals(delTime, info.getDeletionTimestamp());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14c8b4ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 2991c0c..7980a80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DeletionTask;
import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
@@ -260,10 +261,10 @@ public abstract class BaseContainerManagerTest {
protected DeletionService createDeletionService() {
return new DeletionService(exec) {
@Override
- public void delete(String user, Path subDir, Path... baseDirs) {
+ public void delete(DeletionTask deletionTask) {
// Don't do any deletions.
- LOG.info("Psuedo delete: user - " + user + ", subDir - " + subDir
- + ", baseDirs - " + Arrays.asList(baseDirs));
+ LOG.info("Psuedo delete: user - " + user
+ + ", type - " + deletionTask.getDeletionTaskType());
};
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14c8b4ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/FileDeletionMatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/FileDeletionMatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/FileDeletionMatcher.java
new file mode 100644
index 0000000..faad456
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/FileDeletionMatcher.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.mockito.ArgumentMatcher;
+
+import java.util.List;
+
+/**
+ * ArgumentMatcher to check the arguments of the {@link FileDeletionTask}.
+ */
+public class FileDeletionMatcher extends ArgumentMatcher<FileDeletionTask> {
+
+ private final DeletionService delService;
+ private final String user;
+ private final Path subDirIncludes;
+ private final List<Path> baseDirIncludes;
+
+ public FileDeletionMatcher(DeletionService delService, String user,
+ Path subDirIncludes, List<Path> baseDirIncludes) {
+ this.delService = delService;
+ this.user = user;
+ this.subDirIncludes = subDirIncludes;
+ this.baseDirIncludes = baseDirIncludes;
+ }
+
+ @Override
+ public boolean matches(Object o) {
+ FileDeletionTask fd = (FileDeletionTask) o;
+ if (fd.getUser() == null && user != null) {
+ return false;
+ } else if (fd.getUser() != null && user == null) {
+ return false;
+ } else if (fd.getUser() != null && user != null) {
+ return fd.getUser().equals(user);
+ }
+ if (!comparePaths(fd.getSubDir(), subDirIncludes.getName())) {
+ return false;
+ }
+ if (baseDirIncludes == null && fd.getBaseDirs() != null) {
+ return false;
+ } else if (baseDirIncludes != null && fd.getBaseDirs() == null) {
+ return false;
+ } else if (baseDirIncludes != null && fd.getBaseDirs() != null) {
+ if (baseDirIncludes.size() != fd.getBaseDirs().size()) {
+ return false;
+ }
+ for (int i = 0; i < baseDirIncludes.size(); i++) {
+ if (!comparePaths(fd.getBaseDirs().get(i),
+ baseDirIncludes.get(i).getName())) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ public boolean comparePaths(Path p1, String p2) {
+ if (p1 == null && p2 != null) {
+ return false;
+ } else if (p1 != null && p2 == null) {
+ return false;
+ } else if (p1 != null && p2 != null) {
+ return p1.toUri().getPath().contains(p2.toString());
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14c8b4ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestFileDeletionTask.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestFileDeletionTask.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestFileDeletionTask.java
new file mode 100644
index 0000000..fd2e4fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestFileDeletionTask.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Test the attributes of the {@link FileDeletionTask} class.
+ */
+public class TestFileDeletionTask {
+
+ private static final int ID = 0;
+ private static final String USER = "user";
+ private static final Path SUBDIR = new Path("subdir");
+ private static final Path BASEDIR = new Path("basedir");
+
+ private List<Path> baseDirs = new ArrayList<>();
+ private DeletionService deletionService;
+ private FileDeletionTask deletionTask;
+
+ @Before
+ public void setUp() throws Exception {
+ deletionService = mock(DeletionService.class);
+ baseDirs.add(BASEDIR);
+ deletionTask = new FileDeletionTask(ID, deletionService, USER, SUBDIR,
+ baseDirs);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ baseDirs.clear();
+ }
+
+ @Test
+ public void testGetUser() throws Exception {
+ assertEquals(USER, deletionTask.getUser());
+ }
+
+ @Test
+ public void testGetSubDir() throws Exception {
+ assertEquals(SUBDIR, deletionTask.getSubDir());
+ }
+
+ @Test
+ public void testGetBaseDirs() throws Exception {
+ assertEquals(1, deletionTask.getBaseDirs().size());
+ assertEquals(baseDirs, deletionTask.getBaseDirs());
+ }
+
+ @Test
+ public void testConvertDeletionTaskToProto() throws Exception {
+ DeletionServiceDeleteTaskProto proto =
+ deletionTask.convertDeletionTaskToProto();
+ assertEquals(ID, proto.getId());
+ assertEquals(USER, proto.getUser());
+ assertEquals(SUBDIR, new Path(proto.getSubdir()));
+ assertEquals(BASEDIR, new Path(proto.getBasedirs(0)));
+ assertEquals(1, proto.getBasedirsCount());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14c8b4ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
index 2874acb..6cab593 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.argThat;
import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.mock;
@@ -54,6 +55,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceLocalizedEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionMatcher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent;
@@ -823,7 +825,8 @@ public class TestLocalResourcesTrackerImpl {
Path rPath = tracker.getPathForLocalization(req1, base_path,
delService);
Assert.assertFalse(lfs.util().exists(rPath));
- verify(delService, times(1)).delete(eq(user), eq(conflictPath));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, user, conflictPath, null)));
} finally {
lfs.delete(base_path, true);
if (dispatcher != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14c8b4ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index 89cbeb4..d863c6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -31,7 +31,6 @@ import static org.mockito.Matchers.any;
import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.isA;
-import static org.mockito.Matchers.isNull;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
@@ -68,6 +67,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionMatcher;
import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
import org.junit.Assert;
import org.apache.commons.io.FileUtils;
@@ -1066,7 +1066,8 @@ public class TestResourceLocalizationService {
verify(containerBus, times(3)).handle(argThat(matchesContainerLoc));
// Verify deletion of localization token.
- verify(delService).delete((String)isNull(), eq(localizationTokenPath));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, null, localizationTokenPath, null)));
} finally {
spyService.stop();
dispatcher.stop();
@@ -1340,8 +1341,8 @@ public class TestResourceLocalizationService {
Thread.sleep(50);
}
// Verify if downloading resources were submitted for deletion.
- verify(delService).delete(eq(user), (Path) eq(null),
- argThat(new DownloadingPathsMatcher(paths)));
+ verify(delService, times(2)).delete(argThat(new FileDeletionMatcher(
+ delService, user, null, new ArrayList<>(paths))));
LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(
LocalResourceVisibility.PRIVATE, "user0", appId);
@@ -2753,15 +2754,19 @@ public class TestResourceLocalizationService {
for (int i = 0; i < containerLocalDirs.size(); ++i) {
if (i == 2) {
try {
- verify(delService).delete(user, containerLocalDirs.get(i));
- verify(delService).delete(null, nmLocalContainerDirs.get(i));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, user, containerLocalDirs.get(i), null)));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, null, nmLocalContainerDirs.get(i), null)));
Assert.fail("deletion attempts for invalid dirs");
} catch (Throwable e) {
continue;
}
} else {
- verify(delService).delete(user, containerLocalDirs.get(i));
- verify(delService).delete(null, nmLocalContainerDirs.get(i));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, user, containerLocalDirs.get(i), null)));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, null, nmLocalContainerDirs.get(i), null)));
}
}
@@ -2802,15 +2807,19 @@ public class TestResourceLocalizationService {
for (int i = 0; i < containerLocalDirs.size(); ++i) {
if (i == 3) {
try {
- verify(delService).delete(user, containerLocalDirs.get(i));
- verify(delService).delete(null, nmLocalContainerDirs.get(i));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, user, containerLocalDirs.get(i), null)));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, null, nmLocalContainerDirs.get(i), null)));
Assert.fail("deletion attempts for invalid dirs");
} catch (Throwable e) {
continue;
}
} else {
- verify(delService).delete(user, appLocalDirs.get(i));
- verify(delService).delete(null, nmLocalAppDirs.get(i));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, user, containerLocalDirs.get(i), null)));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, null, nmLocalContainerDirs.get(i), null)));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14c8b4ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java
index 097146b..b4bd9d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java
@@ -42,17 +42,16 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionTask;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
-import org.mockito.Matchers;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -311,16 +310,18 @@ public class TestAppLogAggregatorImpl {
@Override
public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
Set<String> paths = new HashSet<>();
- Object[] args = invocationOnMock.getArguments();
- for(int i = 2; i < args.length; i++) {
- Path path = (Path) args[i];
- paths.add(path.toUri().getRawPath());
+ Object[] tasks = invocationOnMock.getArguments();
+ for(int i = 0; i < tasks.length; i++) {
+ FileDeletionTask task = (FileDeletionTask) tasks[i];
+ for (Path path: task.getBaseDirs()) {
+ paths.add(path.toUri().getRawPath());
+ }
}
verifyFilesToDelete(expectedPathsForDeletion, paths);
return null;
}
}).doNothing().when(deletionServiceWithExpectedFiles).delete(
- any(String.class), any(Path.class), Matchers.<Path>anyVararg());
+ any(FileDeletionTask.class));
return deletionServiceWithExpectedFiles;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14c8b4ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index bc1b4b0..37fe77a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -120,6 +120,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionMatcher;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionTask;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.TestNonAggregatingLogHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
@@ -218,8 +220,10 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
// ensure filesystems were closed
verify(logAggregationService).closeFileSystems(
any(UserGroupInformation.class));
- verify(delSrvc).delete(eq(user), eq((Path) null),
- eq(new Path(app1LogDir.getAbsolutePath())));
+ List<Path> dirList = new ArrayList<>();
+ dirList.add(new Path(app1LogDir.toURI()));
+ verify(delSrvc, times(2)).delete(argThat(new FileDeletionMatcher(
+ delSrvc, user, null, dirList)));
String containerIdStr = container11.toString();
File containerLogDir = new File(app1LogDir, containerIdStr);
@@ -333,7 +337,9 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
logAggregationService.stop();
delSrvc.stop();
// Aggregated logs should not be deleted if not uploaded.
- verify(delSrvc, times(0)).delete(user, null);
+ FileDeletionTask deletionTask = new FileDeletionTask(delSrvc, user, null,
+ null);
+ verify(delSrvc, times(0)).delete(deletionTask);
}
@Test
@@ -815,8 +821,9 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
assertEquals(0, logAggregationService.getNumAggregators());
// local log dir shouldn't be deleted given log aggregation cannot
// continue due to aggregated log dir creation failure on remoteFS.
- verify(spyDelSrvc, never()).delete(eq(user), any(Path.class),
- Mockito.<Path>anyVararg());
+ FileDeletionTask deletionTask = new FileDeletionTask(spyDelSrvc, user,
+ null, null);
+ verify(spyDelSrvc, never()).delete(deletionTask);
verify(logAggregationService).closeFileSystems(
any(UserGroupInformation.class));
// make sure local log dir is not deleted in case log aggregation
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14c8b4ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
index ec3757e..7a4ea88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
@@ -22,12 +22,14 @@ import static org.junit.Assert.assertFalse;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.io.File;
@@ -36,6 +38,7 @@ import java.io.IOException;
import java.io.NotSerializableException;
import java.io.ObjectInputStream;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
@@ -66,6 +69,7 @@ import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionMatcher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent;
@@ -531,8 +535,8 @@ public class TestNonAggregatingLogHandler {
boolean matched = false;
while (!matched && System.currentTimeMillis() < verifyStartTime + timeout) {
try {
- verify(delService).delete(eq(user), (Path) eq(null),
- Mockito.argThat(new DeletePathsMatcher(matchPaths)));
+ verify(delService, times(1)).delete(argThat(new FileDeletionMatcher(
+ delService, user, null, Arrays.asList(matchPaths))));
matched = true;
} catch (WantedButNotInvoked e) {
notInvokedException = e;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org