You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2018/02/01 01:52:57 UTC
hadoop git commit: YARN-7816. Allow same application name submitted
by multiple users. (Contributed by Gour Saha)
Repository: hadoop
Updated Branches:
refs/heads/trunk 5a725bb88 -> 0bee3849e
YARN-7816. Allow same application name submitted by multiple users. (Contributed by Gour Saha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0bee3849
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0bee3849
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0bee3849
Branch: refs/heads/trunk
Commit: 0bee3849e323bf71925024992f9e655aee2d75f9
Parents: 5a725bb
Author: Eric Yang <ey...@apache.org>
Authored: Wed Jan 31 20:51:40 2018 -0500
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Jan 31 20:51:40 2018 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/http/HttpServer2.java | 2 +-
.../yarn/service/client/ServiceClient.java | 4 +
.../hadoop/yarn/service/ServiceTestUtils.java | 5 +
.../yarn/service/TestYarnNativeServices.java | 128 +++++++++++++++++++
4 files changed, 138 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bee3849/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index 65aadf3..7e12640 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -136,7 +136,7 @@ public final class HttpServer2 implements FilterContainer {
public static final String HTTP_MAX_THREADS_KEY = "hadoop.http.max.threads";
public static final String HTTP_TEMP_DIR_KEY = "hadoop.http.temp.dir";
- static final String FILTER_INITIALIZER_PROPERTY
+ public static final String FILTER_INITIALIZER_PROPERTY
= "hadoop.http.filter.initializers";
// The ServletContext attribute where the daemon Configuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bee3849/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index ee6e681..612b8b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -540,6 +540,10 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
request.setApplicationTypes(types);
request.setApplicationTags(tags);
request.setApplicationStates(liveStates);
+ String user = UserGroupInformation.getCurrentUser().getUserName();
+ if (user != null) {
+ request.setUsers(Collections.singleton(user));
+ }
List<ApplicationReport> reports = yarnClient.getApplications(request);
if (!reports.isEmpty()) {
String message = "";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bee3849/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index a70a0c2..1117b76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -165,6 +166,10 @@ public class ServiceTestUtils {
// Disable vmem check to disallow NM killing the container
conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
+ // set auth filters
+ conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
+ "org.apache.hadoop.security.AuthenticationFilterInitializer,"
+ + "org.apache.hadoop.security.HttpCrossOriginFilterInitializer");
// setup zk cluster
zkCluster = new TestingCluster(1);
zkCluster.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bee3849/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 5067ffc..b6126bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -22,6 +22,7 @@ import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -34,6 +35,7 @@ import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.client.ServiceClient;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -49,6 +51,7 @@ import java.util.*;
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
/**
* End to end tests to test deploying services with MiniYarnCluster and a in-JVM
@@ -158,6 +161,109 @@ public class TestYarnNativeServices extends ServiceTestUtils {
client.actionDestroy(exampleApp.getName());
}
+ @Test(timeout = 200000)
+ public void testCreateServiceSameNameDifferentUser() throws Exception {
+ String sameAppName = "same-name";
+ String userA = "usera";
+ String userB = "userb";
+
+ setupInternal(NUM_NMS);
+ ServiceClient client = createClient();
+ String origBasePath = getConf().get(YARN_SERVICE_BASE_PATH);
+
+ Service userAApp = new Service();
+ userAApp.setName(sameAppName);
+ userAApp.addComponent(createComponent("comp", 1, "sleep 1000"));
+ Service userBApp = new Service();
+ userBApp.setName(sameAppName);
+ userBApp.addComponent(createComponent("comp", 1, "sleep 1000"));
+
+ File userABasePath = null, userBBasePath = null;
+ try {
+ userABasePath = new File(origBasePath, userA);
+ userABasePath.mkdirs();
+ getConf().set(YARN_SERVICE_BASE_PATH, userABasePath.getAbsolutePath());
+ client.actionCreate(userAApp);
+ waitForServiceToBeStarted(client, userAApp);
+
+ userBBasePath = new File(origBasePath, userB);
+ userBBasePath.mkdirs();
+ getConf().set(YARN_SERVICE_BASE_PATH, userBBasePath.getAbsolutePath());
+ client.actionBuild(userBApp);
+ } catch (Exception e) {
+ Assert
+ .fail("Exception should not be thrown - " + e.getLocalizedMessage());
+ } finally {
+ if (userABasePath != null) {
+ getConf().set(YARN_SERVICE_BASE_PATH, userABasePath.getAbsolutePath());
+ client.actionStop(sameAppName, true);
+ client.actionDestroy(sameAppName);
+ }
+ if (userBBasePath != null) {
+ getConf().set(YARN_SERVICE_BASE_PATH, userBBasePath.getAbsolutePath());
+ client.actionDestroy(sameAppName);
+ }
+ }
+
+ // Need to extend this test to validate that different users can create
+ // apps of exact same name. So far only create followed by build is tested.
+ // Need to test create followed by create.
+ }
+
+ @Test(timeout = 200000)
+ public void testCreateServiceSameNameSameUser() throws Exception {
+ String sameAppName = "same-name";
+ String user = UserGroupInformation.getCurrentUser().getUserName();
+ System.setProperty("user.name", user);
+
+ setupInternal(NUM_NMS);
+ ServiceClient client = createClient();
+
+ Service appA = new Service();
+ appA.setName(sameAppName);
+ appA.addComponent(createComponent("comp", 1, "sleep 1000"));
+ Service appB = new Service();
+ appB.setName(sameAppName);
+ appB.addComponent(createComponent("comp", 1, "sleep 1000"));
+
+ try {
+ client.actionBuild(appA);
+ client.actionBuild(appB);
+ } catch (Exception e) {
+ String expectedMsg = "Service Instance dir already exists:";
+ if (e.getLocalizedMessage() != null) {
+ Assert.assertThat(e.getLocalizedMessage(),
+ CoreMatchers.containsString(expectedMsg));
+ } else {
+ Assert.fail("Message cannot be null. It has to say - " + expectedMsg);
+ }
+ } finally {
+ // cleanup
+ client.actionDestroy(sameAppName);
+ }
+
+ try {
+ client.actionCreate(appA);
+ waitForServiceToBeStarted(client, appA);
+
+ client.actionCreate(appB);
+ waitForServiceToBeStarted(client, appB);
+ } catch (Exception e) {
+ String expectedMsg = "Failed to create service " + sameAppName
+ + ", because it already exists.";
+ if (e.getLocalizedMessage() != null) {
+ Assert.assertThat(e.getLocalizedMessage(),
+ CoreMatchers.containsString(expectedMsg));
+ } else {
+ Assert.fail("Message cannot be null. It has to say - " + expectedMsg);
+ }
+ } finally {
+ // cleanup
+ client.actionStop(sameAppName, true);
+ client.actionDestroy(sameAppName);
+ }
+ }
+
// Test to verify recovery of SeviceMaster after RM is restarted.
// 1. Create an example service.
// 2. Restart RM.
@@ -369,6 +475,28 @@ public class TestYarnNativeServices extends ServiceTestUtils {
}, 2000, 200000);
}
+ /**
+ * Wait until service is started. It does not have to reach a stable state.
+ *
+ * @param client
+ * @param exampleApp
+ * @throws TimeoutException
+ * @throws InterruptedException
+ */
+ private void waitForServiceToBeStarted(ServiceClient client,
+ Service exampleApp) throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ Service retrievedApp = client.getStatus(exampleApp.getName());
+ System.out.println(retrievedApp);
+ return retrievedApp.getState() == ServiceState.STARTED;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }, 2000, 200000);
+ }
+
private ServiceClient createClient() throws Exception {
ServiceClient client = new ServiceClient() {
@Override protected Path addJarResource(String appName,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org