You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/05/30 06:26:47 UTC
svn commit: r1128990 - in /hadoop/mapreduce/branches/MR-279: ./
yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/
yar...
Author: vinodkv
Date: Mon May 30 04:26:46 2011
New Revision: 1128990
URL: http://svn.apache.org/viewvc?rev=1128990&view=rev
Log:
Fixing and reneabling TestContainerTokenSecretManager.
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1128990&r1=1128989&r2=1128990&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Mon May 30 04:26:46 2011
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Fixing and reneabling TestContainerTokenSecretManager. (vinodkv)
+
Set correct version of avro-maven-plugin that is available in
apache maven repositories. (vinodkv)
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java?rev=1128990&r1=1128989&r2=1128990&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerToken.java Mon May 30 04:26:46 2011
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.yarn.api.records;
import java.nio.ByteBuffer;
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1128990&r1=1128989&r2=1128990&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Mon May 30 04:26:46 2011
@@ -448,7 +448,8 @@ public class FifoScheduler implements Re
node.getHttpAddress(), capability);
// If security is enabled, send the container-tokens too.
if (UserGroupInformation.isSecurityEnabled()) {
- ContainerToken containerToken = recordFactory.newRecordInstance(ContainerToken.class);
+ ContainerToken containerToken =
+ recordFactory.newRecordInstance(ContainerToken.class);
ContainerTokenIdentifier tokenidentifier =
new ContainerTokenIdentifier(container.getId(),
container.getContainerManagerAddress(), container.getResource());
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java?rev=1128990&r1=1128989&r2=1128990&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java Mon May 30 04:26:46 2011
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server;
import static org.junit.Assert.fail;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
@@ -35,9 +38,11 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.SecurityInfo;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -56,11 +61,13 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -73,23 +80,35 @@ import org.apache.hadoop.yarn.security.C
import org.apache.hadoop.yarn.security.SchedulerSecurityInfo;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.junit.Ignore;
+import org.junit.BeforeClass;
import org.junit.Test;
-@Ignore
public class TestContainerTokenSecretManager {
private static Log LOG = LogFactory
.getLog(TestContainerTokenSecretManager.class);
- private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+ private static final RecordFactory recordFactory = RecordFactoryProvider
+ .getRecordFactory(null);
+ private static FileContext localFS = null;
+ private static final File localDir = new File("target",
+ TestContainerTokenSecretManager.class.getName() + "-localDir")
+ .getAbsoluteFile();
+
+ @BeforeClass
+ public static void setup() throws AccessControlException,
+ FileNotFoundException, UnsupportedFileSystemException, IOException {
+ localFS = FileContext.getLocalFSFileContext();
+ localFS.delete(new Path(localDir.getAbsolutePath()), true);
+ localDir.mkdir();
+ }
@Test
public void test() throws IOException, InterruptedException {
- final ContainerId containerID = recordFactory.newRecordInstance(ContainerId.class);
- ApplicationId appID = recordFactory.newRecordInstance(ApplicationId.class);
+
+ final ApplicationId appID = recordFactory.newRecordInstance(ApplicationId.class);
appID.setClusterTimestamp(1234);
appID.setId(5);
- containerID.setAppId(appID);
+
final Configuration conf = new Configuration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
@@ -106,9 +125,11 @@ public class TestContainerTokenSecretMan
final YarnRPC yarnRPC = YarnRPC.create(conf);
// Submit an application
- ApplicationSubmissionContext appSubmissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
- appSubmissionContext.setApplicationId(containerID.getAppId());
- appSubmissionContext.setMasterCapability(recordFactory.newRecordInstance(Resource.class));
+ ApplicationSubmissionContext appSubmissionContext =
+ recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
+ appSubmissionContext.setApplicationId(appID);
+ appSubmissionContext.setMasterCapability(recordFactory
+ .newRecordInstance(Resource.class));
appSubmissionContext.getMasterCapability().setMemory(1024);
// appSubmissionContext.resources = new HashMap<String, URL>();
appSubmissionContext.setUser("testUser");
@@ -116,26 +137,35 @@ public class TestContainerTokenSecretMan
// appSubmissionContext.command = new ArrayList<String>();
appSubmissionContext.addCommand("sleep");
appSubmissionContext.addCommand("100");
- URL yarnUrlForJobSubmitDir =
+
+ // TODO: Use a resource to work around bugs. Today NM doesn't create local
+ // app-dirs if there are no file to download!!
+ File file = new File(localDir.getAbsolutePath(), "testFile");
+ FileWriter tmpFile = new FileWriter(file);
+ tmpFile.write("testing");
+ tmpFile.close();
+ URL testFileURL =
ConverterUtils.getYarnUrlFromPath(FileContext.getFileContext()
- .makeQualified(new Path("testPath")));
- appSubmissionContext.setResource(
- YARNApplicationConstants.JOB_SUBMIT_DIR, yarnUrlForJobSubmitDir);
+ .makeQualified(new Path(localDir.getAbsolutePath(), "testFile")));
+ LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
+ rsrc.setResource(testFileURL);
+ rsrc.setSize(file.length());
+ rsrc.setTimestamp(file.lastModified());
+ rsrc.setType(LocalResourceType.FILE);
+ rsrc.setVisibility(LocalResourceVisibility.PRIVATE);
+ appSubmissionContext.setResourceTodo("testFile", rsrc);
resourceManager.getApplicationsManager().submitApplication(
appSubmissionContext);
// Wait till container gets allocated for AM
int waitCounter = 0;
Application app =
- resourceManager.getApplicationsManager().getApplication(
- containerID.getAppId());
+ resourceManager.getApplicationsManager().getApplication(appID);
while (app.getState() != ApplicationState.LAUNCHED && waitCounter <= 20) {
Thread.sleep(1000);
LOG.info("Waiting for AM to be allocated a container. Current state is "
+ app.getState());
- app =
- resourceManager.getApplicationsManager().getApplication(
- containerID.getAppId());
+ app = resourceManager.getApplicationsManager().getApplication(appID);
}
Assert.assertTrue(ApplicationState.PENDING != app.getState());
@@ -149,7 +179,7 @@ public class TestContainerTokenSecretMan
final InetSocketAddress schedulerAddr =
NetUtils.createSocketAddr(schedulerAddressString);
ApplicationTokenIdentifier appTokenIdentifier =
- new ApplicationTokenIdentifier(containerID.getAppId());
+ new ApplicationTokenIdentifier(appID);
ApplicationTokenSecretManager appTokenSecretManager =
new ApplicationTokenSecretManager();
appTokenSecretManager.setMasterKey(ApplicationTokenSecretManager
@@ -179,11 +209,11 @@ public class TestContainerTokenSecretMan
.newRecordInstance(RegisterApplicationMasterRequest.class);
ApplicationMaster applicationMaster = recordFactory
.newRecordInstance(ApplicationMaster.class);
- applicationMaster.setApplicationId(containerID.getAppId());
+ applicationMaster.setApplicationId(appID);
applicationMaster.setState(ApplicationState.RUNNING);
ApplicationStatus status =
recordFactory.newRecordInstance(ApplicationStatus.class);
- status.setApplicationId(containerID.getAppId());
+ status.setApplicationId(appID);
applicationMaster.setStatus(status);
request.setApplicationMaster(applicationMaster);
scheduler.registerApplicationMaster(request);
@@ -200,7 +230,8 @@ public class TestContainerTokenSecretMan
ask.add(rr);
ArrayList<Container> release = new ArrayList<Container>();
- AllocateRequest allocateRequest = recordFactory.newRecordInstance(AllocateRequest.class);
+ AllocateRequest allocateRequest =
+ recordFactory.newRecordInstance(AllocateRequest.class);
allocateRequest.setApplicationStatus(status);
allocateRequest.addAllAsks(ask);
allocateRequest.addAllReleases(release);
@@ -215,7 +246,8 @@ public class TestContainerTokenSecretMan
status.setResponseId(status.getResponseId() + 1);
allocateRequest.setApplicationStatus(status);
allocatedContainers =
- scheduler.allocate(allocateRequest).getAMResponse().getContainerList();
+ scheduler.allocate(allocateRequest).getAMResponse()
+ .getContainerList();
}
Assert.assertNotNull("Container is not allocted!", allocatedContainers);
@@ -240,10 +272,17 @@ public class TestContainerTokenSecretMan
public Void run() {
ContainerManager client =
(ContainerManager) yarnRPC.getProxy(ContainerManager.class,
- NetUtils.createSocketAddr(allocatedContainer.getContainerManagerAddress()
- ), conf);
+ NetUtils.createSocketAddr(allocatedContainer
+ .getContainerManagerAddress()), conf);
try {
- GetContainerStatusRequest request = recordFactory.newRecordInstance(GetContainerStatusRequest.class);
+ LOG.info("Going to make a getContainerStatus() legal request");
+ GetContainerStatusRequest request =
+ recordFactory
+ .newRecordInstance(GetContainerStatusRequest.class);
+ ContainerId containerID =
+ recordFactory.newRecordInstance(ContainerId.class);
+ containerID.setAppId(appID);
+ containerID.setId(1);
request.setContainerId(containerID);
client.getContainerStatus(request);
} catch (YarnRemoteException e) {
@@ -277,12 +316,37 @@ public class TestContainerTokenSecretMan
maliceUser.doAs(new PrivilegedAction<Void>() {
@Override
public Void run() {
+ ContainerManager client =
+ (ContainerManager) yarnRPC.getProxy(ContainerManager.class,
+ NetUtils.createSocketAddr(allocatedContainer
+ .getContainerManagerAddress()), conf);
+ ContainerId containerID;
+
+ LOG.info("Going to contact NM: ilLegal request");
+ GetContainerStatusRequest request =
+ recordFactory
+ .newRecordInstance(GetContainerStatusRequest.class);
+ containerID =
+ recordFactory.newRecordInstance(ContainerId.class);
+ containerID.setAppId(appID);
+ containerID.setId(1);
+ request.setContainerId(containerID);
try {
- yarnRPC.getProxy(ContainerManager.class, NetUtils
- .createSocketAddr(allocatedContainer.getContainerManagerAddress()), conf);
- fail("Connection initiation with illegally modified tokens is expected to fail.");
- } catch (YarnException e) {
- LOG.info("Error", e);
+ client.getContainerStatus(request);
+ fail("Connection initiation with illegally modified "
+ + "tokens is expected to fail.");
+ } catch (YarnRemoteException e) {
+ LOG.error("Got exception", e);
+ fail("Cannot get a YARN remote exception as " +
+ "it will indicate RPC success");
+ } catch (Exception e) {
+ Assert.assertEquals(
+ java.lang.reflect.UndeclaredThrowableException.class
+ .getCanonicalName(), e.getClass().getCanonicalName());
+ Assert
+ .assertEquals(
+ "DIGEST-MD5: digest response format violation. Mismatched response.",
+ e.getCause().getCause().getMessage());
}
return null;
}