You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ra...@apache.org on 2011/04/28 09:58:17 UTC
svn commit: r1097345 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
Author: ravigummadi
Date: Thu Apr 28 07:58:17 2011
New Revision: 1097345
URL: http://svn.apache.org/viewvc?rev=1097345&view=rev
Log:
MAPREDUCE-2417. Fix Gridmix in RoundRobinUserResolver mode to map testing/proxy users to unique users in a trace.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1097345&r1=1097344&r2=1097345&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Apr 28 07:58:17 2011
@@ -80,6 +80,9 @@ Trunk (unreleased changes)
BUG FIXES
+ MAPREDUCE-2417. Fix Gridmix in RoundRobinUserResolver mode to
+ map testing/proxy users to unique users in a trace.
+
MAPREDUCE-2307. Exception thrown in Jobtracker logs, when the Scheduler
configured is FairScheduler. (Devaraj K via matei)
Modified: hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java?rev=1097345&r1=1097344&r2=1097345&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java Thu Apr 28 07:58:17 2011
@@ -38,8 +38,13 @@ public class RoundRobinUserResolver impl
private int uidx = 0;
private List<UserGroupInformation> users = Collections.emptyList();
- private final HashMap<UserGroupInformation,UserGroupInformation> usercache =
- new HashMap<UserGroupInformation,UserGroupInformation>();
+
+ /**
+ * Mapping between user names of original cluster and UGIs of proxy users of
+ * simulated cluster
+ */
+ private final HashMap<String,UserGroupInformation> usercache =
+ new HashMap<String,UserGroupInformation>();
/**
* Userlist assumes one UGI per line, each UGI matching
@@ -89,11 +94,12 @@ public class RoundRobinUserResolver impl
@Override
public synchronized boolean setTargetUsers(URI userloc, Configuration conf)
throws IOException {
+ uidx = 0;
users = parseUserList(userloc, conf);
if (users.size() == 0) {
throw new IOException(buildEmptyUsersErrorMsg(userloc));
}
- usercache.keySet().retainAll(users);
+ usercache.clear();
return true;
}
@@ -105,20 +111,13 @@ public class RoundRobinUserResolver impl
@Override
public synchronized UserGroupInformation getTargetUgi(
UserGroupInformation ugi) {
- UserGroupInformation ret = usercache.get(ugi);
- if (null == ret) {
- ret = users.get(uidx++ % users.size());
- usercache.put(ugi, ret);
- }
- UserGroupInformation val = null;
- try {
- val =
- UserGroupInformation.createProxyUser(ret.getUserName(),
- UserGroupInformation.getLoginUser());
- } catch (IOException e) {
- LOG.error("Error while creating the proxy user " ,e);
+ // UGI of proxy user
+ UserGroupInformation targetUGI = usercache.get(ugi.getUserName());
+ if (null == targetUGI) {
+ targetUGI = users.get(uidx++ % users.size());
+ usercache.put(ugi.getUserName(), targetUGI);
}
- return val;
+ return targetUGI;
}
/**
Modified: hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java?rev=1097345&r1=1097344&r2=1097345&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java Thu Apr 28 07:58:17 2011
@@ -117,25 +117,37 @@ public class TestUserResolve {
// Create user resource file with valid content
writeUserList(usersFilePath,
- "user0,groupA,groupB,groupC\nuser1,groupA,groupC\n"
- + "user2,groupB\nuser3,groupA,groupB,groupC\n");
+ "user0,groupA,groupB,groupC\nuser1,groupA,groupC\n");
+ validateValidUsersFile(rslv, userRsrc);
+ }
- // Validate RoundRobinUserResolver for the case of
- // user resource file with valid content.
- assertTrue(rslv.setTargetUsers(new URI(usersFilePath.toString()), conf));
+ // Validate RoundRobinUserResolver for the case of
+ // user resource file with valid content.
+ private void validateValidUsersFile(UserResolver rslv, URI userRsrc)
+ throws IOException {
+ assertTrue(rslv.setTargetUsers(userRsrc, conf));
UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser("hfre0");
assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
assertEquals("user1",
rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre1"))
.getUserName());
- assertEquals("user2",
+ assertEquals("user0",
rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre2"))
.getUserName());
assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
- assertEquals("user3",
+ assertEquals("user1",
rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre3"))
.getUserName());
- assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
+ // Verify if same user comes again, its mapped user name should be
+ // correct even though UGI is constructed again.
+ assertEquals("user0", rslv.getTargetUgi(
+ UserGroupInformation.createRemoteUser("hfre0")).getUserName());
+ assertEquals("user0",
+ rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre5"))
+ .getUserName());
+ assertEquals("user0",
+ rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre0"))
+ .getUserName());
}
@Test