You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ra...@apache.org on 2011/04/29 05:49:59 UTC
svn commit: r1097679 - in /hadoop/mapreduce/trunk: ./
src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/
src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/
src/docs/src/documentation/content/xdocs/
Author: ravigummadi
Date: Fri Apr 29 03:49:58 2011
New Revision: 1097679
URL: http://svn.apache.org/viewvc?rev=1097679&view=rev
Log:
MAPREDUCE-2416. Remove the restriction of specifying group names in users-list file for Gridmix in RoundRobinUserResolver mode.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/gridmix.xml
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1097679&r1=1097678&r2=1097679&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Apr 29 03:49:58 2011
@@ -83,6 +83,9 @@ Trunk (unreleased changes)
BUG FIXES
+ MAPREDUCE-2416. Remove the restriction of specifying group names in
+ users-list file for Gridmix in RoundRobinUserResolver mode.
+
MAPREDUCE-2417. Fix Gridmix in RoundRobinUserResolver mode to
map testing/proxy users to unique users in a trace.
Modified: hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java?rev=1097679&r1=1097678&r2=1097679&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/RoundRobinUserResolver.java Fri Apr 29 03:49:58 2011
@@ -47,8 +47,9 @@ public class RoundRobinUserResolver impl
new HashMap<String,UserGroupInformation>();
/**
- * Userlist assumes one UGI per line, each UGI matching
- * <username>,<group>[,group]*
+ * Userlist assumes one user per line.
+ * Each line in users-list-file is of the form <username>[,group]*
+ * <br> Group names are ignored(they are not parsed at all).
*/
private List<UserGroupInformation> parseUserList(URI userUri,
Configuration conf)
@@ -60,35 +61,40 @@ public class RoundRobinUserResolver impl
final Path userloc = new Path(userUri.toString());
final Text rawUgi = new Text();
final FileSystem fs = userloc.getFileSystem(conf);
- final ArrayList<UserGroupInformation> ret = new ArrayList();
+ final ArrayList<UserGroupInformation> ugiList =
+ new ArrayList<UserGroupInformation>();
LineReader in = null;
try {
- final ArrayList<String> groups = new ArrayList();
in = new LineReader(fs.open(userloc));
- while (in.readLine(rawUgi) > 0) {
+ while (in.readLine(rawUgi) > 0) {//line is of the form username[,group]*
+ // e is end position of user name in this line
int e = rawUgi.find(",");
- if (e <= 0) {
+ if (rawUgi.getLength() == 0 || e == 0) {
throw new IOException("Missing username: " + rawUgi);
}
+ if (e == -1) {
+ e = rawUgi.getLength();
+ }
final String username = Text.decode(rawUgi.getBytes(), 0, e);
- int s = e;
- while ((e = rawUgi.find(",", ++s)) != -1) {
- groups.add(Text.decode(rawUgi.getBytes(), s, e - s));
- s = e;
+ UserGroupInformation ugi = null;
+ try {
+ ugi = UserGroupInformation.createProxyUser(username,
+ UserGroupInformation.getLoginUser());
+ } catch (IOException ioe) {
+ LOG.error("Error while creating a proxy user " ,ioe);
}
- groups.add(Text.decode(rawUgi.getBytes(), s, rawUgi.getLength() - s));
- if (groups.size() == 0) {
- throw new IOException("Missing groups: " + rawUgi);
+ if (ugi != null) {
+ ugiList.add(ugi);
}
- ret.add(UserGroupInformation.createRemoteUser(username));
+ // No need to parse groups, even if they exist. Go to next line
}
} finally {
if (in != null) {
in.close();
}
}
- return ret;
+ return ugiList;
}
@Override
@@ -113,7 +119,7 @@ public class RoundRobinUserResolver impl
UserGroupInformation ugi) {
// UGI of proxy user
UserGroupInformation targetUGI = usercache.get(ugi.getUserName());
- if (null == targetUGI) {
+ if (targetUGI == null) {
targetUGI = users.get(uidx++ % users.size());
usercache.put(ugi.getUserName(), targetUGI);
}
Modified: hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java?rev=1097679&r1=1097678&r2=1097679&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestUserResolve.java Fri Apr 29 03:49:58 2011
@@ -115,10 +115,21 @@ public class TestUserResolve {
RoundRobinUserResolver.buildEmptyUsersErrorMsg(userRsrc);
validateBadUsersFile(rslv, userRsrc, expectedErrorMsg);
- // Create user resource file with valid content
+ // Create user resource file with valid content like older users list file
+ // with usernames and groups
writeUserList(usersFilePath,
"user0,groupA,groupB,groupC\nuser1,groupA,groupC\n");
validateValidUsersFile(rslv, userRsrc);
+
+ // Create user resource file with valid content with
+ // usernames with groups and without groups
+ writeUserList(usersFilePath, "user0,groupA,groupB\nuser1,");
+ validateValidUsersFile(rslv, userRsrc);
+
+ // Create user resource file with valid content with
+ // usernames without groups
+ writeUserList(usersFilePath, "user0\nuser1");
+ validateValidUsersFile(rslv, userRsrc);
}
// Validate RoundRobinUserResolver for the case of
@@ -129,15 +140,16 @@ public class TestUserResolve {
UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser("hfre0");
assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
assertEquals("user1",
- rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre1"))
- .getUserName());
+ rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre1"))
+ .getUserName());
assertEquals("user0",
- rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre2"))
- .getUserName());
+ rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre2"))
+ .getUserName());
assertEquals("user0", rslv.getTargetUgi(ugi1).getUserName());
assertEquals("user1",
- rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre3"))
- .getUserName());
+ rslv.getTargetUgi(UserGroupInformation.createRemoteUser("hfre3"))
+ .getUserName());
+
// Verify if same user comes again, its mapped user name should be
// correct even though UGI is constructed again.
assertEquals("user0", rslv.getTargetUgi(
Modified: hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/gridmix.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/gridmix.xml?rev=1097679&r1=1097678&r2=1097679&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/gridmix.xml (original)
+++ hadoop/mapreduce/trunk/src/docs/src/documentation/content/xdocs/gridmix.xml Fri Apr 29 03:49:58 2011
@@ -472,28 +472,27 @@ hadoop jar <gridmix-jar> org.apach
</table>
<p>If the parameter <code>gridmix.user.resolve.class</code> is set to
<code>org.apache.hadoop.mapred.gridmix.RoundRobinUserResolver</code>,
- we need to define a users-list file with a list of test users and groups.
+ we need to define a users-list file with a list of test users.
This is specified using the <code>-users</code> option to GridMix.</p>
<note>
Specifying a users-list file using the <code>-users</code> option is
mandatory when using the round-robin user-resolver. Other user-resolvers
ignore this option.
</note>
- <p>A users-list file has one user-group-information (UGI) per line, each
- UGI of the format:</p>
+ <p>A users-list file has one user per line, each line of the format:</p>
<source>
- <username>,<group>[,group]*
+ <username>
</source>
<p>For example:</p>
<source>
- user1,group1
- user2,group2,group3
- user3,group3,group4
+ user1
+ user2
+ user3
</source>
<p>In the above example we have defined three users <code>user1</code>,
- <code>user2</code> and <code>user3</code> with their respective groups.
+ <code>user2</code> and <code>user3</code>.
Now we would associate each unique user in the trace to the above users
- defined in round-robin fashion. For example, if traces users are
+ defined in round-robin fashion. For example, if trace's users are
<code>tuser1</code>, <code>tuser2</code>, <code>tuser3</code>,
<code>tuser4</code> and <code>tuser5</code>, then the mappings would
be:</p>
@@ -504,6 +503,10 @@ hadoop jar <gridmix-jar> org.apach
tuser4 -> user1
tuser5 -> user2
</source>
+ <p>For backward compatibility reasons, each line of users-list file can
+ contain username followed by groupnames in the form username[,group]*.
+ The groupnames will be ignored by Gridmix.
+ </p>
</section>
<section id="assumptions">
<title>Simplifying Assumptions</title>