You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by dd...@apache.org on 2010/06/23 02:00:04 UTC
svn commit: r957086 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/JobTracker.java
src/java/org/apache/hadoop/mapred/tools/MRAdmin.java
src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java
Author: ddas
Date: Wed Jun 23 00:00:04 2010
New Revision: 957086
URL: http://svn.apache.org/viewvc?rev=957086&view=rev
Log:
MAPREDUCE-1855. Makes the refresh methods (for groups and proxy users) independent of the client side configuration. Contributed by Boris Shkolnik.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=957086&r1=957085&r2=957086&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Jun 23 00:00:04 2010
@@ -161,6 +161,9 @@ Release 0.21.0 - Unreleased
MAPREDUCE-1683. Remove JNI calls from ClusterStatus cstr. (Arun Murthy and
Luke Lu via cdouglas)
+ MAPREDUCE-1855. Makes the refresh methods (for groups and proxy users)
+ independent of the client side configuration. (Boris Shkolnik via ddas)
+
NEW FEATURES
MAPREDUCE-1774. Large-scale Automated Framework (Sharad Agarwal, Sreekanth
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=957086&r1=957085&r2=957086&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Wed Jun 23 00:00:04 2010
@@ -4449,18 +4449,18 @@ public class JobTracker implements MRCon
}
@Override
- public void refreshSuperUserGroupsConfiguration(Configuration conf) {
+ public void refreshSuperUserGroupsConfiguration() {
LOG.info("Refreshing superuser proxy groups mapping ");
- ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
+ ProxyUsers.refreshSuperUserGroupsConfiguration();
}
@Override
- public void refreshUserToGroupsMappings(Configuration conf) throws IOException {
+ public void refreshUserToGroupsMappings() throws IOException {
LOG.info("Refreshing all user-to-groups mappings. Requested by user: " +
UserGroupInformation.getCurrentUser().getShortUserName());
- Groups.getUserToGroupsMappingService(conf).refresh();
+ Groups.getUserToGroupsMappingService().refresh();
}
private boolean perTaskMemoryConfigurationSetOnJT() {
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java?rev=957086&r1=957085&r2=957086&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/tools/MRAdmin.java Wed Jun 23 00:00:04 2010
@@ -195,7 +195,7 @@ public class MRAdmin extends Configured
RefreshUserMappingsProtocol.class));
// Refresh the user-to-groups mappings
- refreshProtocol.refreshSuperUserGroupsConfiguration(conf);
+ refreshProtocol.refreshSuperUserGroupsConfiguration();
return 0;
}
@@ -227,7 +227,7 @@ public class MRAdmin extends Configured
RefreshUserMappingsProtocol.class));
// Refresh the user-to-groups mappings
- refreshProtocol.refreshUserToGroupsMappings(conf);
+ refreshProtocol.refreshUserToGroupsMappings();
return 0;
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java?rev=957086&r1=957085&r2=957086&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/TestMapredGroupMappingServiceRefresh.java Wed Jun 23 00:00:04 2010
@@ -25,8 +25,13 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.PrintWriter;
import java.net.URI;
+import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -36,8 +41,10 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.tools.MRAdmin;
@@ -52,6 +59,7 @@ public class TestMapredGroupMappingServi
private MiniDFSCluster cluster;
JobConf config;
private static long groupRefreshTimeoutSec = 2;
+ private String tempResource = null;
private static final Log LOG = LogFactory
.getLog(TestMapredGroupMappingServiceRefresh.class);
@@ -98,6 +106,7 @@ public class TestMapredGroupMappingServi
3, null, null, config);
config.set(JTConfig.JT_IPC_ADDRESS, "localhost:"+miniMRCluster.getJobTrackerPort());
+ ProxyUsers.refreshSuperUserGroupsConfiguration(config);
}
@After
@@ -105,6 +114,10 @@ public class TestMapredGroupMappingServi
if(cluster!=null) {
cluster.shutdown();
}
+ if(tempResource!=null) {
+ File f = new File(tempResource);
+ f.delete();
+ }
}
@Test
@@ -162,6 +175,7 @@ public class TestMapredGroupMappingServi
config.set(userKeyGroups, "gr3,gr4,gr5"); // superuser can proxy for this group
config.set(userKeyHosts,"127.0.0.1");
+ ProxyUsers.refreshSuperUserGroupsConfiguration(config);
UserGroupInformation ugi1 = mock(UserGroupInformation.class);
UserGroupInformation ugi2 = mock(UserGroupInformation.class);
@@ -199,16 +213,16 @@ public class TestMapredGroupMappingServi
fail("first auth for " + ugi2.getShortUserName() + " should've succeeded: " + e.getLocalizedMessage());
}
+ // refresh will look at configuration on the server side
+ // add additional resource with the new value
+ // so the server side will pick it up
+ String rsrc = "testRefreshSuperUserGroupsConfiguration_rsrc.xml";
+ addNewConfigResource(rsrc, userKeyGroups, "gr2", userKeyHosts, "127.0.0.1");
+
MRAdmin admin = new MRAdmin(config);
String [] args = new String[]{"-refreshSuperUserGroupsConfiguration"};
- NameNode nn = cluster.getNameNode();
- Configuration conf = new Configuration(config);
- conf.set(userKeyGroups, "gr2"); // superuser can proxy for this group
- admin.setConf(conf);
admin.run(args);
- //check after...
-
try {
ProxyUsers.authorize(ugi2, "127.0.0.1", config);
fail("second auth for " + ugi2.getShortUserName() + " should've failed ");
@@ -224,5 +238,26 @@ public class TestMapredGroupMappingServi
fail("second auth for " + ugi1.getShortUserName() + " should've succeeded: " + e.getLocalizedMessage());
}
}
+
+ private void addNewConfigResource(String rsrcName, String keyGroup,
+ String groups, String keyHosts, String hosts) throws FileNotFoundException {
+ // location for temp resource should be in CLASSPATH
+ URL url = config.getResource("mapred-site.xml");
+ Path p = new Path(url.getPath());
+ Path dir = p.getParent();
+ tempResource = dir.toString() + "/" + rsrcName;
+
+
+ String newResource =
+ "<configuration>"+
+ "<property><name>" + keyGroup + "</name><value>"+groups+"</value></property>" +
+ "<property><name>" + keyHosts + "</name><value>"+hosts+"</value></property>" +
+ "</configuration>";
+ PrintWriter writer = new PrintWriter(new FileOutputStream(tempResource));
+ writer.println(newResource);
+ writer.close();
+
+ Configuration.addDefaultResource(rsrcName);
+ }
}