You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ss...@apache.org on 2012/01/03 19:58:58 UTC
svn commit: r1226916 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/fs/viewfs/
src/test/java/org/apache/hadoop/fs/viewfs/
Author: sseth
Date: Tue Jan 3 18:58:57 2012
New Revision: 1226916
URL: http://svn.apache.org/viewvc?rev=1226916&view=rev
Log:
HADOOP-7933. Add a getDelegationTokens api to FileSystem which checks for known tokens in the passed Credentials object. (sseth)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1226916&r1=1226915&r2=1226916&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jan 3 18:58:57 2012
@@ -202,6 +202,9 @@ Release 0.23.1 - Unreleased
HADOOP-7504. Add the missing Ganglia31 opts to hadoop-metrics.properties as a comment. (harsh)
+ HADOOP-7933. Add a getDelegationTokens api to FileSystem which checks
+ for known tokens in the passed Credentials object. (sseth)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1226916&r1=1226915&r2=1226916&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Tue Jan 3 18:58:57 2012
@@ -47,6 +47,7 @@ import org.apache.hadoop.conf.Configured
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -393,6 +394,40 @@ public abstract class FileSystem extends
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
return new ArrayList<Token<?>>(0);
}
+
+ /**
+ * @see #getDelegationTokens(String)
+ * This is similar to getDelegationTokens, with the added restriction that if
+ * a token is already present in the passed Credentials object - that token
+ * is returned instead of a new delegation token.
+ *
+ * If the token is found to be cached in the Credentials object, this API does
+ * not verify the token validity or the passed in renewer.
+ *
+ *
+ * @param renewer the account name that is allowed to renew the token.
+ * @param credentials a Credentials object containing already knowing
+ * delegationTokens.
+ * @return a list of delegation tokens.
+ * @throws IOException
+ */
+ @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
+ public List<Token<?>> getDelegationTokens(String renewer,
+ Credentials credentials) throws IOException {
+ List<Token<?>> allTokens = getDelegationTokens(renewer);
+ List<Token<?>> newTokens = new ArrayList<Token<?>>();
+ if (allTokens != null) {
+ for (Token<?> token : allTokens) {
+ Token<?> knownToken = credentials.getToken(token.getService());
+ if (knownToken == null) {
+ newTokens.add(token);
+ } else {
+ newTokens.add(knownToken);
+ }
+ }
+ }
+ return newTokens;
+ }
/** create a file with the provided permission
* The permission of the file is set to be the provided permission as in
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1226916&r1=1226915&r2=1226916&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Tue Jan 3 18:58:57 2012
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
@@ -388,4 +389,11 @@ public class FilterFileSystem extends Fi
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
return fs.getDelegationTokens(renewer);
}
-}
+
+ @Override
+ // FileSystem
+ public List<Token<?>> getDelegationTokens(String renewer,
+ Credentials credentials) throws IOException {
+ return fs.getDelegationTokens(renewer, credentials);
+ }
+}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java?rev=1226916&r1=1226915&r2=1226916&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java Tue Jan 3 18:58:57 2012
@@ -24,7 +24,9 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.StringTokenizer;
import java.util.Map.Entry;
@@ -45,7 +47,9 @@ import org.apache.hadoop.fs.UnsupportedF
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.InodeTree.INode;
import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
@@ -495,7 +499,40 @@ public class ViewFileSystem extends File
}
return result;
}
-
+
+ @Override
+ public List<Token<?>> getDelegationTokens(String renewer,
+ Credentials credentials) throws IOException {
+ List<InodeTree.MountPoint<FileSystem>> mountPoints =
+ fsState.getMountPoints();
+ int initialListSize = 0;
+ for (InodeTree.MountPoint<FileSystem> im : mountPoints) {
+ initialListSize += im.target.targetDirLinkList.length;
+ }
+ Set<String> seenServiceNames = new HashSet<String>();
+ List<Token<?>> result = new ArrayList<Token<?>>(initialListSize);
+ for (int i = 0; i < mountPoints.size(); ++i) {
+ String serviceName =
+ mountPoints.get(i).target.targetFileSystem.getCanonicalServiceName();
+ if (seenServiceNames.contains(serviceName)) {
+ continue;
+ }
+ seenServiceNames.add(serviceName);
+ Token<?> knownToken = credentials.getToken(new Text(serviceName));
+ if (knownToken != null) {
+ result.add(knownToken);
+ } else {
+ List<Token<?>> tokens =
+ mountPoints.get(i).target.targetFileSystem
+ .getDelegationTokens(renewer);
+ if (tokens != null) {
+ result.addAll(tokens);
+ }
+ }
+ }
+ return result;
+ }
+
/*
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java?rev=1226916&r1=1226915&r2=1226916&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java Tue Jan 3 18:58:57 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.viewfs.Confi
import org.apache.hadoop.fs.viewfs.ViewFileSystem;
import org.apache.hadoop.fs.viewfs.ViewFileSystem.MountPoint;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.junit.After;
import org.junit.Assert;
@@ -89,6 +90,16 @@ public class ViewFileSystemBaseTest {
// Set up the defaultMT in the config with our mount point links
//Configuration conf = new Configuration();
conf = ViewFileSystemTestSetup.configWithViewfsScheme();
+ setupMountPoints();
+ fsView = FileSystem.get(FsConstants.VIEWFS_URI, conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ fsTarget.delete(FileSystemTestHelper.getTestRootPath(fsTarget), true);
+ }
+
+ void setupMountPoints() {
ConfigUtil.addLink(conf, "/user", new Path(targetTestRoot,"user").toUri());
ConfigUtil.addLink(conf, "/user2", new Path(targetTestRoot,"user").toUri());
ConfigUtil.addLink(conf, "/data", new Path(targetTestRoot,"data").toUri());
@@ -100,20 +111,17 @@ public class ViewFileSystemBaseTest {
new Path(targetTestRoot,"missingTarget").toUri());
ConfigUtil.addLink(conf, "/linkToAFile",
new Path(targetTestRoot,"aFile").toUri());
-
- fsView = FileSystem.get(FsConstants.VIEWFS_URI, conf);
- }
-
- @After
- public void tearDown() throws Exception {
- fsTarget.delete(FileSystemTestHelper.getTestRootPath(fsTarget), true);
}
@Test
public void testGetMountPoints() {
ViewFileSystem viewfs = (ViewFileSystem) fsView;
MountPoint[] mountPoints = viewfs.getMountPoints();
- Assert.assertEquals(7, mountPoints.length);
+ Assert.assertEquals(getExpectedMountPoints(), mountPoints.length);
+ }
+
+ int getExpectedMountPoints() {
+ return 7;
}
/**
@@ -125,9 +133,46 @@ public class ViewFileSystemBaseTest {
public void testGetDelegationTokens() throws IOException {
List<Token<?>> delTokens =
fsView.getDelegationTokens("sanjay");
- Assert.assertEquals(0, delTokens.size());
+ Assert.assertEquals(getExpectedDelegationTokenCount(), delTokens.size());
}
+ int getExpectedDelegationTokenCount() {
+ return 0;
+ }
+
+ @Test
+ public void testGetDelegationTokensWithCredentials() throws IOException {
+ Credentials credentials = new Credentials();
+ List<Token<?>> delTokens =
+ fsView.getDelegationTokens("sanjay", credentials);
+
+ int expectedTokenCount = getExpectedDelegationTokenCountWithCredentials();
+
+ Assert.assertEquals(expectedTokenCount, delTokens.size());
+ for (int i = 0; i < expectedTokenCount / 2; i++) {
+ Token<?> token = delTokens.get(i);
+ credentials.addToken(token.getService(), token);
+ }
+
+ List<Token<?>> delTokens2 =
+ fsView.getDelegationTokens("sanjay", credentials);
+ Assert.assertEquals(expectedTokenCount, delTokens2.size());
+
+ for (int i = 0; i < delTokens2.size(); i++) {
+ for (int j = 0; j < delTokens.size(); j++) {
+ if (delTokens.get(j) == delTokens2.get(i)) {
+ delTokens.remove(j);
+ break;
+ }
+ }
+ }
+ Assert.assertEquals(expectedTokenCount / 2, delTokens.size());
+ }
+
+ int getExpectedDelegationTokenCountWithCredentials() {
+ return 0;
+ }
+
@Test
public void testBasicPaths() {
Assert.assertEquals(FsConstants.VIEWFS_URI,
@@ -340,7 +385,7 @@ public class ViewFileSystemBaseTest {
FileStatus[] dirPaths = fsView.listStatus(new Path("/"));
FileStatus fs;
- Assert.assertEquals(6, dirPaths.length);
+ Assert.assertEquals(getExpectedDirPaths(), dirPaths.length);
fs = FileSystemTestHelper.containsPath(fsView, "/user", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
@@ -372,6 +417,10 @@ public class ViewFileSystemBaseTest {
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
}
+ int getExpectedDirPaths() {
+ return 6;
+ }
+
@Test
public void testListOnMountTargetDirs() throws IOException {
FileStatus[] dirPaths = fsView.listStatus(new Path("/data"));