You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by da...@apache.org on 2012/08/17 16:05:12 UTC
svn commit: r1374271 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/fs/viewfs/
src/main/java/org/apache/hadoop/security/
src/test/java/org/apache/hadoop/fs/ src/test...
Author: daryn
Date: Fri Aug 17 14:05:11 2012
New Revision: 1374271
URL: http://svn.apache.org/viewvc?rev=1374271&view=rev
Log:
HADOOP-7967. Need generalized multi-token filesystem support (daryn)
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemTokens.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 14:05:11 2012
@@ -806,6 +806,8 @@ Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
+ HADOOP-7967. Need generalized multi-token filesystem support (daryn)
+
NEW FEATURES
IMPROVEMENTS
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java Fri Aug 17 14:05:11 2012
@@ -21,6 +21,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
@@ -217,6 +218,6 @@ public abstract class DelegateToFileSyst
@Override //AbstractFileSystem
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
- return fsImpl.getDelegationTokens(renewer);
+ return Arrays.asList(fsImpl.addDelegationTokens(renewer, null));
}
}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java Fri Aug 17 14:05:11 2012
@@ -110,7 +110,11 @@ public class DelegationTokenRenewer<T ex
fs.getRenewToken().renew(fs.getConf());
} catch (IOException ie) {
try {
- fs.setDelegationToken(fs.getDelegationTokens(null).get(0));
+ Token<?>[] tokens = fs.addDelegationTokens(null, null);
+ if (tokens.length == 0) {
+ throw new IOException("addDelegationTokens returned no tokens");
+ }
+ fs.setDelegationToken(tokens[0]);
} catch (IOException ie2) {
throw new IOException("Can't renew or get new delegation token ", ie);
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Fri Aug 17 14:05:11 2012
@@ -48,6 +48,7 @@ import org.apache.hadoop.conf.Configured
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
@@ -57,6 +58,8 @@ import org.apache.hadoop.util.Progressab
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
+import com.google.common.annotations.VisibleForTesting;
+
/****************************************************************
* An abstract base class for a fairly generic filesystem. It
* may be implemented as a distributed filesystem, or as a "local"
@@ -222,15 +225,25 @@ public abstract class FileSystem extends
/**
* Get a canonical service name for this file system. The token cache is
- * the only user of this value, and uses it to lookup this filesystem's
- * service tokens. The token cache will not attempt to acquire tokens if the
- * service is null.
+ * the only user of the canonical service name, and uses it to lookup this
+ * filesystem's service tokens.
+ * If file system provides a token of its own then it must have a canonical
+ * name, otherwise canonical name can be null.
+ *
+ * Default Impl: If the file system has child file systems
+ * (such as an embedded file system) then it is assumed that the fs has no
+ * tokens of its own and hence returns a null name; otherwise a service
+ * name is built using Uri and port.
+ *
* @return a service string that uniquely identifies this file system, null
* if the filesystem does not implement tokens
* @see SecurityUtil#buildDTServiceName(URI, int)
*/
+ @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
public String getCanonicalServiceName() {
- return SecurityUtil.buildDTServiceName(getUri(), getDefaultPort());
+ return (getChildFileSystems() == null)
+ ? SecurityUtil.buildDTServiceName(getUri(), getDefaultPort())
+ : null;
}
/** @deprecated call #getUri() instead.*/
@@ -396,68 +409,95 @@ public abstract class FileSystem extends
}
/**
- * Deprecated - use @link {@link #getDelegationTokens(String)}
* Get a new delegation token for this file system.
+ * This is an internal method that should have been declared protected
+ * but wasn't historically.
+ * Callers should use {@link #addDelegationTokens(String, Credentials)}
+ *
* @param renewer the account name that is allowed to renew the token.
* @return a new delegation token
* @throws IOException
*/
- @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
- @Deprecated
+ @InterfaceAudience.Private()
public Token<?> getDelegationToken(String renewer) throws IOException {
return null;
}
/**
- * Get one or more delegation tokens associated with the filesystem. Normally
- * a file system returns a single delegation token. A file system that manages
- * multiple file systems underneath, could return set of delegation tokens for
- * all the file systems it manages.
+ * Obtain all delegation tokens used by this FileSystem that are not
+ * already present in the given Credentials. Existing tokens will neither
+ * be verified as valid nor having the given renewer. Missing tokens will
+ * be acquired and added to the given Credentials.
+ *
+ * Default Impl: works for simple fs with its own token
+ * and also for an embedded fs whose tokens are those of its
+ * children file system (i.e. the embedded fs has not tokens of its
+ * own).
*
- * @param renewer the account name that is allowed to renew the token.
+ * @param renewer the user allowed to renew the delegation tokens
+ * @param credentials cache in which to add new delegation tokens
* @return list of new delegation tokens
- * If delegation tokens not supported then return a list of size zero.
* @throws IOException
*/
- @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
- public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
- return new ArrayList<Token<?>>(0);
+ @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
+ public Token<?>[] addDelegationTokens(
+ final String renewer, Credentials credentials) throws IOException {
+ if (credentials == null) {
+ credentials = new Credentials();
+ }
+ final List<Token<?>> tokens = new ArrayList<Token<?>>();
+ collectDelegationTokens(renewer, credentials, tokens);
+ return tokens.toArray(new Token<?>[tokens.size()]);
}
/**
- * @see #getDelegationTokens(String)
- * This is similar to getDelegationTokens, with the added restriction that if
- * a token is already present in the passed Credentials object - that token
- * is returned instead of a new delegation token.
- *
- * If the token is found to be cached in the Credentials object, this API does
- * not verify the token validity or the passed in renewer.
- *
- *
- * @param renewer the account name that is allowed to renew the token.
- * @param credentials a Credentials object containing already knowing
- * delegationTokens.
- * @return a list of delegation tokens.
+ * Recursively obtain the tokens for this FileSystem and all descended
+ * FileSystems as determined by getChildFileSystems().
+ * @param renewer the user allowed to renew the delegation tokens
+ * @param credentials cache in which to add the new delegation tokens
+ * @param tokens list in which to add acquired tokens
* @throws IOException
*/
- @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
- public List<Token<?>> getDelegationTokens(String renewer,
- Credentials credentials) throws IOException {
- List<Token<?>> allTokens = getDelegationTokens(renewer);
- List<Token<?>> newTokens = new ArrayList<Token<?>>();
- if (allTokens != null) {
- for (Token<?> token : allTokens) {
- Token<?> knownToken = credentials.getToken(token.getService());
- if (knownToken == null) {
- newTokens.add(token);
- } else {
- newTokens.add(knownToken);
+ private void collectDelegationTokens(final String renewer,
+ final Credentials credentials,
+ final List<Token<?>> tokens)
+ throws IOException {
+ final String serviceName = getCanonicalServiceName();
+ // Collect token of the this filesystem and then of its embedded children
+ if (serviceName != null) { // fs has token, grab it
+ final Text service = new Text(serviceName);
+ Token<?> token = credentials.getToken(service);
+ if (token == null) {
+ token = getDelegationToken(renewer);
+ if (token != null) {
+ tokens.add(token);
+ credentials.addToken(service, token);
}
}
}
- return newTokens;
+ // Now collect the tokens from the children
+ final FileSystem[] children = getChildFileSystems();
+ if (children != null) {
+ for (final FileSystem fs : children) {
+ fs.collectDelegationTokens(renewer, credentials, tokens);
+ }
+ }
}
+ /**
+ * Get all the immediate child FileSystems embedded in this FileSystem.
+ * It does not recurse and get grand children. If a FileSystem
+ * has multiple child FileSystems, then it should return a unique list
+ * of those FileSystems. Default is to return null to signify no children.
+ *
+ * @return FileSystems used by this FileSystem
+ */
+ @InterfaceAudience.LimitedPrivate({ "HDFS" })
+ @VisibleForTesting
+ public FileSystem[] getChildFileSystems() {
+ return null;
+ }
+
/** create a file with the provided permission
* The permission of the file is set to be the provided permission as in
* setPermission, not permission&~umask
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Fri Aug 17 14:05:11 2012
@@ -22,15 +22,11 @@ import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.EnumSet;
-import java.util.List;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
/****************************************************************
@@ -428,25 +424,7 @@ public class FilterFileSystem extends Fi
}
@Override // FileSystem
- public String getCanonicalServiceName() {
- return fs.getCanonicalServiceName();
- }
-
- @Override // FileSystem
- @SuppressWarnings("deprecation")
- public Token<?> getDelegationToken(String renewer) throws IOException {
- return fs.getDelegationToken(renewer);
- }
-
- @Override // FileSystem
- public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
- return fs.getDelegationTokens(renewer);
- }
-
- @Override
- // FileSystem
- public List<Token<?>> getDelegationTokens(String renewer,
- Credentials credentials) throws IOException {
- return fs.getDelegationTokens(renewer, credentials);
+ public FileSystem[] getChildFileSystems() {
+ return new FileSystem[]{fs};
}
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java Fri Aug 17 14:05:11 2012
@@ -23,7 +23,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -49,11 +49,8 @@ import org.apache.hadoop.fs.UnsupportedF
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.InodeTree.INode;
import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Time;
@@ -236,11 +233,6 @@ public class ViewFileSystem extends File
}
@Override
- public String getCanonicalServiceName() {
- return null;
- }
-
- @Override
public URI getUri() {
return myUri;
}
@@ -549,6 +541,18 @@ public class ViewFileSystem extends File
}
}
+ @Override
+ public FileSystem[] getChildFileSystems() {
+ List<InodeTree.MountPoint<FileSystem>> mountPoints =
+ fsState.getMountPoints();
+ Set<FileSystem> children = new HashSet<FileSystem>();
+ for (InodeTree.MountPoint<FileSystem> mountPoint : mountPoints) {
+ FileSystem targetFs = mountPoint.target.targetFileSystem;
+ children.addAll(Arrays.asList(targetFs.getChildFileSystems()));
+ }
+ return children.toArray(new FileSystem[]{});
+ }
+
public MountPoint[] getMountPoints() {
List<InodeTree.MountPoint<FileSystem>> mountPoints =
fsState.getMountPoints();
@@ -561,59 +565,6 @@ public class ViewFileSystem extends File
return result;
}
-
- @Override
- public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
- List<InodeTree.MountPoint<FileSystem>> mountPoints =
- fsState.getMountPoints();
- int initialListSize = 0;
- for (InodeTree.MountPoint<FileSystem> im : mountPoints) {
- initialListSize += im.target.targetDirLinkList.length;
- }
- List<Token<?>> result = new ArrayList<Token<?>>(initialListSize);
- for ( int i = 0; i < mountPoints.size(); ++i ) {
- List<Token<?>> tokens =
- mountPoints.get(i).target.targetFileSystem.getDelegationTokens(renewer);
- if (tokens != null) {
- result.addAll(tokens);
- }
- }
- return result;
- }
-
- @Override
- public List<Token<?>> getDelegationTokens(String renewer,
- Credentials credentials) throws IOException {
- List<InodeTree.MountPoint<FileSystem>> mountPoints =
- fsState.getMountPoints();
- int initialListSize = 0;
- for (InodeTree.MountPoint<FileSystem> im : mountPoints) {
- initialListSize += im.target.targetDirLinkList.length;
- }
- Set<String> seenServiceNames = new HashSet<String>();
- List<Token<?>> result = new ArrayList<Token<?>>(initialListSize);
- for (int i = 0; i < mountPoints.size(); ++i) {
- String serviceName =
- mountPoints.get(i).target.targetFileSystem.getCanonicalServiceName();
- if (serviceName == null || seenServiceNames.contains(serviceName)) {
- continue;
- }
- seenServiceNames.add(serviceName);
- Token<?> knownToken = credentials.getToken(new Text(serviceName));
- if (knownToken != null) {
- result.add(knownToken);
- } else {
- List<Token<?>> tokens =
- mountPoints.get(i).target.targetFileSystem
- .getDelegationTokens(renewer);
- if (tokens != null) {
- result.addAll(tokens);
- }
- }
- }
- return result;
- }
-
/*
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java Fri Aug 17 14:05:11 2012
@@ -57,6 +57,20 @@ public class Credentials implements Writ
new HashMap<Text, Token<? extends TokenIdentifier>>();
/**
+ * Create an empty credentials instance
+ */
+ public Credentials() {
+ }
+
+ /**
+ * Create a copy of the given credentials
+ * @param credentials to copy
+ */
+ public Credentials(Credentials credentials) {
+ this.addAll(credentials);
+ }
+
+ /**
* Returns the key bytes for the alias
* @param alias the alias for the key
* @return key for this alias
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java Fri Aug 17 14:05:11 2012
@@ -24,8 +24,10 @@ import java.util.Random;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.token.Token;
import org.junit.Assert;
import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
/**
* Helper class for unit tests.
@@ -218,4 +220,39 @@ public final class FileSystemTestHelper
}
Assert.assertEquals(aFs.makeQualified(new Path(path)), s.getPath());
}
+
+ /**
+ * Class to enable easier mocking of a FileSystem
+ * Use getRawFileSystem to retrieve the mock
+ */
+ public static class MockFileSystem extends FilterFileSystem {
+ public MockFileSystem() {
+ // it's a bit ackward to mock ourselves, but it allows the visibility
+ // of methods to be increased
+ super(mock(MockFileSystem.class));
+ }
+ @Override
+ public MockFileSystem getRawFileSystem() {
+ return (MockFileSystem) super.getRawFileSystem();
+
+ }
+ // these basic methods need to directly propagate to the mock to be
+ // more transparent
+ @Override
+ public void initialize(URI uri, Configuration conf) throws IOException {
+ fs.initialize(uri, conf);
+ }
+ @Override
+ public String getCanonicalServiceName() {
+ return fs.getCanonicalServiceName();
+ }
+ @Override
+ public FileSystem[] getChildFileSystems() {
+ return fs.getChildFileSystems();
+ }
+ @Override // publicly expose for mocking
+ public Token<?> getDelegationToken(String renewer) throws IOException {
+ return fs.getDelegationToken(renewer);
+ }
+ }
}
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemTokens.java?rev=1374271&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemTokens.java (added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemTokens.java Fri Aug 17 14:05:11 2012
@@ -0,0 +1,279 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.*;
+import static org.mockito.Mockito.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class TestFileSystemTokens {
+ private static String renewer = "renewer!";
+
+ @Test
+ public void testFsWithNoToken() throws Exception {
+ MockFileSystem fs = createFileSystemForServiceName(null);
+ Credentials credentials = new Credentials();
+
+ fs.addDelegationTokens(renewer, credentials);
+ verifyTokenFetch(fs, false);
+ assertEquals(0, credentials.numberOfTokens());
+ }
+
+ @Test
+ public void testFsWithToken() throws Exception {
+ Text service = new Text("singleTokenFs");
+ MockFileSystem fs = createFileSystemForServiceName(service);
+ Credentials credentials = new Credentials();
+
+ fs.addDelegationTokens(renewer, credentials);
+ verifyTokenFetch(fs, true);
+
+ assertEquals(1, credentials.numberOfTokens());
+ assertNotNull(credentials.getToken(service));
+ }
+
+ @Test
+ public void testFsWithTokenExists() throws Exception {
+ Credentials credentials = new Credentials();
+ Text service = new Text("singleTokenFs");
+ MockFileSystem fs = createFileSystemForServiceName(service);
+ Token<?> token = mock(Token.class);
+ credentials.addToken(service, token);
+
+ fs.addDelegationTokens(renewer, credentials);
+ verifyTokenFetch(fs, false);
+
+ assertEquals(1, credentials.numberOfTokens());
+ assertSame(token, credentials.getToken(service));
+ }
+
+ @Test
+ public void testFsWithChildTokens() throws Exception {
+ Credentials credentials = new Credentials();
+ Text service1 = new Text("singleTokenFs1");
+ Text service2 = new Text("singleTokenFs2");
+
+ MockFileSystem fs1 = createFileSystemForServiceName(service1);
+ MockFileSystem fs2 = createFileSystemForServiceName(service2);
+ MockFileSystem fs3 = createFileSystemForServiceName(null);
+ MockFileSystem multiFs =
+ createFileSystemForServiceName(null, fs1, fs2, fs3);
+
+ multiFs.addDelegationTokens(renewer, credentials);
+ verifyTokenFetch(multiFs, false); // has no tokens of own, only child tokens
+ verifyTokenFetch(fs1, true);
+ verifyTokenFetch(fs2, true);
+ verifyTokenFetch(fs3, false);
+
+ assertEquals(2, credentials.numberOfTokens());
+ assertNotNull(credentials.getToken(service1));
+ assertNotNull(credentials.getToken(service2));
+ }
+
+ @Test
+ public void testFsWithDuplicateChildren() throws Exception {
+ Credentials credentials = new Credentials();
+ Text service = new Text("singleTokenFs1");
+
+ MockFileSystem fs = createFileSystemForServiceName(service);
+ MockFileSystem multiFs =
+ createFileSystemForServiceName(null, fs, new FilterFileSystem(fs));
+
+ multiFs.addDelegationTokens(renewer, credentials);
+ verifyTokenFetch(multiFs, false);
+ verifyTokenFetch(fs, true);
+
+ assertEquals(1, credentials.numberOfTokens());
+ assertNotNull(credentials.getToken(service));
+ }
+
+ @Test
+ public void testFsWithDuplicateChildrenTokenExists() throws Exception {
+ Credentials credentials = new Credentials();
+ Text service = new Text("singleTokenFs1");
+ Token<?> token = mock(Token.class);
+ credentials.addToken(service, token);
+
+ MockFileSystem fs = createFileSystemForServiceName(service);
+ MockFileSystem multiFs =
+ createFileSystemForServiceName(null, fs, new FilterFileSystem(fs));
+
+ multiFs.addDelegationTokens(renewer, credentials);
+ verifyTokenFetch(multiFs, false);
+ verifyTokenFetch(fs, false);
+
+ assertEquals(1, credentials.numberOfTokens());
+ assertSame(token, credentials.getToken(service));
+ }
+
+ @Test
+ public void testFsWithChildTokensOneExists() throws Exception {
+ Credentials credentials = new Credentials();
+ Text service1 = new Text("singleTokenFs1");
+ Text service2 = new Text("singleTokenFs2");
+ Token<?> token = mock(Token.class);
+ credentials.addToken(service2, token);
+
+ MockFileSystem fs1 = createFileSystemForServiceName(service1);
+ MockFileSystem fs2 = createFileSystemForServiceName(service2);
+ MockFileSystem fs3 = createFileSystemForServiceName(null);
+ MockFileSystem multiFs = createFileSystemForServiceName(null, fs1, fs2, fs3);
+
+ multiFs.addDelegationTokens(renewer, credentials);
+ verifyTokenFetch(multiFs, false);
+ verifyTokenFetch(fs1, true);
+ verifyTokenFetch(fs2, false); // we had added its token to credentials
+ verifyTokenFetch(fs3, false);
+
+ assertEquals(2, credentials.numberOfTokens());
+ assertNotNull(credentials.getToken(service1));
+ assertSame(token, credentials.getToken(service2));
+ }
+
+ @Test
+ public void testFsWithMyOwnAndChildTokens() throws Exception {
+ Credentials credentials = new Credentials();
+ Text service1 = new Text("singleTokenFs1");
+ Text service2 = new Text("singleTokenFs2");
+ Text myService = new Text("multiTokenFs");
+ Token<?> token = mock(Token.class);
+ credentials.addToken(service2, token);
+
+ MockFileSystem fs1 = createFileSystemForServiceName(service1);
+ MockFileSystem fs2 = createFileSystemForServiceName(service2);
+ MockFileSystem multiFs = createFileSystemForServiceName(myService, fs1, fs2);
+
+ multiFs.addDelegationTokens(renewer, credentials);
+ verifyTokenFetch(multiFs, true); // its own token and also of its children
+ verifyTokenFetch(fs1, true);
+ verifyTokenFetch(fs2, false); // we had added its token to credentials
+
+ assertEquals(3, credentials.numberOfTokens());
+ assertNotNull(credentials.getToken(myService));
+ assertNotNull(credentials.getToken(service1));
+ assertNotNull(credentials.getToken(service2));
+ }
+
+
+ @Test
+ public void testFsWithMyOwnExistsAndChildTokens() throws Exception {
+ Credentials credentials = new Credentials();
+ Text service1 = new Text("singleTokenFs1");
+ Text service2 = new Text("singleTokenFs2");
+ Text myService = new Text("multiTokenFs");
+ Token<?> token = mock(Token.class);
+ credentials.addToken(myService, token);
+
+ MockFileSystem fs1 = createFileSystemForServiceName(service1);
+ MockFileSystem fs2 = createFileSystemForServiceName(service2);
+ MockFileSystem multiFs = createFileSystemForServiceName(myService, fs1, fs2);
+
+ multiFs.addDelegationTokens(renewer, credentials);
+ verifyTokenFetch(multiFs, false); // we had added its token to credentials
+ verifyTokenFetch(fs1, true);
+ verifyTokenFetch(fs2, true);
+
+ assertEquals(3, credentials.numberOfTokens());
+ assertSame(token, credentials.getToken(myService));
+ assertNotNull(credentials.getToken(service1));
+ assertNotNull(credentials.getToken(service2));
+ }
+
+ @Test
+ public void testFsWithNestedDuplicatesChildren() throws Exception {
+ Credentials credentials = new Credentials();
+ Text service1 = new Text("singleTokenFs1");
+ Text service2 = new Text("singleTokenFs2");
+ Text service4 = new Text("singleTokenFs4");
+ Text multiService = new Text("multiTokenFs");
+ Token<?> token2 = mock(Token.class);
+ credentials.addToken(service2, token2);
+
+ MockFileSystem fs1 = createFileSystemForServiceName(service1);
+ MockFileSystem fs1B = createFileSystemForServiceName(service1);
+ MockFileSystem fs2 = createFileSystemForServiceName(service2);
+ MockFileSystem fs3 = createFileSystemForServiceName(null);
+ MockFileSystem fs4 = createFileSystemForServiceName(service4);
+ // now let's get dirty! ensure dup tokens aren't fetched even when
+ // repeated and dupped in a nested fs. fs4 is a real test of the drill
+ // down: multi-filter-multi-filter-filter-fs4.
+ MockFileSystem multiFs = createFileSystemForServiceName(multiService,
+ fs1, fs1B, fs2, fs2, new FilterFileSystem(fs3),
+ new FilterFileSystem(new FilterFileSystem(fs4)));
+ MockFileSystem superMultiFs = createFileSystemForServiceName(null,
+ fs1, fs1B, fs1, new FilterFileSystem(fs3), new FilterFileSystem(multiFs));
+ superMultiFs.addDelegationTokens(renewer, credentials);
+ verifyTokenFetch(superMultiFs, false); // does not have its own token
+ verifyTokenFetch(multiFs, true); // has its own token
+ verifyTokenFetch(fs1, true);
+ verifyTokenFetch(fs2, false); // we had added its token to credentials
+ verifyTokenFetch(fs3, false); // has no tokens
+ verifyTokenFetch(fs4, true);
+
+ assertEquals(4, credentials.numberOfTokens()); //fs1+fs2+fs4+multifs (fs3=0)
+ assertNotNull(credentials.getToken(service1));
+ assertNotNull(credentials.getToken(service2));
+ assertSame(token2, credentials.getToken(service2));
+ assertNotNull(credentials.getToken(multiService));
+ assertNotNull(credentials.getToken(service4));
+ }
+
+ public static MockFileSystem createFileSystemForServiceName(
+ final Text service, final FileSystem... children) throws IOException {
+ final MockFileSystem fs = new MockFileSystem();
+ final MockFileSystem mockFs = fs.getRawFileSystem();
+ if (service != null) {
+ when(mockFs.getCanonicalServiceName()).thenReturn(service.toString());
+ when(mockFs.getDelegationToken(any(String.class))).thenAnswer(
+ new Answer<Token<?>>() {
+ @Override
+ public Token<?> answer(InvocationOnMock invocation) throws Throwable {
+ Token<?> token = new Token<TokenIdentifier>();
+ token.setService(service);
+ return token;
+ }
+ });
+ }
+ when(mockFs.getChildFileSystems()).thenReturn(children);
+ return fs;
+ }
+
+ // check that canonical name was requested, if renewer is not null that
+ // a token was requested, and that child fs was invoked
+ private void verifyTokenFetch(MockFileSystem fs, boolean expected) throws IOException {
+ verify(fs.getRawFileSystem(), atLeast(1)).getCanonicalServiceName();
+ if (expected) {
+ verify(fs.getRawFileSystem()).getDelegationToken(renewer);
+ } else {
+ verify(fs.getRawFileSystem(), never()).getDelegationToken(any(String.class));
+ }
+ verify(fs.getRawFileSystem(), atLeast(1)).getChildFileSystems();
+ }
+}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java Fri Aug 17 14:05:11 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
import org.junit.BeforeClass;
@@ -185,6 +186,10 @@ public class TestFilterFileSystem {
public boolean cancelDeleteOnExit(Path f) throws IOException {
return false;
}
+ public Token<?>[] addDelegationTokens(String renewer, Credentials creds)
+ throws IOException {
+ return null;
+ }
public String getScheme() {
return "dontcheck";
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java Fri Aug 17 14:05:11 2012
@@ -22,10 +22,18 @@ import static org.junit.Assert.*;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-
+import java.util.Arrays;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.junit.BeforeClass;
import org.junit.Test;
/**
@@ -38,6 +46,29 @@ import org.junit.Test;
public class TestViewFileSystemDelegationTokenSupport {
private static final String MOUNT_TABLE_NAME = "vfs-cluster";
+ static Configuration conf;
+ static FileSystem viewFs;
+ static FakeFileSystem fs1;
+ static FakeFileSystem fs2;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ conf = ViewFileSystemTestSetup.createConfig();
+ fs1 = setupFileSystem(new URI("fs1:///"), FakeFileSystem.class);
+ fs2 = setupFileSystem(new URI("fs2:///"), FakeFileSystem.class);
+ viewFs = FileSystem.get(FsConstants.VIEWFS_URI, conf);
+ }
+
+ static FakeFileSystem setupFileSystem(URI uri, Class<? extends FileSystem> clazz)
+ throws Exception {
+ String scheme = uri.getScheme();
+ conf.set("fs."+scheme+".impl", clazz.getName());
+ FakeFileSystem fs = (FakeFileSystem)FileSystem.get(uri, conf);
+ // mount each fs twice, will later ensure 1 token/fs
+ ConfigUtil.addLink(conf, "/mounts/"+scheme+"-one", fs.getUri());
+ ConfigUtil.addLink(conf, "/mounts/"+scheme+"-two", fs.getUri());
+ return fs;
+ }
/**
* Regression test for HADOOP-8408.
@@ -69,4 +100,92 @@ public class TestViewFileSystemDelegatio
assertNull(serviceName);
}
+ @Test
+ public void testGetChildFileSystems() throws Exception {
+ assertNull(fs1.getChildFileSystems());
+ assertNull(fs2.getChildFileSystems());
+ List<FileSystem> children = Arrays.asList(viewFs.getChildFileSystems());
+ assertEquals(2, children.size());
+ assertTrue(children.contains(fs1));
+ assertTrue(children.contains(fs2));
+ }
+
+ @Test
+ public void testAddDelegationTokens() throws Exception {
+ Credentials creds = new Credentials();
+ Token<?> fs1Tokens[] = addTokensWithCreds(fs1, creds);
+ assertEquals(1, fs1Tokens.length);
+ assertEquals(1, creds.numberOfTokens());
+ Token<?> fs2Tokens[] = addTokensWithCreds(fs2, creds);
+ assertEquals(1, fs2Tokens.length);
+ assertEquals(2, creds.numberOfTokens());
+
+ Credentials savedCreds = creds;
+ creds = new Credentials();
+
+ // should get the same set of tokens as explicitly fetched above
+ Token<?> viewFsTokens[] = viewFs.addDelegationTokens("me", creds);
+ assertEquals(2, viewFsTokens.length);
+ assertTrue(creds.getAllTokens().containsAll(savedCreds.getAllTokens()));
+ assertEquals(savedCreds.numberOfTokens(), creds.numberOfTokens());
+ // should get none, already have all tokens
+ viewFsTokens = viewFs.addDelegationTokens("me", creds);
+ assertEquals(0, viewFsTokens.length);
+ assertTrue(creds.getAllTokens().containsAll(savedCreds.getAllTokens()));
+ assertEquals(savedCreds.numberOfTokens(), creds.numberOfTokens());
+ }
+
+ Token<?>[] addTokensWithCreds(FileSystem fs, Credentials creds) throws Exception {
+ Credentials savedCreds;
+
+ savedCreds = new Credentials(creds);
+ Token<?> tokens[] = fs.addDelegationTokens("me", creds);
+ // test that we got the token we wanted, and that creds were modified
+ assertEquals(1, tokens.length);
+ assertEquals(fs.getCanonicalServiceName(), tokens[0].getService().toString());
+ assertTrue(creds.getAllTokens().contains(tokens[0]));
+ assertTrue(creds.getAllTokens().containsAll(savedCreds.getAllTokens()));
+ assertEquals(savedCreds.numberOfTokens()+1, creds.numberOfTokens());
+
+ // shouldn't get any new tokens since already in creds
+ savedCreds = new Credentials(creds);
+ Token<?> tokenRefetch[] = fs.addDelegationTokens("me", creds);
+ assertEquals(0, tokenRefetch.length);
+ assertTrue(creds.getAllTokens().containsAll(savedCreds.getAllTokens()));
+ assertEquals(savedCreds.numberOfTokens(), creds.numberOfTokens());
+
+ return tokens;
+ }
+
+ static class FakeFileSystem extends RawLocalFileSystem {
+ URI uri;
+
+ public void initialize(URI name, Configuration conf) throws IOException {
+ this.uri = name;
+ }
+
+ @Override
+ public Path getInitialWorkingDirectory() {
+ return new Path("/"); // ctor calls getUri before the uri is inited...
+ }
+
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getCanonicalServiceName() {
+ return String.valueOf(this.getUri()+"/"+this.hashCode());
+ }
+
+ @Override
+ public Token<?> getDelegationToken(String renewer) throws IOException {
+ Token<?> token = new Token<TokenIdentifier>();
+ token.setService(new Text(getCanonicalServiceName()));
+ return token;
+ }
+
+ @Override
+ public void close() {}
+ }
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java?rev=1374271&r1=1374270&r2=1374271&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java Fri Aug 17 14:05:11 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.fs.viewfs;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
@@ -137,9 +138,9 @@ public class ViewFileSystemBaseTest {
*/
@Test
public void testGetDelegationTokens() throws IOException {
- List<Token<?>> delTokens =
- fsView.getDelegationTokens("sanjay");
- Assert.assertEquals(getExpectedDelegationTokenCount(), delTokens.size());
+ Token<?>[] delTokens =
+ fsView.addDelegationTokens("sanjay", new Credentials());
+ Assert.assertEquals(getExpectedDelegationTokenCount(), delTokens.length);
}
int getExpectedDelegationTokenCount() {
@@ -150,29 +151,20 @@ public class ViewFileSystemBaseTest {
public void testGetDelegationTokensWithCredentials() throws IOException {
Credentials credentials = new Credentials();
List<Token<?>> delTokens =
- fsView.getDelegationTokens("sanjay", credentials);
+ Arrays.asList(fsView.addDelegationTokens("sanjay", credentials));
int expectedTokenCount = getExpectedDelegationTokenCountWithCredentials();
Assert.assertEquals(expectedTokenCount, delTokens.size());
+ Credentials newCredentials = new Credentials();
for (int i = 0; i < expectedTokenCount / 2; i++) {
Token<?> token = delTokens.get(i);
- credentials.addToken(token.getService(), token);
+ newCredentials.addToken(token.getService(), token);
}
List<Token<?>> delTokens2 =
- fsView.getDelegationTokens("sanjay", credentials);
- Assert.assertEquals(expectedTokenCount, delTokens2.size());
-
- for (int i = 0; i < delTokens2.size(); i++) {
- for (int j = 0; j < delTokens.size(); j++) {
- if (delTokens.get(j) == delTokens2.get(i)) {
- delTokens.remove(j);
- break;
- }
- }
- }
- Assert.assertEquals((expectedTokenCount + 1) / 2, delTokens.size());
+ Arrays.asList(fsView.addDelegationTokens("sanjay", newCredentials));
+ Assert.assertEquals((expectedTokenCount + 1) / 2, delTokens2.size());
}
int getExpectedDelegationTokenCountWithCredentials() {