You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:09:10 UTC
svn commit: r1077379 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
core/org/apache/hadoop/fs/ core/org/apache/hadoop/ipc/
core/org/apache/hadoop/security/ test/org/apache/hadoop/filecache/
test/org/apache/hadoop/fs/ test/org/apache/had...
Author: omalley
Date: Fri Mar 4 04:09:10 2011
New Revision: 1077379
URL: http://svn.apache.org/viewvc?rev=1077379&view=rev
Log:
commit 788db205ba6cf03f72c2e83162e6fd43b470a429
Author: Owen O'Malley <om...@apache.org>
Date: Sat Apr 10 00:56:02 2010 -0700
Revert HADOOP-6670 (UGI.equals, hashCode) since it fails scale testing.
This reverts commit dee59d8ebfde0cac7fe2ca41fb887c07a8e233f3.
+++ b/YAHOO-CHANGES.txt
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFileSystem.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestUserGroupInformation.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/FileSystem.java?rev=1077379&r1=1077378&r2=1077379&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/FileSystem.java Fri Mar 4 04:09:10 2011
@@ -228,10 +228,7 @@ public abstract class FileSystem extends
* @throws IOException
*/
public static void closeAll() throws IOException {
- LOG.debug("Starting clear of FileSystem cache with " + CACHE.size() +
- " elements.");
CACHE.closeAll();
- LOG.debug("Done clearing cache");
}
/** Make sure that a path specifies a FileSystem. */
@@ -1260,7 +1257,6 @@ public abstract class FileSystem extends
// delete all files that were marked as delete-on-exit.
processDeleteOnExit();
CACHE.remove(this.key, this);
- LOG.debug("Removing filesystem for " + getUri());
}
/** Return the total size of all files in the filesystem.*/
@@ -1386,7 +1382,6 @@ public abstract class FileSystem extends
private static FileSystem createFileSystem(URI uri, Configuration conf
) throws IOException {
Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
- LOG.debug("Creating filesystem for " + uri);
if (clazz == null) {
throw new IOException("No FileSystem for scheme: " + uri.getScheme());
}
@@ -1490,14 +1485,6 @@ public abstract class FileSystem extends
return "("+ugi.toString() + ")@" + scheme + "://" + authority;
}
}
-
- /**
- * Get the number of file systems in the cache.
- * @return the number of cached file systems
- */
- int size() {
- return map.size();
- }
}
public static final class Statistics {
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Client.java?rev=1077379&r1=1077378&r2=1077379&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Client.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Client.java Fri Mar 4 04:09:10 2011
@@ -1043,8 +1043,8 @@ public class Client {
if (obj instanceof ConnectionId) {
ConnectionId id = (ConnectionId) obj;
return address.equals(id.address) && protocol == id.protocol &&
- ((ticket != null && ticket.equals(id.ticket)) ||
- (ticket == id.ticket));
+ ticket == id.ticket;
+ //Note : ticket is a ref comparision.
}
return false;
}
@@ -1052,7 +1052,7 @@ public class Client {
@Override
public int hashCode() {
return (address.hashCode() + PRIME * System.identityHashCode(protocol)) ^
- (ticket == null ? 0 : ticket.hashCode());
+ System.identityHashCode(ticket);
}
}
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java?rev=1077379&r1=1077378&r2=1077379&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java Fri Mar 4 04:09:10 2011
@@ -931,7 +931,7 @@ public class UserGroupInformation {
} else if (o == null || getClass() != o.getClass()) {
return false;
} else {
- return subject == ((UserGroupInformation) o).subject;
+ return subject.equals(((UserGroupInformation) o).subject);
}
}
@@ -940,7 +940,7 @@ public class UserGroupInformation {
*/
@Override
public int hashCode() {
- return System.identityHashCode(subject);
+ return subject.hashCode();
}
/**
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java?rev=1077379&r1=1077378&r2=1077379&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java Fri Mar 4 04:09:10 2011
@@ -297,57 +297,18 @@ public class TestTrackerDistributedCache
if (!canRun()) {
return;
}
- checkLocalizedPath(true);
- checkLocalizedPath(false);
+ checkLocalizedPath("true");
+ checkLocalizedPath("false");
}
- private void appendStringArray(StringBuilder buffer, String[] data) {
- if (data != null && data.length != 0) {
- buffer.append(data[0]);
- for(int i=1; i < data.length; i++) {
- buffer.append(',');
- buffer.append(data[i]);
- }
- }
- }
-
- private void appendUriArray(StringBuilder buffer, URI[] data) {
- if (data != null && data.length != 0) {
- buffer.append(data[0]);
- for(int i=1; i < data.length; i++) {
- buffer.append(',');
- buffer.append(data[i]);
- }
- }
- }
-
- private void dumpState(Configuration conf1) throws IOException {
- StringBuilder buf = new StringBuilder();
- buf.append("\nFiles:");
- appendUriArray(buf, DistributedCache.getCacheFiles(conf1));
- buf.append("\nArchives:");
- appendUriArray(buf, DistributedCache.getCacheArchives(conf1));
- buf.append("\nFile Visible:");
- appendStringArray(buf, TrackerDistributedCacheManager.getFileVisibilities
- (conf1));
- buf.append("\nArchive Visible:");
- appendStringArray(buf, TrackerDistributedCacheManager.getArchiveVisibilities
- (conf1));
- buf.append("\nFile timestamps:");
- appendStringArray(buf, DistributedCache.getFileTimestamps(conf1));
- buf.append("\nArchive timestamps:");
- appendStringArray(buf, DistributedCache.getArchiveTimestamps(conf1));
- LOG.info("state = " + buf.toString());
- }
-
- private void checkLocalizedPath(boolean visibility)
+ private void checkLocalizedPath(String visibility)
throws IOException, LoginException, InterruptedException {
TrackerDistributedCacheManager manager =
new TrackerDistributedCacheManager(conf, taskController);
String userName = getJobOwnerName();
File workDir = new File(TEST_ROOT_DIR, "workdir");
Path cacheFile = new Path(TEST_ROOT_DIR, "fourthcachefile");
- if (visibility) {
+ if ("true".equals(visibility)) {
createPublicTempFile(cacheFile);
} else {
createPrivateTempFile(cacheFile);
@@ -358,7 +319,6 @@ public class TestTrackerDistributedCache
DistributedCache.addCacheFile(cacheFile.toUri(), conf1);
TrackerDistributedCacheManager.determineTimestamps(conf1);
TrackerDistributedCacheManager.determineCacheVisibilities(conf1);
- dumpState(conf1);
// Task localizing for job
TaskDistributedCacheManager handle = manager
@@ -368,7 +328,7 @@ public class TestTrackerDistributedCache
TaskTracker.getPublicDistributedCacheDir());
TaskDistributedCacheManager.CacheFile c = handle.getCacheFiles().get(0);
String distCacheDir;
- if (visibility) {
+ if ("true".equals(visibility)) {
distCacheDir = TaskTracker.getPublicDistributedCacheDir();
} else {
distCacheDir = TaskTracker.getPrivateDistributedCacheDir(userName);
@@ -377,19 +337,19 @@ public class TestTrackerDistributedCache
manager.getLocalCache(cacheFile.toUri(), conf1, distCacheDir,
fs.getFileStatus(cacheFile), false,
c.timestamp, new Path(TEST_ROOT_DIR), false,
- visibility);
+ Boolean.parseBoolean(visibility));
assertTrue("Cache file didn't get localized in the expected directory. " +
"Expected localization to happen within " +
ROOT_MAPRED_LOCAL_DIR + "/" + distCacheDir +
", but was localized at " +
localizedPath, localizedPath.toString().contains(distCacheDir));
- if (visibility) {
+ if ("true".equals(visibility)) {
checkPublicFilePermissions(new Path[]{localizedPath});
} else {
checkFilePermissions(new Path[]{localizedPath});
}
}
-
+
/**
* Check proper permissions on the cache files
*
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFileSystem.java?rev=1077379&r1=1077378&r2=1077379&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFileSystem.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFileSystem.java Fri Mar 4 04:09:10 2011
@@ -647,26 +647,28 @@ public class TestFileSystem extends Test
assertNotSame(fsA, fsB);
Token<T> t1 = mock(Token.class);
- UserGroupInformation ugiA2 = UserGroupInformation.createRemoteUser("foo");
+ ugiA = UserGroupInformation.createRemoteUser("foo");
+ ugiA.addToken(t1);
- fsA = ugiA2.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
public FileSystem run() throws Exception {
return FileSystem.get(new URI("cachedfile://a"), conf);
}
});
- // Although the users in the UGI are same, they have different subjects
- // and so are different.
+ //Although the users in the UGI are same, ugiA has tokens in it, and
+ //we should end up with different filesystems corresponding to the two UGIs
assertNotSame(fsA, fsA1);
+ ugiA = UserGroupInformation.createRemoteUser("foo");
ugiA.addToken(t1);
- fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ fsA1 = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
public FileSystem run() throws Exception {
return FileSystem.get(new URI("cachedfile://a"), conf);
}
});
- // Make sure that different UGI's with the same subject lead to the same
- // file system.
+ //Now the users in the UGI are the same, and they also have the same token.
+ //We should have the same filesystem for both
assertSame(fsA, fsA1);
}
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestUserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestUserGroupInformation.java?rev=1077379&r1=1077378&r2=1077379&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestUserGroupInformation.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestUserGroupInformation.java Fri Mar 4 04:09:10 2011
@@ -156,17 +156,10 @@ public class TestUserGroupInformation {
UserGroupInformation.createUserForTesting(USER_NAME, GROUP_NAMES);
assertEquals(uugi, uugi);
- // The subjects should be different, so this should fail
- UserGroupInformation ugi2 =
- UserGroupInformation.createUserForTesting(USER_NAME, GROUP_NAMES);
- assertFalse(uugi.equals(ugi2));
- assertFalse(uugi.hashCode() == ugi2.hashCode());
-
- // two ugi that have the same subject need to be equal
- UserGroupInformation ugi3 = new UserGroupInformation(uugi.getSubject());
- assertEquals(uugi, ugi3);
- assertEquals(uugi.hashCode(), ugi3.hashCode());
-
+ // The subjects should be equal, so this should work
+ assertTrue(uugi.equals(
+ UserGroupInformation.createUserForTesting
+ (USER_NAME, GROUP_NAMES)));
// ensure that different UGI with the same subject are equal
assertEquals(uugi, new UserGroupInformation(uugi.getSubject()));
}
@@ -179,8 +172,8 @@ public class TestUserGroupInformation {
"RealUser", GROUP_NAMES);
UserGroupInformation proxyUgi1 = UserGroupInformation.createProxyUser(
USER_NAME, realUgi1);
- UserGroupInformation proxyUgi2 =
- new UserGroupInformation( proxyUgi1.getSubject());
+ UserGroupInformation proxyUgi2 = UserGroupInformation.createProxyUser(
+ USER_NAME, realUgi2);
UserGroupInformation remoteUgi = UserGroupInformation.createRemoteUser(USER_NAME);
assertEquals(proxyUgi1, proxyUgi2);
assertFalse(remoteUgi.equals(proxyUgi1));
@@ -291,16 +284,16 @@ public class TestUserGroupInformation {
return null;
}
});
- UserGroupInformation proxyUgi2 =
- new UserGroupInformation(proxyUgi.getSubject());
+ UserGroupInformation proxyUgi2 = UserGroupInformation.createProxyUser(
+ "proxy", ugi);
proxyUgi2.setAuthenticationMethod(AuthenticationMethod.PROXY);
Assert.assertEquals(proxyUgi, proxyUgi2);
// Equality should work if authMethod is null
UserGroupInformation realugi = UserGroupInformation.getCurrentUser();
UserGroupInformation proxyUgi3 = UserGroupInformation.createProxyUser(
"proxyAnother", realugi);
- UserGroupInformation proxyUgi4 =
- new UserGroupInformation(proxyUgi3.getSubject());
+ UserGroupInformation proxyUgi4 = UserGroupInformation.createProxyUser(
+ "proxyAnother", realugi);
Assert.assertEquals(proxyUgi3, proxyUgi4);
}
}