You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:48:35 UTC
svn commit: r1077725 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
hdfs/org/apache/hadoop/hdfs/protocol/
hdfs/org/apache/hadoop/hdfs/security/token/block/
hdfs/org/apache/hadoop/hdfs/server/protocol/
test/org/apache/hadoop/hdfs/securit...
Author: omalley
Date: Fri Mar 4 04:48:35 2011
New Revision: 1077725
URL: http://svn.apache.org/viewvc?rev=1077725&view=rev
Log:
commit 3bd8328161d6b16b48d4abf29914e8289ae8e375
Author: Jakob Homan <jh...@yahoo-inc.com>
Date: Wed Sep 22 14:55:04 2010 -0700
Modulate HDFS-1353 to not bump the RPC protocol versions.
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1077725&r1=1077724&r2=1077725&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Fri Mar 4 04:48:35 2011
@@ -50,9 +50,12 @@ public interface ClientProtocol extends
* Compared to the previous version the following changes have been introduced:
* (Only the latest change is reflected.
* The log of historical changes can be retrieved from the svn).
- * 62: Remove getBlockLocations optimization
+ * 61: Serialized format of BlockTokenIdentifier changed to contain
+ * multiple blocks within a single BlockTokenIdentifier
+ *
+ * (bumped to 61 to bring in line with trunk)
*/
- public static final long versionID = 62L;
+ public static final long versionID = 61L;
///////////////////////////////////////
// File contents
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java?rev=1077725&r1=1077724&r2=1077725&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java Fri Mar 4 04:48:35 2011
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.security.
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Arrays;
import java.util.EnumSet;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
@@ -35,20 +36,23 @@ public class BlockTokenIdentifier extend
private long expiryDate;
private int keyId;
private String userId;
- private long blockId;
+ private long [] blockIds;
private EnumSet<AccessMode> modes;
private byte [] cache;
public BlockTokenIdentifier() {
- this(null, 0l, EnumSet.noneOf(AccessMode.class));
+ this(null, new long [] {}, EnumSet.noneOf(AccessMode.class));
}
- public BlockTokenIdentifier(String userId, long blockId,
+ public BlockTokenIdentifier(String userId, long [] blockIds,
EnumSet<AccessMode> modes) {
+ if(blockIds == null)
+ throw new IllegalArgumentException("blockIds can't be null");
this.cache = null;
this.userId = userId;
- this.blockId = blockId;
+ this.blockIds = Arrays.copyOf(blockIds, blockIds.length);
+ Arrays.sort(this.blockIds);
this.modes = modes == null ? EnumSet.noneOf(AccessMode.class) : modes;
}
@@ -60,7 +64,7 @@ public class BlockTokenIdentifier extend
@Override
public UserGroupInformation getUser() {
if (userId == null || "".equals(userId)) {
- return UserGroupInformation.createRemoteUser(Long.toString(blockId));
+ return UserGroupInformation.createRemoteUser(Arrays.toString(blockIds));
}
return UserGroupInformation.createRemoteUser(userId);
}
@@ -87,8 +91,25 @@ public class BlockTokenIdentifier extend
return userId;
}
- public long getBlockId() {
- return blockId;
+ /**
+ * Return sorted array of blockIds this {@link BlockTokenIdentifier} includes
+ */
+ public long [] getBlockIds() {
+ return blockIds;
+ }
+
+ /**
+ * Is specified blockId included in this BlockTokenIdentifier?
+ */
+ public boolean isBlockIncluded(long blockId) {
+ switch(blockIds.length) {
+ case 1:
+ return blockIds[0] == blockId;
+ case 2:
+ return (blockIds[0] == blockId) || (blockIds[1] == blockId);
+ default:
+ return Arrays.binarySearch(blockIds, blockId) >= 0;
+ }
}
public EnumSet<AccessMode> getAccessModes() {
@@ -99,7 +120,7 @@ public class BlockTokenIdentifier extend
public String toString() {
return "block_token_identifier (expiryDate=" + this.getExpiryDate()
+ ", keyId=" + this.getKeyId() + ", userId=" + this.getUserId()
- + ", blockIds=" + blockId + ", access modes="
+ + ", blockIds=" + Arrays.toString(blockIds) + ", access modes="
+ this.getAccessModes() + ")";
}
@@ -116,7 +137,7 @@ public class BlockTokenIdentifier extend
BlockTokenIdentifier that = (BlockTokenIdentifier) obj;
return this.expiryDate == that.expiryDate && this.keyId == that.keyId
&& isEqual(this.userId, that.userId)
- && this.blockId == that.blockId
+ && Arrays.equals(this.blockIds, that.blockIds)
&& isEqual(this.modes, that.modes);
}
return false;
@@ -124,7 +145,7 @@ public class BlockTokenIdentifier extend
/** {@inheritDoc} */
public int hashCode() {
- return (int) expiryDate ^ keyId ^ (int)blockId ^ modes.hashCode()
+ return (int) expiryDate ^ keyId ^ Arrays.hashCode(blockIds) ^ modes.hashCode()
^ (userId == null ? 0 : userId.hashCode());
}
@@ -133,7 +154,9 @@ public class BlockTokenIdentifier extend
expiryDate = WritableUtils.readVLong(in);
keyId = WritableUtils.readVInt(in);
userId = WritableUtils.readString(in);
- blockId = WritableUtils.readVLong(in);
+ blockIds = new long[WritableUtils.readVInt(in)];
+ for(int i = 0; i < blockIds.length; i++)
+ blockIds[i] = WritableUtils.readVLong(in);
int length = WritableUtils.readVInt(in);
for (int i = 0; i < length; i++) {
modes.add(WritableUtils.readEnum(in, AccessMode.class));
@@ -144,7 +167,9 @@ public class BlockTokenIdentifier extend
WritableUtils.writeVLong(out, expiryDate);
WritableUtils.writeVInt(out, keyId);
WritableUtils.writeString(out, userId);
- WritableUtils.writeVLong(out, blockId);
+ WritableUtils.writeVInt(out, blockIds.length);
+ for(int i = 0; i < blockIds.length; i++)
+ WritableUtils.writeVLong(out, blockIds[i]);
WritableUtils.writeVInt(out, modes.size());
for (AccessMode aMode : modes) {
WritableUtils.writeEnum(out, aMode);
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java?rev=1077725&r1=1077724&r2=1077725&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java Fri Mar 4 04:48:35 2011
@@ -174,16 +174,29 @@ public class BlockTokenSecretManager ext
/** Generate an block token for current user */
public Token<BlockTokenIdentifier> generateToken(Block block,
EnumSet<AccessMode> modes) throws IOException {
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- String userID = (ugi == null ? null : ugi.getShortUserName());
- return generateToken(userID, block, modes);
+ return generateToken(new long [] { block.getBlockId() }, modes);
}
/** Generate a block token for a specified user */
public Token<BlockTokenIdentifier> generateToken(String userId, Block block,
EnumSet<AccessMode> modes) throws IOException {
- BlockTokenIdentifier id = new BlockTokenIdentifier(userId,
- block.getBlockId(), modes);
+ return generateToken(userId, new long [] { block.getBlockId() }, modes);
+ }
+
+ /** Generate a block token for the current user based on a collection
+ * of blockIds
+ */
+ public Token<BlockTokenIdentifier> generateToken(long[] blockIds,
+ EnumSet<AccessMode> modes) throws IOException {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String userID = (ugi == null ? null : ugi.getShortUserName());
+ return generateToken(userID, blockIds, modes);
+ }
+
+ /** Generate a block token based on a collection of blockIds */
+ public Token<BlockTokenIdentifier> generateToken(String userID,
+ long[] blockIds, EnumSet<AccessMode> modes) {
+ BlockTokenIdentifier id = new BlockTokenIdentifier(userID, blockIds, modes);
return new Token<BlockTokenIdentifier>(id, this);
}
@@ -202,7 +215,7 @@ public class BlockTokenSecretManager ext
throw new InvalidToken("Block token with " + id.toString()
+ " doesn't belong to user " + userId);
}
- if (id.getBlockId() != block.getBlockId()) {
+ if (!id.isBlockIncluded(block.getBlockId())) {
throw new InvalidToken("Block token with " + id.toString()
+ " doesn't apply to block " + block);
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java?rev=1077725&r1=1077724&r2=1077725&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java Fri Mar 4 04:48:35 2011
@@ -41,9 +41,12 @@ import org.apache.hadoop.security.Kerber
clientPrincipal = DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY)
public interface DatanodeProtocol extends VersionedProtocol {
/**
- * 26: Remove getBlockLocations optimization
+ * 25: Serialized format of BlockTokenIdentifier changed to contain
+ * multiple blocks within a single BlockTokenIdentifier
+ *
+ * (bumped to 25 to bring in line with trunk)
*/
- public static final long versionID = 26L;
+ public static final long versionID = 25L;
// error code
final static int NOTIFY = 0;
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=1077725&r1=1077724&r2=1077725&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java Fri Mar 4 04:48:35 2011
@@ -18,21 +18,11 @@
package org.apache.hadoop.hdfs.security.token.block;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.Arrays;
import java.util.EnumSet;
import java.util.Set;
@@ -40,8 +30,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.io.TestWritable;
@@ -56,7 +46,18 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.log4j.Level;
+
import org.junit.Test;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -111,7 +112,7 @@ public class TestBlockToken {
LOG.info("Got: " + id.toString());
assertTrue("Received BlockTokenIdentifier is wrong", ident.equals(id));
sm.checkAccess(id, null, block, BlockTokenSecretManager.AccessMode.WRITE);
- result = new LocatedBlock(new Block(id.getBlockId()), null);
+ result = new LocatedBlock(new Block(id.getBlockIds()[0]), null);
}
return result;
}
@@ -224,4 +225,29 @@ public class TestBlockToken {
}
}
}
+
+ @Test
+ public void collectionOfBlocksActsSanely() {
+ final long[][] testBlockIds = new long [][] {{99l, 7l, -32l, 0l},
+ {},
+ {42l},
+ {-5235l, 2352}};
+ final long [] notBlockIds = new long [] { 32l, 1l, -23423423l};
+
+ for(long [] bids : testBlockIds) {
+ BlockTokenIdentifier bti = new BlockTokenIdentifier("Madame Butterfly",
+ bids, EnumSet.noneOf(BlockTokenSecretManager.AccessMode.class));
+
+ for(long bid : bids) assertTrue(bti.isBlockIncluded(bid));
+
+ for(long nbid : notBlockIds) assertFalse(bti.isBlockIncluded(nbid));
+
+ // BlockTokenIdentifiers maintain a sorted array of the block Ids.
+ long[] sorted = Arrays.copyOf(bids, bids.length);
+ Arrays.sort(sorted);
+
+ assertTrue(Arrays.toString(bids)+" doesn't equal "+Arrays.toString(sorted),
+ Arrays.equals(bti.getBlockIds(), sorted));
+ }
+ }
}