You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by jg...@apache.org on 2010/09/04 02:41:31 UTC
svn commit: r992515 - in /hadoop/hdfs/trunk: ./
src/java/org/apache/hadoop/hdfs/protocol/
src/java/org/apache/hadoop/hdfs/security/token/block/
src/java/org/apache/hadoop/hdfs/server/namenode/
src/java/org/apache/hadoop/hdfs/server/protocol/ src/test/h...
Author: jghoman
Date: Sat Sep 4 00:41:30 2010
New Revision: 992515
URL: http://svn.apache.org/viewvc?rev=992515&view=rev
Log:
HDFS-1353. Remove most of getBlockLocation optimization.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Sat Sep 4 00:41:30 2010
@@ -234,6 +234,8 @@ Trunk (unreleased changes)
HDFS-1355. ant veryclean (clean-cache) doesn't clean enough.
(Luke Lu via jghoman)
+ HDFS-1353. Remove most of getBlockLocation optimization. (jghoman)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Sat Sep 4 00:41:30 2010
@@ -68,9 +68,9 @@ public interface ClientProtocol extends
* Compared to the previous version the following changes have been introduced:
* (Only the latest change is reflected.
* The log of historical changes can be retrieved from the svn).
- * 62: Allow iterative getListinng piggyback block locations.
+ * 63: remove getBlockLocations optimization
*/
- public static final long versionID = 62L;
+ public static final long versionID = 63L;
///////////////////////////////////////
// File contents
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java Sat Sep 4 00:41:30 2010
@@ -21,7 +21,6 @@ package org.apache.hadoop.hdfs.security.
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.Arrays;
import java.util.EnumSet;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -38,23 +37,20 @@ public class BlockTokenIdentifier extend
private long expiryDate;
private int keyId;
private String userId;
- private long [] blockIds;
+ private long blockId;
private EnumSet<AccessMode> modes;
private byte [] cache;
public BlockTokenIdentifier() {
- this(null, new long [] {}, EnumSet.noneOf(AccessMode.class));
+ this(null, 0, EnumSet.noneOf(AccessMode.class));
}
- public BlockTokenIdentifier(String userId, long [] blockIds,
+ public BlockTokenIdentifier(String userId, long blockId,
EnumSet<AccessMode> modes) {
- if(blockIds == null)
- throw new IllegalArgumentException("blockIds can't be null");
this.cache = null;
this.userId = userId;
- this.blockIds = Arrays.copyOf(blockIds, blockIds.length);
- Arrays.sort(this.blockIds);
+ this.blockId = blockId;
this.modes = modes == null ? EnumSet.noneOf(AccessMode.class) : modes;
}
@@ -66,7 +62,7 @@ public class BlockTokenIdentifier extend
@Override
public UserGroupInformation getUser() {
if (userId == null || "".equals(userId)) {
- return UserGroupInformation.createRemoteUser(Arrays.toString(blockIds));
+ return UserGroupInformation.createRemoteUser(Long.toString(blockId));
}
return UserGroupInformation.createRemoteUser(userId);
}
@@ -93,27 +89,10 @@ public class BlockTokenIdentifier extend
return userId;
}
- /**
- * Return sorted array of blockIds this {@link BlockTokenIdentifier} includes
- */
- public long [] getBlockIds() {
- return blockIds;
+ public long getBlockId() {
+ return blockId;
}
-
- /**
- * Is specified blockId included in this BlockTokenIdentifier?
- */
- public boolean isBlockIncluded(long blockId) {
- switch(blockIds.length) {
- case 1:
- return blockIds[0] == blockId;
- case 2:
- return (blockIds[0] == blockId) || (blockIds[1] == blockId);
- default:
- return Arrays.binarySearch(blockIds, blockId) >= 0;
- }
- }
-
+
public EnumSet<AccessMode> getAccessModes() {
return modes;
}
@@ -122,7 +101,7 @@ public class BlockTokenIdentifier extend
public String toString() {
return "block_token_identifier (expiryDate=" + this.getExpiryDate()
+ ", keyId=" + this.getKeyId() + ", userId=" + this.getUserId()
- + ", blockId=" + Arrays.toString(blockIds) + ", access modes="
+ + ", blockId=" + this.getBlockId() + ", access modes="
+ this.getAccessModes() + ")";
}
@@ -138,8 +117,7 @@ public class BlockTokenIdentifier extend
if (obj instanceof BlockTokenIdentifier) {
BlockTokenIdentifier that = (BlockTokenIdentifier) obj;
return this.expiryDate == that.expiryDate && this.keyId == that.keyId
- && isEqual(this.userId, that.userId)
- && Arrays.equals(this.blockIds, that.blockIds)
+ && isEqual(this.userId, that.userId) && this.blockId == that.blockId
&& isEqual(this.modes, that.modes);
}
return false;
@@ -147,18 +125,16 @@ public class BlockTokenIdentifier extend
/** {@inheritDoc} */
public int hashCode() {
- return (int) expiryDate ^ keyId ^ Arrays.hashCode(blockIds) ^ modes.hashCode()
+ return (int) expiryDate ^ keyId ^ (int) blockId ^ modes.hashCode()
^ (userId == null ? 0 : userId.hashCode());
}
public void readFields(DataInput in) throws IOException {
- cache = null;
+ this.cache = null;
expiryDate = WritableUtils.readVLong(in);
keyId = WritableUtils.readVInt(in);
userId = WritableUtils.readString(in);
- blockIds = new long[WritableUtils.readVInt(in)];
- for(int i = 0; i < blockIds.length; i++)
- blockIds[i] = WritableUtils.readVLong(in);
+ blockId = WritableUtils.readVLong(in);
int length = WritableUtils.readVInt(in);
for (int i = 0; i < length; i++) {
modes.add(WritableUtils.readEnum(in, AccessMode.class));
@@ -169,9 +145,7 @@ public class BlockTokenIdentifier extend
WritableUtils.writeVLong(out, expiryDate);
WritableUtils.writeVInt(out, keyId);
WritableUtils.writeString(out, userId);
- WritableUtils.writeVInt(out, blockIds.length);
- for(int i = 0; i < blockIds.length; i++)
- WritableUtils.writeVLong(out, blockIds[i]);
+ WritableUtils.writeVLong(out, blockId);
WritableUtils.writeVInt(out, modes.size());
for (AccessMode aMode : modes) {
WritableUtils.writeEnum(out, aMode);
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java Sat Sep 4 00:41:30 2010
@@ -176,29 +176,16 @@ public class BlockTokenSecretManager ext
/** Generate an block token for current user */
public Token<BlockTokenIdentifier> generateToken(Block block,
EnumSet<AccessMode> modes) throws IOException {
- return generateToken(new long [] { block.getBlockId() }, modes);
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String userID = (ugi == null ? null : ugi.getShortUserName());
+ return generateToken(userID, block, modes);
}
/** Generate a block token for a specified user */
public Token<BlockTokenIdentifier> generateToken(String userId, Block block,
EnumSet<AccessMode> modes) throws IOException {
- return generateToken(userId, new long [] { block.getBlockId() }, modes);
- }
-
- /** Generate a block token for the current user based on a collection
- * of blockIds
- */
- public Token<BlockTokenIdentifier> generateToken(long[] blockIds,
- EnumSet<AccessMode> modes) throws IOException {
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- String userID = (ugi == null ? null : ugi.getShortUserName());
- return generateToken(userID, blockIds, modes);
- }
-
- /** Generate a block token based on a collection of blockIds */
- public Token<BlockTokenIdentifier> generateToken(String userID,
- long[] blockIds, EnumSet<AccessMode> modes) {
- BlockTokenIdentifier id = new BlockTokenIdentifier(userID, blockIds, modes);
+ BlockTokenIdentifier id = new BlockTokenIdentifier(userId, block
+ .getBlockId(), modes);
return new Token<BlockTokenIdentifier>(id, this);
}
@@ -217,7 +204,7 @@ public class BlockTokenSecretManager ext
throw new InvalidToken("Block token with " + id.toString()
+ " doesn't belong to user " + userId);
}
- if (!id.isBlockIncluded(block.getBlockId())) {
+ if (id.getBlockId() != block.getBlockId()) {
throw new InvalidToken("Block token with " + id.toString()
+ " doesn't apply to block " + block);
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Sat Sep 4 00:41:30 2010
@@ -384,7 +384,7 @@ public class BlockManager {
}
List<LocatedBlock> getBlockLocations(BlockInfo[] blocks, long offset,
- long length, int nrBlocksToReturn) throws IOException {
+ long length, int nrBlocksToReturn, boolean needBlockToken) throws IOException {
int curBlk = 0;
long curPos = 0, blkSize = 0;
int nrBlocks = (blocks[0].getNumBytes() == 0) ? 0 : blocks.length;
@@ -403,7 +403,7 @@ public class BlockManager {
long endOff = offset + length;
List<LocatedBlock> results = new ArrayList<LocatedBlock>(blocks.length);
do {
- results.add(getBlockLocation(blocks[curBlk], curPos));
+ results.add(getBlockLocation(blocks[curBlk], curPos, needBlockToken));
curPos += blocks[curBlk].getNumBytes();
curBlk++;
} while (curPos < endOff
@@ -412,13 +412,14 @@ public class BlockManager {
return results;
}
- /** @return a LocatedBlock for the given block */
- LocatedBlock getBlockLocation(final BlockInfo blk, final long pos
+ /** @param needBlockToken
+ * @return a LocatedBlock for the given block */
+ LocatedBlock getBlockLocation(final BlockInfo blk, final long pos, boolean needBlockToken
) throws IOException {
if (!blk.isComplete()) {
final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)blk;
final DatanodeDescriptor[] locations = uc.getExpectedLocations();
- return namesystem.createLocatedBlock(uc, locations, pos, false);
+ return namesystem.createLocatedBlock(uc, locations, pos, false, needBlockToken);
}
// get block locations
@@ -444,7 +445,7 @@ public class BlockManager {
machines[j++] = d;
}
}
- return namesystem.createLocatedBlock(blk, machines, pos, isCorrupt);
+ return namesystem.createLocatedBlock(blk, machines, pos, isCorrupt, needBlockToken);
}
/**
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sat Sep 4 00:41:30 2010
@@ -24,7 +24,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.*;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.*;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
@@ -799,44 +798,32 @@ public class FSNamesystem implements FSC
} else {
final long n = inode.computeFileSize(false);
final List<LocatedBlock> locatedblocks = blockManager.getBlockLocations(
- blocks, offset, length, Integer.MAX_VALUE);
+ blocks, offset, length, Integer.MAX_VALUE, needBlockToken);
final BlockInfo last = inode.getLastBlock();
if (LOG.isDebugEnabled()) {
LOG.debug("last = " + last);
}
-
- if(isBlockTokenEnabled && needBlockToken) {
- setBlockTokens(locatedblocks);
- }
-
+
if (last.isComplete()) {
return new LocatedBlocks(n, inode.isUnderConstruction(), locatedblocks,
- blockManager.getBlockLocation(last, n-last.getNumBytes()), true);
+ blockManager.getBlockLocation(last, n-last.getNumBytes(), needBlockToken), true);
} else {
return new LocatedBlocks(n, inode.isUnderConstruction(), locatedblocks,
- blockManager.getBlockLocation(last, n), false);
+ blockManager.getBlockLocation(last, n, needBlockToken), false);
}
}
}
- /** Generate a combined block token for all the blocks to be returned. */
- private void setBlockTokens(List<LocatedBlock> locatedBlocks) throws IOException {
- long [] blockIds = new long[locatedBlocks.size()];
- for(int i = 0; i < blockIds.length; i++) {
- blockIds[i] = locatedBlocks.get(i).getBlock().getBlockId();
- }
-
- Token<BlockTokenIdentifier> token =
- blockTokenSecretManager.generateToken(blockIds,
- EnumSet.of(BlockTokenSecretManager.AccessMode.READ));
-
- for(LocatedBlock l : locatedBlocks) l.setBlockToken(token);
- }
-
- /** Create a LocatedBlock. */
+ /** Create a LocatedBlock.
+ * @param needBlockToken */
LocatedBlock createLocatedBlock(final Block b, final DatanodeInfo[] locations,
- final long offset, final boolean corrupt) throws IOException {
- return new LocatedBlock(b, locations, offset, corrupt);
+ final long offset, final boolean corrupt, boolean needBlockToken) throws IOException {
+ final LocatedBlock lb = new LocatedBlock(b, locations, offset, corrupt);
+ if (isBlockTokenEnabled && needBlockToken) {
+ lb.setBlockToken(blockTokenSecretManager.generateToken(b,
+ EnumSet.of(BlockTokenSecretManager.AccessMode.READ)));
+ }
+ return lb;
}
/**
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java Sat Sep 4 00:41:30 2010
@@ -44,9 +44,9 @@ import org.apache.avro.reflect.Nullable;
@InterfaceAudience.Private
public interface DatanodeProtocol extends VersionedProtocol {
/**
- * 25: HDFS-1081. Performance optimization on getBlocksLocation()
+ * 26: remove getBlockLocations optimization
*/
- public static final long versionID = 25L;
+ public static final long versionID = 26;
// error code
final static int NOTIFY = 0;
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=992515&r1=992514&r2=992515&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java Sat Sep 4 00:41:30 2010
@@ -22,7 +22,6 @@ import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.util.Arrays;
import java.util.EnumSet;
import java.util.Set;
@@ -103,16 +102,15 @@ public class TestBlockToken {
Set<TokenIdentifier> tokenIds = UserGroupInformation.getCurrentUser()
.getTokenIdentifiers();
assertEquals("Only one BlockTokenIdentifier expected", 1, tokenIds.size());
- long [] result = {0};
+ long result = 0;
for (TokenIdentifier tokenId : tokenIds) {
BlockTokenIdentifier id = (BlockTokenIdentifier) tokenId;
LOG.info("Got: " + id.toString());
assertTrue("Received BlockTokenIdentifier is wrong", ident.equals(id));
sm.checkAccess(id, null, block, BlockTokenSecretManager.AccessMode.WRITE);
- result = id.getBlockIds();
+ result = id.getBlockId();
}
- assertEquals("Got more than one block back", 1, result.length);
- return result[0];
+ return result;
}
}
@@ -223,28 +221,4 @@ public class TestBlockToken {
}
}
- @Test
- public void collectionOfBlocksActsSanely() {
- final long[][] testBlockIds = new long [][] {{99l, 7l, -32l, 0l},
- {},
- {42l},
- {-5235l, 2352}};
- final long [] notBlockIds = new long [] { 32l, 1l, -23423423l};
-
- for(long [] bids : testBlockIds) {
- BlockTokenIdentifier bti = new BlockTokenIdentifier("Madame Butterfly",
- bids, EnumSet.noneOf(BlockTokenSecretManager.AccessMode.class));
-
- for(long bid : bids) assertTrue(bti.isBlockIncluded(bid));
-
- for(long nbid : notBlockIds) assertFalse(bti.isBlockIncluded(nbid));
-
- // BlockTokenIdentifiers maintain a sorted array of the block Ids.
- long[] sorted = Arrays.copyOf(bids, bids.length);
- Arrays.sort(sorted);
-
- assertTrue(Arrays.toString(bids)+" doesn't equal "+Arrays.toString(sorted),
- Arrays.equals(bti.getBlockIds(), sorted));
- }
- }
}