You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2013/04/24 20:59:08 UTC
svn commit: r1471598 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/common/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/server/blockma...
Author: suresh
Date: Wed Apr 24 18:59:08 2013
New Revision: 1471598
URL: http://svn.apache.org/r1471598
Log:
HDFS-4346. Merge change r1428167 and r1428560 from trunk
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1471598&r1=1471597&r2=1471598&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Apr 24 18:59:08 2013
@@ -81,6 +81,9 @@ Release 2.0.5-beta - UNRELEASED
HDFS-4209. Clean up the addNode/addChild/addChildNoQuotaCheck methods in
FSDirectory and INodeDirectory. (szetszwo)
+ HDFS-4346. Add SequentialNumber as a base class for INodeId and
+ GenerationStamp. (szetszwo)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java?rev=1471598&r1=1471597&r2=1471598&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java Wed Apr 24 18:59:08 2013
@@ -17,19 +17,18 @@
*/
package org.apache.hadoop.hdfs.server.common;
-import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.SequentialNumber;
/****************************************************************
* A GenerationStamp is a Hadoop FS primitive, identified by a long.
****************************************************************/
@InterfaceAudience.Private
-public class GenerationStamp implements Comparable<GenerationStamp> {
+public class GenerationStamp extends SequentialNumber {
/**
- * The first valid generation stamp.
+ * The last reserved generation stamp.
*/
- public static final long FIRST_VALID_STAMP = 1000L;
+ public static final long LAST_RESERVED_STAMP = 1000L;
/**
* Generation stamp of blocks that pre-date the introduction
@@ -37,62 +36,10 @@ public class GenerationStamp implements
*/
public static final long GRANDFATHER_GENERATION_STAMP = 0;
- private AtomicLong genstamp = new AtomicLong();
-
/**
- * Create a new instance, initialized to FIRST_VALID_STAMP.
+ * Create a new instance, initialized to {@link #LAST_RESERVED_STAMP}.
*/
public GenerationStamp() {
- this(GenerationStamp.FIRST_VALID_STAMP);
- }
-
- /**
- * Create a new instance, initialized to the specified value.
- */
- GenerationStamp(long stamp) {
- genstamp.set(stamp);
- }
-
- /**
- * Returns the current generation stamp
- */
- public long getStamp() {
- return genstamp.get();
- }
-
- /**
- * Sets the current generation stamp
- */
- public void setStamp(long stamp) {
- genstamp.set(stamp);
- }
-
- /**
- * First increments the counter and then returns the stamp
- */
- public long nextStamp() {
- return genstamp.incrementAndGet();
- }
-
- @Override // Comparable
- public int compareTo(GenerationStamp that) {
- long stamp1 = this.genstamp.get();
- long stamp2 = that.genstamp.get();
- return stamp1 < stamp2 ? -1 :
- stamp1 > stamp2 ? 1 : 0;
- }
-
- @Override // Object
- public boolean equals(Object o) {
- if (!(o instanceof GenerationStamp)) {
- return false;
- }
- return compareTo((GenerationStamp)o) == 0;
- }
-
- @Override // Object
- public int hashCode() {
- long stamp = genstamp.get();
- return (int) (stamp^(stamp>>>32));
+ super(LAST_RESERVED_STAMP);
}
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1471598&r1=1471597&r2=1471598&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Apr 24 18:59:08 2013
@@ -381,25 +381,29 @@ public class FSNamesystem implements Nam
private INodeId inodeId;
/**
- * Set the last allocated inode id when fsimage is loaded or editlog is
- * applied.
- * @throws IOException
+ * Set the last allocated inode id when fsimage or editlog is loaded.
*/
public void resetLastInodeId(long newValue) throws IOException {
- inodeId.resetLastInodeId(newValue);
+ try {
+ inodeId.skipTo(newValue);
+ } catch(IllegalStateException ise) {
+ throw new IOException(ise);
+ }
}
/** Should only be used for tests to reset to any value */
void resetLastInodeIdWithoutChecking(long newValue) {
- inodeId.resetLastInodeIdWithoutChecking(newValue);
+ inodeId.setCurrentValue(newValue);
}
+ /** @return the last inode ID. */
public long getLastInodeId() {
- return inodeId.getLastInodeId();
+ return inodeId.getCurrentValue();
}
+ /** Allocate a new inode ID. */
public long allocateNewInodeId() {
- return inodeId.allocateNewInodeId();
+ return inodeId.nextValue();
}
/**
@@ -408,9 +412,9 @@ public class FSNamesystem implements Nam
void clear() {
dir.reset();
dtSecretManager.reset();
- generationStamp.setStamp(GenerationStamp.FIRST_VALID_STAMP);
+ generationStamp.setCurrentValue(GenerationStamp.LAST_RESERVED_STAMP);
leaseManager.removeAllLeases();
- inodeId.resetLastInodeIdWithoutChecking(INodeId.LAST_RESERVED_ID);
+ inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID);
}
@VisibleForTesting
@@ -2561,8 +2565,7 @@ public class FSNamesystem implements Nam
assert hasWriteLock();
Block b = new Block(getFSImage().getUniqueBlockId(), 0, 0);
// Increment the generation stamp for every new block.
- nextGenerationStamp();
- b.setGenerationStamp(getGenerationStamp());
+ b.setGenerationStamp(nextGenerationStamp());
return b;
}
@@ -4810,14 +4813,14 @@ public class FSNamesystem implements Nam
* Sets the generation stamp for this filesystem
*/
void setGenerationStamp(long stamp) {
- generationStamp.setStamp(stamp);
+ generationStamp.setCurrentValue(stamp);
}
/**
* Gets the generation stamp for this filesystem
*/
long getGenerationStamp() {
- return generationStamp.getStamp();
+ return generationStamp.getCurrentValue();
}
/**
@@ -4829,7 +4832,7 @@ public class FSNamesystem implements Nam
throw new SafeModeException(
"Cannot get next generation stamp", safeMode);
}
- long gs = generationStamp.nextStamp();
+ final long gs = generationStamp.nextValue();
getEditLog().logGenerationStamp(gs);
// NB: callers sync the log
return gs;
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java?rev=1471598&r1=1471597&r2=1471598&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java Wed Apr 24 18:59:08 2013
@@ -17,16 +17,14 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.SequentialNumber;
/**
* An id which uniquely identifies an inode
*/
@InterfaceAudience.Private
-class INodeId implements Comparable<INodeId> {
+class INodeId extends SequentialNumber {
/**
* The last reserved inode id. Reserve id 1 to 1000 for potential future
* usage. The id won't be recycled and is not expected to wrap around in a
@@ -40,66 +38,7 @@ class INodeId implements Comparable<INod
*/
public static final long GRANDFATHER_INODE_ID = 0;
- private AtomicLong lastInodeId = new AtomicLong();
-
- /**
- * Create a new instance, initialized to LAST_RESERVED_ID.
- */
INodeId() {
- lastInodeId.set(INodeId.LAST_RESERVED_ID);
- }
-
- /**
- * Set the last allocated inode id when fsimage is loaded or editlog is
- * applied.
- * @throws IOException
- */
- void resetLastInodeId(long newValue) throws IOException {
- if (newValue < getLastInodeId()) {
- throw new IOException(
- "Can't reset lastInodeId to be less than its current value "
- + getLastInodeId() + ", newValue=" + newValue);
- }
-
- lastInodeId.set(newValue);
- }
-
- void resetLastInodeIdWithoutChecking(long newValue) {
- lastInodeId.set(newValue);
- }
-
- long getLastInodeId() {
- return lastInodeId.get();
- }
-
- /**
- * First increment the counter and then get the id.
- */
- long allocateNewInodeId() {
- return lastInodeId.incrementAndGet();
- }
-
- @Override
- // Comparable
- public int compareTo(INodeId that) {
- long id1 = this.getLastInodeId();
- long id2 = that.getLastInodeId();
- return id1 < id2 ? -1 : id1 > id2 ? 1 : 0;
- }
-
- @Override
- // Object
- public boolean equals(Object o) {
- if (!(o instanceof INodeId)) {
- return false;
- }
- return compareTo((INodeId) o) == 0;
- }
-
- @Override
- // Object
- public int hashCode() {
- long id = getLastInodeId();
- return (int) (id ^ (id >>> 32));
+ super(LAST_RESERVED_ID);
}
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java?rev=1471598&r1=1471597&r2=1471598&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java Wed Apr 24 18:59:08 2013
@@ -56,7 +56,7 @@ public class TestBlockInfo {
LOG.info("Building block list...");
for (int i = 0; i < MAX_BLOCKS; i++) {
- blockList.add(new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP));
+ blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP));
blockInfoList.add(new BlockInfo(blockList.get(i), 3));
dd.addBlock(blockInfoList.get(i));
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java?rev=1471598&r1=1471597&r2=1471598&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java Wed Apr 24 18:59:08 2013
@@ -54,7 +54,7 @@ public class TestComputeInvalidateWork {
for (int i=0; i<nodes.length; i++) {
for(int j=0; j<3*blockInvalidateLimit+1; j++) {
Block block = new Block(i*(blockInvalidateLimit+1)+j, 0,
- GenerationStamp.FIRST_VALID_STAMP);
+ GenerationStamp.LAST_RESERVED_STAMP);
bm.addToInvalidates(block, nodes[i]);
}
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java?rev=1471598&r1=1471597&r2=1471598&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java Wed Apr 24 18:59:08 2013
@@ -44,7 +44,7 @@ public class TestDatanodeDescriptor {
DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor();
ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS);
for (int i=0; i<MAX_BLOCKS; i++) {
- blockList.add(new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP));
+ blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP));
}
dd.addBlocksToBeInvalidated(blockList);
Block[] bc = dd.getInvalidateBlocks(MAX_LIMIT);
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java?rev=1471598&r1=1471597&r2=1471598&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java Wed Apr 24 18:59:08 2013
@@ -75,7 +75,8 @@ public class TestHeartbeatHandling {
synchronized(hm) {
for (int i=0; i<MAX_REPLICATE_BLOCKS; i++) {
dd.addBlockToBeReplicated(
- new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP), ONE_TARGET);
+ new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP),
+ ONE_TARGET);
}
DatanodeCommand[] cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd,
namesystem).getCommands();
@@ -85,7 +86,7 @@ public class TestHeartbeatHandling {
ArrayList<Block> blockList = new ArrayList<Block>(MAX_INVALIDATE_BLOCKS);
for (int i=0; i<MAX_INVALIDATE_BLOCKS; i++) {
- blockList.add(new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP));
+ blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP));
}
dd.addBlocksToBeInvalidated(blockList);
cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem)
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java?rev=1471598&r1=1471597&r2=1471598&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java Wed Apr 24 18:59:08 2013
@@ -54,7 +54,7 @@ public class CreateEditsLog {
static final String EDITS_DIR = "/tmp/EditsLogOut";
static String edits_dir = EDITS_DIR;
static final public long BLOCK_GENERATION_STAMP =
- GenerationStamp.FIRST_VALID_STAMP;
+ GenerationStamp.LAST_RESERVED_STAMP;
static void addFiles(FSEditLog editLog, int numFiles, short replication,
int blocksPerFile, long startingBlockId,