You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/10/20 02:34:01 UTC
svn commit: r1400351 - in
/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/
src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/native/
src/main/...
Author: szetszwo
Date: Sat Oct 20 00:33:57 2012
New Revision: 1400351
URL: http://svn.apache.org/viewvc?rev=1400351&view=rev
Log:
Merge r1399946 through r1400349 from trunk.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1399946-1400349
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1400351&r1=1400350&r2=1400351&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Sat Oct 20 00:33:57 2012
@@ -388,7 +388,7 @@ Release 2.0.3-alpha - Unreleased
HDFS-4037. Rename the getReplication() method in BlockCollection to
getBlockReplication(). (szetszwo)
- HDFS-4036. Remove "throw UnresolvedLinkException" from
+ HDFS-4036. Remove "throws UnresolvedLinkException" from
FSDirectory.unprotectedAddFile(..). (Jing Zhao via szetszwo)
HDFS-2946. HA: Put a cap on the number of completed edits files retained
@@ -405,6 +405,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4053. Increase the default block size. (eli)
+ HDFS-4088. Remove "throws QuotaExceededException" from an
+ INodeDirectoryWithQuota constructor. (szetszwo)
+
OPTIMIZATIONS
BUG FIXES
@@ -474,6 +477,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4055. TestAuditLogs is flaky. (Binglin Chang via eli)
+ HDFS-4072. On file deletion remove corresponding blocks pending
+ replications. (Jing Zhao via suresh)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1400351&r1=1400350&r2=1400351&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/pom.xml Sat Oct 20 00:33:57 2012
@@ -318,6 +318,9 @@ http://maven.apache.org/xsd/maven-4.0.0.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
+ <configuration>
+ <skipTests>false</skipTests>
+ </configuration>
<executions>
<execution>
<id>create-protobuf-generated-sources-directory</id>
@@ -539,58 +542,47 @@ http://maven.apache.org/xsd/maven-4.0.0.
<build>
<plugins>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
<executions>
<execution>
- <id>define-classpath</id>
- <phase>process-resources</phase>
+ <id>make</id>
+ <phase>compile</phase>
<goals><goal>run</goal></goals>
<configuration>
- <exportAntProperties>true</exportAntProperties>
- <tests>
- <property name="test.classpath" refid="maven.test.classpath"/>
- </tests>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.hadoop.cmake.maven.ng</groupId>
- <artifactId>cmake-ng</artifactId>
- <executions>
- <execution>
- <id>cmake-compile</id>
- <goals><goal>compile</goal></goals>
- <configuration>
- <target>all</target>
- <source>${basedir}/src</source>
- <vars>
- <GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH>
- <JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
- <REQUIRE_FUSE>${require.fuse}</REQUIRE_FUSE>
- <REQUIRE_LIBWEBHDFS>${require.libwebhdfs}</REQUIRE_LIBWEBHDFS>
- </vars>
- </configuration>
- </execution>
- <execution>
- <id>test_libhdfs_threaded</id>
- <goals><goal>test</goal></goals>
- <configuration>
- <binary>${project.build.directory}/native/test_libhdfs_threaded</binary>
- <env><CLASSPATH>${test.classpath}</CLASSPATH></env>
- <timeout>300</timeout>
- <results>${project.build.directory}/results</results>
+ <target>
+ <mkdir dir="${project.build.directory}/native"/>
+ <exec executable="cmake" dir="${project.build.directory}/native"
+ failonerror="true">
+ <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse}"/>
+ </exec>
+ <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
+ <arg line="VERBOSE=1"/>
+ </exec>
+ </target>
</configuration>
</execution>
<execution>
- <id>test_native_mini_dfs</id>
- <goals><goal>test</goal></goals>
+ <id>native_tests</id>
+ <phase>test</phase>
+ <goals><goal>run</goal></goals>
<configuration>
- <binary>${project.build.directory}/native/test_native_mini_dfs</binary>
- <env><CLASSPATH>${test.classpath}</CLASSPATH></env>
- <timeout>300</timeout>
- <results>${project.build.directory}/results</results>
+ <target>
+ <property name="compile_classpath" refid="maven.compile.classpath"/>
+ <property name="test_classpath" refid="maven.test.classpath"/>
+ <exec executable="sh" failonerror="true" dir="${project.build.directory}/native/">
+ <arg value="-c"/>
+ <arg value="[ x$SKIPTESTS = xtrue ] || ${project.build.directory}/native/test_libhdfs_threaded"/>
+ <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
+ <env key="SKIPTESTS" value="${skipTests}"/>
+ </exec>
+ <exec executable="sh" failonerror="true" dir="${project.build.directory}/native/">
+ <arg value="-c"/>
+ <arg value="[ x$SKIPTESTS = xtrue ] || ${project.build.directory}/native/test_libhdfs_threaded"/>
+ <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
+ <env key="SKIPTESTS" value="${skipTests}"/>
+ </exec>
+ </target>
</configuration>
</execution>
</executions>
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1399946-1400349
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1400351&r1=1400350&r2=1400351&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Sat Oct 20 00:33:57 2012
@@ -288,7 +288,7 @@ public class BlockManager {
}
private static BlockTokenSecretManager createBlockTokenSecretManager(
- final Configuration conf) throws IOException {
+ final Configuration conf) {
final boolean isEnabled = conf.getBoolean(
DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY,
DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
@@ -1260,7 +1260,7 @@ public class BlockManager {
// Move the block-replication into a "pending" state.
// The reason we use 'pending' is so we can retry
// replications that fail after an appropriate amount of time.
- pendingReplications.add(block, targets.length);
+ pendingReplications.increment(block, targets.length);
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug(
"BLOCK* block " + block
@@ -1306,8 +1306,11 @@ public class BlockManager {
/**
* Choose target datanodes according to the replication policy.
- * @throws IOException if the number of targets < minimum replication.
- * @see BlockPlacementPolicy#chooseTarget(String, int, DatanodeDescriptor, HashMap, long)
+ *
+ * @throws IOException
+ * if the number of targets < minimum replication.
+ * @see BlockPlacementPolicy#chooseTarget(String, int, DatanodeDescriptor,
+ * List, boolean, HashMap, long)
*/
public DatanodeDescriptor[] chooseTarget(final String src,
final int numOfReplicas, final DatanodeDescriptor client,
@@ -1811,7 +1814,7 @@ assert storedBlock.findDatanode(dn) < 0
/**
* Queue the given reported block for later processing in the
- * standby node. {@see PendingDataNodeMessages}.
+ * standby node. @see PendingDataNodeMessages.
* @param reason a textual reason to report in the debug logs
*/
private void queueReportedBlock(DatanodeDescriptor dn, Block block,
@@ -1976,14 +1979,15 @@ assert storedBlock.findDatanode(dn) < 0
}
/**
- * Faster version of {@link addStoredBlock()}, intended for use with
- * initial block report at startup. If not in startup safe mode, will
- * call standard addStoredBlock().
- * Assumes this method is called "immediately" so there is no need to
- * refresh the storedBlock from blocksMap.
- * Doesn't handle underReplication/overReplication, or worry about
+ * Faster version of
+ * {@link #addStoredBlock(BlockInfo, DatanodeDescriptor, DatanodeDescriptor, boolean)}
+ * , intended for use with initial block report at startup. If not in startup
+ * safe mode, will call standard addStoredBlock(). Assumes this method is
+ * called "immediately" so there is no need to refresh the storedBlock from
+ * blocksMap. Doesn't handle underReplication/overReplication, or worry about
* pendingReplications or corruptReplicas, because it's in startup safe mode.
* Doesn't log every block, because there are typically millions of them.
+ *
* @throws IOException
*/
private void addStoredBlockImmediate(BlockInfo storedBlock,
@@ -2505,7 +2509,7 @@ assert storedBlock.findDatanode(dn) < 0
//
// Modify the blocks->datanode map and node's map.
//
- pendingReplications.remove(block);
+ pendingReplications.decrement(block);
processAndHandleReportedBlock(node, block, ReplicaState.FINALIZED,
delHintNode);
}
@@ -2641,7 +2645,7 @@ assert storedBlock.findDatanode(dn) < 0
}
/**
- * Simpler, faster form of {@link countNodes()} that only returns the number
+ * Simpler, faster form of {@link #countNodes(Block)} that only returns the number
* of live nodes. If in startup safemode (or its 30-sec extension period),
* then it gains speed by ignoring issues of excess replicas or nodes
* that are decommissioned or in process of becoming decommissioned.
@@ -2790,6 +2794,8 @@ assert storedBlock.findDatanode(dn) < 0
addToInvalidates(block);
corruptReplicas.removeFromCorruptReplicasMap(block);
blocksMap.removeBlock(block);
+ // Remove the block from pendingReplications
+ pendingReplications.remove(block);
if (postponedMisreplicatedBlocks.remove(block)) {
postponedMisreplicatedBlocksCount--;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java?rev=1400351&r1=1400350&r2=1400351&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java Sat Oct 20 00:33:57 2012
@@ -72,7 +72,7 @@ class PendingReplicationBlocks {
/**
* Add a block to the list of pending Replications
*/
- void add(Block block, int numReplicas) {
+ void increment(Block block, int numReplicas) {
synchronized (pendingReplications) {
PendingBlockInfo found = pendingReplications.get(block);
if (found == null) {
@@ -89,7 +89,7 @@ class PendingReplicationBlocks {
* Decrement the number of pending replication requests
* for this block.
*/
- void remove(Block block) {
+ void decrement(Block block) {
synchronized (pendingReplications) {
PendingBlockInfo found = pendingReplications.get(block);
if (found != null) {
@@ -104,6 +104,16 @@ class PendingReplicationBlocks {
}
}
+ /**
+ * Remove the record about the given block from pendingReplications.
+ * @param block The given block whose pending replication requests need to be
+ * removed
+ */
+ void remove(Block block) {
+ synchronized (pendingReplications) {
+ pendingReplications.remove(block);
+ }
+ }
public void clear() {
synchronized (pendingReplications) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java?rev=1400351&r1=1400350&r2=1400351&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java Sat Oct 20 00:33:57 2012
@@ -42,16 +42,14 @@ public class INodeDirectoryWithQuota ext
super(other);
INode.DirCounts counts = new INode.DirCounts();
other.spaceConsumedInTree(counts);
- this.nsCount= counts.getNsCount();
+ this.nsCount = counts.getNsCount();
this.diskspace = counts.getDsCount();
setQuota(nsQuota, dsQuota);
}
/** constructor with no quota verification */
- INodeDirectoryWithQuota(
- PermissionStatus permissions, long modificationTime,
- long nsQuota, long dsQuota)
- {
+ INodeDirectoryWithQuota(PermissionStatus permissions, long modificationTime,
+ long nsQuota, long dsQuota) {
super(permissions, modificationTime);
this.nsQuota = nsQuota;
this.dsQuota = dsQuota;
@@ -59,9 +57,8 @@ public class INodeDirectoryWithQuota ext
}
/** constructor with no quota verification */
- INodeDirectoryWithQuota(String name, PermissionStatus permissions,
- long nsQuota, long dsQuota)
- {
+ INodeDirectoryWithQuota(String name, PermissionStatus permissions,
+ long nsQuota, long dsQuota) {
super(name, permissions);
this.nsQuota = nsQuota;
this.dsQuota = dsQuota;
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1399946-1400349
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1399946-1400349
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1399946-1400349
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1399946-1400349
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1399946-1400349
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java?rev=1400351&r1=1400350&r2=1400351&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java Sat Oct 20 00:33:57 2012
@@ -20,14 +20,30 @@ package org.apache.hadoop.hdfs.server.bl
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.junit.Test;
/**
- * This class tests the internals of PendingReplicationBlocks.java
+ * This class tests the internals of PendingReplicationBlocks.java,
+ * as well as how PendingReplicationBlocks acts in BlockManager
*/
public class TestPendingReplication {
final static int TIMEOUT = 3; // 3 seconds
+ private static final int DFS_REPLICATION_INTERVAL = 1;
+ // Number of datanodes in the cluster
+ private static final int DATANODE_COUNT = 5;
@Test
public void testPendingReplication() {
@@ -40,7 +56,7 @@ public class TestPendingReplication {
//
for (int i = 0; i < 10; i++) {
Block block = new Block(i, i, 0);
- pendingReplications.add(block, i);
+ pendingReplications.increment(block, i);
}
assertEquals("Size of pendingReplications ",
10, pendingReplications.size());
@@ -50,15 +66,15 @@ public class TestPendingReplication {
// remove one item and reinsert it
//
Block blk = new Block(8, 8, 0);
- pendingReplications.remove(blk); // removes one replica
+ pendingReplications.decrement(blk); // removes one replica
assertEquals("pendingReplications.getNumReplicas ",
7, pendingReplications.getNumReplicas(blk));
for (int i = 0; i < 7; i++) {
- pendingReplications.remove(blk); // removes all replicas
+ pendingReplications.decrement(blk); // removes all replicas
}
assertTrue(pendingReplications.size() == 9);
- pendingReplications.add(blk, 8);
+ pendingReplications.increment(blk, 8);
assertTrue(pendingReplications.size() == 10);
//
@@ -86,7 +102,7 @@ public class TestPendingReplication {
for (int i = 10; i < 15; i++) {
Block block = new Block(i, i, 0);
- pendingReplications.add(block, i);
+ pendingReplications.increment(block, i);
}
assertTrue(pendingReplications.size() == 15);
@@ -116,4 +132,70 @@ public class TestPendingReplication {
}
pendingReplications.stop();
}
+
+ /**
+ * Test if BlockManager can correctly remove corresponding pending records
+ * when a file is deleted
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testPendingAndInvalidate() throws Exception {
+ final Configuration CONF = new HdfsConfiguration();
+ CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
+ CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+ DFS_REPLICATION_INTERVAL);
+ CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFS_REPLICATION_INTERVAL);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(
+ DATANODE_COUNT).build();
+ cluster.waitActive();
+
+ FSNamesystem namesystem = cluster.getNamesystem();
+ BlockManager bm = namesystem.getBlockManager();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ try {
+ // 1. create a file
+ Path filePath = new Path("/tmp.txt");
+ DFSTestUtil.createFile(fs, filePath, 1024, (short) 3, 0L);
+
+ // 2. disable the heartbeats
+ for (DataNode dn : cluster.getDataNodes()) {
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
+ }
+
+ // 3. mark a couple of blocks as corrupt
+ LocatedBlock block = NameNodeAdapter.getBlockLocations(
+ cluster.getNameNode(), filePath.toString(), 0, 1).get(0);
+ cluster.getNamesystem().writeLock();
+ try {
+ bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
+ "TEST");
+ bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[1],
+ "TEST");
+ } finally {
+ cluster.getNamesystem().writeUnlock();
+ }
+ BlockManagerTestUtil.computeAllPendingWork(bm);
+ BlockManagerTestUtil.updateState(bm);
+ assertEquals(bm.getPendingReplicationBlocksCount(), 1L);
+ assertEquals(bm.pendingReplications.getNumReplicas(block.getBlock()
+ .getLocalBlock()), 2);
+
+ // 4. delete the file
+ fs.delete(filePath, true);
+ // retry at most 10 times, each time sleep for 1s. Note that 10s is much
+ // less than the default pending record timeout (5~10min)
+ int retries = 10;
+ long pendingNum = bm.getPendingReplicationBlocksCount();
+ while (pendingNum != 0 && retries-- > 0) {
+ Thread.sleep(1000); // let NN do the deletion
+ BlockManagerTestUtil.updateState(bm);
+ pendingNum = bm.getPendingReplicationBlocksCount();
+ }
+ assertEquals(pendingNum, 0L);
+ } finally {
+ cluster.shutdown();
+ }
+ }
}