You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cm...@apache.org on 2014/03/11 23:41:05 UTC
svn commit: r1576533 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/client/
src/main/java/org/apache/hadoop/hdfs/server/datanode/
src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ src/ma...
Author: cmccabe
Date: Tue Mar 11 22:41:05 2014
New Revision: 1576533
URL: http://svn.apache.org/r1576533
Log:
HDFS-6086. Fix a case where zero-copy or no-checksum reads were not allowed even when the block was cached. (cmccabe)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitReplica.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1576533&r1=1576532&r2=1576533&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Mar 11 22:41:05 2014
@@ -598,6 +598,9 @@ Release 2.4.0 - UNRELEASED
HDFS-6077. Running slive with webhdfs on secure HA cluster fails with unkown
host exception. (jing9)
+ HDFS-6086. Fix a case where zero-copy or no-checksum reads were not allowed
+ even when the block was cached (cmccabe)
+
BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitReplica.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitReplica.java?rev=1576533&r1=1576532&r2=1576533&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitReplica.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitReplica.java Tue Mar 11 22:41:05 2014
@@ -191,7 +191,15 @@ public class ShortCircuitReplica {
if (slot == null) {
return false;
}
- return slot.addAnchor();
+ boolean result = slot.addAnchor();
+ if (LOG.isTraceEnabled()) {
+ if (result) {
+ LOG.trace(this + ": added no-checksum anchor to slot " + slot);
+ } else {
+ LOG.trace(this + ": could not add no-checksum anchor to slot " + slot);
+ }
+ }
+ return result;
}
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java?rev=1576533&r1=1576532&r2=1576533&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java Tue Mar 11 22:41:05 2014
@@ -261,8 +261,10 @@ class DataXceiver extends Receiver imple
"anything but a UNIX domain socket.");
}
if (slotId != null) {
+ boolean isCached = datanode.data.
+ isCached(blk.getBlockPoolId(), blk.getBlockId());
datanode.shortCircuitRegistry.registerSlot(
- ExtendedBlockId.fromExtendedBlock(blk), slotId);
+ ExtendedBlockId.fromExtendedBlock(blk), slotId, isCached);
}
try {
fis = datanode.requestShortCircuitFdsForRead(blk, token, maxVersion);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java?rev=1576533&r1=1576532&r2=1576533&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java Tue Mar 11 22:41:05 2014
@@ -287,12 +287,12 @@ public class ShortCircuitRegistry {
return info;
}
- public synchronized void registerSlot(ExtendedBlockId blockId, SlotId slotId)
- throws InvalidRequestException {
+ public synchronized void registerSlot(ExtendedBlockId blockId, SlotId slotId,
+ boolean isCached) throws InvalidRequestException {
if (!enabled) {
if (LOG.isTraceEnabled()) {
- LOG.trace("registerSlot: ShortCircuitRegistry is " +
- "not enabled.");
+ LOG.trace(this + " can't register a slot because the " +
+ "ShortCircuitRegistry is not enabled.");
}
throw new UnsupportedOperationException();
}
@@ -303,8 +303,17 @@ public class ShortCircuitRegistry {
"registered with shmId " + shmId);
}
Slot slot = shm.registerSlot(slotId.getSlotIdx(), blockId);
+ if (isCached) {
+ slot.makeAnchorable();
+ } else {
+ slot.makeUnanchorable();
+ }
boolean added = slots.put(blockId, slot);
Preconditions.checkState(added);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(this + ": registered " + blockId + " with slot " +
+ slotId + " (isCached=" + isCached + ")");
+ }
}
public synchronized void unregisterSlot(SlotId slotId)
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java?rev=1576533&r1=1576532&r2=1576533&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java Tue Mar 11 22:41:05 2014
@@ -320,6 +320,14 @@ public interface FsDatasetSpi<V extends
*/
public void uncache(String bpid, long[] blockIds);
+ /**
+ * Determine if the specified block is cached.
+ * @param bpid Block pool id
+ * @param blockIds - block id
+ * @returns true if the block is cached
+ */
+ public boolean isCached(String bpid, long blockId);
+
/**
* Check if all the data directories are healthy
* @throws DiskErrorException
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java?rev=1576533&r1=1576532&r2=1576533&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java Tue Mar 11 22:41:05 2014
@@ -473,4 +473,10 @@ public class FsDatasetCache {
public long getNumBlocksCached() {
return numBlocksCached.get();
}
+
+ public synchronized boolean isCached(String bpid, long blockId) {
+ ExtendedBlockId block = new ExtendedBlockId(blockId, bpid);
+ Value val = mappableBlockMap.get(block);
+ return (val != null) && val.state.shouldAdvertise();
+ }
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java?rev=1576533&r1=1576532&r2=1576533&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java Tue Mar 11 22:41:05 2014
@@ -1271,6 +1271,10 @@ class FsDatasetImpl implements FsDataset
}
}
+ public boolean isCached(String bpid, long blockId) {
+ return cacheManager.isCached(bpid, blockId);
+ }
+
@Override // FsDatasetSpi
public synchronized boolean contains(final ExtendedBlock block) {
final long blockId = block.getLocalBlock().getBlockId();
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java?rev=1576533&r1=1576532&r2=1576533&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java Tue Mar 11 22:41:05 2014
@@ -916,9 +916,11 @@ public final class CacheManager {
if (metrics != null) {
metrics.addCacheBlockReport((int) (endTime - startTime));
}
- LOG.info("Processed cache report from "
- + datanodeID + ", blocks: " + blockIds.size()
- + ", processing time: " + (endTime - startTime) + " msecs");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processed cache report from "
+ + datanodeID + ", blocks: " + blockIds.size()
+ + ", processing time: " + (endTime - startTime) + " msecs");
+ }
}
private void processCacheReportImpl(final DatanodeDescriptor datanode,
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java?rev=1576533&r1=1576532&r2=1576533&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java Tue Mar 11 22:41:05 2014
@@ -607,7 +607,7 @@ public class TestEnhancedByteBufferAcces
conf.setLong(DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
DFSTestUtil.roundUpToMultiple(TEST_FILE_LENGTH, 4096));
MiniDFSCluster cluster = null;
- ByteBuffer result = null;
+ ByteBuffer result = null, result2 = null;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
FsDatasetSpi<?> fsd = cluster.getDataNodes().get(0).getFSDataset();
@@ -644,9 +644,22 @@ public class TestEnhancedByteBufferAcces
} catch (UnsupportedOperationException e) {
Assert.fail("expected to be able to read cached file via zero-copy");
}
- // Verify result
Assert.assertArrayEquals(Arrays.copyOfRange(original, 0,
BLOCK_SIZE), byteBufferToArray(result));
+ // Test that files opened after the cache operation has finished
+ // still get the benefits of zero-copy (regression test for HDFS-6086)
+ FSDataInputStream fsIn2 = fs.open(TEST_PATH);
+ try {
+ result2 = fsIn2.read(null, TEST_FILE_LENGTH,
+ EnumSet.noneOf(ReadOption.class));
+ } catch (UnsupportedOperationException e) {
+ Assert.fail("expected to be able to read cached file via zero-copy");
+ }
+ Assert.assertArrayEquals(Arrays.copyOfRange(original, 0,
+ BLOCK_SIZE), byteBufferToArray(result2));
+ fsIn2.releaseBuffer(result2);
+ fsIn2.close();
+
// check that the replica is anchored
final ExtendedBlock firstBlock =
DFSTestUtil.getFirstBlock(fs, TEST_PATH);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1576533&r1=1576532&r2=1576533&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Tue Mar 11 22:41:05 2014
@@ -641,6 +641,11 @@ public class SimulatedFSDataset implemen
"SimulatedFSDataset does not support uncache operation!");
}
+ @Override // FSDatasetSpi
+ public boolean isCached(String bpid, long blockId) {
+ return false;
+ }
+
private BInfo getBInfo(final ExtendedBlock b) {
final Map<Block, BInfo> map = blockMap.get(b.getBlockPoolId());
return map == null? null: map.get(b.getLocalBlock());